Warning, /include/llvm/ProfileData/InstrProfData.inc is written in an unsupported language. File is not indexed.
0001 /*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\
0002 |*
0003 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
0004 |* See https://llvm.org/LICENSE.txt for license information.
0005 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
0006 |*
0007 \*===----------------------------------------------------------------------===*/
0008 /*
0009 * This is the main file that defines all the data structure, signature,
0010 * constant literals that are shared across profiling runtime library,
0011 * compiler (instrumentation), and host tools (reader/writer). The entities
0012 * defined in this file affect the profile runtime ABI, the raw profile format,
0013 * or both.
0014 *
0015 * The file has two identical copies. The primary copy lives in LLVM and
0016 * the other one sits in compiler-rt/lib/profile directory. To make changes
0017 * in this file, first modify the primary copy and copy it over to compiler-rt.
0018 * Testing of any change in this file can start only after the two copies are
0019 * synced up.
0020 *
0021 * The first part of the file includes macros that defines types, names, and
0022 * initializers for the member fields of the core data structures. The field
0023 * declarations for one structure is enabled by defining the field activation
0024 * macro associated with that structure. Only one field activation record
0025 * can be defined at one time and the rest definitions will be filtered out by
0026 * the preprocessor.
0027 *
0028 * Examples of how the template is used to instantiate structure definition:
0029 * 1. To declare a structure:
0030 *
0031 * struct ProfData {
0032 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
0033 * Type Name;
0034 * #include "llvm/ProfileData/InstrProfData.inc"
0035 * };
0036 *
0037 * 2. To construct LLVM type arrays for the struct type:
0038 *
0039 * Type *DataTypes[] = {
0040 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
0041 * LLVMType,
0042 * #include "llvm/ProfileData/InstrProfData.inc"
0043 * };
0044 *
0045 * 4. To construct constant array for the initializers:
0046 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
0047 * Initializer,
0048 * Constant *ConstantVals[] = {
0049 * #include "llvm/ProfileData/InstrProfData.inc"
0050 * };
0051 *
0052 *
0053 * The second part of the file includes definitions all other entities that
0054 * are related to runtime ABI and format. When no field activation macro is
0055 * defined, this file can be included to introduce the definitions.
0056 *
0057 \*===----------------------------------------------------------------------===*/
0058
0059 /* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in
0060 * the compiler runtime. */
0061 #ifndef INSTR_PROF_VISIBILITY
0062 #define INSTR_PROF_VISIBILITY
0063 #endif
0064
0065 // clang-format off:consider re-enabling clang-format if auto-formatted C macros
0066 // are readable (e.g., after `issue #82426` is fixed)
0067 /* INSTR_PROF_DATA start. */
0068 /* Definition of member fields of the per-function control structure. */
0069 #ifndef INSTR_PROF_DATA
0070 #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer)
0071 #else
0072 #define INSTR_PROF_DATA_DEFINED
0073 #endif
0074 INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
0075 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
0076 IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName()))))
0077 INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
0078 ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
0079 Inc->getHash()->getZExtValue()))
0080 INSTR_PROF_DATA(const IntPtrT, IntPtrTy, CounterPtr, RelativeCounterPtr)
0081 INSTR_PROF_DATA(const IntPtrT, IntPtrTy, BitmapPtr, RelativeBitmapPtr)
0082 /* This is used to map function pointers for the indirect call targets to
0083 * function name hashes during the conversion from raw to merged profile
0084 * data.
0085 */
0086 INSTR_PROF_DATA(const IntPtrT, llvm::PointerType::getUnqual(Ctx), FunctionPointer, \
0087 FunctionAddr)
0088 INSTR_PROF_DATA(IntPtrT, llvm::PointerType::getUnqual(Ctx), Values, \
0089 ValuesPtrExpr)
0090 INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \
0091 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters))
0092 INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
0093 ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) \
0094 INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \
0095 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumBitmapBytes))
0096 #undef INSTR_PROF_DATA
0097 /* INSTR_PROF_DATA end. */
0098
0099 /* For a virtual table object, record the name hash to associate profiled
0100 * addresses with global variables, and record {starting address, size in bytes}
0101 * to map the profiled virtual table (which usually have an offset from the
0102 * starting address) back to a virtual table object. */
0103 #ifndef INSTR_PROF_VTABLE_DATA
0104 #define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer)
0105 #else
0106 #define INSTR_PROF_VTABLE_DATA_DEFINED
0107 #endif
0108 INSTR_PROF_VTABLE_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), \
0109 VTableNameHash, ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
0110 IndexedInstrProf::ComputeHash(PGOVTableName)))
0111 INSTR_PROF_VTABLE_DATA(const IntPtrT, llvm::PointerType::getUnqual(Ctx), \
0112 VTablePointer, VTableAddr)
0113 INSTR_PROF_VTABLE_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), VTableSize, \
0114 ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \
0115 VTableSizeVal))
0116 #undef INSTR_PROF_VTABLE_DATA
0117 /* INSTR_PROF_VTABLE_DATA end. */
0118
0119 /* This is an internal data structure used by value profiler. It
0120 * is defined here to allow serialization code sharing by LLVM
0121 * to be used in unit test.
0122 *
0123 * typedef struct ValueProfNode {
0124 * // InstrProfValueData VData;
0125 * uint64_t Value;
0126 * uint64_t Count;
0127 * struct ValueProfNode *Next;
0128 * } ValueProfNode;
0129 */
0130 /* INSTR_PROF_VALUE_NODE start. */
0131 #ifndef INSTR_PROF_VALUE_NODE
0132 #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer)
0133 #else
0134 #define INSTR_PROF_DATA_DEFINED
0135 #endif
0136 INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Value, \
0137 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
0138 INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \
0139 ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
0140 INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::PointerType::getUnqual(Ctx), Next, \
0141 ConstantInt::get(llvm::PointerType::getUnqual(Ctx), 0))
0142 #undef INSTR_PROF_VALUE_NODE
0143 /* INSTR_PROF_VALUE_NODE end. */
0144
0145 /* INSTR_PROF_RAW_HEADER start */
0146 /* Definition of member fields of the raw profile header data structure. */
0147 /* Please update llvm/docs/InstrProfileFormat.rst as appropriate when updating
0148 raw profile format. */
0149 #ifndef INSTR_PROF_RAW_HEADER
0150 #define INSTR_PROF_RAW_HEADER(Type, Name, Initializer)
0151 #else
0152 #define INSTR_PROF_DATA_DEFINED
0153 #endif
0154 INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic())
0155 INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version())
0156 INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL))
0157 INSTR_PROF_RAW_HEADER(uint64_t, NumData, NumData)
0158 INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters)
0159 INSTR_PROF_RAW_HEADER(uint64_t, NumCounters, NumCounters)
0160 INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters)
0161 INSTR_PROF_RAW_HEADER(uint64_t, NumBitmapBytes, NumBitmapBytes)
0162 INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterBitmapBytes, PaddingBytesAfterBitmapBytes)
0163 INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize)
0164 INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta,
0165 (uintptr_t)CountersBegin - (uintptr_t)DataBegin)
0166 INSTR_PROF_RAW_HEADER(uint64_t, BitmapDelta,
0167 (uintptr_t)BitmapBegin - (uintptr_t)DataBegin)
0168 INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
0169 INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables)
0170 INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize)
0171 INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
0172 #undef INSTR_PROF_RAW_HEADER
0173 /* INSTR_PROF_RAW_HEADER end */
0174
0175 /* VALUE_PROF_FUNC_PARAM start */
0176 /* Definition of parameter types of the runtime API used to do value profiling
0177 * for a given value site.
0178 */
0179 #ifndef VALUE_PROF_FUNC_PARAM
0180 #define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType)
0181 #define INSTR_PROF_COMMA
0182 #else
0183 #define INSTR_PROF_DATA_DEFINED
0184 #define INSTR_PROF_COMMA ,
0185 #endif
0186 VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \
0187 INSTR_PROF_COMMA
0188 VALUE_PROF_FUNC_PARAM(void *, Data, PointerType::getUnqual(Ctx)) INSTR_PROF_COMMA
0189 VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
0190 #undef VALUE_PROF_FUNC_PARAM
0191 #undef INSTR_PROF_COMMA
0192 /* VALUE_PROF_FUNC_PARAM end */
0193
0194 /* VALUE_PROF_KIND start */
0195 #ifndef VALUE_PROF_KIND
0196 #define VALUE_PROF_KIND(Enumerator, Value, Descr)
0197 #else
0198 #define INSTR_PROF_DATA_DEFINED
0199 #endif
0200 /* For indirect function call value profiling, the addresses of the target
0201 * functions are profiled by the instrumented code. The target addresses are
0202 * written in the raw profile data and converted to target function name's MD5
0203 * hash by the profile reader during deserialization. Typically, this happens
0204 * when the raw profile data is read during profile merging.
0205 *
0206 * For this remapping the ProfData is used. ProfData contains both the function
0207 * name hash and the function address.
0208 */
0209 VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target")
0210 /* For memory intrinsic functions size profiling. */
0211 VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size")
0212 /* For virtual table address profiling, the address point of the virtual table
0213 * (i.e., the address contained in objects pointing to a virtual table) are
0214 * profiled. Note this may not be the address of the per C++ class virtual table
0215 * object (e.g., there might be an offset).
0216 *
0217 * The profiled addresses are stored in raw profile, together with the following
0218 * two types of information.
0219 * 1. The (starting and ending) addresses of per C++ class virtual table objects.
0220 * 2. The (compressed) virtual table object names.
0221 * RawInstrProfReader converts profiled virtual table addresses to virtual table
0222 * objects' MD5 hash.
0223 */
0224 VALUE_PROF_KIND(IPVK_VTableTarget, 2, "The profiled address point of the vtable")
0225 /* These two kinds must be the last to be
0226 * declared. This is to make sure the string
0227 * array created with the template can be
0228 * indexed with the kind value.
0229 */
0230 VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first")
0231 VALUE_PROF_KIND(IPVK_Last, IPVK_VTableTarget, "last")
0232
0233 #undef VALUE_PROF_KIND
0234 /* VALUE_PROF_KIND end */
0235
0236 #undef COVMAP_V2_OR_V3
0237 #ifdef COVMAP_V2
0238 #define COVMAP_V2_OR_V3
0239 #endif
0240 #ifdef COVMAP_V3
0241 #define COVMAP_V2_OR_V3
0242 #endif
0243
0244 /* COVMAP_FUNC_RECORD start */
0245 /* Definition of member fields of the function record structure in coverage
0246 * map.
0247 */
0248 #ifndef COVMAP_FUNC_RECORD
0249 #define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer)
0250 #else
0251 #define INSTR_PROF_DATA_DEFINED
0252 #endif
0253 #ifdef COVMAP_V1
0254 COVMAP_FUNC_RECORD(const IntPtrT, llvm::PointerType::getUnqual(Ctx), \
0255 NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \
0256 llvm::PointerType::getUnqual(Ctx)))
0257 COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \
0258 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \
0259 NameValue.size()))
0260 #endif
0261 #ifdef COVMAP_V2_OR_V3
0262 COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
0263 llvm::ConstantInt::get( \
0264 llvm::Type::getInt64Ty(Ctx), NameHash))
0265 #endif
0266 COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \
0267 llvm::ConstantInt::get( \
0268 llvm::Type::getInt32Ty(Ctx), CoverageMapping.size()))
0269 COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
0270 llvm::ConstantInt::get( \
0271 llvm::Type::getInt64Ty(Ctx), FuncHash))
0272 #ifdef COVMAP_V3
0273 COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FilenamesRef, \
0274 llvm::ConstantInt::get( \
0275 llvm::Type::getInt64Ty(Ctx), FilenamesRef))
0276 COVMAP_FUNC_RECORD(const char, \
0277 llvm::ArrayType::get(llvm::Type::getInt8Ty(Ctx), \
0278 CoverageMapping.size()), \
0279 CoverageMapping,
0280 llvm::ConstantDataArray::getRaw( \
0281 CoverageMapping, CoverageMapping.size(), \
0282 llvm::Type::getInt8Ty(Ctx)))
0283 #endif
0284 #undef COVMAP_FUNC_RECORD
0285 /* COVMAP_FUNC_RECORD end. */
0286
0287 /* COVMAP_HEADER start */
0288 /* Definition of member fields of coverage map header.
0289 */
0290 #ifndef COVMAP_HEADER
0291 #define COVMAP_HEADER(Type, LLVMType, Name, Initializer)
0292 #else
0293 #define INSTR_PROF_DATA_DEFINED
0294 #endif
0295 COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \
0296 llvm::ConstantInt::get(Int32Ty, NRecords))
0297 COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \
0298 llvm::ConstantInt::get(Int32Ty, FilenamesSize))
0299 COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \
0300 llvm::ConstantInt::get(Int32Ty, CoverageMappingSize))
0301 COVMAP_HEADER(uint32_t, Int32Ty, Version, \
0302 llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion))
0303 #undef COVMAP_HEADER
0304 /* COVMAP_HEADER end. */
0305
0306 /* COVINIT_FUNC start */
0307 #ifndef COVINIT_FUNC
0308 #define COVINIT_FUNC(Type, LLVMType, Name, Initializer)
0309 #else
0310 #define INSTR_PROF_DATA_DEFINED
0311 #endif
0312 COVINIT_FUNC(IntPtrT, llvm::PointerType::getUnqual(Ctx), WriteoutFunction, \
0313 WriteoutF)
0314 COVINIT_FUNC(IntPtrT, llvm::PointerType::getUnqual(Ctx), ResetFunction, \
0315 ResetF)
0316 #undef COVINIT_FUNC
0317 /* COVINIT_FUNC end */
0318
0319 #ifdef INSTR_PROF_SECT_ENTRY
0320 #define INSTR_PROF_DATA_DEFINED
0321 INSTR_PROF_SECT_ENTRY(IPSK_data, \
0322 INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \
0323 INSTR_PROF_DATA_COFF, "__DATA,")
0324 INSTR_PROF_SECT_ENTRY(IPSK_cnts, \
0325 INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \
0326 INSTR_PROF_CNTS_COFF, "__DATA,")
0327 INSTR_PROF_SECT_ENTRY(IPSK_bitmap, \
0328 INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON), \
0329 INSTR_PROF_BITS_COFF, "__DATA,")
0330 INSTR_PROF_SECT_ENTRY(IPSK_name, \
0331 INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \
0332 INSTR_PROF_NAME_COFF, "__DATA,")
0333 INSTR_PROF_SECT_ENTRY(IPSK_vname, \
0334 INSTR_PROF_QUOTE(INSTR_PROF_VNAME_COMMON), \
0335 INSTR_PROF_VNAME_COFF, "__DATA,")
0336 INSTR_PROF_SECT_ENTRY(IPSK_vals, \
0337 INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \
0338 INSTR_PROF_VALS_COFF, "__DATA,")
0339 INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \
0340 INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \
0341 INSTR_PROF_VNODES_COFF, "__DATA,")
0342 INSTR_PROF_SECT_ENTRY(IPSK_vtab, \
0343 INSTR_PROF_QUOTE(INSTR_PROF_VTAB_COMMON), \
0344 INSTR_PROF_VTAB_COFF, "__DATA,")
0345 INSTR_PROF_SECT_ENTRY(IPSK_covmap, \
0346 INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \
0347 INSTR_PROF_COVMAP_COFF, "__LLVM_COV,")
0348 INSTR_PROF_SECT_ENTRY(IPSK_covfun, \
0349 INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \
0350 INSTR_PROF_COVFUN_COFF, "__LLVM_COV,")
0351 INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \
0352 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \
0353 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,")
0354 INSTR_PROF_SECT_ENTRY(IPSK_covdata, \
0355 INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON), \
0356 INSTR_PROF_COVDATA_COFF, "__LLVM_COV,")
0357 INSTR_PROF_SECT_ENTRY(IPSK_covname, \
0358 INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON), \
0359 INSTR_PROF_COVNAME_COFF, "__LLVM_COV,")
0360 INSTR_PROF_SECT_ENTRY(IPSK_covinit, \
0361 INSTR_PROF_QUOTE(INSTR_PROF_COVINIT_COMMON), \
0362 INSTR_PROF_COVINIT_COFF, "__LLVM_COV,")
0363
0364 #undef INSTR_PROF_SECT_ENTRY
0365 #endif
0366
0367
0368 #ifdef INSTR_PROF_VALUE_PROF_DATA
0369 #define INSTR_PROF_DATA_DEFINED
0370
0371 #define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255
0372 /*!
0373 * This is the header of the data structure that defines the on-disk
0374 * layout of the value profile data of a particular kind for one function.
0375 */
0376 typedef struct ValueProfRecord {
0377 /* The kind of the value profile record. */
0378 uint32_t Kind;
0379 /*
0380 * The number of value profile sites. It is guaranteed to be non-zero;
0381 * otherwise the record for this kind won't be emitted.
0382 */
0383 uint32_t NumValueSites;
0384 /*
0385 * The first element of the array that stores the number of profiled
0386 * values for each value site. The size of the array is NumValueSites.
0387 * Since NumValueSites is greater than zero, there is at least one
0388 * element in the array.
0389 */
0390 uint8_t SiteCountArray[1];
0391
0392 /*
0393 * The fake declaration is for documentation purpose only.
0394 * Align the start of next field to be on 8 byte boundaries.
0395 uint8_t Padding[X];
0396 */
0397
0398 /* The array of value profile data. The size of the array is the sum
0399 * of all elements in SiteCountArray[].
0400 InstrProfValueData ValueData[];
0401 */
0402
0403 #ifdef __cplusplus
0404 /*!
0405 * Return the number of value sites.
0406 */
0407 uint32_t getNumValueSites() const { return NumValueSites; }
0408 /*!
0409 * Read data from this record and save it to Record.
0410 */
0411 void deserializeTo(InstrProfRecord &Record,
0412 InstrProfSymtab *SymTab);
0413 /*
0414 * In-place byte swap:
0415 * Do byte swap for this instance. \c Old is the original order before
0416 * the swap, and \c New is the New byte order.
0417 */
0418 void swapBytes(llvm::endianness Old, llvm::endianness New);
0419 #endif
0420 } ValueProfRecord;
0421
0422 /*!
0423 * Per-function header/control data structure for value profiling
0424 * data in indexed format.
0425 */
0426 typedef struct ValueProfData {
0427 /*
0428 * Total size in bytes including this field. It must be a multiple
0429 * of sizeof(uint64_t).
0430 */
0431 uint32_t TotalSize;
0432 /*
0433 *The number of value profile kinds that has value profile data.
0434 * In this implementation, a value profile kind is considered to
0435 * have profile data if the number of value profile sites for the
0436 * kind is not zero. More aggressively, the implementation can
0437 * choose to check the actual data value: if none of the value sites
0438 * has any profiled values, the kind can be skipped.
0439 */
0440 uint32_t NumValueKinds;
0441
0442 /*
0443 * Following are a sequence of variable length records. The prefix/header
0444 * of each record is defined by ValueProfRecord type. The number of
0445 * records is NumValueKinds.
0446 * ValueProfRecord Record_1;
0447 * ValueProfRecord Record_N;
0448 */
0449
0450 #if __cplusplus
0451 /*!
0452 * Return the total size in bytes of the on-disk value profile data
0453 * given the data stored in Record.
0454 */
0455 static uint32_t getSize(const InstrProfRecord &Record);
0456 /*!
0457 * Return a pointer to \c ValueProfData instance ready to be streamed.
0458 */
0459 static std::unique_ptr<ValueProfData>
0460 serializeFrom(const InstrProfRecord &Record);
0461 /*!
0462 * Check the integrity of the record.
0463 */
0464 Error checkIntegrity();
0465 /*!
0466 * Return a pointer to \c ValueProfileData instance ready to be read.
0467 * All data in the instance are properly byte swapped. The input
0468 * data is assumed to be in little endian order.
0469 */
0470 static Expected<std::unique_ptr<ValueProfData>>
0471 getValueProfData(const unsigned char *SrcBuffer,
0472 const unsigned char *const SrcBufferEnd,
0473 llvm::endianness SrcDataEndianness);
0474 /*!
0475 * Swap byte order from \c Endianness order to host byte order.
0476 */
0477 void swapBytesToHost(llvm::endianness Endianness);
0478 /*!
0479 * Swap byte order from host byte order to \c Endianness order.
0480 */
0481 void swapBytesFromHost(llvm::endianness Endianness);
0482 /*!
0483 * Return the total size of \c ValueProfileData.
0484 */
0485 uint32_t getSize() const { return TotalSize; }
0486 /*!
0487 * Read data from this data and save it to \c Record.
0488 */
0489 void deserializeTo(InstrProfRecord &Record,
0490 InstrProfSymtab *SymTab);
0491 void operator delete(void *ptr) { ::operator delete(ptr); }
0492 #endif
0493 } ValueProfData;
0494
0495 /*
0496 * The closure is designed to abstact away two types of value profile data:
0497 * - InstrProfRecord which is the primary data structure used to
0498 * represent profile data in host tools (reader, writer, and profile-use)
0499 * - value profile runtime data structure suitable to be used by C
0500 * runtime library.
0501 *
0502 * Both sources of data need to serialize to disk/memory-buffer in common
0503 * format: ValueProfData. The abstraction allows compiler-rt's raw profiler
0504 * writer to share the same format and code with indexed profile writer.
0505 *
0506 * For documentation of the member methods below, refer to corresponding methods
0507 * in class InstrProfRecord.
0508 */
0509 typedef struct ValueProfRecordClosure {
0510 const void *Record;
0511 uint32_t (*GetNumValueKinds)(const void *Record);
0512 uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind);
0513 uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind);
0514 uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S);
0515
0516 /*
0517 * After extracting the value profile data from the value profile record,
0518 * this method is used to map the in-memory value to on-disk value. If
0519 * the method is null, value will be written out untranslated.
0520 */
0521 uint64_t (*RemapValueData)(uint32_t, uint64_t Value);
0522 void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K,
0523 uint32_t S);
0524 ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes);
0525 } ValueProfRecordClosure;
0526
0527 INSTR_PROF_VISIBILITY ValueProfRecord *
0528 getFirstValueProfRecord(ValueProfData *VPD);
0529 INSTR_PROF_VISIBILITY ValueProfRecord *
0530 getValueProfRecordNext(ValueProfRecord *VPR);
0531 INSTR_PROF_VISIBILITY InstrProfValueData *
0532 getValueProfRecordValueData(ValueProfRecord *VPR);
0533 INSTR_PROF_VISIBILITY uint32_t
0534 getValueProfRecordHeaderSize(uint32_t NumValueSites);
0535
0536 #undef INSTR_PROF_VALUE_PROF_DATA
0537 #endif /* INSTR_PROF_VALUE_PROF_DATA */
0538
0539
0540 #ifdef INSTR_PROF_COMMON_API_IMPL
0541 #define INSTR_PROF_DATA_DEFINED
0542 #ifdef __cplusplus
0543 #define INSTR_PROF_INLINE inline
0544 #define INSTR_PROF_NULLPTR nullptr
0545 #else
0546 #define INSTR_PROF_INLINE
0547 #define INSTR_PROF_NULLPTR NULL
0548 #endif
0549
0550 #ifndef offsetof
0551 #define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
0552 #endif
0553
0554 // clang-format on
0555
0556 /*!
0557 * Return the \c ValueProfRecord header size including the
0558 * padding bytes.
0559 */
0560 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t
0561 getValueProfRecordHeaderSize(uint32_t NumValueSites) {
0562 uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) +
0563 sizeof(uint8_t) * NumValueSites;
0564 /* Round the size to multiple of 8 bytes. */
0565 Size = (Size + 7) & ~7;
0566 return Size;
0567 }
0568
0569 /*!
0570 * Return the total size of the value profile record including the
0571 * header and the value data.
0572 */
0573 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t
0574 getValueProfRecordSize(uint32_t NumValueSites, uint32_t NumValueData) {
0575 return getValueProfRecordHeaderSize(NumValueSites) +
0576 sizeof(InstrProfValueData) * NumValueData;
0577 }
0578
0579 /*!
0580 * Return the pointer to the start of value data array.
0581 */
0582 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE InstrProfValueData *
0583 getValueProfRecordValueData(ValueProfRecord *This) {
0584 return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize(
0585 This->NumValueSites));
0586 }
0587
0588 /*!
0589 * Return the total number of value data for \c This record.
0590 */
0591 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t
0592 getValueProfRecordNumValueData(ValueProfRecord *This) {
0593 uint32_t NumValueData = 0;
0594 uint32_t I;
0595 for (I = 0; I < This->NumValueSites; I++)
0596 NumValueData += This->SiteCountArray[I];
0597 return NumValueData;
0598 }
0599
0600 /*!
0601 * Use this method to advance to the next \c This \c ValueProfRecord.
0602 */
0603 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE ValueProfRecord *
0604 getValueProfRecordNext(ValueProfRecord *This) {
0605 uint32_t NumValueData = getValueProfRecordNumValueData(This);
0606 return (ValueProfRecord *)((char *)This +
0607 getValueProfRecordSize(This->NumValueSites,
0608 NumValueData));
0609 }
0610
0611 /*!
0612 * Return the first \c ValueProfRecord instance.
0613 */
0614 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE ValueProfRecord *
0615 getFirstValueProfRecord(ValueProfData *This) {
0616 return (ValueProfRecord *)((char *)This + sizeof(ValueProfData));
0617 }
0618
0619 /* Closure based interfaces. */
0620
0621 /*!
0622 * Return the total size in bytes of the on-disk value profile data
0623 * given the data stored in Record.
0624 */
0625 INSTR_PROF_VISIBILITY uint32_t
0626 getValueProfDataSize(ValueProfRecordClosure *Closure) {
0627 uint32_t Kind;
0628 uint32_t TotalSize = sizeof(ValueProfData);
0629 const void *Record = Closure->Record;
0630
0631 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
0632 uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind);
0633 if (!NumValueSites)
0634 continue;
0635 TotalSize += getValueProfRecordSize(NumValueSites,
0636 Closure->GetNumValueData(Record, Kind));
0637 }
0638 return TotalSize;
0639 }
0640
0641 /*!
0642 * Extract value profile data of a function for the profile kind \c ValueKind
0643 * from the \c Closure and serialize the data into \c This record instance.
0644 */
0645 INSTR_PROF_VISIBILITY void
0646 serializeValueProfRecordFrom(ValueProfRecord *This,
0647 ValueProfRecordClosure *Closure,
0648 uint32_t ValueKind, uint32_t NumValueSites) {
0649 uint32_t S;
0650 const void *Record = Closure->Record;
0651 This->Kind = ValueKind;
0652 This->NumValueSites = NumValueSites;
0653 InstrProfValueData *DstVD = getValueProfRecordValueData(This);
0654
0655 for (S = 0; S < NumValueSites; S++) {
0656 uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S);
0657 This->SiteCountArray[S] = ND;
0658 Closure->GetValueForSite(Record, DstVD, ValueKind, S);
0659 DstVD += ND;
0660 }
0661 }
0662
0663 /*!
0664 * Extract value profile data of a function from the \c Closure
0665 * and serialize the data into \c DstData if it is not NULL or heap
0666 * memory allocated by the \c Closure's allocator method. If \c
0667 * DstData is not null, the caller is expected to set the TotalSize
0668 * in DstData.
0669 */
0670 INSTR_PROF_VISIBILITY ValueProfData *
0671 serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
0672 ValueProfData *DstData) {
0673 uint32_t Kind;
0674 uint32_t TotalSize =
0675 DstData ? DstData->TotalSize : getValueProfDataSize(Closure);
0676
0677 ValueProfData *VPD =
0678 DstData ? DstData : Closure->AllocValueProfData(TotalSize);
0679
0680 VPD->TotalSize = TotalSize;
0681 VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record);
0682 ValueProfRecord *VR = getFirstValueProfRecord(VPD);
0683 for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
0684 uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind);
0685 if (!NumValueSites)
0686 continue;
0687 serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites);
0688 VR = getValueProfRecordNext(VR);
0689 }
0690 return VPD;
0691 }
0692
0693 #undef INSTR_PROF_COMMON_API_IMPL
0694 #endif /* INSTR_PROF_COMMON_API_IMPL */
0695
0696 /*============================================================================*/
0697
0698 // clang-format off:consider re-enabling clang-format if auto-formatted C macros
0699 // are readable (e.g., after `issue #82426` is fixed)
0700 #ifndef INSTR_PROF_DATA_DEFINED
0701
0702 #ifndef INSTR_PROF_DATA_INC
0703 #define INSTR_PROF_DATA_INC
0704
0705 /* Helper macros. */
0706 #define INSTR_PROF_SIMPLE_QUOTE(x) #x
0707 #define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x)
0708 #define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y
0709 #define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y)
0710
0711 /* Magic number to detect file format and endianness.
0712 * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0,
0713 * so that utilities, like strings, don't grab it as a string. 129 is also
0714 * invalid UTF-8, and high enough to be interesting.
0715 * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR"
0716 * for 32-bit platforms.
0717 */
0718 #define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
0719 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
0720 (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129
0721 #define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
0722 (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \
0723 (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
0724
0725 /* Raw profile format version (start from 1). */
0726 #define INSTR_PROF_RAW_VERSION 10
0727 /* Indexed profile format version (start from 1). */
0728 #define INSTR_PROF_INDEX_VERSION 12
0729 /* Coverage mapping format version (start from 0). */
0730 #define INSTR_PROF_COVMAP_VERSION 6
0731
0732 /* Profile version is always of type uint64_t. Reserve the upper 32 bits in the
0733 * version for other variants of profile. We set the 8th most significant bit
0734 * (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation
0735 * generated profile, and 0 if this is a Clang FE generated profile.
0736 * 1 in bit 57 indicates there are context-sensitive records in the profile.
0737 * The 54th bit indicates whether to always instrument loop entry blocks.
0738 * The 58th bit indicates whether to always instrument function entry blocks.
0739 * The 59th bit indicates whether to use debug info to correlate profiles.
0740 * The 60th bit indicates single byte coverage instrumentation.
0741 * The 61st bit indicates function entry instrumentation only.
0742 * The 62nd bit indicates whether memory profile information is present.
0743 * The 63rd bit indicates if this is a temporal profile.
0744 */
0745 #define VARIANT_MASKS_ALL 0xffffffff00000000ULL
0746 #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
0747 #define VARIANT_MASK_INSTR_LOOP_ENTRIES (0x1ULL << 55)
0748 #define VARIANT_MASK_IR_PROF (0x1ULL << 56)
0749 #define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
0750 #define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58)
0751 #define VARIANT_MASK_DBG_CORRELATE (0x1ULL << 59)
0752 #define VARIANT_MASK_BYTE_COVERAGE (0x1ULL << 60)
0753 #define VARIANT_MASK_FUNCTION_ENTRY_ONLY (0x1ULL << 61)
0754 #define VARIANT_MASK_MEMPROF (0x1ULL << 62)
0755 #define VARIANT_MASK_TEMPORAL_PROF (0x1ULL << 63)
0756 #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
0757 #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
0758 #define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias
0759 #define INSTR_PROF_PROFILE_BITMAP_BIAS_VAR __llvm_profile_bitmap_bias
0760 #define INSTR_PROF_PROFILE_SET_TIMESTAMP __llvm_profile_set_timestamp
0761 #define INSTR_PROF_PROFILE_SAMPLING_VAR __llvm_profile_sampling
0762
0763 /* The variable that holds the name of the profile data
0764 * specified via command line. */
0765 #define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename
0766
0767 /* section name strings common to all targets other
0768 than WIN32 */
0769 #define INSTR_PROF_DATA_COMMON __llvm_prf_data
0770 #define INSTR_PROF_NAME_COMMON __llvm_prf_names
0771 #define INSTR_PROF_VNAME_COMMON __llvm_prf_vns
0772 #define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts
0773 #define INSTR_PROF_BITS_COMMON __llvm_prf_bits
0774 #define INSTR_PROF_VALS_COMMON __llvm_prf_vals
0775 #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
0776 #define INSTR_PROF_VTAB_COMMON __llvm_prf_vtab
0777 #define INSTR_PROF_COVMAP_COMMON __llvm_covmap
0778 #define INSTR_PROF_COVFUN_COMMON __llvm_covfun
0779 #define INSTR_PROF_COVDATA_COMMON __llvm_covdata
0780 #define INSTR_PROF_COVNAME_COMMON __llvm_covnames
0781 #define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile
0782 #define INSTR_PROF_COVINIT_COMMON __llvm_covinit
0783
0784 /* Windows section names. Because these section names contain dollar characters,
0785 * they must be quoted.
0786 */
0787 #define INSTR_PROF_DATA_COFF ".lprfd$M"
0788 #define INSTR_PROF_NAME_COFF ".lprfn$M"
0789 #define INSTR_PROF_VNAME_COFF ".lprfvn$M"
0790 #define INSTR_PROF_CNTS_COFF ".lprfc$M"
0791 #define INSTR_PROF_BITS_COFF ".lprfb$M"
0792 #define INSTR_PROF_VALS_COFF ".lprfv$M"
0793 #define INSTR_PROF_VNODES_COFF ".lprfnd$M"
0794 #define INSTR_PROF_VTAB_COFF ".lprfvt$M"
0795 #define INSTR_PROF_COVMAP_COFF ".lcovmap$M"
0796 #define INSTR_PROF_COVFUN_COFF ".lcovfun$M"
0797 /* Since cov data and cov names sections are not allocated, we don't need to
0798 * access them at runtime.
0799 */
0800 #define INSTR_PROF_COVDATA_COFF ".lcovd"
0801 #define INSTR_PROF_COVNAME_COFF ".lcovn"
0802 #define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M"
0803
0804 // FIXME: Placeholder for Windows. Windows currently does not initialize
0805 // the GCOV functions in the runtime.
0806 #define INSTR_PROF_COVINIT_COFF ".lcovd$M"
0807
0808 #ifdef _WIN32
0809 /* Runtime section names and name strings. */
0810 #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF
0811 #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF
0812 #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF
0813 #define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_BITS_COFF
0814 #define INSTR_PROF_VTAB_SECT_NAME INSTR_PROF_VTAB_COFF
0815 #define INSTR_PROF_VNAME_SECT_NAME INSTR_PROF_VNAME_COFF
0816 /* Array of pointers. Each pointer points to a list
0817 * of value nodes associated with one value site.
0818 */
0819 #define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF
0820 /* Value profile nodes section. */
0821 #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF
0822 #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF
0823 #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF
0824 #define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_COVDATA_COFF
0825 #define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_COVNAME_COFF
0826 #define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF
0827 #define INSTR_PROF_COVINIT_SECT_NAME INSTR_PROF_COVINIT_COFF
0828 #else
0829 /* Runtime section names and name strings. */
0830 #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON)
0831 #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON)
0832 #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON)
0833 #define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON)
0834 #define INSTR_PROF_VTAB_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VTAB_COMMON)
0835 #define INSTR_PROF_VNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNAME_COMMON)
0836 /* Array of pointers. Each pointer points to a list
0837 * of value nodes associated with one value site.
0838 */
0839 #define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON)
0840 /* Value profile nodes section. */
0841 #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON)
0842 #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON)
0843 #define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON)
0844 #define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON)
0845 #define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON)
0846 /* Order file instrumentation. */
0847 #define INSTR_PROF_ORDERFILE_SECT_NAME \
0848 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON)
0849 #define INSTR_PROF_COVINIT_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVINIT_COMMON)
0850 #endif
0851
0852 #define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer
0853 #define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \
0854 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME)
0855 #define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx
0856 #define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \
0857 INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME)
0858
0859 /* Macros to define start/stop section symbol for a given
0860 * section on Linux. For instance
0861 * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will
0862 * expand to __start___llvm_prof_data
0863 */
0864 #define INSTR_PROF_SECT_START(Sect) \
0865 INSTR_PROF_CONCAT(__start_,Sect)
0866 #define INSTR_PROF_SECT_STOP(Sect) \
0867 INSTR_PROF_CONCAT(__stop_,Sect)
0868
0869 /* Value Profiling API linkage name. */
0870 #define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
0871 #define INSTR_PROF_VALUE_PROF_FUNC_STR \
0872 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
0873 #define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop
0874 #define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \
0875 INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC)
0876
0877 /* InstrProfile per-function control data alignment. */
0878 #define INSTR_PROF_DATA_ALIGNMENT 8
0879
0880 /* The data structure that represents a tracked value by the
0881 * value profiler.
0882 */
0883 typedef struct InstrProfValueData {
0884 /* Profiled value. */
0885 uint64_t Value;
0886 /* Number of times the value appears in the training run. */
0887 uint64_t Count;
0888 } InstrProfValueData;
0889
0890 #endif /* INSTR_PROF_DATA_INC */
0891
0892 #ifndef INSTR_ORDER_FILE_INC
0893 /* The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). */
0894 #define INSTR_ORDER_FILE_BUFFER_SIZE 131072
0895 #define INSTR_ORDER_FILE_BUFFER_BITS 17
0896 #define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff
0897 #endif /* INSTR_ORDER_FILE_INC */
0898 #else
0899 #undef INSTR_PROF_DATA_DEFINED
0900 #endif
0901
0902 #undef COVMAP_V2_OR_V3
0903
0904 #ifdef INSTR_PROF_VALUE_PROF_MEMOP_API
0905
0906 #ifdef __cplusplus
0907 #define INSTR_PROF_INLINE inline
0908 #else
0909 #define INSTR_PROF_INLINE
0910 #endif
0911
0912 /* The value range buckets (22 buckets) for the memop size value profiling looks
0913 * like:
0914 *
0915 * [0, 0]
0916 * [1, 1]
0917 * [2, 2]
0918 * [3, 3]
0919 * [4, 4]
0920 * [5, 5]
0921 * [6, 6]
0922 * [7, 7]
0923 * [8, 8]
0924 * [9, 15]
0925 * [16, 16]
0926 * [17, 31]
0927 * [32, 32]
0928 * [33, 63]
0929 * [64, 64]
0930 * [65, 127]
0931 * [128, 128]
0932 * [129, 255]
0933 * [256, 256]
0934 * [257, 511]
0935 * [512, 512]
0936 * [513, UINT64_MAX]
0937 *
0938 * Each range has a 'representative value' which is the lower end value of the
0939 * range and used to store in the runtime profile data records and the VP
0940 * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127].
0941 */
0942 #define INSTR_PROF_NUM_BUCKETS 22
0943
0944 /*
0945 * Clz and Popcount. This code was copied from
0946 * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and
0947 * llvm/include/llvm/Support/MathExtras.h.
0948 */
0949 #if defined(_MSC_VER) && !defined(__clang__)
0950
0951 #include <intrin.h>
0952 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
0953 int InstProfClzll(unsigned long long X) {
0954 unsigned long LeadZeroIdx = 0;
0955 #if !defined(_M_ARM64) && !defined(_M_X64)
0956 // Scan the high 32 bits.
0957 if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32)))
0958 return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset
0959 // from the MSB.
0960 // Scan the low 32 bits.
0961 if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X)))
0962 return (int)(63 - LeadZeroIdx);
0963 #else
0964 if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx;
0965 #endif
0966 return 64;
0967 }
0968 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
0969 int InstProfPopcountll(unsigned long long X) {
0970 // This code originates from https://reviews.llvm.org/rG30626254510f.
0971 unsigned long long v = X;
0972 v = v - ((v >> 1) & 0x5555555555555555ULL);
0973 v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
0974 v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
0975 return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56);
0976 }
0977
0978 #else
0979
0980 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
0981 int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); }
0982 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
0983 int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); }
0984
0985 #endif /* defined(_MSC_VER) && !defined(__clang__) */
0986
0987 // clang-format on
0988
0989 /* Map an (observed) memop size value to the representative value of its range.
0990 * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */
0991 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t
0992 InstrProfGetRangeRepValue(uint64_t Value) {
0993 if (Value <= 8)
0994 // The first ranges are individually tracked. Use the value as is.
0995 return Value;
0996 else if (Value >= 513)
0997 // The last range is mapped to its lowest value.
0998 return 513;
0999 else if (InstProfPopcountll(Value) == 1)
1000 // If it's a power of two, use it as is.
1001 return Value;
1002 else
1003 // Otherwise, take to the previous power of two + 1.
1004 return (UINT64_C(1) << (64 - InstProfClzll(Value) - 1)) + 1;
1005 }
1006
1007 /* Return true if the range that an (observed) memop size value belongs to has
1008 * only a single value in the range. For example, 0 -> true, 8 -> true, 10 ->
1009 * false, 64 -> true, 100 -> false, 513 -> false. */
1010 INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned
1011 InstrProfIsSingleValRange(uint64_t Value) {
1012 if (Value <= 8)
1013 // The first ranges are individually tracked.
1014 return 1;
1015 else if (InstProfPopcountll(Value) == 1)
1016 // If it's a power of two, there's only one value.
1017 return 1;
1018 else
1019 // Otherwise, there's more than one value in the range.
1020 return 0;
1021 }
1022
1023 #endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */
