Global variables as type descriptors

.clang-tidy

@@ -2,7 +2,6 @@
 Checks:          'clang-diagnostic-*,clang-analyzer-*,-*,performance-*,modernize-*,llvm-namespace-comment,google-explicit-constructor,bugprone-*,misc-*,readability-*, -readability-named-parameter,-modernize-use-trailing-return-type,-modernize-use-using'
 WarningsAsErrors: ''
 HeaderFilterRegex: ''
-AnalyzeTemporaryDtors: false
 CheckOptions:
   - key:             google-readability-braces-around-statements.ShortStatementLines
     value:           '1'

.github/workflows/basic-ci.yml

@@ -12,10 +12,22 @@ env:
 
 jobs:
   format-check:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
+
+      - run: |
+          wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
+          echo "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-18 main" | sudo tee /etc/apt/sources.list.d/llvm-18.list
+
+      - name: Update apt
+        run: sudo apt-get update
+
+      - name: Install clang-format
+        run: |
+          sudo apt-get remove clang-format-*
+          sudo apt-get install -t llvm-toolchain-noble-18 clang-format-18
 
       - name: Format source code
         run: |
@@ -24,7 +36,7 @@ jobs:
             -a \( -name "*.c" -o -name "*.cpp" -o -name "*.h" \) \
             -not -path "*/lulesh/*" -not -path "*/CallSite.h" \
             -print0 \
-            | xargs -0 clang-format-14 -i
+            | xargs -0 clang-format-18 -i
 
       - name: Format check
         run: |
@@ -35,7 +47,7 @@ jobs:
     runs-on: ubuntu-22.04
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - uses: codespell-project/actions-codespell@v2
 
   lit-suite:
@@ -73,7 +85,7 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - name: LLVM apt
         if: ${{ matrix.llvm-version == 19 }}

.github/workflows/ext-ci.yml

@@ -31,10 +31,10 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - name: Checkout test-bench
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           repository: tudasc/typeart-bench
           ssh-key: ${{ secrets.AUTH_SSH_CI_EXT }}
@@ -129,10 +129,10 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - name: Checkout AD test-bench
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           repository: ahueck/typeart-ad-benchmarks
           ssh-key: ${{ secrets.AUTH_SSH_CI_EXT_AD }}
@@ -215,10 +215,10 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - name: Checkout OMP test-bench
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           repository: tudasc/typeart-bench
           ssh-key: ${{ secrets.AUTH_SSH_CI_EXT }}

demo/Makefile

@@ -3,7 +3,7 @@
 MPIRUN ?= mpirun
 MPICC ?= typeart-mpicc # instead of standard mpicc
 
-all: libtool.so 01_ex 02_ex toy toy-stack
+all: libtool.so demo demo_broken
 
 libtool.so: tool.c
 	$(MPICC) -shared -fPIC $< -o $@

externals/abseil/CMakeLists.txt

@@ -3,7 +3,7 @@ set(ABSL_PROPAGATE_CXX_STD ON)
 FetchContent_Declare(
   cpp-abseil
   GIT_REPOSITORY https://github.com/abseil/abseil-cpp.git
-  GIT_TAG 20240722.1
+  GIT_TAG 20250814.1
   GIT_SHALLOW 1
 )
 

lib/passes/CMakeLists.txt

@@ -12,6 +12,8 @@ set(PASS_SOURCES
   instrumentation/MemOpArgCollector.cpp
   instrumentation/MemOpInstrumentation.cpp
   instrumentation/Instrumentation.cpp
+  instrumentation/TypeIDProvider.cpp
+  instrumentation/CallBackFunctionInserter.cpp
   TypeARTConfiguration.cpp
   Commandline.cpp
 )

lib/passes/Commandline.cpp

@@ -15,6 +15,7 @@
 #include "analysis/MemInstFinder.h"
 #include "configuration/Configuration.h"
 #include "configuration/EnvironmentConfiguration.h"
+#include "instrumentation/TypeIDProvider.h"
 #include "support/ConfigurationBase.h"
 #include "support/Logger.h"
 #include "typegen/TypeGenerator.h"
@@ -83,6 +84,15 @@ static cl::opt<ConfigStdArgTypes::stack_ty> cl_typeart_instrument_stack(Commandl
                                                                         cl::init(ConfigStdArgValues::stack),
                                                                         cl::cat(typeart_category));
 
+static cl::opt<typeart::TypeSerializationImplementation> cl_typeart_type_serialization(
+    CommandlineStdArgs::type_serialization, cl::desc(ConfigStdArgDescriptions::type_serialization),
+    cl::values(clEnumValN(typeart::TypeSerializationImplementation::FILE, "file", "File based type serialization"),
+               clEnumValN(typeart::TypeSerializationImplementation::INLINE, "inline",
+                          "Type descriptors through global variables"),
+               clEnumValN(typeart::TypeSerializationImplementation::HYBRID, "hybrid",
+                          "Type descriptors through global variables except for C/C++ built-in types")),
+    cl::Hidden, cl::init(typeart::TypeSerializationImplementation::FILE), cl::cat(typeart_category));
+
 static cl::opt<ConfigStdArgTypes::stack_lifetime_ty> cl_typeart_instrument_stack_lifetime(
     CommandlineStdArgs::stack_lifetime, cl::desc(ConfigStdArgDescriptions::stack_lifetime),
     cl::init(ConfigStdArgValues::stack_lifetime), cl::cat(typeart_category));
@@ -198,6 +208,7 @@ CommandLineOptions::CommandLineOptions() {
       make_entry(ConfigStdArgs::heap, cl_typeart_instrument_heap),
       make_entry(ConfigStdArgs::global, cl_typeart_instrument_global),
       make_entry(ConfigStdArgs::stack, cl_typeart_instrument_stack),
+      make_entry(ConfigStdArgs::type_serialization, cl_typeart_type_serialization),
       make_entry(ConfigStdArgs::stack_lifetime, cl_typeart_instrument_stack_lifetime),
       make_entry(ConfigStdArgs::typegen, cl_typeart_typegen_implementation),
       make_entry(ConfigStdArgs::filter, cl_typeart_call_filter),
@@ -217,6 +228,7 @@ CommandLineOptions::CommandLineOptions() {
       make_occurr_entry(ConfigStdArgs::heap, cl_typeart_instrument_heap),
       make_occurr_entry(ConfigStdArgs::global, cl_typeart_instrument_global),
       make_occurr_entry(ConfigStdArgs::stack, cl_typeart_instrument_stack),
+      make_occurr_entry(ConfigStdArgs::type_serialization, cl_typeart_type_serialization),
       make_occurr_entry(ConfigStdArgs::stack_lifetime, cl_typeart_instrument_stack_lifetime),
       make_occurr_entry(ConfigStdArgs::typegen, cl_typeart_typegen_implementation),
       make_occurr_entry(ConfigStdArgs::filter, cl_typeart_call_filter),

lib/passes/TypeARTPass.cpp

@@ -19,9 +19,11 @@
 #include "configuration/PassBuilderUtil.h"
 #include "configuration/PassConfiguration.h"
 #include "configuration/TypeARTOptions.h"
+#include "instrumentation/CallBackFunctionInserter.h"
 #include "instrumentation/MemOpArgCollector.h"
 #include "instrumentation/MemOpInstrumentation.h"
 #include "instrumentation/TypeARTFunctions.h"
+#include "instrumentation/TypeIDProvider.h"
 #include "support/ConfigurationBase.h"
 #include "support/Logger.h"
 #include "support/ModuleDumper.h"
@@ -48,7 +50,9 @@
 
 #include <cassert>
 #include <cstddef>
+#include <llvm/Config/llvm-config.h>
 #include <llvm/Support/Error.h>
+#include <memory>
 #include <optional>
 #include <sstream>
 #include <string>
@@ -95,26 +99,10 @@ class TypeArtPass : public llvm::PassInfoMixin<TypeArtPass> {
   std::optional<config::TypeARTConfigOptions> pass_opts{std::nullopt};
   std::unique_ptr<config::Configuration> pass_config;
 
-  struct TypeArtFunc {
-    const std::string name;
-    llvm::Value* f{nullptr};
-  };
-
-  TypeArtFunc typeart_alloc{"__typeart_alloc"};
-  TypeArtFunc typeart_alloc_global{"__typeart_alloc_global"};
-  TypeArtFunc typeart_alloc_stack{"__typeart_alloc_stack"};
-  TypeArtFunc typeart_free{"__typeart_free"};
-  TypeArtFunc typeart_leave_scope{"__typeart_leave_scope"};
-
-  TypeArtFunc typeart_alloc_omp        = typeart_alloc;
-  TypeArtFunc typeart_alloc_stacks_omp = typeart_alloc_stack;
-  TypeArtFunc typeart_free_omp         = typeart_free;
-  TypeArtFunc typeart_leave_scope_omp  = typeart_leave_scope;
-
   std::unique_ptr<analysis::MemInstFinder> meminst_finder;
   std::unique_ptr<TypeGenerator> typeManager;
   InstrumentationHelper instrumentation_helper;
-  TAFunctions functions;
+  std::unique_ptr<TAFunctionQuery> functions;
   std::unique_ptr<InstrumentationContext> instrumentation_context;
 
   const config::Configuration& configuration() const {
@@ -168,32 +156,48 @@ class TypeArtPass : public llvm::PassInfoMixin<TypeArtPass> {
 
     instrumentation_helper.setModule(m);
     ModuleData mdata{&m};
-    typeManager->registerModule(mdata);
+    const auto has_cu_types = typeManager->registerModule(mdata);
+
+    declareInstrumentationFunctions(m);
+    {
+      auto type_id_handler = get_type_id_handler(m, &typeManager->getTypeDatabase(), configuration(), functions.get());
+      // const bool heap   = configuration()[config::ConfigStdArgs::heap];
+      if (has_cu_types) {
+        LOG_DEBUG("Registering compilation unit types list")
+        type_id_handler->registerModule(mdata);
+      }
 
-    auto arg_collector =
-        std::make_unique<MemOpArgCollector>(configuration(), typeManager.get(), instrumentation_helper);
-    // const bool instrument_stack_lifetime = configuration()[config::ConfigStdArgs::stack_lifetime];
-    auto mem_instrument = std::make_unique<MemOpInstrumentation>(configuration(), functions, instrumentation_helper);
-    instrumentation_context =
-        std::make_unique<InstrumentationContext>(std::move(arg_collector), std::move(mem_instrument));
+      auto arg_collector =
+          std::make_unique<MemOpArgCollector>(configuration(), typeManager.get(), instrumentation_helper);
+      // const bool instrument_stack_lifetime = configuration()[config::ConfigStdArgs::stack_lifetime];
+      auto cb_provider    = make_callback_inserter(configuration(), std::move(type_id_handler), functions.get());
+      auto mem_instrument = std::make_unique<MemOpInstrumentation>(configuration(), functions.get(),
+                                                                   instrumentation_helper, std::move(cb_provider));
 
+      instrumentation_context =
+          std::make_unique<InstrumentationContext>(std::move(arg_collector), std::move(mem_instrument));
+    }
     return true;
   }
 
   bool doFinalization() {
     /*
      * Persist the accumulated type definition information for this module.
      */
-    const std::string types_file = configuration()[config::ConfigStdArgs::types];
-    LOG_DEBUG("Writing type file to " << types_file);
-
-    const auto [stored, error] = typeManager->store();
-    if (stored) {
-      LOG_DEBUG("Success!");
-    } else {
-      LOG_FATAL("Failed writing type config to " << types_file << ". Reason: " << error.message());
+    // TODO: inline/hybrid types not supported in non-opaque mode
+    const bool emit_type_file_always     = bool(LLVM_VERSION_MAJOR < 15);
+    TypeSerializationImplementation mode = configuration()[config::ConfigStdArgs::type_serialization];
+    if (emit_type_file_always || mode == TypeSerializationImplementation::FILE) {
+      const std::string types_file = configuration()[config::ConfigStdArgs::types];
+      LOG_DEBUG("Writing type file to " << types_file);
+
+      const auto [stored, error] = typeManager->store();
+      if (stored) {
+        LOG_DEBUG("Success!");
+      } else {
+        LOG_FATAL("Failed writing type config to " << types_file << ". Reason: " << error.message());
+      }
     }
-
     const bool print_stats = configuration()[config::ConfigStdArgs::stats];
     if (print_stats) {
       auto& out = llvm::errs();
@@ -203,30 +207,7 @@ class TypeArtPass : public llvm::PassInfoMixin<TypeArtPass> {
   }
 
   void declareInstrumentationFunctions(Module& m) {
-    // Remove this return if problems come up during compilation
-    if (typeart_alloc_global.f != nullptr && typeart_alloc_stack.f != nullptr && typeart_alloc.f != nullptr &&
-        typeart_free.f != nullptr && typeart_leave_scope.f != nullptr) {
-      return;
-    }
-
-    TAFunctionDeclarator decl(m, instrumentation_helper, functions);
-
-    auto alloc_arg_types      = instrumentation_helper.make_parameters(IType::ptr, IType::type_id, IType::extent);
-    auto free_arg_types       = instrumentation_helper.make_parameters(IType::ptr);
-    auto leavescope_arg_types = instrumentation_helper.make_parameters(IType::stack_count);
-
-    typeart_alloc.f        = decl.make_function(IFunc::heap, typeart_alloc.name, alloc_arg_types);
-    typeart_alloc_stack.f  = decl.make_function(IFunc::stack, typeart_alloc_stack.name, alloc_arg_types);
-    typeart_alloc_global.f = decl.make_function(IFunc::global, typeart_alloc_global.name, alloc_arg_types);
-    typeart_free.f         = decl.make_function(IFunc::free, typeart_free.name, free_arg_types);
-    typeart_leave_scope.f  = decl.make_function(IFunc::scope, typeart_leave_scope.name, leavescope_arg_types);
-
-    typeart_alloc_omp.f = decl.make_function(IFunc::heap_omp, typeart_alloc_omp.name, alloc_arg_types, true);
-    typeart_alloc_stacks_omp.f =
-        decl.make_function(IFunc::stack_omp, typeart_alloc_stacks_omp.name, alloc_arg_types, true);
-    typeart_free_omp.f = decl.make_function(IFunc::free_omp, typeart_free_omp.name, free_arg_types, true);
-    typeart_leave_scope_omp.f =
-        decl.make_function(IFunc::scope_omp, typeart_leave_scope_omp.name, leavescope_arg_types, true);
+    functions = declare_instrumentation_functions(m, configuration());
   }
 
   void printStats(llvm::raw_ostream& out) {
@@ -292,7 +273,7 @@ class TypeArtPass : public llvm::PassInfoMixin<TypeArtPass> {
     const bool instrument_global = configuration()[config::ConfigStdArgs::global];
     bool globals_were_instrumented{false};
     if (instrument_global) {
-      declareInstrumentationFunctions(m);
+      // declareInstrumentationFunctions(m);
 
       const auto& globalsList = meminst_finder->getModuleGlobals();
       if (!globalsList.empty()) {
@@ -322,7 +303,7 @@ class TypeArtPass : public llvm::PassInfoMixin<TypeArtPass> {
 
     // FIXME this is required when "PassManagerBuilder::EP_OptimizerLast" is used as the function (constant) pointer are
     // nullpointer/invalidated
-    declareInstrumentationFunctions(*f.getParent());
+    // declareInstrumentationFunctions(*f.getParent());
 
     bool mod{false};
     //  auto& c = f.getContext();
@@ -364,7 +345,7 @@ class LegacyTypeArtPass : public llvm::ModulePass {
  public:
   static char ID;  // NOLINT
 
-  LegacyTypeArtPass() : ModulePass(ID){};
+  LegacyTypeArtPass() : ModulePass(ID) {};
 
   bool doInitialization(llvm::Module&) override;
 
@@ -399,17 +380,19 @@ llvm::PassPluginLibraryInfo getTypeartPassPluginInfo() {
   using namespace llvm;
   return {LLVM_PLUGIN_API_VERSION, "TypeART", LLVM_VERSION_STRING, [](PassBuilder& pass_builder) {
             pass_builder.registerPipelineStartEPCallback([](auto& MPM, OptimizationLevel) {
-              auto parameters = typeart::util::pass::parsePassParameters(typeart::config::pass::parse_typeart_config,
-                                                                         "typeart<heap;stats>", "typeart");
+              auto parameters =
+                  typeart::util::pass::parsePassParameters(typeart::config::pass::parse_typeart_config,
+                                                           "typeart<heap;stats;type-serialization=hybrid>", "typeart");
               if (!parameters) {
                 LOG_FATAL("Error parsing heap params: " << parameters.takeError())
                 return;
               }
               MPM.addPass(typeart::pass::TypeArtPass(typeart::pass::TypeArtPass(parameters.get())));
             });
             pass_builder.registerOptimizerLastEPCallback([](auto& MPM, OptimizationLevel) {
-              auto parameters = typeart::util::pass::parsePassParameters(typeart::config::pass::parse_typeart_config,
-                                                                         "typeart<no-heap;stack;stats>", "typeart");
+              auto parameters = typeart::util::pass::parsePassParameters(
+                  typeart::config::pass::parse_typeart_config, "typeart<no-heap;stack;stats;type-serialization=hybrid>",
+                  "typeart");
               if (!parameters) {
                 LOG_FATAL("Error parsing stack params: " << parameters.takeError())
                 return;

lib/passes/analysis/MemInstFinder.cpp

@@ -193,7 +193,7 @@ bool MemInstFinderPass::runOnModule(Module& module) {
                         }
 
                         if (util::starts_with_any_of(name, "llvm.", "__llvm_gcov", "__llvm_gcda", "__profn", "___asan",
-                                                     "__msan", "__tsan")) {
+                                                     "__msan", "__tsan", "__typeart", "_typeart")) {
                           LOG_DEBUG("Prefixed matched on " << name)
                           return true;
                         }

lib/passes/configuration/EnvironmentConfiguration.cpp

@@ -16,6 +16,7 @@
 #include "OptionsUtil.h"
 #include "PassConfiguration.h"
 #include "configuration/TypeARTOptions.h"
+#include "instrumentation/TypeIDProvider.h"
 #include "support/ConfigurationBase.h"
 #include "support/Logger.h"
 #include "support/Util.h"
@@ -113,6 +114,9 @@ EnvironmentFlagsOptions::EnvironmentFlagsOptions() {
                                                EnvironmentStdArgsValues::global),
       make_entry<ConfigStdArgTypes::stack_ty>(ConfigStdArgs::stack, EnvironmentStdArgs::stack,
                                               EnvironmentStdArgsValues::stack),
+      make_entry<typeart::TypeSerializationImplementation>(ConfigStdArgs::type_serialization,
+                                                           EnvironmentStdArgs::type_serialization,
+                                                           EnvironmentStdArgsValues::type_serialization),
       make_entry<ConfigStdArgTypes::stack_lifetime_ty>(
           ConfigStdArgs::stack_lifetime, EnvironmentStdArgs::stack_lifetime, EnvironmentStdArgsValues::stack_lifetime),
       make_entry<typeart::TypegenImplementation>(ConfigStdArgs::typegen, EnvironmentStdArgs::typegen,
@@ -148,6 +152,7 @@ EnvironmentFlagsOptions::EnvironmentFlagsOptions() {
       make_occurr_entry(ConfigStdArgs::heap, EnvironmentStdArgs::heap),
       make_occurr_entry(ConfigStdArgs::global, EnvironmentStdArgs::global),
       make_occurr_entry(ConfigStdArgs::stack, EnvironmentStdArgs::stack),
+      make_occurr_entry(ConfigStdArgs::type_serialization, EnvironmentStdArgs::type_serialization),
       make_occurr_entry(ConfigStdArgs::stack_lifetime, EnvironmentStdArgs::stack_lifetime),
       make_occurr_entry(ConfigStdArgs::typegen, EnvironmentStdArgs::typegen),
       make_occurr_entry(ConfigStdArgs::filter, EnvironmentStdArgs::filter),