Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
openSUSE:Factory:RISCV
Mesa
24300-llvmpipe-add-an-implementation-with-llvm-...
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File 24300-llvmpipe-add-an-implementation-with-llvm-orcjit.patch of Package Mesa
From 744d1e9fb2ff50fed3e2804e01b5867c774a32f0 Mon Sep 17 00:00:00 2001 From: Alex Fan <alex.fan.q@gmail.com> Date: Tue, 28 Jun 2022 15:51:24 +0000 Subject: [PATCH] llvmpipe: add an implementation with llvm orcjit no per-func perf nor disasm is implemented yet. disk cache is also left empty --- src/gallium/auxiliary/draw/draw_context.c | 4 + src/gallium/auxiliary/draw/draw_llvm.c | 94 +- src/gallium/auxiliary/draw/draw_llvm.h | 27 +- src/gallium/auxiliary/gallivm/lp_bld_coro.c | 6 +- src/gallium/auxiliary/gallivm/lp_bld_coro.h | 8 + src/gallium/auxiliary/gallivm/lp_bld_init.c | 9 +- src/gallium/auxiliary/gallivm/lp_bld_init.h | 41 +- .../auxiliary/gallivm/lp_bld_init_orc.cpp | 892 ++++++++++++++++++ src/gallium/auxiliary/meson.build | 2 +- src/gallium/drivers/llvmpipe/lp_context.c | 14 + src/gallium/drivers/llvmpipe/lp_context.h | 4 + src/gallium/drivers/llvmpipe/lp_screen.c | 4 + src/gallium/drivers/llvmpipe/lp_state_cs.c | 19 + src/gallium/drivers/llvmpipe/lp_state_cs.h | 3 + src/gallium/drivers/llvmpipe/lp_state_fs.c | 33 + src/gallium/drivers/llvmpipe/lp_state_fs.h | 6 + .../llvmpipe/lp_state_fs_linear_llvm.c | 4 + src/gallium/drivers/llvmpipe/lp_state_setup.c | 9 + src/gallium/drivers/llvmpipe/lp_state_setup.h | 3 + src/gallium/drivers/llvmpipe/lp_test_arit.c | 19 + src/gallium/drivers/llvmpipe/lp_test_blend.c | 33 +- src/gallium/drivers/llvmpipe/lp_test_conv.c | 33 +- src/gallium/drivers/llvmpipe/lp_test_format.c | 50 +- ...est_printf.c => lp_test_lookup_multiple.c} | 52 +- src/gallium/drivers/llvmpipe/lp_test_printf.c | 31 +- .../drivers/llvmpipe/lp_texture_handle.c | 28 +- src/gallium/drivers/llvmpipe/meson.build | 2 +- 27 files changed, 1396 insertions(+), 34 deletions(-) create mode 100644 src/gallium/auxiliary/gallivm/lp_bld_init_orc.cpp copy src/gallium/drivers/llvmpipe/{lp_test_printf.c => lp_test_lookup_multiple.c} (69%) diff --git a/src/gallium/auxiliary/draw/draw_context.c b/src/gallium/auxiliary/draw/draw_context.c index 58e8aced4e4..83ce386b846 100644 --- a/src/gallium/auxiliary/draw/draw_context.c +++ b/src/gallium/auxiliary/draw/draw_context.c @@ -77,7 +77,11 @@ draw_create_context(struct pipe_context *pipe, void *context, #if DRAW_LLVM_AVAILABLE if (try_llvm && draw_get_option_use_llvm()) { +#if GALLIVM_USE_ORCJIT == 1 + draw->llvm = draw_llvm_create(draw, (LLVMOrcThreadSafeContextRef)context); +#else draw->llvm = draw_llvm_create(draw, (LLVMContextRef)context); +#endif } #endif diff --git a/src/gallium/auxiliary/draw/draw_llvm.c b/src/gallium/auxiliary/draw/draw_llvm.c index 36ff317c85d..8e9f42dbb84 100644 --- a/src/gallium/auxiliary/draw/draw_llvm.c +++ b/src/gallium/auxiliary/draw/draw_llvm.c @@ -381,8 +381,13 @@ get_vertex_header_ptr_type(struct draw_llvm_variant *variant) /** * Create per-context LLVM info. */ +#if GALLIVM_USE_ORCJIT == 1 +struct draw_llvm * +draw_llvm_create(struct draw_context *draw, LLVMOrcThreadSafeContextRef context) +#else struct draw_llvm * draw_llvm_create(struct draw_context *draw, LLVMContextRef context) +#endif { struct draw_llvm *llvm; @@ -395,6 +400,16 @@ draw_llvm_create(struct draw_context *draw, LLVMContextRef context) llvm->draw = draw; +#if GALLIVM_USE_ORCJIT == 1 + llvm->_ts_context = context; + if (!llvm->_ts_context) { + llvm->_ts_context = LLVMOrcCreateNewThreadSafeContext(); + llvm->context_owned = true; + } + if (!llvm->_ts_context) + goto fail; + llvm->context = LLVMOrcThreadSafeContextGetContext(llvm->_ts_context); +#else llvm->context = context; if (!llvm->context) { llvm->context = LLVMContextCreate(); @@ -407,6 +422,7 @@ draw_llvm_create(struct draw_context *draw, LLVMContextRef context) } if (!llvm->context) goto fail; +#endif llvm->nr_variants = 0; list_inithead(&llvm->vs_variants_list.list); @@ -434,9 +450,16 @@ fail: void draw_llvm_destroy(struct draw_llvm *llvm) { +#if GALLIVM_USE_ORCJIT == 1 + if (llvm->context_owned) + LLVMOrcDisposeThreadSafeContext(llvm->_ts_context); + llvm->_ts_context = NULL; + llvm->context = NULL; +#else if (llvm->context_owned) LLVMContextDispose(llvm->context); llvm->context = NULL; +#endif /* XXX free other draw_llvm data? */ FREE(llvm); @@ -510,7 +533,11 @@ draw_llvm_create_variant(struct draw_llvm *llvm, if (!cached.data_size) needs_caching = true; } +#if GALLIVM_USE_ORCJIT == 1 + variant->gallivm = gallivm_create(module_name, llvm->_ts_context, &cached); +#else variant->gallivm = gallivm_create(module_name, llvm->context, &cached); +#endif create_vs_jit_types(variant); @@ -529,8 +556,13 @@ draw_llvm_create_variant(struct draw_llvm *llvm, gallivm_compile_module(variant->gallivm); +#if GALLIVM_USE_ORCJIT == 1 + variant->jit_func = (draw_jit_vert_func) + gallivm_jit_function(variant->gallivm, variant->function_name); +#else variant->jit_func = (draw_jit_vert_func) gallivm_jit_function(variant->gallivm, variant->function); +#endif if (needs_caching) llvm->draw->disk_cache_insert_shader(llvm->draw->disk_cache_cookie, @@ -1633,6 +1665,10 @@ draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant) variant_func = LLVMAddFunction(gallivm->module, func_name, func_type); variant->function = variant_func; +#if GALLIVM_USE_ORCJIT == 1 + variant->function_name = MALLOC(strlen(func_name)+1); + strcpy(variant->function_name, func_name); +#endif LLVMSetFunctionCallConv(variant_func, LLVMCCallConv); for (i = 0; i < num_arg_types; ++i) @@ -2249,6 +2285,11 @@ draw_llvm_destroy_variant(struct draw_llvm_variant *variant) variant->shader->variants_cached--; list_del(&variant->list_item_global.list); llvm->nr_variants--; + +#if GALLIVM_USE_ORCJIT == 1 + if(variant->function_name) + FREE(variant->function_name); +#endif FREE(variant); } @@ -2358,6 +2399,10 @@ draw_gs_llvm_generate(struct draw_llvm *llvm, variant_func = LLVMAddFunction(gallivm->module, func_name, func_type); variant->function = variant_func; +#if GALLIVM_USE_ORCJIT == 1 + variant->function_name = MALLOC(strlen(func_name)+1); + strcpy(variant->function_name, func_name); +#endif LLVMSetFunctionCallConv(variant_func, LLVMCCallConv); @@ -2524,7 +2569,11 @@ draw_gs_llvm_create_variant(struct draw_llvm *llvm, if (!cached.data_size) needs_caching = true; } +#if GALLIVM_USE_ORCJIT == 1 + variant->gallivm = gallivm_create(module_name, llvm->_ts_context, &cached); +#else variant->gallivm = gallivm_create(module_name, llvm->context, &cached); +#endif create_gs_jit_types(variant); @@ -2535,8 +2584,13 @@ draw_gs_llvm_create_variant(struct draw_llvm *llvm, gallivm_compile_module(variant->gallivm); +#if GALLIVM_USE_ORCJIT == 1 + variant->jit_func = (draw_gs_jit_func) + gallivm_jit_function(variant->gallivm, variant->function_name); +#else variant->jit_func = (draw_gs_jit_func) gallivm_jit_function(variant->gallivm, variant->function); +#endif if (needs_caching) llvm->draw->disk_cache_insert_shader(llvm->draw->disk_cache_cookie, @@ -2569,6 +2623,10 @@ draw_gs_llvm_destroy_variant(struct draw_gs_llvm_variant *variant) variant->shader->variants_cached--; list_del(&variant->list_item_global.list); llvm->nr_gs_variants--; +#if GALLIVM_USE_ORCJIT == 1 + if(variant->function_name) + FREE(variant->function_name); +#endif FREE(variant); } @@ -2942,6 +3000,10 @@ draw_tcs_llvm_generate(struct draw_llvm *llvm, variant_coro = LLVMAddFunction(gallivm->module, func_name_coro, coro_func_type); variant->function = variant_func; +#if GALLIVM_USE_ORCJIT == 1 + variant->function_name = MALLOC(strlen(func_name)+1); + strcpy(variant->function_name, func_name); +#endif LLVMSetFunctionCallConv(variant_func, LLVMCCallConv); LLVMSetFunctionCallConv(variant_coro, LLVMCCallConv); @@ -3177,8 +3239,11 @@ draw_tcs_llvm_create_variant(struct draw_llvm *llvm, if (!cached.data_size) needs_caching = true; } - +#if GALLIVM_USE_ORCJIT == 1 + variant->gallivm = gallivm_create(module_name, llvm->_ts_context, &cached); +#else variant->gallivm = gallivm_create(module_name, llvm->context, &cached); +#endif create_tcs_jit_types(variant); @@ -3189,10 +3254,16 @@ draw_tcs_llvm_create_variant(struct draw_llvm *llvm, draw_tcs_llvm_generate(llvm, variant); +#if GALLIVM_USE_ORCJIT == 1 + gallivm_compile_module(variant->gallivm); + variant->jit_func = (draw_tcs_jit_func) + gallivm_jit_function(variant->gallivm, variant->function_name); +#else gallivm_compile_module(variant->gallivm); variant->jit_func = (draw_tcs_jit_func) gallivm_jit_function(variant->gallivm, variant->function); +#endif if (needs_caching) llvm->draw->disk_cache_insert_shader(llvm->draw->disk_cache_cookie, @@ -3225,6 +3296,10 @@ draw_tcs_llvm_destroy_variant(struct draw_tcs_llvm_variant *variant) variant->shader->variants_cached--; list_del(&variant->list_item_global.list); llvm->nr_tcs_variants--; +#if GALLIVM_USE_ORCJIT == 1 + if(variant->function_name) + FREE(variant->function_name); +#endif FREE(variant); } @@ -3507,6 +3582,10 @@ draw_tes_llvm_generate(struct draw_llvm *llvm, variant_func = LLVMAddFunction(gallivm->module, func_name, func_type); variant->function = variant_func; +#if GALLIVM_USE_ORCJIT == 1 + variant->function_name = MALLOC(strlen(func_name)+1); + strcpy(variant->function_name, func_name); +#endif LLVMSetFunctionCallConv(variant_func, LLVMCCallConv); for (i = 0; i < ARRAY_SIZE(arg_types); ++i) @@ -3697,7 +3776,11 @@ draw_tes_llvm_create_variant(struct draw_llvm *llvm, if (!cached.data_size) needs_caching = true; } +#if GALLIVM_USE_ORCJIT == 1 + variant->gallivm = gallivm_create(module_name, llvm->_ts_context, &cached); +#else variant->gallivm = gallivm_create(module_name, llvm->context, &cached); +#endif create_tes_jit_types(variant); @@ -3713,8 +3796,13 @@ draw_tes_llvm_create_variant(struct draw_llvm *llvm, gallivm_compile_module(variant->gallivm); +#if GALLIVM_USE_ORCJIT == 1 + variant->jit_func = (draw_tes_jit_func) + gallivm_jit_function(variant->gallivm, variant->function_name); +#else variant->jit_func = (draw_tes_jit_func) gallivm_jit_function(variant->gallivm, variant->function); +#endif if (needs_caching) llvm->draw->disk_cache_insert_shader(llvm->draw->disk_cache_cookie, @@ -3747,6 +3835,10 @@ draw_tes_llvm_destroy_variant(struct draw_tes_llvm_variant *variant) variant->shader->variants_cached--; list_del(&variant->list_item_global.list); llvm->nr_tes_variants--; +#if GALLIVM_USE_ORCJIT == 1 + if(variant->function_name) + FREE(variant->function_name); +#endif FREE(variant); } diff --git a/src/gallium/auxiliary/draw/draw_llvm.h b/src/gallium/auxiliary/draw/draw_llvm.h index 65199feab1b..c2d7e49089e 100644 --- a/src/gallium/auxiliary/draw/draw_llvm.h +++ b/src/gallium/auxiliary/draw/draw_llvm.h @@ -42,6 +42,11 @@ #include "pipe/p_context.h" #include "util/list.h" +#define GALLIVM_USE_ORCJIT 1 + +#if GALLIVM_USE_ORCJIT == 1 +#include <llvm-c/Orc.h> +#endif struct draw_llvm; struct llvm_vertex_shader; @@ -398,6 +403,9 @@ struct draw_llvm_variant LLVMTypeRef vertex_header_ptr_type; LLVMValueRef function; +#if GALLIVM_USE_ORCJIT == 1 + char *function_name; +#endif draw_jit_vert_func jit_func; struct llvm_vertex_shader *shader; @@ -431,6 +439,9 @@ struct draw_gs_llvm_variant LLVMValueRef io_ptr; LLVMValueRef num_prims; LLVMValueRef function; +#if GALLIVM_USE_ORCJIT == 1 + char *function_name; +#endif draw_gs_jit_func jit_func; struct llvm_geometry_shader *shader; @@ -457,6 +468,9 @@ struct draw_tcs_llvm_variant /* LLVMValueRef io_ptr; */ LLVMValueRef num_prims; LLVMValueRef function; +#if GALLIVM_USE_ORCJIT == 1 + char *function_name; +#endif draw_tcs_jit_func jit_func; struct llvm_tess_ctrl_shader *shader; @@ -487,6 +501,9 @@ struct draw_tes_llvm_variant LLVMValueRef io_ptr; LLVMValueRef num_prims; LLVMValueRef function; +#if GALLIVM_USE_ORCJIT == 1 + char *function_name; +#endif draw_tes_jit_func jit_func; struct llvm_tess_eval_shader *shader; @@ -539,6 +556,9 @@ struct draw_llvm { struct draw_context *draw; LLVMContextRef context; +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcThreadSafeContextRef _ts_context; +#endif bool context_owned; struct draw_vs_jit_context vs_jit_context; @@ -584,8 +604,13 @@ llvm_tess_eval_shader(struct draw_tess_eval_shader *tes) return (struct llvm_tess_eval_shader *)tes; } +#if GALLIVM_USE_ORCJIT == 1 struct draw_llvm * -draw_llvm_create(struct draw_context *draw, LLVMContextRef llvm_context); +draw_llvm_create(struct draw_context *draw, LLVMOrcThreadSafeContextRef context); +#else +struct draw_llvm * +draw_llvm_create(struct draw_context *draw, LLVMContextRef context); +#endif void draw_llvm_destroy(struct draw_llvm *llvm); diff --git a/src/gallium/auxiliary/gallivm/lp_bld_coro.c b/src/gallium/auxiliary/gallivm/lp_bld_coro.c index 0214dcf6742..f5a2b31b6d9 100644 --- a/src/gallium/auxiliary/gallivm/lp_bld_coro.c +++ b/src/gallium/auxiliary/gallivm/lp_bld_coro.c @@ -156,12 +156,14 @@ coro_free(char *ptr) void lp_build_coro_add_malloc_hooks(struct gallivm_state *gallivm) { +#if GALLIVM_USE_ORCJIT == 0 assert(gallivm->engine); +#endif assert(gallivm->coro_malloc_hook); assert(gallivm->coro_free_hook); - LLVMAddGlobalMapping(gallivm->engine, gallivm->coro_malloc_hook, coro_malloc); - LLVMAddGlobalMapping(gallivm->engine, gallivm->coro_free_hook, coro_free); + gallivm_add_global_mapping(gallivm, gallivm->coro_malloc_hook, coro_malloc); + gallivm_add_global_mapping(gallivm, gallivm->coro_free_hook, coro_free); } void lp_build_coro_declare_malloc_hooks(struct gallivm_state *gallivm) diff --git a/src/gallium/auxiliary/gallivm/lp_bld_coro.h b/src/gallium/auxiliary/gallivm/lp_bld_coro.h index 2fbaecc3152..4ee6ea54793 100644 --- a/src/gallium/auxiliary/gallivm/lp_bld_coro.h +++ b/src/gallium/auxiliary/gallivm/lp_bld_coro.h @@ -26,6 +26,10 @@ #ifndef LP_BLD_CORO_H #define LP_BLD_CORO_H +#ifdef __cplusplus +extern "C" { +#endif + #include <stdbool.h> #include "util/compiler.h" #include "gallivm/lp_bld.h" @@ -84,4 +88,8 @@ static inline void lp_build_coro_add_presplit(LLVMValueRef coro) #endif } +#ifdef __cplusplus +} +#endif + #endif diff --git a/src/gallium/auxiliary/gallivm/lp_bld_init.c b/src/gallium/auxiliary/gallivm/lp_bld_init.c index cd2108f3a08..fc025441e97 100644 --- a/src/gallium/auxiliary/gallivm/lp_bld_init.c +++ b/src/gallium/auxiliary/gallivm/lp_bld_init.c @@ -540,6 +540,12 @@ gallivm_verify_function(struct gallivm_state *gallivm, } } +void +gallivm_add_global_mapping(struct gallivm_state *gallivm, LLVMValueRef sym, void* addr) +{ + LLVMAddGlobalMapping(gallivm->engine, sym, addr); +} + void lp_init_clock_hook(struct gallivm_state *gallivm) { if (gallivm->get_time_hook) @@ -670,7 +676,8 @@ gallivm_compile_module(struct gallivm_state *gallivm) ++gallivm->compiled; lp_init_printf_hook(gallivm); - LLVMAddGlobalMapping(gallivm->engine, gallivm->debug_printf_hook, debug_printf); + if (gallivm->debug_printf_hook) + gallivm_add_global_mapping(gallivm, gallivm->debug_printf_hook, debug_printf); lp_init_clock_hook(gallivm); LLVMAddGlobalMapping(gallivm->engine, gallivm->get_time_hook, os_time_get_nano); diff --git a/src/gallium/auxiliary/gallivm/lp_bld_init.h b/src/gallium/auxiliary/gallivm/lp_bld_init.h index 418921cb7ad..294d6085e6d 100644 --- a/src/gallium/auxiliary/gallivm/lp_bld_init.h +++ b/src/gallium/auxiliary/gallivm/lp_bld_init.h @@ -29,11 +29,16 @@ #ifndef LP_BLD_INIT_H #define LP_BLD_INIT_H +#define GALLIVM_USE_ORCJIT 1 #include "util/compiler.h" #include "util/u_pointer.h" // for func_pointer #include "lp_bld.h" +#if GALLIVM_USE_ORCJIT == 1 +#include <llvm-c/Orc.h> +#else #include <llvm-c/ExecutionEngine.h> +#endif #ifdef __cplusplus extern "C" { @@ -44,18 +49,27 @@ struct gallivm_state { char *module_name; LLVMModuleRef module; - LLVMExecutionEngineRef engine; LLVMTargetDataRef target; +#if GALLIVM_USE_ORCJIT == 1 + /* own this->module */ + LLVMOrcThreadSafeContextRef _ts_context; + /* each module is in its own jitdylib */ + LLVMOrcJITDylibRef _per_module_jd; +#else + LLVMExecutionEngineRef engine; #if GALLIVM_USE_NEW_PASS == 0 LLVMPassManagerRef passmgr; #if GALLIVM_HAVE_CORO == 1 LLVMPassManagerRef cgpassmgr; #endif #endif - LLVMContextRef context; - LLVMBuilderRef builder; LLVMMCJITMemoryManagerRef memorymgr; struct lp_generated_code *code; +#endif + + LLVMContextRef context; + LLVMBuilderRef builder; + struct lp_cached_code *cache; unsigned compiled; LLVMValueRef coro_malloc_hook; @@ -77,10 +91,15 @@ lp_build_init_native_width(void); bool lp_build_init(void); - +#if GALLIVM_USE_ORCJIT == 1 +struct gallivm_state * +gallivm_create(const char *name, LLVMOrcThreadSafeContextRef context, + struct lp_cached_code *cache); +#else struct gallivm_state * gallivm_create(const char *name, LLVMContextRef context, struct lp_cached_code *cache); +#endif void gallivm_destroy(struct gallivm_state *gallivm); @@ -93,11 +112,25 @@ gallivm_verify_function(struct gallivm_state *gallivm, LLVMValueRef func); void +gallivm_add_global_mapping(struct gallivm_state *gallivm, LLVMValueRef sym, void* addr); + +/** + * for ORCJIT, after this function gets called, all access and modification to + * module and any structure associated to it should be avoided, + * as module has been moved into ORCJIT and may be recycled + */ +void gallivm_compile_module(struct gallivm_state *gallivm); +#if GALLIVM_USE_ORCJIT == 1 +func_pointer +gallivm_jit_function(struct gallivm_state *gallivm, + const char *func_name); +#else func_pointer gallivm_jit_function(struct gallivm_state *gallivm, LLVMValueRef func); +#endif unsigned gallivm_get_perf_flags(void); diff --git a/src/gallium/auxiliary/gallivm/lp_bld_init_orc.cpp b/src/gallium/auxiliary/gallivm/lp_bld_init_orc.cpp new file mode 100644 index 00000000000..45673e8633a --- /dev/null +++ b/src/gallium/auxiliary/gallivm/lp_bld_init_orc.cpp @@ -0,0 +1,892 @@ +#include "util/detect.h" +#include "util/compiler.h" +#include "util/macros.h" +#include "util/u_cpu_detect.h" +#include "util/u_debug.h" +#include "util/os_time.h" +#include "lp_bld.h" +#include "lp_bld_debug.h" +#include "lp_bld_init.h" +#include "lp_bld_coro.h" +#include "lp_bld_printf.h" + +#include <llvm/Config/llvm-config.h> +#include <llvm-c/Core.h> +#include <llvm-c/Orc.h> +#include <llvm-c/LLJIT.h> +#include <llvm-c/TargetMachine.h> +#include <llvm-c/Support.h> + +#include <llvm-c/Analysis.h> +#include <llvm-c/BitWriter.h> +#if GALLIVM_USE_NEW_PASS == 1 +#include <llvm-c/Transforms/PassBuilder.h> +#elif GALLIVM_HAVE_CORO == 1 +#include <llvm-c/Transforms/Scalar.h> +#if LLVM_VERSION_MAJOR >= 7 +#include <llvm-c/Transforms/Utils.h> +#endif +#if LLVM_VERSION_MAJOR <= 8 && (DETECT_ARCH_AARCH64 || DETECT_ARCH_ARM || DETECT_ARCH_S390 || DETECT_ARCH_MIPS64) +#include <llvm-c/Transforms/IPO.h> +#endif +#include <llvm-c/Transforms/Coroutines.h> +#endif + +#include <llvm/ADT/StringMap.h> +#include <llvm/ExecutionEngine/Orc/LLJIT.h> +#include <llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h> +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include <llvm/Target/TargetMachine.h> +#include <llvm/Support/TargetSelect.h> +#include <llvm/Support/Casting.h> +#include <llvm/TargetParser/Host.h> +#include <llvm/Support/CBindingWrapping.h> +#if LLVM_USE_INTEL_JITEVENTS +#include <llvm/ExecutionEngine/JITEventListener.h> +#endif + +/* conflict with ObjectLinkingLayer.h */ +#include "util/u_memory.h" + +#if (defined(_WIN32) && LLVM_VERSION_MAJOR >= 15) +/* use ObjectLinkingLayer (JITLINK backend) */ +#define USE_JITLINK +#endif +/* else use old RTDyldObjectLinkingLayer (RuntimeDyld backend) */ + +unsigned lp_native_vector_width; + +unsigned gallivm_perf = 0; + +static const struct debug_named_value lp_bld_perf_flags[] = { + { "brilinear", GALLIVM_PERF_BRILINEAR, "enable brilinear optimization" }, + { "rho_approx", GALLIVM_PERF_RHO_APPROX, "enable rho_approx optimization" }, + { "no_quad_lod", GALLIVM_PERF_NO_QUAD_LOD, "disable quad_lod optimization" }, + { "no_aos_sampling", GALLIVM_PERF_NO_AOS_SAMPLING, "disable aos sampling optimization" }, + { "nopt", GALLIVM_PERF_NO_OPT, "disable optimization passes to speed up shader compilation" }, + DEBUG_NAMED_VALUE_END +}; + +unsigned gallivm_debug = 0; + +static const struct debug_named_value lp_bld_debug_flags[] = { + { "tgsi", GALLIVM_DEBUG_TGSI, NULL }, + { "ir", GALLIVM_DEBUG_IR, NULL }, + { "asm", GALLIVM_DEBUG_ASM, NULL }, + { "perf", GALLIVM_DEBUG_PERF, NULL }, + { "gc", GALLIVM_DEBUG_GC, NULL }, +/* Don't allow setting DUMP_BC for release builds, since writing the files may be an issue with setuid. */ +#ifdef DEBUG + { "dumpbc", GALLIVM_DEBUG_DUMP_BC, NULL }, +#endif + DEBUG_NAMED_VALUE_END +}; + +DEBUG_GET_ONCE_FLAGS_OPTION(gallivm_debug, "GALLIVM_DEBUG", lp_bld_debug_flags, 0) + +struct lp_cached_code { + void *data; + size_t data_size; + bool dont_cache; + void *jit_obj_cache; +}; + +namespace { + +class LLVMEnsureMultithreaded { +public: + LLVMEnsureMultithreaded() + { + if (!LLVMIsMultithreaded()) { + LLVMStartMultithreaded(); + } + } +}; + +LLVMEnsureMultithreaded lLVMEnsureMultithreaded; + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(llvm::orc::ThreadSafeContext, + LLVMOrcThreadSafeContextRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(llvm::orc::IRTransformLayer, + LLVMOrcIRTransformLayerRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(llvm::orc::JITDylib, LLVMOrcJITDylibRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(llvm::orc::JITTargetMachineBuilder, + LLVMOrcJITTargetMachineBuilderRef) +LLVMTargetMachineRef wrap(const llvm::TargetMachine *P) { + return reinterpret_cast<LLVMTargetMachineRef>(const_cast<llvm::TargetMachine*>(P)); +} + +llvm::ExitOnError ExitOnErr; + +inline const char* get_module_name(LLVMModuleRef mod) { + using llvm::Module; + return llvm::unwrap(mod)->getModuleIdentifier().c_str(); +} + +once_flag init_lpjit_once_flag = ONCE_FLAG_INIT; + +/* A JIT singleton built upon LLJIT */ +class LPJit +{ +public: + static LPJit* get_instance() { + call_once(&init_lpjit_once_flag, init_lpjit); + return jit; + } + + gallivm_state *find_gallivm_state(LLVMModuleRef mod) { +#if DEBUG + using llvm::Module; + auto I = gallivm_modules.find(llvm::unwrap(mod)->getModuleIdentifier()); + if (I == gallivm_modules.end()) { + debug_printf("No gallivm state found for module: %s", get_module_name(mod)); + return NULL; + } + return I->second; +#endif + return NULL; + } + + static char *get_unique_name(const char* name) { + LPJit* jit = get_instance(); + size_t size = name == NULL? 16: strlen(name) + 16; + char *name_uniq = (char *)MALLOC(size); + if (!name_uniq) { + return NULL; + } + do { + snprintf(name_uniq, size, "%s_%u", name, jit->jit_dylib_count++); + } while(jit->lljit->getExecutionSession().getJITDylibByName(name_uniq)); + return name_uniq; + } + + static LLVMOrcJITDylibRef create_jit_dylib(const char * name) { + using llvm::orc::JITDylib; + LPJit* jit = get_instance(); + JITDylib& tmp = ExitOnErr(jit->lljit->createJITDylib(name)); + return wrap(&tmp); + } + + static void register_gallivm_state(gallivm_state *gallivm) { +#if DEBUG + LPJit* jit = get_instance(); + jit->gallivm_modules[gallivm->module_name] = gallivm; +#endif + } + + static void deregister_gallivm_state(gallivm_state *gallivm) { +#if DEBUG + LPJit* jit = get_instance(); + (void)jit->gallivm_modules.erase(gallivm->module_name); +#endif + } + + static void add_ir_module_to_jd( + LLVMOrcThreadSafeContextRef ts_context, + LLVMModuleRef mod, + LLVMOrcJITDylibRef jd) { + using llvm::Module; + using llvm::orc::ThreadSafeModule; + using llvm::orc::JITDylib; + ThreadSafeModule tsm( + std::unique_ptr<Module>(llvm::unwrap(mod)), *::unwrap(ts_context)); + ExitOnErr(get_instance()->lljit->addIRModule( + *::unwrap(jd), std::move(tsm) + )); + } + + static void add_mapping_to_jd( + LLVMValueRef sym, + void *addr, + LLVMOrcJITDylibRef jd) { +#if LLVM_VERSION_MAJOR >= 17 + using llvm::orc::ExecutorAddr; + using llvm::JITSymbolFlags; +#else + using llvm::JITEvaluatedSymbol; +#endif + using llvm::orc::ExecutionSession; + using llvm::orc::JITDylib; + using llvm::orc::SymbolMap; + JITDylib* JD = ::unwrap(jd); + auto& es = LPJit::get_instance()->lljit->getExecutionSession(); + auto name = es.intern(llvm::unwrap(sym)->getName()); + SymbolMap map(1); +#if LLVM_VERSION_MAJOR >= 17 + map[name] = {ExecutorAddr::fromPtr(addr), + JITSymbolFlags::Exported}; +#else + map[name] = JITEvaluatedSymbol::fromPointer(addr); +#endif + auto munit = llvm::orc::absoluteSymbols(map); + llvm::cantFail(JD->define(std::move(munit))); + } + + static void *lookup_in_jd( + const char *func_name, + LLVMOrcJITDylibRef jd) { + using llvm::orc::JITDylib; + using llvm::JITEvaluatedSymbol; + using llvm::orc::ExecutorAddr; + JITDylib* JD = ::unwrap(jd); + auto func = ExitOnErr(LPJit::get_instance()->lljit->lookup(*JD, func_name)); +#if LLVM_VERSION_MAJOR >= 15 + return func.toPtr<void *>(); +#else + return (void *)(func.getAddress()); +#endif + } + + static void remove_jd(LLVMOrcJITDylibRef jd) { + using llvm::orc::ExecutionSession; + using llvm::orc::JITDylib; + auto& es = LPJit::get_instance()->lljit->getExecutionSession(); + ExitOnErr(es.removeJITDylib(* ::unwrap(jd))); + } + + LLVMTargetMachineRef tm; + +private: + LPJit(); + ~LPJit() = default; + LPJit(const LPJit&) = delete; + LPJit& operator=(const LPJit&) = delete; + + static void init_native_targets(); + llvm::orc::JITTargetMachineBuilder create_jtdb(); + + static void init_lpjit() { + jit = new LPJit; + } + static LPJit* jit; + + std::unique_ptr<llvm::orc::LLJIT> lljit; + /* avoid name conflict */ + unsigned jit_dylib_count; + +#if DEBUG + /* map from module name to gallivm_state */ + llvm::StringMap<gallivm_state *> gallivm_modules; +#endif +}; + +LLVMErrorRef module_transform(void *Ctx, LLVMModuleRef mod) { + int64_t time_begin = 0; + if (gallivm_debug & GALLIVM_DEBUG_PERF) + time_begin = os_time_get(); + +#if GALLIVM_USE_NEW_PASS == 1 + char passes[1024]; + passes[0] = 0; + LLVMTargetMachineRef tm = LPJit::get_instance()->tm; + + /* + * there should be some way to combine these two pass runs but I'm not seeing it, + * at the time of writing. + */ + strcpy(passes, "default<O0>"); + + LLVMPassBuilderOptionsRef opts = LLVMCreatePassBuilderOptions(); + LLVMRunPasses(mod, passes, tm, opts); + + if (!(gallivm_perf & GALLIVM_PERF_NO_OPT)) + strcpy(passes, "sroa,early-cse,simplifycfg,reassociate,mem2reg,instsimplify,instcombine"); + else + strcpy(passes, "mem2reg"); + + LLVMRunPasses(mod, passes, tm, opts); + LLVMDisposePassBuilderOptions(opts); + +#else /* GALLIVM_USE_NEW_PASS */ + LLVMPassManagerRef passmgr = LLVMCreateFunctionPassManagerForModule(mod); + +#if GALLIVM_HAVE_CORO == 1 + LLVMPassManagerRef cgpassmgr = LLVMCreatePassManager(); +#endif + +#if GALLIVM_HAVE_CORO == 1 +#if LLVM_VERSION_MAJOR <= 8 && (DETECT_ARCH_AARCH64 || DETECT_ARCH_ARM || DETECT_ARCH_S390 || DETECT_ARCH_MIPS64) + LLVMAddArgumentPromotionPass(cgpassmgr); + LLVMAddFunctionAttrsPass(cgpassmgr); +#endif + LLVMAddCoroEarlyPass(cgpassmgr); + LLVMAddCoroSplitPass(cgpassmgr); + LLVMAddCoroElidePass(cgpassmgr); +#endif + + if ((gallivm_perf & GALLIVM_PERF_NO_OPT) == 0) { + /* + * TODO: Evaluate passes some more - keeping in mind + * both quality of generated code and compile times. + */ + /* + * NOTE: if you change this, don't forget to change the output + * with GALLIVM_DEBUG_DUMP_BC in gallivm_compile_module. + */ + LLVMAddScalarReplAggregatesPass(passmgr); + LLVMAddEarlyCSEPass(passmgr); + LLVMAddCFGSimplificationPass(passmgr); + /* + * FIXME: LICM is potentially quite useful. However, for some + * rather crazy shaders the compile time can reach _hours_ per shader, + * due to licm implying lcssa (since llvm 3.5), which can take forever. + * Even for sane shaders, the cost of licm is rather high (and not just + * due to lcssa, licm itself too), though mostly only in cases when it + * can actually move things, so having to disable it is a pity. + * LLVMAddLICMPass(passmgr); + */ + LLVMAddReassociatePass(passmgr); + LLVMAddPromoteMemoryToRegisterPass(passmgr); +#if LLVM_VERSION_MAJOR <= 11 + LLVMAddConstantPropagationPass(passmgr); +#else + LLVMAddInstructionSimplifyPass(passmgr); +#endif + LLVMAddInstructionCombiningPass(passmgr); + LLVMAddGVNPass(passmgr); + } + else { + /* We need at least this pass to prevent the backends to fail in + * unexpected ways. + */ + LLVMAddPromoteMemoryToRegisterPass(passmgr); + } +#if GALLIVM_HAVE_CORO == 1 + LLVMAddCoroCleanupPass(passmgr); + + LLVMRunPassManager(cgpassmgr, mod); +#endif + /* Run optimization passes */ + LLVMInitializeFunctionPassManager(passmgr); + LLVMValueRef func; + func = LLVMGetFirstFunction(mod); + while (func) { + if (0) { + debug_printf("optimizing func %s...\n", LLVMGetValueName(func)); + } + + /* Disable frame pointer omission on debug/profile builds */ + /* XXX: And workaround http://llvm.org/PR21435 */ +#if defined(DEBUG) || defined(PROFILE) || DETECT_ARCH_X86 || DETECT_ARCH_X86_64 + LLVMAddTargetDependentFunctionAttr(func, "no-frame-pointer-elim", "true"); + LLVMAddTargetDependentFunctionAttr(func, "no-frame-pointer-elim-non-leaf", "true"); +#endif + + LLVMRunFunctionPassManager(passmgr, func); + func = LLVMGetNextFunction(func); + } + LLVMFinalizeFunctionPassManager(passmgr); +#if GALLIVM_HAVE_CORO == 1 + LLVMDisposePassManager(cgpassmgr); +#endif + LLVMDisposePassManager(passmgr); +#endif /* GALLIVM_USE_NEW_PASS */ + if (gallivm_debug & GALLIVM_DEBUG_PERF) { + int64_t time_end = os_time_get(); + int time_msec = (int)((time_end - time_begin) / 1000); + + const char *module_name = get_module_name(mod); + debug_printf("optimizing module %s took %d msec\n", + module_name, time_msec); + } + + return LLVMErrorSuccess; +} + +LLVMErrorRef module_transform_wrapper( + void *Ctx, LLVMOrcThreadSafeModuleRef *ModInOut, + LLVMOrcMaterializationResponsibilityRef MR) { + return LLVMOrcThreadSafeModuleWithModuleDo(*ModInOut, *module_transform, Ctx); +} + +LPJit* LPJit::jit = NULL; + +LPJit::LPJit() :jit_dylib_count(0) { + using namespace llvm::orc; + gallivm_debug = debug_get_option_gallivm_debug(); + + gallivm_perf = debug_get_flags_option("GALLIVM_PERF", lp_bld_perf_flags, 0 ); + + init_native_targets(); + JITTargetMachineBuilder JTMB = create_jtdb(); + tm = wrap(ExitOnErr(JTMB.createTargetMachine()).release()); + + /* Create an LLJIT instance with an ObjectLinkingLayer (JITLINK) + * or RuntimeDyld as the base layer. + * intel & perf listeners are not supported by ObjectLinkingLayer yet + */ + lljit = ExitOnErr( + LLJITBuilder() + .setJITTargetMachineBuilder(std::move(JTMB)) +#ifdef USE_JITLINK + .setObjectLinkingLayerCreator( + [&](ExecutionSession &ES, const llvm::Triple &TT) { + return std::make_unique<ObjectLinkingLayer>( + ES, ExitOnErr(llvm::jitlink::InProcessMemoryManager::Create())); + }) +#else +#if LLVM_USE_INTEL_JITEVENTS + .RegisterJITEventListener( + llvm::JITEventListener::createIntelJITEventListener()) +#endif +#endif + .create()); + + LLVMOrcIRTransformLayerRef TL = wrap(&lljit->getIRTransformLayer()); + LLVMOrcIRTransformLayerSetTransform(TL, *module_transform_wrapper, NULL); +} + +void LPJit::init_native_targets() { + // If we have a native target, initialize it to ensure it is linked in and + // usable by the JIT. + llvm::InitializeNativeTarget(); + + llvm::InitializeNativeTargetAsmPrinter(); + + llvm::InitializeNativeTargetDisassembler(); +#if DEBUG + { + char *env_llc_options = getenv("GALLIVM_LLC_OPTIONS"); + if (env_llc_options) { + char *option; + char *options[64] = {(char *) "llc"}; // Warning without cast + int n; + for (n = 0, option = strtok(env_llc_options, " "); option; n++, option = strtok(NULL, " ")) { + options[n + 1] = option; + } + if (gallivm_debug & (GALLIVM_DEBUG_IR | GALLIVM_DEBUG_ASM | GALLIVM_DEBUG_DUMP_BC)) { + debug_printf("llc additional options (%d):\n", n); + for (int i = 1; i <= n; i++) + debug_printf("\t%s\n", options[i]); + debug_printf("\n"); + } + LLVMParseCommandLineOptions(n + 1, options, NULL); + } + } +#endif + + if (util_get_cpu_caps()->has_avx2 || util_get_cpu_caps()->has_avx) { + ::lp_native_vector_width = 256; + } else { + /* Leave it at 128, even when no SIMD extensions are available. + * Really needs to be a multiple of 128 so can fit 4 floats. + */ + ::lp_native_vector_width = 128; + } + +#if DETECT_ARCH_PPC_64 + /* Set the NJ bit in VSCR to 0 so denormalized values are handled as + * specified by IEEE standard (PowerISA 2.06 - Section 6.3). This guarantees + * that some rounding and half-float to float handling does not round + * incorrectly to 0. + * XXX: should eventually follow same logic on all platforms. + * Right now denorms get explicitly disabled (but elsewhere) for x86, + * whereas ppc64 explicitly enables them... + */ + if (util_get_cpu_caps()->has_altivec) { + unsigned short mask[] = { 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, + 0xFFFF, 0xFFFF, 0xFFFE, 0xFFFF }; + __asm ( + "mfvscr %%v1\n" + "vand %0,%%v1,%0\n" + "mtvscr %0" + : + : "r" (*mask) + ); + } +#endif +} + +llvm::orc::JITTargetMachineBuilder LPJit::create_jtdb() { + using namespace llvm; + using orc::JITTargetMachineBuilder; + +#if defined(_WIN32) && LLVM_VERSION_MAJOR < 15 + /* + * JITLink works on Windows, but only through ELF object format. + * + * XXX: We could use `LLVM_HOST_TRIPLE "-elf"` but LLVM_HOST_TRIPLE has + * different strings for MinGW/MSVC, so better play it safe and be + * explicit. + */ +# ifdef _WIN64 + JITTargetMachineBuilder JTMB((Triple("x86_64-pc-win32-elf"))); +# else + JITTargetMachineBuilder JTMB((Triple("i686-pc-win32-elf"))); +# endif +#else + /* + * llvm::sys::getProcessTriple() is bogus. It returns the host LLVM was + * compiled on. Be careful when doing cross compilation + */ + JITTargetMachineBuilder JTMB((Triple(sys::getProcessTriple()))); +#endif + + TargetOptions options; + /** + * LLVM 3.1+ haven't more "extern unsigned llvm::StackAlignmentOverride" and + * friends for configuring code generation options, like stack alignment. + */ +#if DETECT_ARCH_X86 && LLVM_VERSION_MAJOR < 13 + options.StackAlignmentOverride = 4; +#endif + + JTMB.setOptions(options); + + std::vector<std::string> MAttrs; + +#if LLVM_VERSION_MAJOR >= 4 && (DETECT_ARCH_X86 || DETECT_ARCH_X86_64 || DETECT_ARCH_ARM) + /* llvm-3.3+ implements sys::getHostCPUFeatures for Arm + * and llvm-3.7+ for x86, which allows us to enable/disable + * code generation based on the results of cpuid on these + * architectures. + */ + StringMap<bool> features; + sys::getHostCPUFeatures(features); + + for (StringMapIterator<bool> f = features.begin(); + f != features.end(); + ++f) { + MAttrs.push_back(((*f).second ? "+" : "-") + (*f).first().str()); + } +#elif DETECT_ARCH_X86 || DETECT_ARCH_X86_64 + /* + * We need to unset attributes because sometimes LLVM mistakenly assumes + * certain features are present given the processor name. + * + * https://bugs.freedesktop.org/show_bug.cgi?id=92214 + * http://llvm.org/PR25021 + * http://llvm.org/PR19429 + * http://llvm.org/PR16721 + */ + MAttrs.push_back(util_get_cpu_caps()->has_sse ? "+sse" : "-sse" ); + MAttrs.push_back(util_get_cpu_caps()->has_sse2 ? "+sse2" : "-sse2" ); + MAttrs.push_back(util_get_cpu_caps()->has_sse3 ? "+sse3" : "-sse3" ); + MAttrs.push_back(util_get_cpu_caps()->has_ssse3 ? "+ssse3" : "-ssse3" ); + MAttrs.push_back(util_get_cpu_caps()->has_sse4_1 ? "+sse4.1" : "-sse4.1"); + MAttrs.push_back(util_get_cpu_caps()->has_sse4_2 ? "+sse4.2" : "-sse4.2"); + /* + * AVX feature is not automatically detected from CPUID by the X86 target + * yet, because the old (yet default) JIT engine is not capable of + * emitting the opcodes. On newer llvm versions it is and at least some + * versions (tested with 3.3) will emit avx opcodes without this anyway. + */ + MAttrs.push_back(util_get_cpu_caps()->has_avx ? "+avx" : "-avx"); + MAttrs.push_back(util_get_cpu_caps()->has_f16c ? "+f16c" : "-f16c"); + MAttrs.push_back(util_get_cpu_caps()->has_fma ? "+fma" : "-fma"); + MAttrs.push_back(util_get_cpu_caps()->has_avx2 ? "+avx2" : "-avx2"); + /* disable avx512 and all subvariants */ + MAttrs.push_back("-avx512cd"); + MAttrs.push_back("-avx512er"); + MAttrs.push_back("-avx512f"); + MAttrs.push_back("-avx512pf"); + MAttrs.push_back("-avx512bw"); + MAttrs.push_back("-avx512dq"); + MAttrs.push_back("-avx512vl"); +#endif +#if DETECT_ARCH_ARM + if (!util_get_cpu_caps()->has_neon) { + MAttrs.push_back("-neon"); + MAttrs.push_back("-crypto"); + MAttrs.push_back("-vfp2"); + } +#endif + +#if DETECT_ARCH_PPC + MAttrs.push_back(util_get_cpu_caps()->has_altivec ? "+altivec" : "-altivec"); +#if (LLVM_VERSION_MAJOR < 4) + /* + * Make sure VSX instructions are disabled + * See LLVM bugs: + * https://llvm.org/bugs/show_bug.cgi?id=25503#c7 (fixed in 3.8.1) + * https://llvm.org/bugs/show_bug.cgi?id=26775 (fixed in 3.8.1) + * https://llvm.org/bugs/show_bug.cgi?id=33531 (fixed in 4.0) + * https://llvm.org/bugs/show_bug.cgi?id=34647 (llc performance on certain unusual shader IR; intro'd in 4.0, pending as of 5.0) + */ + if (util_get_cpu_caps()->has_altivec) { + MAttrs.push_back("-vsx"); + } +#else + /* + * Bug 25503 is fixed, by the same fix that fixed + * bug 26775, in versions of LLVM later than 3.8 (starting with 3.8.1). + * BZ 33531 actually comprises more than one bug, all of + * which are fixed in LLVM 4.0. + * + * With LLVM 4.0 or higher: + * Make sure VSX instructions are ENABLED (if supported), unless + * VSX instructions are explicitly enabled/disabled via GALLIVM_VSX=1 or 0. + */ + if (util_get_cpu_caps()->has_altivec) { + MAttrs.push_back(util_get_cpu_caps()->has_vsx ? "+vsx" : "-vsx"); + } +#endif +#endif + +#if DETECT_ARCH_MIPS64 + MAttrs.push_back(util_get_cpu_caps()->has_msa ? "+msa" : "-msa"); + /* MSA requires a 64-bit FPU register file */ + MAttrs.push_back("+fp64"); +#endif + + JTMB.addFeatures(MAttrs); + + if (gallivm_debug & (GALLIVM_DEBUG_IR | GALLIVM_DEBUG_ASM | GALLIVM_DEBUG_DUMP_BC)) { + int n = MAttrs.size(); + if (n > 0) { + debug_printf("llc -mattr option(s): "); + for (int i = 0; i < n; i++) + debug_printf("%s%s", MAttrs[i].c_str(), (i < n - 1) ? "," : ""); + debug_printf("\n"); + } + } + + std::string MCPU = llvm::sys::getHostCPUName().str(); + /* + * Note that the MAttrs set above will be sort of ignored (since we should + * not set any which would not be set by specifying the cpu anyway). + * It ought to be safe though since getHostCPUName() should include bits + * not only from the cpu but environment as well (for instance if it's safe + * to use avx instructions which need OS support). According to + * http://llvm.org/bugs/show_bug.cgi?id=19429 however if I understand this + * right it may be necessary to specify older cpu (or disable mattrs) though + * when not using MCJIT so no instructions are generated which the old JIT + * can't handle. Not entirely sure if we really need to do anything yet. + * + * Not sure if the above is also the case for ORCJIT, but we need set CPU + * manually since we don't use JITTargetMachineBuilder::detectHost() + */ + +#if DETECT_ARCH_PPC_64 + /* + * Large programs, e.g. gnome-shell and firefox, may tax the addressability + * of the Medium code model once dynamically generated JIT-compiled shader + * programs are linked in and relocated. Yet the default code model as of + * LLVM 8 is Medium or even Small. + * The cost of changing from Medium to Large is negligible: + * - an additional 8-byte pointer stored immediately before the shader entrypoint; + * - change an add-immediate (addis) instruction to a load (ld). + */ + JTMB.setCodeModel(CodeModel::Large); + +#if UTIL_ARCH_LITTLE_ENDIAN + /* + * Versions of LLVM prior to 4.0 lacked a table entry for "POWER8NVL", + * resulting in (big-endian) "generic" being returned on + * little-endian Power8NVL systems. The result was that code that + * attempted to load the least significant 32 bits of a 64-bit quantity + * from memory loaded the wrong half. This resulted in failures in some + * Piglit tests, e.g. + * .../arb_gpu_shader_fp64/execution/conversion/frag-conversion-explicit-double-uint + */ + if (MCPU == "generic") + MCPU = "pwr8"; +#endif +#endif + +#if DETECT_ARCH_MIPS64 + /* + * ls3a4000 CPU and ls2k1000 SoC is a mips64r5 compatible with MSA SIMD + * instruction set implemented, while ls3a3000 is mips64r2 compatible + * only. getHostCPUName() return "generic" on all loongson + * mips CPU currently. So we override the MCPU to mips64r5 if MSA is + * implemented, feedback to mips64r2 for all other ordinary mips64 cpu. + */ + if (MCPU == "generic") + MCPU = util_get_cpu_caps()->has_msa ? "mips64r5" : "mips64r2"; +#endif + + JTMB.setCPU(MCPU); + if (gallivm_debug & (GALLIVM_DEBUG_IR | GALLIVM_DEBUG_ASM | GALLIVM_DEBUG_DUMP_BC)) { + debug_printf("llc -mcpu option: %s\n", MCPU.c_str()); + } + + return JTMB; +} + + +} /* Anonymous namespace */ + +unsigned +lp_build_init_native_width(void) +{ + // Default to 256 until we're confident llvmpipe with 512 is as correct and not slower than 256 + lp_native_vector_width = MIN2(util_get_cpu_caps()->max_vector_bits, 256); + assert(lp_native_vector_width); + + lp_native_vector_width = debug_get_num_option("LP_NATIVE_VECTOR_WIDTH", lp_native_vector_width); + assert(lp_native_vector_width); + + return lp_native_vector_width; +} + +bool +lp_build_init(void) +{ + lp_build_init_native_width(); + (void)LPJit::get_instance(); + return true; +} + +bool +init_gallivm_state(struct gallivm_state *gallivm, const char *name, + LLVMOrcThreadSafeContextRef context, struct lp_cached_code *cache) +{ + assert(!gallivm->context); + assert(!gallivm->_ts_context); + assert(!gallivm->module); + + if (!lp_build_init()) + return false; + + // cache is not implemented + gallivm->cache = cache; + if (gallivm->cache) + gallivm->cache->data_size = 0; + + gallivm->_ts_context = context; + gallivm->context = LLVMOrcThreadSafeContextGetContext(context); + + gallivm->module_name = LPJit::get_unique_name(name); + gallivm->module = LLVMModuleCreateWithNameInContext(gallivm->module_name, + gallivm->context); +#if DETECT_ARCH_X86 + lp_set_module_stack_alignment_override(gallivm->module, 4); +#endif + gallivm->builder = LLVMCreateBuilderInContext(gallivm->context); + gallivm->_per_module_jd = LPJit::create_jit_dylib(gallivm->module_name); + + gallivm->target = LLVMCreateTargetDataLayout(LPJit::get_instance()->tm); + + lp_build_coro_declare_malloc_hooks(gallivm); + + return true; +} + +struct gallivm_state * +gallivm_create(const char *name, LLVMOrcThreadSafeContextRef context, + struct lp_cached_code *cache){ + struct gallivm_state *gallivm; + + gallivm = CALLOC_STRUCT(gallivm_state); + if (gallivm) { + if (!init_gallivm_state(gallivm, name, context, cache)) { + FREE(gallivm); + gallivm = NULL; + } + } + + assert(gallivm != NULL); + return gallivm; +} + +void +gallivm_destroy(struct gallivm_state *gallivm) +{ + LPJit::remove_jd(gallivm->_per_module_jd); + gallivm->_per_module_jd = nullptr; +} + +void +gallivm_free_ir(struct gallivm_state *gallivm) +{ + if (gallivm->module) + LLVMDisposeModule(gallivm->module); + FREE(gallivm->module_name); + + if (gallivm->target) { + LLVMDisposeTargetData(gallivm->target); + } + + if (gallivm->builder) + LLVMDisposeBuilder(gallivm->builder); + + gallivm->target = NULL; + gallivm->module=NULL; + gallivm->module_name=NULL; + gallivm->builder=NULL; + gallivm->context=NULL; + gallivm->_ts_context=NULL; + gallivm->cache=NULL; + LPJit::deregister_gallivm_state(gallivm); +} + +void +gallivm_verify_function(struct gallivm_state *gallivm, + LLVMValueRef func) +{ + /* Verify the LLVM IR. If invalid, dump and abort */ +#ifdef DEBUG + if (LLVMVerifyFunction(func, LLVMPrintMessageAction)) { + lp_debug_dump_value(func); + assert(0); + return; + } +#endif + + if (gallivm_debug & GALLIVM_DEBUG_IR) { + /* Print the LLVM IR to stderr */ + lp_debug_dump_value(func); + debug_printf("\n"); + } + return; +} + +void +gallivm_add_global_mapping(struct gallivm_state *gallivm, LLVMValueRef sym, void* addr) +{ + LPJit::add_mapping_to_jd(sym, addr, gallivm->_per_module_jd); +} + +void lp_init_clock_hook(struct gallivm_state *gallivm) +{ + if (gallivm->get_time_hook) + return; + + LLVMTypeRef get_time_type = LLVMFunctionType(LLVMInt64TypeInContext(gallivm->context), NULL, 0, 1); + gallivm->get_time_hook = LLVMAddFunction(gallivm->module, "get_time_hook", get_time_type); +} + +void +gallivm_compile_module(struct gallivm_state *gallivm) +{ + lp_init_printf_hook(gallivm); + gallivm_add_global_mapping(gallivm, gallivm->debug_printf_hook, (void *)debug_printf); + + lp_init_clock_hook(gallivm); + gallivm_add_global_mapping(gallivm, gallivm->get_time_hook, (void *)os_time_get_nano); + + lp_build_coro_add_malloc_hooks(gallivm); + + LPJit::add_ir_module_to_jd(gallivm->_ts_context, gallivm->module, + gallivm->_per_module_jd); + /* ownership of module is now transferred into orc jit, + * disallow modifying it + */ + LPJit::register_gallivm_state(gallivm); + gallivm->module=nullptr; + + /* defer compilation till first lookup by gallivm_jit_function */ +} + +func_pointer +gallivm_jit_function(struct gallivm_state *gallivm, + const char *func_name) +{ + return pointer_to_func( + LPJit::lookup_in_jd(func_name, gallivm->_per_module_jd)); +} + +unsigned +gallivm_get_perf_flags(void){ + return gallivm_perf; +} + +void +lp_set_module_stack_alignment_override(LLVMModuleRef MRef, unsigned align) +{ +#if LLVM_VERSION_MAJOR >= 13 + llvm::Module *M = llvm::unwrap(MRef); + M->setOverrideStackAlignment(align); +#endif +} diff --git a/src/gallium/auxiliary/meson.build b/src/gallium/auxiliary/meson.build index 96b0272c69b..f1e77242cdb 100644 --- a/src/gallium/auxiliary/meson.build +++ b/src/gallium/auxiliary/meson.build @@ -347,7 +347,7 @@ if draw_with_llvm 'gallivm/lp_bld_gather.c', 'gallivm/lp_bld_gather.h', 'gallivm/lp_bld.h', - 'gallivm/lp_bld_init.c', + 'gallivm/lp_bld_init_orc.cpp', 'gallivm/lp_bld_init.h', 'gallivm/lp_bld_intr.c', 'gallivm/lp_bld_intr.h', diff --git a/src/gallium/drivers/llvmpipe/lp_context.c b/src/gallium/drivers/llvmpipe/lp_context.c index 8e5e8ced3fe..8fd4f8a468a 100644 --- a/src/gallium/drivers/llvmpipe/lp_context.c +++ b/src/gallium/drivers/llvmpipe/lp_context.c @@ -108,7 +108,11 @@ llvmpipe_destroy(struct pipe_context *pipe) llvmpipe_sampler_matrix_destroy(llvmpipe); #ifndef USE_GLOBAL_LLVM_CONTEXT +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcDisposeThreadSafeContext(llvmpipe->context); +#else LLVMContextDispose(llvmpipe->context); +#endif #endif llvmpipe->context = NULL; @@ -258,15 +262,25 @@ llvmpipe_create_context(struct pipe_screen *screen, void *priv, #ifdef USE_GLOBAL_LLVM_CONTEXT llvmpipe->context = LLVMGetGlobalContext(); +#else +#if GALLIVM_USE_ORCJIT == 1 + llvmpipe->context = LLVMOrcCreateNewThreadSafeContext(); #else llvmpipe->context = LLVMContextCreate(); +#endif #endif if (!llvmpipe->context) goto fail; +#if GALLIVM_USE_ORCJIT == 1 +#if LLVM_VERSION_MAJOR == 15 + LLVMContextSetOpaquePointers(LLVMOrcThreadSafeContextGetContext(llvmpipe->context), false); +#endif +#else #if LLVM_VERSION_MAJOR == 15 LLVMContextSetOpaquePointers(llvmpipe->context, false); +#endif #endif /* diff --git a/src/gallium/drivers/llvmpipe/lp_context.h b/src/gallium/drivers/llvmpipe/lp_context.h index 7dd6a3f8596..d30aa89db33 100644 --- a/src/gallium/drivers/llvmpipe/lp_context.h +++ b/src/gallium/drivers/llvmpipe/lp_context.h @@ -190,7 +190,11 @@ struct llvmpipe_context { unsigned render_cond_offset; /** The LLVMContext to use for LLVM related work */ +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcThreadSafeContextRef context; +#else LLVMContextRef context; +#endif int max_global_buffers; struct pipe_resource **global_buffers; diff --git a/src/gallium/drivers/llvmpipe/lp_screen.c b/src/gallium/drivers/llvmpipe/lp_screen.c index ad3d66424e1..514217dc954 100644 --- a/src/gallium/drivers/llvmpipe/lp_screen.c +++ b/src/gallium/drivers/llvmpipe/lp_screen.c @@ -950,7 +950,11 @@ static void lp_disk_cache_create(struct llvmpipe_screen *screen) { struct mesa_sha1 ctx; +#if GALLIVM_USE_ORCJIT == 1 + unsigned gallivm_perf = 0; +#else unsigned gallivm_perf = gallivm_get_perf_flags(); +#endif unsigned char sha1[20]; char cache_id[20 * 2 + 1]; _mesa_sha1_init(&ctx); diff --git a/src/gallium/drivers/llvmpipe/lp_state_cs.c b/src/gallium/drivers/llvmpipe/lp_state_cs.c index 46cc5ffb06f..e84f39f1d83 100644 --- a/src/gallium/drivers/llvmpipe/lp_state_cs.c +++ b/src/gallium/drivers/llvmpipe/lp_state_cs.c @@ -397,6 +397,10 @@ generate_compute(struct llvmpipe_context *lp, lp_build_coro_add_presplit(coro); variant->function = function; +#if GALLIVM_USE_ORCJIT == 1 + variant->function_name = MALLOC(strlen(func_name)+1); + strcpy(variant->function_name, func_name); +#endif for (i = 0; i < CS_ARG_MAX - !is_mesh; ++i) { if (LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind) { @@ -1006,6 +1010,10 @@ llvmpipe_remove_cs_shader_variant(struct llvmpipe_context *lp, lp->nr_cs_variants--; lp->nr_cs_instrs -= variant->nr_instrs; +#if GALLIVM_USE_ORCJIT == 1 + if(variant->function_name) + FREE(variant->function_name); +#endif FREE(variant); } @@ -1264,12 +1272,23 @@ generate_variant(struct llvmpipe_context *lp, generate_compute(lp, shader, variant); +#if GALLIVM_USE_ORCJIT == 1 +/* module has been moved into ORCJIT after gallivm_compile_module */ + lp_build_coro_add_malloc_hooks(variant->gallivm); + variant->nr_instrs += lp_build_count_ir_module(variant->gallivm->module); + + gallivm_compile_module(variant->gallivm); + + variant->jit_function = (lp_jit_cs_func) + gallivm_jit_function(variant->gallivm, variant->function_name); +#else gallivm_compile_module(variant->gallivm); variant->nr_instrs += lp_build_count_ir_module(variant->gallivm->module); variant->jit_function = (lp_jit_cs_func) gallivm_jit_function(variant->gallivm, variant->function); +#endif if (needs_caching) { lp_disk_cache_insert_shader(screen, &cached, ir_sha1_cache_key); diff --git a/src/gallium/drivers/llvmpipe/lp_state_cs.h b/src/gallium/drivers/llvmpipe/lp_state_cs.h index 11d2e2cd825..d697213530a 100644 --- a/src/gallium/drivers/llvmpipe/lp_state_cs.h +++ b/src/gallium/drivers/llvmpipe/lp_state_cs.h @@ -91,6 +91,9 @@ struct lp_compute_shader_variant LLVMTypeRef jit_vertex_header_ptr_type; LLVMTypeRef jit_prim_type; LLVMValueRef function; +#if GALLIVM_USE_ORCJIT == 1 + char* function_name; +#endif lp_jit_cs_func jit_function; /* Total number of LLVM instructions generated */ diff --git a/src/gallium/drivers/llvmpipe/lp_state_fs.c b/src/gallium/drivers/llvmpipe/lp_state_fs.c index 21312aa98e2..3155e81d26b 100644 --- a/src/gallium/drivers/llvmpipe/lp_state_fs.c +++ b/src/gallium/drivers/llvmpipe/lp_state_fs.c @@ -3207,6 +3207,10 @@ generate_fragment(struct llvmpipe_context *lp, LLVMSetFunctionCallConv(function, LLVMCCallConv); variant->function[partial_mask] = function; +#if GALLIVM_USE_ORCJIT == 1 + variant->function_name[partial_mask] = MALLOC(strlen(func_name)+1); + strcpy(variant->function_name[partial_mask], func_name); +#endif /* XXX: need to propagate noalias down into color param now we are * passing a pointer-to-pointer? @@ -3913,20 +3917,37 @@ generate_variant(struct llvmpipe_context *lp, * Compile everything */ +#if GALLIVM_USE_ORCJIT == 1 +/* module has been moved into ORCJIT after gallivm_compile_module */ + variant->nr_instrs += lp_build_count_ir_module(variant->gallivm->module); + + gallivm_compile_module(variant->gallivm); +#else gallivm_compile_module(variant->gallivm); variant->nr_instrs += lp_build_count_ir_module(variant->gallivm->module); +#endif if (variant->function[RAST_EDGE_TEST]) { variant->jit_function[RAST_EDGE_TEST] = (lp_jit_frag_func) +#if GALLIVM_USE_ORCJIT == 1 + gallivm_jit_function(variant->gallivm, + variant->function_name[RAST_EDGE_TEST]); +#else gallivm_jit_function(variant->gallivm, variant->function[RAST_EDGE_TEST]); +#endif } if (variant->function[RAST_WHOLE]) { variant->jit_function[RAST_WHOLE] = (lp_jit_frag_func) +#if GALLIVM_USE_ORCJIT == 1 + gallivm_jit_function(variant->gallivm, + variant->function_name[RAST_WHOLE]); +#else gallivm_jit_function(variant->gallivm, variant->function[RAST_WHOLE]); +#endif } else if (!variant->jit_function[RAST_WHOLE]) { variant->jit_function[RAST_WHOLE] = (lp_jit_frag_func) variant->jit_function[RAST_EDGE_TEST]; @@ -3935,7 +3956,11 @@ generate_variant(struct llvmpipe_context *lp, if (linear_pipeline) { if (variant->linear_function) { variant->jit_linear_llvm = (lp_jit_linear_llvm_func) +#if GALLIVM_USE_ORCJIT == 1 + gallivm_jit_function(variant->gallivm, variant->linear_function_name); +#else gallivm_jit_function(variant->gallivm, variant->linear_function); +#endif } /* @@ -4111,6 +4136,14 @@ llvmpipe_destroy_shader_variant(struct llvmpipe_context *lp, { gallivm_destroy(variant->gallivm); lp_fs_reference(lp, &variant->shader, NULL); +#if GALLIVM_USE_ORCJIT == 1 + if (variant->function_name[RAST_EDGE_TEST]) + FREE(variant->function_name[RAST_EDGE_TEST]); + if (variant->function_name[RAST_WHOLE]) + FREE(variant->function_name[RAST_WHOLE]); + if (variant->linear_function_name) + FREE(variant->linear_function_name); +#endif FREE(variant); } diff --git a/src/gallium/drivers/llvmpipe/lp_state_fs.h b/src/gallium/drivers/llvmpipe/lp_state_fs.h index 195cdd5960b..533bf90e402 100644 --- a/src/gallium/drivers/llvmpipe/lp_state_fs.h +++ b/src/gallium/drivers/llvmpipe/lp_state_fs.h @@ -168,6 +168,9 @@ struct lp_fragment_shader_variant LLVMTypeRef jit_linear_textures_type; LLVMValueRef function[2]; // [RAST_WHOLE], [RAST_EDGE_TEST] +#if GALLIVM_USE_ORCJIT == 1 + char* function_name[2]; +#endif lp_jit_frag_func jit_function[2]; // [RAST_WHOLE], [RAST_EDGE_TEST] @@ -177,6 +180,9 @@ struct lp_fragment_shader_variant /* Functions within the linear path: */ LLVMValueRef linear_function; +#if GALLIVM_USE_ORCJIT == 1 + char* linear_function_name; +#endif lp_jit_linear_llvm_func jit_linear_llvm; /* Bitmask to say what cbufs are unswizzled */ diff --git a/src/gallium/drivers/llvmpipe/lp_state_fs_linear_llvm.c b/src/gallium/drivers/llvmpipe/lp_state_fs_linear_llvm.c index c64f5cf2c34..4859d65e1cf 100644 --- a/src/gallium/drivers/llvmpipe/lp_state_fs_linear_llvm.c +++ b/src/gallium/drivers/llvmpipe/lp_state_fs_linear_llvm.c @@ -289,6 +289,10 @@ llvmpipe_fs_variant_linear_llvm(struct llvmpipe_context *lp, LLVMSetFunctionCallConv(function, LLVMCCallConv); variant->linear_function = function; +#if GALLIVM_USE_ORCJIT == 1 + variant->linear_function_name = MALLOC(strlen(func_name)+1); + strcpy(variant->linear_function_name, func_name); +#endif /* XXX: need to propagate noalias down into color param now we are * passing a pointer-to-pointer? diff --git a/src/gallium/drivers/llvmpipe/lp_state_setup.c b/src/gallium/drivers/llvmpipe/lp_state_setup.c index 2f4bbdf7a88..7a7e472d36d 100644 --- a/src/gallium/drivers/llvmpipe/lp_state_setup.c +++ b/src/gallium/drivers/llvmpipe/lp_state_setup.c @@ -688,6 +688,10 @@ generate_setup_variant(struct lp_setup_variant_key *key, arg_types, ARRAY_SIZE(arg_types), 0); variant->function = LLVMAddFunction(gallivm->module, func_name, func_type); +#if GALLIVM_USE_ORCJIT == 1 + variant->function_name = MALLOC(strlen(func_name)+1); + strcpy(variant->function_name, func_name); +#endif if (!variant->function) goto fail; @@ -731,8 +735,13 @@ generate_setup_variant(struct lp_setup_variant_key *key, gallivm_compile_module(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + variant->jit_function = (lp_jit_setup_triangle) + gallivm_jit_function(gallivm, variant->function_name); +#else variant->jit_function = (lp_jit_setup_triangle) gallivm_jit_function(gallivm, variant->function); +#endif if (!variant->jit_function) goto fail; diff --git a/src/gallium/drivers/llvmpipe/lp_state_setup.h b/src/gallium/drivers/llvmpipe/lp_state_setup.h index ef208937396..1e9c6dbd65c 100644 --- a/src/gallium/drivers/llvmpipe/lp_state_setup.h +++ b/src/gallium/drivers/llvmpipe/lp_state_setup.h @@ -66,6 +66,9 @@ struct lp_setup_variant { * assembly. */ LLVMValueRef function; +#if GALLIVM_USE_ORCJIT == 1 + char *function_name; +#endif /* The actual generated setup function: */ diff --git a/src/gallium/drivers/llvmpipe/lp_test_arit.c b/src/gallium/drivers/llvmpipe/lp_test_arit.c index fd8489f8358..aff12d7ea40 100644 --- a/src/gallium/drivers/llvmpipe/lp_test_arit.c +++ b/src/gallium/drivers/llvmpipe/lp_test_arit.c @@ -417,7 +417,11 @@ test_unary(unsigned verbose, FILE *fp, const struct unary_test_t *test, unsigned { char test_name[128]; snprintf(test_name, sizeof test_name, "%s.v%u", test->name, length); +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcThreadSafeContextRef context; +#else LLVMContextRef context; +#endif struct gallivm_state *gallivm; LLVMValueRef test_func; unary_func_t test_func_jit; @@ -433,9 +437,16 @@ test_unary(unsigned verbose, FILE *fp, const struct unary_test_t *test, unsigned in[i] = 1.0; } +#if GALLIVM_USE_ORCJIT == 1 + context = LLVMOrcCreateNewThreadSafeContext(); +#if LLVM_VERSION_MAJOR == 15 + LLVMContextSetOpaquePointers(LLVMOrcThreadSafeContextGetContext(context), false); +#endif +#else context = LLVMContextCreate(); #if LLVM_VERSION_MAJOR == 15 LLVMContextSetOpaquePointers(context, false); +#endif #endif gallivm = gallivm_create("test_module", context, NULL); @@ -443,7 +454,11 @@ test_unary(unsigned verbose, FILE *fp, const struct unary_test_t *test, unsigned gallivm_compile_module(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + test_func_jit = (unary_func_t) gallivm_jit_function(gallivm, test_name); +#else test_func_jit = (unary_func_t) gallivm_jit_function(gallivm, test_func); +#endif gallivm_free_ir(gallivm); @@ -512,7 +527,11 @@ test_unary(unsigned verbose, FILE *fp, const struct unary_test_t *test, unsigned } gallivm_destroy(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcDisposeThreadSafeContext(context); +#else LLVMContextDispose(context); +#endif align_free(in); align_free(out); diff --git a/src/gallium/drivers/llvmpipe/lp_test_blend.c b/src/gallium/drivers/llvmpipe/lp_test_blend.c index 7851bf29dd6..fbfd672bb17 100644 --- a/src/gallium/drivers/llvmpipe/lp_test_blend.c +++ b/src/gallium/drivers/llvmpipe/lp_test_blend.c @@ -131,16 +131,24 @@ dump_blend_type(FILE *fp, } +#if GALLIVM_USE_ORCJIT == 1 +static const char * +add_blend_test(struct gallivm_state *gallivm, + const struct pipe_blend_state *blend, + struct lp_type type) +#else static LLVMValueRef add_blend_test(struct gallivm_state *gallivm, const struct pipe_blend_state *blend, struct lp_type type) +#endif { LLVMModuleRef module = gallivm->module; LLVMContextRef context = gallivm->context; LLVMTypeRef vec_type; LLVMTypeRef args[5]; LLVMValueRef func; + const char *func_name = "test"; LLVMValueRef src_ptr; LLVMValueRef src1_ptr; LLVMValueRef dst_ptr; @@ -160,7 +168,7 @@ add_blend_test(struct gallivm_state *gallivm, vec_type = lp_build_vec_type(gallivm, type); args[4] = args[3] = args[2] = args[1] = args[0] = LLVMPointerType(vec_type, 0); - func = LLVMAddFunction(module, "test", LLVMFunctionType(LLVMVoidTypeInContext(context), args, 5, 0)); + func = LLVMAddFunction(module, func_name, LLVMFunctionType(LLVMVoidTypeInContext(context), args, 5, 0)); LLVMSetFunctionCallConv(func, LLVMCCallConv); src_ptr = LLVMGetParam(func, 0); src1_ptr = LLVMGetParam(func, 1); @@ -188,7 +196,11 @@ add_blend_test(struct gallivm_state *gallivm, gallivm_verify_function(gallivm, func); +#if GALLIVM_USE_ORCJIT == 1 + return func_name; +#else return func; +#endif } @@ -437,9 +449,17 @@ test_one(unsigned verbose, const struct pipe_blend_state *blend, struct lp_type type) { +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcThreadSafeContextRef context; +#else LLVMContextRef context; +#endif struct gallivm_state *gallivm; +#if GALLIVM_USE_ORCJIT == 1 + const char *func; +#else LLVMValueRef func = NULL; +#endif blend_test_ptr_t blend_test_ptr; bool success; const unsigned n = LP_TEST_NUM_SAMPLES; @@ -451,9 +471,16 @@ test_one(unsigned verbose, if (verbose >= 1) dump_blend_type(stdout, blend, type); +#if GALLIVM_USE_ORCJIT == 1 + context = LLVMOrcCreateNewThreadSafeContext(); +#if LLVM_VERSION_MAJOR == 15 + LLVMContextSetOpaquePointers(LLVMOrcThreadSafeContextGetContext(context), false); +#endif +#else context = LLVMContextCreate(); #if LLVM_VERSION_MAJOR == 15 LLVMContextSetOpaquePointers(context, false); +#endif #endif gallivm = gallivm_create("test_module", context, NULL); @@ -584,7 +611,11 @@ test_one(unsigned verbose, write_tsv_row(fp, blend, type, cycles_avg, success); gallivm_destroy(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcDisposeThreadSafeContext(context); +#else LLVMContextDispose(context); +#endif return success; } diff --git a/src/gallium/drivers/llvmpipe/lp_test_conv.c b/src/gallium/drivers/llvmpipe/lp_test_conv.c index 9fb64486f11..a7257d08e7f 100644 --- a/src/gallium/drivers/llvmpipe/lp_test_conv.c +++ b/src/gallium/drivers/llvmpipe/lp_test_conv.c @@ -96,16 +96,24 @@ dump_conv_types(FILE *fp, } +#if GALLIVM_USE_ORCJIT == 1 +static const char * +add_conv_test(struct gallivm_state *gallivm, + struct lp_type src_type, unsigned num_srcs, + struct lp_type dst_type, unsigned num_dsts) +#else static LLVMValueRef add_conv_test(struct gallivm_state *gallivm, struct lp_type src_type, unsigned num_srcs, struct lp_type dst_type, unsigned num_dsts) +#endif { LLVMModuleRef module = gallivm->module; LLVMContextRef context = gallivm->context; LLVMBuilderRef builder = gallivm->builder; LLVMTypeRef args[2]; LLVMValueRef func; + const char *func_name = "test"; LLVMValueRef src_ptr; LLVMValueRef dst_ptr; LLVMBasicBlockRef block; @@ -118,7 +126,7 @@ add_conv_test(struct gallivm_state *gallivm, args[0] = LLVMPointerType(src_vec_type, 0); args[1] = LLVMPointerType(dst_vec_type, 0); - func = LLVMAddFunction(module, "test", + func = LLVMAddFunction(module, func_name, LLVMFunctionType(LLVMVoidTypeInContext(context), args, 2, 0)); LLVMSetFunctionCallConv(func, LLVMCCallConv); @@ -146,7 +154,11 @@ add_conv_test(struct gallivm_state *gallivm, gallivm_verify_function(gallivm, func); +#if GALLIVM_USE_ORCJIT == 1 + return func_name; +#else return func; +#endif } @@ -157,9 +169,17 @@ test_one(unsigned verbose, struct lp_type src_type, struct lp_type dst_type) { +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcThreadSafeContextRef context; +#else LLVMContextRef context; +#endif struct gallivm_state *gallivm; +#if GALLIVM_USE_ORCJIT == 1 + const char *func; +#else LLVMValueRef func = NULL; +#endif conv_test_ptr_t conv_test_ptr; bool success; const unsigned n = LP_TEST_NUM_SAMPLES; @@ -222,9 +242,16 @@ test_one(unsigned verbose, eps *= 2; } +#if GALLIVM_USE_ORCJIT == 1 + context = LLVMOrcCreateNewThreadSafeContext(); +#if LLVM_VERSION_MAJOR == 15 + LLVMContextSetOpaquePointers(LLVMOrcThreadSafeContextGetContext(context), false); +#endif +#else context = LLVMContextCreate(); #if LLVM_VERSION_MAJOR == 15 LLVMContextSetOpaquePointers(context, false); +#endif #endif gallivm = gallivm_create("test_module", context, NULL); @@ -337,7 +364,11 @@ test_one(unsigned verbose, write_tsv_row(fp, src_type, dst_type, cycles_avg, success); gallivm_destroy(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcDisposeThreadSafeContext(context); +#else LLVMContextDispose(context); +#endif return success; } diff --git a/src/gallium/drivers/llvmpipe/lp_test_format.c b/src/gallium/drivers/llvmpipe/lp_test_format.c index 6d3a60bd8db..ba049a43bbf 100644 --- a/src/gallium/drivers/llvmpipe/lp_test_format.c +++ b/src/gallium/drivers/llvmpipe/lp_test_format.c @@ -79,9 +79,9 @@ static LLVMValueRef add_fetch_rgba_test(struct gallivm_state *gallivm, unsigned verbose, const struct util_format_description *desc, struct lp_type type, - unsigned use_cache) + unsigned use_cache, + char *name) { - char name[256]; LLVMContextRef context = gallivm->context; LLVMModuleRef module = gallivm->module; LLVMBuilderRef builder = gallivm->builder; @@ -96,7 +96,7 @@ add_fetch_rgba_test(struct gallivm_state *gallivm, unsigned verbose, LLVMValueRef rgba; LLVMValueRef cache = NULL; - snprintf(name, sizeof name, "fetch_%s_%s", desc->short_name, + snprintf(name, 64 * sizeof(char), "fetch_%s_%s", desc->short_name, type.floating ? "float" : "unorm8"); args[0] = LLVMPointerType(lp_build_vec_type(gallivm, type), 0); @@ -139,9 +139,14 @@ test_format_float(unsigned verbose, FILE *fp, const struct util_format_description *desc, unsigned use_cache) { +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcThreadSafeContextRef context; +#else LLVMContextRef context; +#endif struct gallivm_state *gallivm; LLVMValueRef fetch = NULL; + char fetch_name[64]; fetch_ptr_t fetch_ptr; alignas(16) uint8_t packed[UTIL_FORMAT_MAX_PACKED_BYTES]; alignas(16) float unpacked[4]; @@ -149,18 +154,29 @@ test_format_float(unsigned verbose, FILE *fp, bool success = true; unsigned i, j, k, l; +#if GALLIVM_USE_ORCJIT == 1 + context = LLVMOrcCreateNewThreadSafeContext(); +#if LLVM_VERSION_MAJOR == 15 + LLVMContextSetOpaquePointers(LLVMOrcThreadSafeContextGetContext(context), false); +#endif +#else context = LLVMContextCreate(); #if LLVM_VERSION_MAJOR == 15 LLVMContextSetOpaquePointers(context, false); +#endif #endif gallivm = gallivm_create("test_module_float", context, NULL); fetch = add_fetch_rgba_test(gallivm, verbose, desc, - lp_float32_vec4_type(), use_cache); + lp_float32_vec4_type(), use_cache, fetch_name); gallivm_compile_module(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + fetch_ptr = (fetch_ptr_t) gallivm_jit_function(gallivm, fetch_name); +#else fetch_ptr = (fetch_ptr_t) gallivm_jit_function(gallivm, fetch); +#endif gallivm_free_ir(gallivm); @@ -228,7 +244,11 @@ test_format_float(unsigned verbose, FILE *fp, } gallivm_destroy(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcDisposeThreadSafeContext(context); +#else LLVMContextDispose(context); +#endif if (fp) write_tsv_row(fp, desc, success); @@ -243,9 +263,14 @@ test_format_unorm8(unsigned verbose, FILE *fp, const struct util_format_description *desc, unsigned use_cache) { +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcThreadSafeContextRef context; +#else LLVMContextRef context; +#endif struct gallivm_state *gallivm; LLVMValueRef fetch = NULL; + char fetch_name[64]; fetch_ptr_t fetch_ptr; alignas(16) uint8_t packed[UTIL_FORMAT_MAX_PACKED_BYTES]; uint8_t unpacked[4]; @@ -253,18 +278,29 @@ test_format_unorm8(unsigned verbose, FILE *fp, bool success = true; unsigned i, j, k, l; +#if GALLIVM_USE_ORCJIT == 1 + context = LLVMOrcCreateNewThreadSafeContext(); +#if LLVM_VERSION_MAJOR == 15 + LLVMContextSetOpaquePointers(LLVMOrcThreadSafeContextGetContext(context), false); +#endif +#else context = LLVMContextCreate(); #if LLVM_VERSION_MAJOR == 15 LLVMContextSetOpaquePointers(context, false); +#endif #endif gallivm = gallivm_create("test_module_unorm8", context, NULL); fetch = add_fetch_rgba_test(gallivm, verbose, desc, - lp_unorm8_vec4_type(), use_cache); + lp_unorm8_vec4_type(), use_cache, fetch_name); gallivm_compile_module(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + fetch_ptr = (fetch_ptr_t) gallivm_jit_function(gallivm, fetch_name); +#else fetch_ptr = (fetch_ptr_t) gallivm_jit_function(gallivm, fetch); +#endif gallivm_free_ir(gallivm); @@ -331,7 +367,11 @@ test_format_unorm8(unsigned verbose, FILE *fp, } gallivm_destroy(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcDisposeThreadSafeContext(context); +#else LLVMContextDispose(context); +#endif if (fp) write_tsv_row(fp, desc, success); diff --git a/src/gallium/drivers/llvmpipe/lp_test_printf.c b/src/gallium/drivers/llvmpipe/lp_test_lookup_multiple.c similarity index 69% copy from src/gallium/drivers/llvmpipe/lp_test_printf.c copy to src/gallium/drivers/llvmpipe/lp_test_lookup_multiple.c index 7e53aa51928..328ed5a1456 100644 --- a/src/gallium/drivers/llvmpipe/lp_test_printf.c +++ b/src/gallium/drivers/llvmpipe/lp_test_lookup_multiple.c @@ -58,18 +58,20 @@ typedef void (*test_printf_t)(int i); static LLVMValueRef -add_printf_test(struct gallivm_state *gallivm) +add_printf_test(struct gallivm_state *gallivm, int n, char *func_name) { LLVMModuleRef module = gallivm->module; LLVMTypeRef args[1] = { LLVMIntTypeInContext(gallivm->context, 32) }; - LLVMValueRef func = LLVMAddFunction(module, "test_printf", LLVMFunctionType(LLVMVoidTypeInContext(gallivm->context), args, 1, 0)); + snprintf(func_name, 64 * sizeof(char), "test_lookup_multiple_%d", n); + LLVMValueRef func = LLVMAddFunction(module, func_name, LLVMFunctionType(LLVMVoidTypeInContext(gallivm->context), args, 1, 0)); LLVMBuilderRef builder = gallivm->builder; LLVMBasicBlockRef block = LLVMAppendBasicBlockInContext(gallivm->context, func, "entry"); LLVMSetFunctionCallConv(func, LLVMCCallConv); LLVMPositionBuilderAtEnd(builder, block); - lp_build_printf(gallivm, "hello, world\n"); + lp_build_printf(gallivm, "hello, world from "); + lp_build_printf(gallivm, func_name); lp_build_printf(gallivm, "print 5 6: %d %d\n", LLVMConstInt(LLVMInt32TypeInContext(gallivm->context), 5, 0), LLVMConstInt(LLVMInt32TypeInContext(gallivm->context), 6, 0)); @@ -86,33 +88,59 @@ add_printf_test(struct gallivm_state *gallivm) UTIL_ALIGN_STACK static bool -test_printf(unsigned verbose, FILE *fp, +test_lookup_multiple(unsigned verbose, FILE *fp, const struct printf_test_case *testcase) { - LLVMContextRef context; struct gallivm_state *gallivm; - LLVMValueRef test; - test_printf_t test_printf_func; + const int N = 10; + LLVMValueRef func[N]; + char func_name[N][64]; + test_printf_t test_lookup_multiple_func[N]; bool success = true; + int i; - context = LLVMContextCreate(); +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcThreadSafeContextRef context = LLVMOrcCreateNewThreadSafeContext(); +#if LLVM_VERSION_MAJOR == 15 + LLVMContextSetOpaquePointers(LLVMOrcThreadSafeContextGetContext(context), false); +#endif +#else + LLVMContextRef context = LLVMContextCreate(); #if LLVM_VERSION_MAJOR == 15 LLVMContextSetOpaquePointers(context, false); +#endif #endif gallivm = gallivm_create("test_module", context, NULL); - test = add_printf_test(gallivm); + for(i = 0; i < N; i++){ + func[i] = add_printf_test(gallivm, i, func_name[i]); + } gallivm_compile_module(gallivm); - test_printf_func = (test_printf_t) gallivm_jit_function(gallivm, test); +#if GALLIVM_USE_ORCJIT == 1 + for(i = 0; i < N; i++){ + test_lookup_multiple_func[i] = (test_printf_t) gallivm_jit_function(gallivm, func_name[i]); + } + (void)func; +#else + for(i = 0; i < N; i++){ + test_lookup_multiple_func[i] = (test_printf_t) gallivm_jit_function(gallivm, func[i]); + } +#endif gallivm_free_ir(gallivm); - test_printf_func(0); + for(i = 0; i < N; i++){ + test_lookup_multiple_func[i](0); + } gallivm_destroy(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcDisposeThreadSafeContext(context); +#else LLVMContextDispose(context); +#endif return success; } @@ -123,7 +151,7 @@ test_all(unsigned verbose, FILE *fp) { bool success = true; - test_printf(verbose, fp, NULL); + test_lookup_multiple(verbose, fp, NULL); return success; } diff --git a/src/gallium/drivers/llvmpipe/lp_test_printf.c b/src/gallium/drivers/llvmpipe/lp_test_printf.c index 7e53aa51928..8df52eaf86c 100644 --- a/src/gallium/drivers/llvmpipe/lp_test_printf.c +++ b/src/gallium/drivers/llvmpipe/lp_test_printf.c @@ -57,12 +57,18 @@ write_tsv_header(FILE *fp) typedef void (*test_printf_t)(int i); +#if GALLIVM_USE_ORCJIT == 1 +static const char * +add_printf_test(struct gallivm_state *gallivm) +#else static LLVMValueRef add_printf_test(struct gallivm_state *gallivm) +#endif { LLVMModuleRef module = gallivm->module; + const char *func_name = "test_printf"; LLVMTypeRef args[1] = { LLVMIntTypeInContext(gallivm->context, 32) }; - LLVMValueRef func = LLVMAddFunction(module, "test_printf", LLVMFunctionType(LLVMVoidTypeInContext(gallivm->context), args, 1, 0)); + LLVMValueRef func = LLVMAddFunction(module, func_name, LLVMFunctionType(LLVMVoidTypeInContext(gallivm->context), args, 1, 0)); LLVMBuilderRef builder = gallivm->builder; LLVMBasicBlockRef block = LLVMAppendBasicBlockInContext(gallivm->context, func, "entry"); @@ -80,7 +86,11 @@ add_printf_test(struct gallivm_state *gallivm) gallivm_verify_function(gallivm, func); +#if GALLIVM_USE_ORCJIT == 1 + return func_name; +#else return func; +#endif } @@ -89,15 +99,30 @@ static bool test_printf(unsigned verbose, FILE *fp, const struct printf_test_case *testcase) { +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcThreadSafeContextRef context; +#else LLVMContextRef context; +#endif struct gallivm_state *gallivm; +#if GALLIVM_USE_ORCJIT == 1 + const char *test; +#else LLVMValueRef test; +#endif test_printf_t test_printf_func; bool success = true; +#if GALLIVM_USE_ORCJIT == 1 + context = LLVMOrcCreateNewThreadSafeContext(); +#if LLVM_VERSION_MAJOR == 15 + LLVMContextSetOpaquePointers(LLVMOrcThreadSafeContextGetContext(context), false); +#endif +#else context = LLVMContextCreate(); #if LLVM_VERSION_MAJOR == 15 LLVMContextSetOpaquePointers(context, false); +#endif #endif gallivm = gallivm_create("test_module", context, NULL); @@ -112,7 +137,11 @@ test_printf(unsigned verbose, FILE *fp, test_printf_func(0); gallivm_destroy(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcDisposeThreadSafeContext(context); +#else LLVMContextDispose(context); +#endif return success; } diff --git a/src/gallium/drivers/llvmpipe/lp_texture_handle.c b/src/gallium/drivers/llvmpipe/lp_texture_handle.c index 72152babcc4..ba49f9f18cc 100644 --- a/src/gallium/drivers/llvmpipe/lp_texture_handle.c +++ b/src/gallium/drivers/llvmpipe/lp_texture_handle.c @@ -202,13 +202,20 @@ llvmpipe_sampler_matrix_destroy(struct llvmpipe_context *ctx) static void * compile_function(struct llvmpipe_context *ctx, struct gallivm_state *gallivm, LLVMValueRef function, +#if GALLIVM_USE_ORCJIT == 1 + const char *function_name, +#endif bool needs_caching, uint8_t cache_key[SHA1_DIGEST_LENGTH]) { gallivm_verify_function(gallivm, function); gallivm_compile_module(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + void *function_ptr = func_to_pointer(gallivm_jit_function(gallivm, function_name)); +#else void *function_ptr = func_to_pointer(gallivm_jit_function(gallivm, function)); +#endif if (needs_caching) lp_disk_cache_insert_shader(llvmpipe_screen(ctx->pipe.screen), gallivm->cache, cache_key); @@ -289,7 +296,8 @@ compile_image_function(struct llvmpipe_context *ctx, struct lp_static_texture_st return NULL; } - LLVMValueRef function = LLVMAddFunction(gallivm->module, "image", function_type); + const char *function_name = "image"; + LLVMValueRef function = LLVMAddFunction(gallivm->module, function_name, function_type); uint32_t arg_index = 0; @@ -336,7 +344,11 @@ compile_image_function(struct llvmpipe_context *ctx, struct lp_static_texture_st free(image_soa); +#if GALLIVM_USE_ORCJIT == 1 + return compile_function(ctx, gallivm, function, function_name, needs_caching, cache_key); +#else return compile_function(ctx, gallivm, function, needs_caching, cache_key); +#endif } static void * @@ -432,7 +444,8 @@ compile_sample_function(struct llvmpipe_context *ctx, struct lp_static_texture_s lp_jit_init_cs_types(&cs); LLVMTypeRef function_type = lp_build_sample_function_type(gallivm, sample_key); - LLVMValueRef function = LLVMAddFunction(gallivm->module, "sample", function_type); + const char *function_name = "sample"; + LLVMValueRef function = LLVMAddFunction(gallivm->module, function_name, function_type); uint32_t arg_index = 0; @@ -484,7 +497,11 @@ compile_sample_function(struct llvmpipe_context *ctx, struct lp_static_texture_s free(sampler_soa); +#if GALLIVM_USE_ORCJIT == 1 + return compile_function(ctx, gallivm, function, function_name, needs_caching, cache_key); +#else return compile_function(ctx, gallivm, function, needs_caching, cache_key); +#endif } static void * @@ -537,7 +554,8 @@ compile_size_function(struct llvmpipe_context *ctx, struct lp_static_texture_sta params.target = PIPE_TEXTURE_CUBE_ARRAY; LLVMTypeRef function_type = lp_build_size_function_type(gallivm, ¶ms); - LLVMValueRef function = LLVMAddFunction(gallivm->module, "size", function_type); + const char *function_name = "size"; + LLVMValueRef function = LLVMAddFunction(gallivm->module, function_name, function_type); uint32_t arg_index = 0; @@ -566,7 +584,11 @@ compile_size_function(struct llvmpipe_context *ctx, struct lp_static_texture_sta free(sampler_soa); +#if GALLIVM_USE_ORCJIT == 1 + return compile_function(ctx, gallivm, function, function_name, needs_caching, cache_key); +#else return compile_function(ctx, gallivm, function, needs_caching, cache_key); +#endif } static void diff --git a/src/gallium/drivers/llvmpipe/meson.build b/src/gallium/drivers/llvmpipe/meson.build index 38ff88980f9..d405c67a2d6 100644 --- a/src/gallium/drivers/llvmpipe/meson.build +++ b/src/gallium/drivers/llvmpipe/meson.build @@ -132,7 +132,7 @@ driver_swrast = declare_dependency( if with_tests and with_gallium_softpipe and draw_with_llvm foreach t : ['lp_test_format', 'lp_test_arit', 'lp_test_blend', - 'lp_test_conv', 'lp_test_printf'] + 'lp_test_conv', 'lp_test_printf', 'lp_test_lookup_multiple'] test( t, executable( -- 2.44.0
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor
