1 | //===-- examples/ParallelJIT/ParallelJIT.cpp - Exercise threaded-safe JIT -===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // Parallel JIT |
10 | // |
11 | // This test program creates two LLVM functions then calls them from three |
12 | // separate threads. It requires the pthreads library. |
13 | // The three threads are created and then block waiting on a condition variable. |
14 | // Once all threads are blocked on the conditional variable, the main thread |
15 | // wakes them up. This complicated work is performed so that all three threads |
16 | // call into the JIT at the same time (or the best possible approximation of the |
17 | // same time). This test had assertion errors until I got the locking right. |
18 | // |
19 | //===----------------------------------------------------------------------===// |
20 | |
21 | #include "llvm/ADT/APInt.h" |
22 | #include "llvm/ADT/STLExtras.h" |
23 | #include "llvm/ExecutionEngine/ExecutionEngine.h" |
24 | #include "llvm/ExecutionEngine/GenericValue.h" |
25 | #include "llvm/ExecutionEngine/MCJIT.h" |
26 | #include "llvm/IR/Argument.h" |
27 | #include "llvm/IR/BasicBlock.h" |
28 | #include "llvm/IR/Constants.h" |
29 | #include "llvm/IR/DerivedTypes.h" |
30 | #include "llvm/IR/Function.h" |
31 | #include "llvm/IR/InstrTypes.h" |
32 | #include "llvm/IR/Instruction.h" |
33 | #include "llvm/IR/Instructions.h" |
34 | #include "llvm/IR/LLVMContext.h" |
35 | #include "llvm/IR/Module.h" |
36 | #include "llvm/IR/Type.h" |
37 | #include "llvm/Support/Casting.h" |
38 | #include "llvm/Support/TargetSelect.h" |
39 | #include <algorithm> |
40 | #include <cassert> |
41 | #include <cstddef> |
42 | #include <cstdint> |
43 | #include <iostream> |
44 | #include <memory> |
45 | #include <vector> |
46 | #include <pthread.h> |
47 | |
48 | using namespace llvm; |
49 | |
50 | static Function* createAdd1(Module *M) { |
51 | LLVMContext &Context = M->getContext(); |
52 | // Create the add1 function entry and insert this entry into module M. The |
53 | // function will have a return type of "int" and take an argument of "int". |
54 | Function *Add1F = |
55 | Function::Create(Ty: FunctionType::get(Result: Type::getInt32Ty(C&: Context), |
56 | Params: {Type::getInt32Ty(C&: Context)}, isVarArg: false), |
57 | Linkage: Function::ExternalLinkage, N: "add1" , M); |
58 | |
59 | // Add a basic block to the function. As before, it automatically inserts |
60 | // because of the last argument. |
61 | BasicBlock *BB = BasicBlock::Create(Context, Name: "EntryBlock" , Parent: Add1F); |
62 | |
63 | // Get pointers to the constant `1'. |
64 | Value *One = ConstantInt::get(Ty: Type::getInt32Ty(C&: Context), V: 1); |
65 | |
66 | // Get pointers to the integer argument of the add1 function... |
67 | assert(Add1F->arg_begin() != Add1F->arg_end()); // Make sure there's an arg |
68 | Argument *ArgX = &*Add1F->arg_begin(); // Get the arg |
69 | ArgX->setName("AnArg" ); // Give it a nice symbolic name for fun. |
70 | |
71 | // Create the add instruction, inserting it into the end of BB. |
72 | Instruction *Add = BinaryOperator::CreateAdd(V1: One, V2: ArgX, Name: "addresult" , BB); |
73 | |
74 | // Create the return instruction and add it to the basic block |
75 | ReturnInst::Create(C&: Context, retVal: Add, InsertAtEnd: BB); |
76 | |
77 | // Now, function add1 is ready. |
78 | return Add1F; |
79 | } |
80 | |
81 | static Function *CreateFibFunction(Module *M) { |
82 | LLVMContext &Context = M->getContext(); |
83 | // Create the fib function and insert it into module M. This function is said |
84 | // to return an int and take an int parameter. |
85 | FunctionType *FibFTy = FunctionType::get(Result: Type::getInt32Ty(C&: Context), |
86 | Params: {Type::getInt32Ty(C&: Context)}, isVarArg: false); |
87 | Function *FibF = |
88 | Function::Create(Ty: FibFTy, Linkage: Function::ExternalLinkage, N: "fib" , M); |
89 | |
90 | // Add a basic block to the function. |
91 | BasicBlock *BB = BasicBlock::Create(Context, Name: "EntryBlock" , Parent: FibF); |
92 | |
93 | // Get pointers to the constants. |
94 | Value *One = ConstantInt::get(Ty: Type::getInt32Ty(C&: Context), V: 1); |
95 | Value *Two = ConstantInt::get(Ty: Type::getInt32Ty(C&: Context), V: 2); |
96 | |
97 | // Get pointer to the integer argument of the add1 function... |
98 | Argument *ArgX = &*FibF->arg_begin(); // Get the arg. |
99 | ArgX->setName("AnArg" ); // Give it a nice symbolic name for fun. |
100 | |
101 | // Create the true_block. |
102 | BasicBlock *RetBB = BasicBlock::Create(Context, Name: "return" , Parent: FibF); |
103 | // Create an exit block. |
104 | BasicBlock *RecurseBB = BasicBlock::Create(Context, Name: "recurse" , Parent: FibF); |
105 | |
106 | // Create the "if (arg < 2) goto exitbb" |
107 | Value *CondInst = new ICmpInst(BB, ICmpInst::ICMP_SLE, ArgX, Two, "cond" ); |
108 | BranchInst::Create(IfTrue: RetBB, IfFalse: RecurseBB, Cond: CondInst, InsertAtEnd: BB); |
109 | |
110 | // Create: ret int 1 |
111 | ReturnInst::Create(C&: Context, retVal: One, InsertAtEnd: RetBB); |
112 | |
113 | // create fib(x-1) |
114 | Value *Sub = BinaryOperator::CreateSub(V1: ArgX, V2: One, Name: "arg" , BB: RecurseBB); |
115 | Value *CallFibX1 = CallInst::Create(Func: FibF, Args: Sub, NameStr: "fibx1" , InsertAtEnd: RecurseBB); |
116 | |
117 | // create fib(x-2) |
118 | Sub = BinaryOperator::CreateSub(V1: ArgX, V2: Two, Name: "arg" , BB: RecurseBB); |
119 | Value *CallFibX2 = CallInst::Create(Func: FibF, Args: Sub, NameStr: "fibx2" , InsertAtEnd: RecurseBB); |
120 | |
121 | // fib(x-1)+fib(x-2) |
122 | Value *Sum = |
123 | BinaryOperator::CreateAdd(V1: CallFibX1, V2: CallFibX2, Name: "addresult" , BB: RecurseBB); |
124 | |
125 | // Create the return instruction and add it to the basic block |
126 | ReturnInst::Create(C&: Context, retVal: Sum, InsertAtEnd: RecurseBB); |
127 | |
128 | return FibF; |
129 | } |
130 | |
131 | struct threadParams { |
132 | ExecutionEngine* EE; |
133 | Function* F; |
134 | int value; |
135 | }; |
136 | |
137 | // We block the subthreads just before they begin to execute: |
138 | // we want all of them to call into the JIT at the same time, |
139 | // to verify that the locking is working correctly. |
140 | class WaitForThreads |
141 | { |
142 | public: |
143 | WaitForThreads() |
144 | { |
145 | n = 0; |
146 | waitFor = 0; |
147 | |
148 | int result = pthread_cond_init( cond: &condition, cond_attr: nullptr ); |
149 | (void)result; |
150 | assert( result == 0 ); |
151 | |
152 | result = pthread_mutex_init( mutex: &mutex, mutexattr: nullptr ); |
153 | assert( result == 0 ); |
154 | } |
155 | |
156 | ~WaitForThreads() |
157 | { |
158 | int result = pthread_cond_destroy( cond: &condition ); |
159 | (void)result; |
160 | assert( result == 0 ); |
161 | |
162 | result = pthread_mutex_destroy( mutex: &mutex ); |
163 | assert( result == 0 ); |
164 | } |
165 | |
166 | // All threads will stop here until another thread calls releaseThreads |
167 | void block() |
168 | { |
169 | int result = pthread_mutex_lock( mutex: &mutex ); |
170 | (void)result; |
171 | assert( result == 0 ); |
172 | n ++; |
173 | //~ std::cout << "block() n " << n << " waitFor " << waitFor << std::endl; |
174 | |
175 | assert( waitFor == 0 || n <= waitFor ); |
176 | if ( waitFor > 0 && n == waitFor ) |
177 | { |
178 | // There are enough threads blocked that we can release all of them |
179 | std::cout << "Unblocking threads from block()" << std::endl; |
180 | unblockThreads(); |
181 | } |
182 | else |
183 | { |
184 | // We just need to wait until someone unblocks us |
185 | result = pthread_cond_wait( cond: &condition, mutex: &mutex ); |
186 | assert( result == 0 ); |
187 | } |
188 | |
189 | // unlock the mutex before returning |
190 | result = pthread_mutex_unlock( mutex: &mutex ); |
191 | assert( result == 0 ); |
192 | } |
193 | |
194 | // If there are num or more threads blocked, it will signal them all |
195 | // Otherwise, this thread blocks until there are enough OTHER threads |
196 | // blocked |
197 | void releaseThreads( size_t num ) |
198 | { |
199 | int result = pthread_mutex_lock( mutex: &mutex ); |
200 | (void)result; |
201 | assert( result == 0 ); |
202 | |
203 | if ( n >= num ) { |
204 | std::cout << "Unblocking threads from releaseThreads()" << std::endl; |
205 | unblockThreads(); |
206 | } |
207 | else |
208 | { |
209 | waitFor = num; |
210 | pthread_cond_wait( cond: &condition, mutex: &mutex ); |
211 | } |
212 | |
213 | // unlock the mutex before returning |
214 | result = pthread_mutex_unlock( mutex: &mutex ); |
215 | assert( result == 0 ); |
216 | } |
217 | |
218 | private: |
219 | void unblockThreads() |
220 | { |
221 | // Reset the counters to zero: this way, if any new threads |
222 | // enter while threads are exiting, they will block instead |
223 | // of triggering a new release of threads |
224 | n = 0; |
225 | |
226 | // Reset waitFor to zero: this way, if waitFor threads enter |
227 | // while threads are exiting, they will block instead of |
228 | // triggering a new release of threads |
229 | waitFor = 0; |
230 | |
231 | int result = pthread_cond_broadcast( cond: &condition ); |
232 | (void)result; |
233 | assert(result == 0); |
234 | } |
235 | |
236 | size_t n; |
237 | size_t waitFor; |
238 | pthread_cond_t condition; |
239 | pthread_mutex_t mutex; |
240 | }; |
241 | |
242 | static WaitForThreads synchronize; |
243 | |
244 | void* callFunc( void* param ) |
245 | { |
246 | struct threadParams* p = (struct threadParams*) param; |
247 | |
248 | // Call the `foo' function with no arguments: |
249 | std::vector<GenericValue> Args(1); |
250 | Args[0].IntVal = APInt(32, p->value); |
251 | |
252 | synchronize.block(); // wait until other threads are at this point |
253 | GenericValue gv = p->EE->runFunction(F: p->F, ArgValues: Args); |
254 | |
255 | return (void*)(intptr_t)gv.IntVal.getZExtValue(); |
256 | } |
257 | |
258 | int main() { |
259 | InitializeNativeTarget(); |
260 | LLVMInitializeNativeAsmPrinter(); |
261 | LLVMContext Context; |
262 | |
263 | // Create some module to put our function into it. |
264 | std::unique_ptr<Module> Owner = std::make_unique<Module>(args: "test" , args&: Context); |
265 | Module *M = Owner.get(); |
266 | |
267 | Function* add1F = createAdd1( M ); |
268 | Function* fibF = CreateFibFunction( M ); |
269 | |
270 | // Now we create the JIT. |
271 | ExecutionEngine* EE = EngineBuilder(std::move(Owner)).create(); |
272 | |
273 | //~ std::cout << "We just constructed this LLVM module:\n\n" << *M; |
274 | //~ std::cout << "\n\nRunning foo: " << std::flush; |
275 | |
276 | // Create one thread for add1 and two threads for fib |
277 | struct threadParams add1 = { .EE: EE, .F: add1F, .value: 1000 }; |
278 | struct threadParams fib1 = { .EE: EE, .F: fibF, .value: 39 }; |
279 | struct threadParams fib2 = { .EE: EE, .F: fibF, .value: 42 }; |
280 | |
281 | pthread_t add1Thread; |
282 | int result = pthread_create( newthread: &add1Thread, attr: nullptr, start_routine: callFunc, arg: &add1 ); |
283 | if ( result != 0 ) { |
284 | std::cerr << "Could not create thread" << std::endl; |
285 | return 1; |
286 | } |
287 | |
288 | pthread_t fibThread1; |
289 | result = pthread_create( newthread: &fibThread1, attr: nullptr, start_routine: callFunc, arg: &fib1 ); |
290 | if ( result != 0 ) { |
291 | std::cerr << "Could not create thread" << std::endl; |
292 | return 1; |
293 | } |
294 | |
295 | pthread_t fibThread2; |
296 | result = pthread_create( newthread: &fibThread2, attr: nullptr, start_routine: callFunc, arg: &fib2 ); |
297 | if ( result != 0 ) { |
298 | std::cerr << "Could not create thread" << std::endl; |
299 | return 1; |
300 | } |
301 | |
302 | synchronize.releaseThreads(num: 3); // wait until other threads are at this point |
303 | |
304 | void* returnValue; |
305 | result = pthread_join( th: add1Thread, thread_return: &returnValue ); |
306 | if ( result != 0 ) { |
307 | std::cerr << "Could not join thread" << std::endl; |
308 | return 1; |
309 | } |
310 | std::cout << "Add1 returned " << intptr_t(returnValue) << std::endl; |
311 | |
312 | result = pthread_join( th: fibThread1, thread_return: &returnValue ); |
313 | if ( result != 0 ) { |
314 | std::cerr << "Could not join thread" << std::endl; |
315 | return 1; |
316 | } |
317 | std::cout << "Fib1 returned " << intptr_t(returnValue) << std::endl; |
318 | |
319 | result = pthread_join( th: fibThread2, thread_return: &returnValue ); |
320 | if ( result != 0 ) { |
321 | std::cerr << "Could not join thread" << std::endl; |
322 | return 1; |
323 | } |
324 | std::cout << "Fib2 returned " << intptr_t(returnValue) << std::endl; |
325 | |
326 | return 0; |
327 | } |
328 | |