1//===-- examples/ParallelJIT/ParallelJIT.cpp - Exercise threaded-safe JIT -===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Parallel JIT
10//
11// This test program creates two LLVM functions then calls them from three
12// separate threads. It requires the pthreads library.
13// The three threads are created and then block waiting on a condition variable.
14// Once all threads are blocked on the conditional variable, the main thread
15// wakes them up. This complicated work is performed so that all three threads
16// call into the JIT at the same time (or the best possible approximation of the
17// same time). This test had assertion errors until I got the locking right.
18//
19//===----------------------------------------------------------------------===//
20
21#include "llvm/ADT/APInt.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/ExecutionEngine/ExecutionEngine.h"
24#include "llvm/ExecutionEngine/GenericValue.h"
25#include "llvm/ExecutionEngine/MCJIT.h"
26#include "llvm/IR/Argument.h"
27#include "llvm/IR/BasicBlock.h"
28#include "llvm/IR/Constants.h"
29#include "llvm/IR/DerivedTypes.h"
30#include "llvm/IR/Function.h"
31#include "llvm/IR/InstrTypes.h"
32#include "llvm/IR/Instruction.h"
33#include "llvm/IR/Instructions.h"
34#include "llvm/IR/LLVMContext.h"
35#include "llvm/IR/Module.h"
36#include "llvm/IR/Type.h"
37#include "llvm/Support/Casting.h"
38#include "llvm/Support/TargetSelect.h"
39#include <algorithm>
40#include <cassert>
41#include <cstddef>
42#include <cstdint>
43#include <iostream>
44#include <memory>
45#include <vector>
46#include <pthread.h>
47
48using namespace llvm;
49
50static Function* createAdd1(Module *M) {
51 LLVMContext &Context = M->getContext();
52 // Create the add1 function entry and insert this entry into module M. The
53 // function will have a return type of "int" and take an argument of "int".
54 Function *Add1F =
55 Function::Create(Ty: FunctionType::get(Result: Type::getInt32Ty(C&: Context),
56 Params: {Type::getInt32Ty(C&: Context)}, isVarArg: false),
57 Linkage: Function::ExternalLinkage, N: "add1", M);
58
59 // Add a basic block to the function. As before, it automatically inserts
60 // because of the last argument.
61 BasicBlock *BB = BasicBlock::Create(Context, Name: "EntryBlock", Parent: Add1F);
62
63 // Get pointers to the constant `1'.
64 Value *One = ConstantInt::get(Ty: Type::getInt32Ty(C&: Context), V: 1);
65
66 // Get pointers to the integer argument of the add1 function...
67 assert(Add1F->arg_begin() != Add1F->arg_end()); // Make sure there's an arg
68 Argument *ArgX = &*Add1F->arg_begin(); // Get the arg
69 ArgX->setName("AnArg"); // Give it a nice symbolic name for fun.
70
71 // Create the add instruction, inserting it into the end of BB.
72 Instruction *Add = BinaryOperator::CreateAdd(V1: One, V2: ArgX, Name: "addresult", BB);
73
74 // Create the return instruction and add it to the basic block
75 ReturnInst::Create(C&: Context, retVal: Add, InsertAtEnd: BB);
76
77 // Now, function add1 is ready.
78 return Add1F;
79}
80
81static Function *CreateFibFunction(Module *M) {
82 LLVMContext &Context = M->getContext();
83 // Create the fib function and insert it into module M. This function is said
84 // to return an int and take an int parameter.
85 FunctionType *FibFTy = FunctionType::get(Result: Type::getInt32Ty(C&: Context),
86 Params: {Type::getInt32Ty(C&: Context)}, isVarArg: false);
87 Function *FibF =
88 Function::Create(Ty: FibFTy, Linkage: Function::ExternalLinkage, N: "fib", M);
89
90 // Add a basic block to the function.
91 BasicBlock *BB = BasicBlock::Create(Context, Name: "EntryBlock", Parent: FibF);
92
93 // Get pointers to the constants.
94 Value *One = ConstantInt::get(Ty: Type::getInt32Ty(C&: Context), V: 1);
95 Value *Two = ConstantInt::get(Ty: Type::getInt32Ty(C&: Context), V: 2);
96
97 // Get pointer to the integer argument of the add1 function...
98 Argument *ArgX = &*FibF->arg_begin(); // Get the arg.
99 ArgX->setName("AnArg"); // Give it a nice symbolic name for fun.
100
101 // Create the true_block.
102 BasicBlock *RetBB = BasicBlock::Create(Context, Name: "return", Parent: FibF);
103 // Create an exit block.
104 BasicBlock *RecurseBB = BasicBlock::Create(Context, Name: "recurse", Parent: FibF);
105
106 // Create the "if (arg < 2) goto exitbb"
107 Value *CondInst = new ICmpInst(BB, ICmpInst::ICMP_SLE, ArgX, Two, "cond");
108 BranchInst::Create(IfTrue: RetBB, IfFalse: RecurseBB, Cond: CondInst, InsertAtEnd: BB);
109
110 // Create: ret int 1
111 ReturnInst::Create(C&: Context, retVal: One, InsertAtEnd: RetBB);
112
113 // create fib(x-1)
114 Value *Sub = BinaryOperator::CreateSub(V1: ArgX, V2: One, Name: "arg", BB: RecurseBB);
115 Value *CallFibX1 = CallInst::Create(Func: FibF, Args: Sub, NameStr: "fibx1", InsertAtEnd: RecurseBB);
116
117 // create fib(x-2)
118 Sub = BinaryOperator::CreateSub(V1: ArgX, V2: Two, Name: "arg", BB: RecurseBB);
119 Value *CallFibX2 = CallInst::Create(Func: FibF, Args: Sub, NameStr: "fibx2", InsertAtEnd: RecurseBB);
120
121 // fib(x-1)+fib(x-2)
122 Value *Sum =
123 BinaryOperator::CreateAdd(V1: CallFibX1, V2: CallFibX2, Name: "addresult", BB: RecurseBB);
124
125 // Create the return instruction and add it to the basic block
126 ReturnInst::Create(C&: Context, retVal: Sum, InsertAtEnd: RecurseBB);
127
128 return FibF;
129}
130
131struct threadParams {
132 ExecutionEngine* EE;
133 Function* F;
134 int value;
135};
136
137// We block the subthreads just before they begin to execute:
138// we want all of them to call into the JIT at the same time,
139// to verify that the locking is working correctly.
140class WaitForThreads
141{
142public:
143 WaitForThreads()
144 {
145 n = 0;
146 waitFor = 0;
147
148 int result = pthread_cond_init( cond: &condition, cond_attr: nullptr );
149 (void)result;
150 assert( result == 0 );
151
152 result = pthread_mutex_init( mutex: &mutex, mutexattr: nullptr );
153 assert( result == 0 );
154 }
155
156 ~WaitForThreads()
157 {
158 int result = pthread_cond_destroy( cond: &condition );
159 (void)result;
160 assert( result == 0 );
161
162 result = pthread_mutex_destroy( mutex: &mutex );
163 assert( result == 0 );
164 }
165
166 // All threads will stop here until another thread calls releaseThreads
167 void block()
168 {
169 int result = pthread_mutex_lock( mutex: &mutex );
170 (void)result;
171 assert( result == 0 );
172 n ++;
173 //~ std::cout << "block() n " << n << " waitFor " << waitFor << std::endl;
174
175 assert( waitFor == 0 || n <= waitFor );
176 if ( waitFor > 0 && n == waitFor )
177 {
178 // There are enough threads blocked that we can release all of them
179 std::cout << "Unblocking threads from block()" << std::endl;
180 unblockThreads();
181 }
182 else
183 {
184 // We just need to wait until someone unblocks us
185 result = pthread_cond_wait( cond: &condition, mutex: &mutex );
186 assert( result == 0 );
187 }
188
189 // unlock the mutex before returning
190 result = pthread_mutex_unlock( mutex: &mutex );
191 assert( result == 0 );
192 }
193
194 // If there are num or more threads blocked, it will signal them all
195 // Otherwise, this thread blocks until there are enough OTHER threads
196 // blocked
197 void releaseThreads( size_t num )
198 {
199 int result = pthread_mutex_lock( mutex: &mutex );
200 (void)result;
201 assert( result == 0 );
202
203 if ( n >= num ) {
204 std::cout << "Unblocking threads from releaseThreads()" << std::endl;
205 unblockThreads();
206 }
207 else
208 {
209 waitFor = num;
210 pthread_cond_wait( cond: &condition, mutex: &mutex );
211 }
212
213 // unlock the mutex before returning
214 result = pthread_mutex_unlock( mutex: &mutex );
215 assert( result == 0 );
216 }
217
218private:
219 void unblockThreads()
220 {
221 // Reset the counters to zero: this way, if any new threads
222 // enter while threads are exiting, they will block instead
223 // of triggering a new release of threads
224 n = 0;
225
226 // Reset waitFor to zero: this way, if waitFor threads enter
227 // while threads are exiting, they will block instead of
228 // triggering a new release of threads
229 waitFor = 0;
230
231 int result = pthread_cond_broadcast( cond: &condition );
232 (void)result;
233 assert(result == 0);
234 }
235
236 size_t n;
237 size_t waitFor;
238 pthread_cond_t condition;
239 pthread_mutex_t mutex;
240};
241
242static WaitForThreads synchronize;
243
244void* callFunc( void* param )
245{
246 struct threadParams* p = (struct threadParams*) param;
247
248 // Call the `foo' function with no arguments:
249 std::vector<GenericValue> Args(1);
250 Args[0].IntVal = APInt(32, p->value);
251
252 synchronize.block(); // wait until other threads are at this point
253 GenericValue gv = p->EE->runFunction(F: p->F, ArgValues: Args);
254
255 return (void*)(intptr_t)gv.IntVal.getZExtValue();
256}
257
258int main() {
259 InitializeNativeTarget();
260 LLVMInitializeNativeAsmPrinter();
261 LLVMContext Context;
262
263 // Create some module to put our function into it.
264 std::unique_ptr<Module> Owner = std::make_unique<Module>(args: "test", args&: Context);
265 Module *M = Owner.get();
266
267 Function* add1F = createAdd1( M );
268 Function* fibF = CreateFibFunction( M );
269
270 // Now we create the JIT.
271 ExecutionEngine* EE = EngineBuilder(std::move(Owner)).create();
272
273 //~ std::cout << "We just constructed this LLVM module:\n\n" << *M;
274 //~ std::cout << "\n\nRunning foo: " << std::flush;
275
276 // Create one thread for add1 and two threads for fib
277 struct threadParams add1 = { .EE: EE, .F: add1F, .value: 1000 };
278 struct threadParams fib1 = { .EE: EE, .F: fibF, .value: 39 };
279 struct threadParams fib2 = { .EE: EE, .F: fibF, .value: 42 };
280
281 pthread_t add1Thread;
282 int result = pthread_create( newthread: &add1Thread, attr: nullptr, start_routine: callFunc, arg: &add1 );
283 if ( result != 0 ) {
284 std::cerr << "Could not create thread" << std::endl;
285 return 1;
286 }
287
288 pthread_t fibThread1;
289 result = pthread_create( newthread: &fibThread1, attr: nullptr, start_routine: callFunc, arg: &fib1 );
290 if ( result != 0 ) {
291 std::cerr << "Could not create thread" << std::endl;
292 return 1;
293 }
294
295 pthread_t fibThread2;
296 result = pthread_create( newthread: &fibThread2, attr: nullptr, start_routine: callFunc, arg: &fib2 );
297 if ( result != 0 ) {
298 std::cerr << "Could not create thread" << std::endl;
299 return 1;
300 }
301
302 synchronize.releaseThreads(num: 3); // wait until other threads are at this point
303
304 void* returnValue;
305 result = pthread_join( th: add1Thread, thread_return: &returnValue );
306 if ( result != 0 ) {
307 std::cerr << "Could not join thread" << std::endl;
308 return 1;
309 }
310 std::cout << "Add1 returned " << intptr_t(returnValue) << std::endl;
311
312 result = pthread_join( th: fibThread1, thread_return: &returnValue );
313 if ( result != 0 ) {
314 std::cerr << "Could not join thread" << std::endl;
315 return 1;
316 }
317 std::cout << "Fib1 returned " << intptr_t(returnValue) << std::endl;
318
319 result = pthread_join( th: fibThread2, thread_return: &returnValue );
320 if ( result != 0 ) {
321 std::cerr << "Could not join thread" << std::endl;
322 return 1;
323 }
324 std::cout << "Fib2 returned " << intptr_t(returnValue) << std::endl;
325
326 return 0;
327}
328

source code of llvm/examples/ParallelJIT/ParallelJIT.cpp