1/****************************************************************************
2**
3** Copyright (C) 2016 The Qt Company Ltd.
4** Contact: https://www.qt.io/licensing/
5**
6** This file is part of the QtTest module of the Qt Toolkit.
7**
8** $QT_BEGIN_LICENSE:LGPL$
9** Commercial License Usage
10** Licensees holding valid commercial Qt licenses may use this file in
11** accordance with the commercial license agreement provided with the
12** Software or, alternatively, in accordance with the terms contained in
13** a written agreement between you and The Qt Company. For licensing terms
14** and conditions see https://www.qt.io/terms-conditions. For further
15** information use the contact form at https://www.qt.io/contact-us.
16**
17** GNU Lesser General Public License Usage
18** Alternatively, this file may be used under the terms of the GNU Lesser
19** General Public License version 3 as published by the Free Software
20** Foundation and appearing in the file LICENSE.LGPL3 included in the
21** packaging of this file. Please review the following information to
22** ensure the GNU Lesser General Public License version 3 requirements
23** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
24**
25** GNU General Public License Usage
26** Alternatively, this file may be used under the terms of the GNU
27** General Public License version 2.0 or (at your option) the GNU General
28** Public license version 3 or any later version approved by the KDE Free
29** Qt Foundation. The licenses are as published by the Free Software
30** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
31** included in the packaging of this file. Please review the following
32** information to ensure the GNU General Public License requirements will
33** be met: https://www.gnu.org/licenses/gpl-2.0.html and
34** https://www.gnu.org/licenses/gpl-3.0.html.
35**
36** $QT_END_LICENSE$
37**
38****************************************************************************/
39
40#include "qxctestlogger_p.h"
41
42#include <QtCore/qstring.h>
43
44#include <QtTest/private/qtestlog_p.h>
45#include <QtTest/private/qtestresult_p.h>
46
47#import <XCTest/XCTest.h>
48
49// ---------------------------------------------------------
50
51@interface XCTestProbe (Private)
52+ (BOOL)isTesting;
53+ (void)runTests:(id)unusedArgument;
54+ (NSString*)testScope;
55+ (BOOL)isInverseTestScope;
56@end
57
58@interface XCTestDriver : NSObject
59+ (XCTestDriver*)sharedTestDriver;
60@property (readonly, assign) NSObject *IDEConnection;
61@end
62
63@interface XCTest (Private)
64- (NSString *)nameForLegacyLogging;
65@end
66
67QT_WARNING_PUSH
68// Ignore XCTestProbe deprecation
69QT_WARNING_DISABLE_DEPRECATED
70
71// ---------------------------------------------------------
72
73@interface QtTestLibWrapper : XCTestCase
74@end
75
76@interface QtTestLibTests : XCTestSuite
77+ (XCTestSuiteRun*)testRun;
78@end
79
80@interface QtTestLibTest : XCTestCase
81@property (nonatomic, retain) NSString* testObjectName;
82@property (nonatomic, retain) NSString* testFunctionName;
83@end
84
85// ---------------------------------------------------------
86
87class ThreadBarriers
88{
89public:
90 enum Barrier {
91 XCTestCanStartTesting,
92 XCTestHaveStarted,
93 QtTestsCanStartTesting,
94 QtTestsHaveCompleted,
95 XCTestsHaveCompleted,
96 BarrierCount
97 };
98
99 static ThreadBarriers *get()
100 {
101 static ThreadBarriers instance;
102 return &instance;
103 }
104
105 static void initialize() { get(); }
106
107 void wait(Barrier barrier) { dispatch_semaphore_wait(barriers[barrier], DISPATCH_TIME_FOREVER); }
108 void signal(Barrier barrier) { dispatch_semaphore_signal(barriers[barrier]); }
109
110private:
111 #define FOREACH_BARRIER(cmd) for (int i = 0; i < BarrierCount; ++i) { cmd }
112
113 ThreadBarriers() { FOREACH_BARRIER(barriers[i] = dispatch_semaphore_create(0);) }
114 ~ThreadBarriers() { FOREACH_BARRIER(dispatch_release(barriers[i]);) }
115
116 dispatch_semaphore_t barriers[BarrierCount];
117};
118
119#define WAIT_FOR_BARRIER(b) ThreadBarriers::get()->wait(ThreadBarriers::b);
120#define SIGNAL_BARRIER(b) ThreadBarriers::get()->signal(ThreadBarriers::b);
121
122// ---------------------------------------------------------
123
124@implementation QtTestLibWrapper
125
126+ (void)load
127{
128 NSAutoreleasePool *autoreleasepool = [[NSAutoreleasePool alloc] init];
129
130 if (![XCTestProbe isTesting])
131 return;
132
133 if (Q_UNLIKELY(!([NSDate timeIntervalSinceReferenceDate] > 0)))
134 qFatal("error: Device date '%s' is bad, likely set to update automatically. Please correct.",
135 [[NSDate date] description].UTF8String);
136
137 XCTestDriver *testDriver = nil;
138 if ([QtTestLibWrapper usingTestManager])
139 testDriver = [XCTestDriver sharedTestDriver];
140
141 // Spawn off task to run test infrastructure on separate thread so that we can
142 // let main() execute like normal on the main thread. The queue will never be
143 // destroyed, so there's no point in trying to keep a proper retain count.
144 dispatch_async(dispatch_queue_create("io.qt.QTestLib.xctest-wrapper", DISPATCH_QUEUE_SERIAL), ^{
145 Q_ASSERT(![NSThread isMainThread]);
146 [XCTestProbe runTests:nil];
147 Q_UNREACHABLE();
148 });
149
150 // Initialize barriers before registering exit handler so that the
151 // semaphores stay alive until after the exit handler completes.
152 ThreadBarriers::initialize();
153
154 // We register an exit handler so that we can intercept when main() completes
155 // and let the XCTest thread finish up. For main() functions that never started
156 // testing using QtTestLib we also need to signal that xcTestsCanStart.
157 atexit_b(^{
158 Q_ASSERT([NSThread isMainThread]);
159
160 // In case not started by startLogging
161 SIGNAL_BARRIER(XCTestCanStartTesting);
162
163 // [XCTestProbe runTests:] ends up calling [XCTestProbe exitTests:] after
164 // all test suites have completed, which calls exit(). We use that to signal
165 // to the main thread that it's free to continue its exit handler.
166 atexit_b(^{
167 Q_ASSERT(![NSThread isMainThread]);
168 SIGNAL_BARRIER(XCTestsHaveCompleted);
169
170 // Block forever so that the main thread does all the cleanup
171 dispatch_semaphore_wait(dispatch_semaphore_create(0), DISPATCH_TIME_FOREVER);
172 });
173
174 SIGNAL_BARRIER(QtTestsHaveCompleted);
175
176 // Ensure XCTest complets the top level tests suite
177 WAIT_FOR_BARRIER(XCTestsHaveCompleted);
178 });
179
180 // Let test driver (Xcode) connection setup complete before continuing
181 if ([[[NSProcessInfo processInfo] arguments] containsObject:@"--use-testmanagerd"]) {
182 while (!testDriver.IDEConnection)
183 [[NSRunLoop mainRunLoop] runMode:NSDefaultRunLoopMode beforeDate:[NSDate distantFuture]];
184 }
185
186 // Wait for our QtTestLib test suite to run before running main
187 WAIT_FOR_BARRIER(QtTestsCanStartTesting);
188
189 // Prevent XCTestProbe from re-launching runTests on application startup
190 [[NSNotificationCenter defaultCenter] removeObserver:[XCTestProbe class]
191 name:[NSString stringWithFormat:@"%@DidFinishLaunchingNotification",
192 #if defined(Q_OS_MACOS)
193 @"NSApplication"
194 #else
195 @"UIApplication"
196 #endif
197 ]
198 object:nil];
199
200 [autoreleasepool release];
201}
202
203+ (QTestLibTests *)defaultTestSuite
204{
205 return [[QtTestLibTests alloc] initWithName:@"QtTestLib"];
206}
207
208+ (BOOL)usingTestManager
209{
210 return [[[NSProcessInfo processInfo] arguments] containsObject:@"--use-testmanagerd"];
211}
212
213@end
214
215// ---------------------------------------------------------
216
217static XCTestSuiteRun *s_qtTestSuiteRun = nullptr;
218
219@implementation QtTestLibTests
220
221- (void)performTest:(XCTestSuiteRun *)testSuiteRun
222{
223 Q_ASSERT(![NSThread isMainThread]);
224
225 Q_ASSERT(!s_qtTestSuiteRun);
226 s_qtTestSuiteRun = testSuiteRun;
227
228 SIGNAL_BARRIER(QtTestsCanStartTesting);
229
230 // Wait for main() to complete, or a QtTestLib test to start, so we
231 // know if we should start the QtTestLib test suite.
232 WAIT_FOR_BARRIER(XCTestCanStartTesting);
233
234 if (QXcodeTestLogger::isActive())
235 [testSuiteRun start];
236
237 SIGNAL_BARRIER(XCTestHaveStarted);
238
239 // All test reporting happens on main thread from now on. Wait until
240 // main() completes before allowing the XCTest thread to continue.
241 WAIT_FOR_BARRIER(QtTestsHaveCompleted);
242
243 if ([testSuiteRun startDate])
244 [testSuiteRun stop];
245}
246
247+ (XCTestSuiteRun*)testRun
248{
249 return s_qtTestSuiteRun;
250}
251
252@end
253
254// ---------------------------------------------------------
255
256@implementation QtTestLibTest
257
258- (instancetype)initWithInvocation:(NSInvocation *)invocation
259{
260 if (self = [super initWithInvocation:invocation]) {
261 // The test object name and function name are used by XCTest after QtTestLib has
262 // reset them, so we need to store them up front for each XCTestCase.
263 self.testObjectName = [NSString stringWithUTF8String:QTestResult::currentTestObjectName()];
264 self.testFunctionName = [NSString stringWithUTF8String:QTestResult::currentTestFunction()];
265 }
266
267 return self;
268}
269
270- (NSString *)testClassName
271{
272 return self.testObjectName;
273}
274
275- (NSString *)testMethodName
276{
277 return self.testFunctionName;
278}
279
280- (NSString *)nameForLegacyLogging
281{
282 NSString *name = [NSString stringWithFormat:@"%@::%@", [self testClassName], [self testMethodName]];
283 if (QTestResult::currentDataTag() || QTestResult::currentGlobalDataTag()) {
284 const char *currentDataTag = QTestResult::currentDataTag() ? QTestResult::currentDataTag() : "";
285 const char *globalDataTag = QTestResult::currentGlobalDataTag() ? QTestResult::currentGlobalDataTag() : "";
286 const char *filler = (currentDataTag[0] && globalDataTag[0]) ? ":" : "";
287 name = [name stringByAppendingString:[NSString stringWithFormat:@"(%s%s%s)",
288 globalDataTag, filler, currentDataTag]];
289 }
290
291 return name;
292}
293
294@end
295
296// ---------------------------------------------------------
297
298bool QXcodeTestLogger::canLogTestProgress()
299{
300 return [XCTestProbe isTesting]; // FIXME: Exclude xctool
301}
302
303int QXcodeTestLogger::parseCommandLineArgument(const char *argument)
304{
305 if (strncmp(argument, "-NS", 3) == 0 || strncmp(argument, "-Apple", 6) == 0)
306 return 2; // -NSTreatUnknownArgumentsAsOpen, -ApplePersistenceIgnoreState, etc, skip argument
307 else if (strcmp(argument, "--use-testmanagerd") == 0)
308 return 2; // Skip UID argument
309 else if (strncmp(argument, "-XCTest", 7) == 0)
310 return 2; // -XCTestInvertScope, -XCTest scope, etc, skip argument
311 else if (strcmp(argument + (strlen(argument) - 7), ".xctest") == 0)
312 return 1; // Skip test bundle
313 else
314 return 0;
315}
316
317// ---------------------------------------------------------
318
319QXcodeTestLogger *QXcodeTestLogger::s_currentTestLogger = 0;
320
321// ---------------------------------------------------------
322
323QXcodeTestLogger::QXcodeTestLogger()
324 : QAbstractTestLogger(0)
325 , m_testRuns([[NSMutableArray<XCTestRun *> arrayWithCapacity:2] retain])
326
327{
328 Q_ASSERT(!s_currentTestLogger);
329 s_currentTestLogger = this;
330}
331
332QXcodeTestLogger::~QXcodeTestLogger()
333{
334 s_currentTestLogger = 0;
335 [m_testRuns release];
336}
337
338void QXcodeTestLogger::startLogging()
339{
340 SIGNAL_BARRIER(XCTestCanStartTesting);
341
342 static dispatch_once_t onceToken;
343 dispatch_once (&onceToken, ^{
344 WAIT_FOR_BARRIER(XCTestHaveStarted);
345 });
346
347 // Scope test object suite under top level QtTestLib test run
348 [m_testRuns addObject:[QtTestLibTests testRun]];
349
350 NSString *suiteName = [NSString stringWithUTF8String:QTestResult::currentTestObjectName()];
351 pushTestRunForTest([XCTestSuite testSuiteWithName:suiteName], true);
352}
353
354void QXcodeTestLogger::stopLogging()
355{
356 popTestRun();
357}
358
359static bool isTestFunctionInActiveScope(const char *function)
360{
361 static NSString *testScope = [XCTestProbe testScope];
362
363 enum TestScope { Unknown, All, None, Self, Selected };
364 static TestScope activeScope = Unknown;
365
366 if (activeScope == Unknown) {
367 if ([testScope isEqualToString:@"All"])
368 activeScope = All;
369 else if ([testScope isEqualToString:@"None"])
370 activeScope = None;
371 else if ([testScope isEqualToString:@"Self"])
372 activeScope = Self;
373 else
374 activeScope = Selected;
375 }
376
377 if (activeScope == All)
378 return true;
379 else if (activeScope == None)
380 return false;
381 else if (activeScope == Self)
382 return true; // Investigate
383
384 Q_ASSERT(activeScope == Selected);
385
386 static NSArray<NSString *> *forcedTests = [@[ @"initTestCase", @"initTestCase_data", @"cleanupTestCase" ] retain];
387 if ([forcedTests containsObject:[NSString stringWithUTF8String:function]])
388 return true;
389
390 static NSArray<NSString *> *testsInScope = [[testScope componentsSeparatedByString:@","] retain];
391 bool inScope = [testsInScope containsObject:[NSString stringWithFormat:@"%s/%s",
392 QTestResult::currentTestObjectName(), function]];
393
394 if ([XCTestProbe isInverseTestScope])
395 inScope = !inScope;
396
397 return inScope;
398}
399
400void QXcodeTestLogger::enterTestFunction(const char *function)
401{
402 if (!isTestFunctionInActiveScope(function))
403 QTestResult::setSkipCurrentTest(true);
404
405 XCTest *test = [QtTestLibTest testCaseWithInvocation:nil];
406 pushTestRunForTest(test, !QTestResult::skipCurrentTest());
407}
408
409void QXcodeTestLogger::leaveTestFunction()
410{
411 popTestRun();
412}
413
414void QXcodeTestLogger::addIncident(IncidentTypes type, const char *description,
415 const char *file, int line)
416{
417 XCTestRun *testRun = [m_testRuns lastObject];
418
419 // The 'expected' argument to recordFailureWithDescription refers to whether
420 // the failure was a regular failed assertion, or an unexpected exception,
421 // so in our case it's always 'YES', and we need to explicitly ignore XFail.
422 if (type == QAbstractTestLogger::XFail) {
423 QTestCharBuffer buf;
424 NSString *testCaseName = [[testRun test] nameForLegacyLogging];
425 QTest::qt_asprintf(&buf, "Test Case '%s' failed expectedly (%s).\n",
426 [testCaseName UTF8String], description);
427 outputString(buf.constData());
428 return;
429 }
430
431 if (type == QAbstractTestLogger::Pass) {
432 // We ignore non-data passes, as we're already reporting that as part of the
433 // normal test case start/stop cycle.
434 if (!(QTestResult::currentDataTag() || QTestResult::currentGlobalDataTag()))
435 return;
436
437 QTestCharBuffer buf;
438 NSString *testCaseName = [[testRun test] nameForLegacyLogging];
439 QTest::qt_asprintf(&buf, "Test Case '%s' passed.\n", [testCaseName UTF8String]);
440 outputString(buf.constData());
441 return;
442 }
443
444 // FIXME: Handle blacklisted tests
445
446 if (!file || !description)
447 return; // Or report?
448
449 [testRun recordFailureWithDescription:[NSString stringWithUTF8String:description]
450 inFile:[NSString stringWithUTF8String:file] atLine:line expected:YES];
451}
452
453void QXcodeTestLogger::addMessage(MessageTypes type, const QString &message,
454 const char *file, int line)
455{
456 QTestCharBuffer buf;
457
458 if (QTestLog::verboseLevel() > 0 && (file && line)) {
459 QTest::qt_asprintf(&buf, "%s:%d: ", file, line);
460 outputString(buf.constData());
461 }
462
463 if (type == QAbstractTestLogger::Skip) {
464 XCTestRun *testRun = [m_testRuns lastObject];
465 NSString *testCaseName = [[testRun test] nameForLegacyLogging];
466 QTest::qt_asprintf(&buf, "Test Case '%s' skipped (%s).\n",
467 [testCaseName UTF8String], message.toUtf8().constData());
468 } else {
469 QTest::qt_asprintf(&buf, "%s\n", message.toUtf8().constData());
470 }
471
472 outputString(buf.constData());
473}
474
475void QXcodeTestLogger::addBenchmarkResult(const QBenchmarkResult &result)
476{
477 Q_UNUSED(result);
478}
479
480void QXcodeTestLogger::pushTestRunForTest(XCTest *test, bool start)
481{
482 XCTestRun *testRun = [[test testRunClass] testRunWithTest:test];
483 [m_testRuns addObject:testRun];
484
485 if (start)
486 [testRun start];
487}
488
489XCTestRun *QXcodeTestLogger::popTestRun()
490{
491 XCTestRun *testRun = [[m_testRuns lastObject] retain];
492 [m_testRuns removeLastObject];
493
494 if ([testRun startDate])
495 [testRun stop];
496
497 [[m_testRuns lastObject] addTestRun:testRun];
498 [testRun release];
499
500 return testRun;
501}
502
503bool QXcodeTestLogger::isActive()
504{
505 return s_currentTestLogger;
506}
507
508QT_WARNING_POP
509