1/****************************************************************************
2**
3** Copyright (C) 2020 The Qt Company Ltd.
4** Contact: https://www.qt.io/licensing/
5**
6** This file is part of the QtTest module of the Qt Toolkit.
7**
8** $QT_BEGIN_LICENSE:LGPL$
9** Commercial License Usage
10** Licensees holding valid commercial Qt licenses may use this file in
11** accordance with the commercial license agreement provided with the
12** Software or, alternatively, in accordance with the terms contained in
13** a written agreement between you and The Qt Company. For licensing terms
14** and conditions see https://www.qt.io/terms-conditions. For further
15** information use the contact form at https://www.qt.io/contact-us.
16**
17** GNU Lesser General Public License Usage
18** Alternatively, this file may be used under the terms of the GNU Lesser
19** General Public License version 3 as published by the Free Software
20** Foundation and appearing in the file LICENSE.LGPL3 included in the
21** packaging of this file. Please review the following information to
22** ensure the GNU Lesser General Public License version 3 requirements
23** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
24**
25** GNU General Public License Usage
26** Alternatively, this file may be used under the terms of the GNU
27** General Public License version 2.0 or (at your option) the GNU General
28** Public license version 3 or any later version approved by the KDE Free
29** Qt Foundation. The licenses are as published by the Free Software
30** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
31** included in the packaging of this file. Please review the following
32** information to ensure the GNU General Public License requirements will
33** be met: https://www.gnu.org/licenses/gpl-2.0.html and
34** https://www.gnu.org/licenses/gpl-3.0.html.
35**
36** $QT_END_LICENSE$
37**
38****************************************************************************/
39
40#include <QtTest/qtestassert.h>
41
42#include <QtTest/private/qtestlog_p.h>
43#include <QtTest/private/qtestresult_p.h>
44#include <QtTest/private/qabstracttestlogger_p.h>
45#include <QtTest/private/qplaintestlogger_p.h>
46#include <QtTest/private/qcsvbenchmarklogger_p.h>
47#include <QtTest/private/qjunittestlogger_p.h>
48#include <QtTest/private/qxmltestlogger_p.h>
49#include <QtTest/private/qteamcitylogger_p.h>
50#include <QtTest/private/qtaptestlogger_p.h>
51#if defined(HAVE_XCTEST)
52#include <QtTest/private/qxctestlogger_p.h>
53#endif
54
55#if defined(Q_OS_DARWIN)
56#include <QtTest/private/qappletestlogger_p.h>
57#endif
58
59#include <QtCore/qatomic.h>
60#include <QtCore/qbytearray.h>
61#include <QtCore/qelapsedtimer.h>
62#include <QtCore/qlist.h>
63#include <QtCore/qvariant.h>
64#if QT_CONFIG(regularexpression)
65#include <QtCore/QRegularExpression>
66#endif
67
68#include <stdlib.h>
69#include <string.h>
70#include <limits.h>
71
72QT_BEGIN_NAMESPACE
73
74static void saveCoverageTool(const char * appname, bool testfailed, bool installedTestCoverage)
75{
76#ifdef __COVERAGESCANNER__
77# if QT_CONFIG(testlib_selfcover)
78 __coveragescanner_teststate(QTestLog::failCount() > 0 ? "FAILED" :
79 QTestLog::passCount() > 0 ? "PASSED" : "SKIPPED");
80# else
81 if (!installedTestCoverage)
82 return;
83 // install again to make sure the filename is correct.
84 // without this, a plugin or similar may have changed the filename.
85 __coveragescanner_install(appname);
86 __coveragescanner_teststate(testfailed ? "FAILED" : "PASSED");
87 __coveragescanner_save();
88 __coveragescanner_testname("");
89 __coveragescanner_clear();
90 unsetenv("QT_TESTCOCOON_ACTIVE");
91# endif // testlib_selfcover
92#else
93 Q_UNUSED(appname);
94 Q_UNUSED(testfailed);
95 Q_UNUSED(installedTestCoverage);
96#endif
97}
98
99static QElapsedTimer elapsedFunctionTime;
100static QElapsedTimer elapsedTotalTime;
101
102#define FOREACH_TEST_LOGGER for (QAbstractTestLogger *logger : *QTest::loggers())
103
104namespace QTest {
105
106 int fails = 0;
107 int passes = 0;
108 int skips = 0;
109 int blacklists = 0;
110
111 struct IgnoreResultList
112 {
113 inline IgnoreResultList(QtMsgType tp, const QVariant &patternIn)
114 : type(tp), pattern(patternIn) {}
115
116 static inline void clearList(IgnoreResultList *&list)
117 {
118 while (list) {
119 IgnoreResultList *current = list;
120 list = list->next;
121 delete current;
122 }
123 }
124
125 static void append(IgnoreResultList *&list, QtMsgType type, const QVariant &patternIn)
126 {
127 QTest::IgnoreResultList *item = new QTest::IgnoreResultList(type, patternIn);
128
129 if (!list) {
130 list = item;
131 return;
132 }
133 IgnoreResultList *last = list;
134 for ( ; last->next; last = last->next) ;
135 last->next = item;
136 }
137
138 static bool stringsMatch(const QString &expected, const QString &actual)
139 {
140 if (expected == actual)
141 return true;
142
143 // ignore an optional whitespace at the end of str
144 // (the space was added automatically by ~QDebug() until Qt 5.3,
145 // so autotests still might expect it)
146 if (expected.endsWith(QLatin1Char(' ')))
147 return actual == QStringView{expected}.left(expected.length() - 1);
148
149 return false;
150 }
151
152 inline bool matches(QtMsgType tp, const QString &message) const
153 {
154 return tp == type
155 && (pattern.userType() == QMetaType::QString ?
156 stringsMatch(pattern.toString(), message) :
157#if QT_CONFIG(regularexpression)
158 pattern.toRegularExpression().match(message).hasMatch());
159#else
160 false);
161#endif
162 }
163
164 QtMsgType type;
165 QVariant pattern;
166 IgnoreResultList *next = nullptr;
167 };
168
169 static IgnoreResultList *ignoreResultList = nullptr;
170
171 Q_GLOBAL_STATIC(QList<QAbstractTestLogger *>, loggers)
172
173 static int verbosity = 0;
174 static int maxWarnings = 2002;
175 static bool installedTestCoverage = true;
176
177 static QtMessageHandler oldMessageHandler;
178
179 static bool handleIgnoredMessage(QtMsgType type, const QString &message)
180 {
181 if (!ignoreResultList)
182 return false;
183 IgnoreResultList *last = nullptr;
184 IgnoreResultList *list = ignoreResultList;
185 while (list) {
186 if (list->matches(type, message)) {
187 // remove the item from the list
188 if (last)
189 last->next = list->next;
190 else if (list->next)
191 ignoreResultList = list->next;
192 else
193 ignoreResultList = nullptr;
194
195 delete list;
196 return true;
197 }
198
199 last = list;
200 list = list->next;
201 }
202 return false;
203 }
204
205 static void messageHandler(QtMsgType type, const QMessageLogContext & context, const QString &message)
206 {
207 static QBasicAtomicInt counter = Q_BASIC_ATOMIC_INITIALIZER(QTest::maxWarnings);
208
209 if (QTestLog::loggerCount() == 0) {
210 // if this goes wrong, something is seriously broken.
211 qInstallMessageHandler(oldMessageHandler);
212 QTEST_ASSERT(QTestLog::loggerCount() != 0);
213 }
214
215 if (handleIgnoredMessage(type, message)) {
216 // the message is expected, so just swallow it.
217 return;
218 }
219
220 if (type != QtFatalMsg) {
221 if (counter.loadRelaxed() <= 0)
222 return;
223
224 if (!counter.deref()) {
225 FOREACH_TEST_LOGGER {
226 logger->addMessage(QAbstractTestLogger::QSystem,
227 QStringLiteral("Maximum amount of warnings exceeded. Use -maxwarnings to override."));
228 }
229 return;
230 }
231 }
232
233 FOREACH_TEST_LOGGER
234 logger->addMessage(type, context, message);
235
236 if (type == QtFatalMsg) {
237 /* Right now, we're inside the custom message handler and we're
238 * being qt_message_output in qglobal.cpp. After we return from
239 * this function, it will proceed with calling exit() and abort()
240 * and hence crash. Therefore, we call these logging functions such
241 * that we wrap up nicely, and in particular produce well-formed XML. */
242 QTestResult::addFailure("Received a fatal error.", "Unknown file", 0);
243 QTestLog::leaveTestFunction();
244 QTestLog::stopLogging();
245 }
246 }
247}
248
249void QTestLog::enterTestFunction(const char* function)
250{
251 elapsedFunctionTime.restart();
252 if (printAvailableTags)
253 return;
254
255 QTEST_ASSERT(function);
256
257 FOREACH_TEST_LOGGER
258 logger->enterTestFunction(function);
259}
260
261void QTestLog::enterTestData(QTestData *data)
262{
263 QTEST_ASSERT(data);
264
265 FOREACH_TEST_LOGGER
266 logger->enterTestData(data);
267}
268
269int QTestLog::unhandledIgnoreMessages()
270{
271 int i = 0;
272 QTest::IgnoreResultList *list = QTest::ignoreResultList;
273 while (list) {
274 ++i;
275 list = list->next;
276 }
277 return i;
278}
279
280void QTestLog::leaveTestFunction()
281{
282 if (printAvailableTags)
283 return;
284
285 FOREACH_TEST_LOGGER
286 logger->leaveTestFunction();
287}
288
289void QTestLog::printUnhandledIgnoreMessages()
290{
291 QString message;
292 QTest::IgnoreResultList *list = QTest::ignoreResultList;
293 while (list) {
294 if (list->pattern.userType() == QMetaType::QString) {
295 message = QStringLiteral("Did not receive message: \"") + list->pattern.toString() + QLatin1Char('"');
296 } else {
297#if QT_CONFIG(regularexpression)
298 message = QStringLiteral("Did not receive any message matching: \"") + list->pattern.toRegularExpression().pattern() + QLatin1Char('"');
299#endif
300 }
301 FOREACH_TEST_LOGGER
302 logger->addMessage(QAbstractTestLogger::Info, message);
303
304 list = list->next;
305 }
306}
307
308void QTestLog::clearIgnoreMessages()
309{
310 QTest::IgnoreResultList::clearList(QTest::ignoreResultList);
311}
312
313void QTestLog::addPass(const char *msg)
314{
315 if (printAvailableTags)
316 return;
317
318 QTEST_ASSERT(msg);
319
320 ++QTest::passes;
321
322 FOREACH_TEST_LOGGER
323 logger->addIncident(QAbstractTestLogger::Pass, msg);
324}
325
326void QTestLog::addFail(const char *msg, const char *file, int line)
327{
328 QTEST_ASSERT(msg);
329
330 ++QTest::fails;
331
332 FOREACH_TEST_LOGGER
333 logger->addIncident(QAbstractTestLogger::Fail, msg, file, line);
334}
335
336void QTestLog::addXFail(const char *msg, const char *file, int line)
337{
338 QTEST_ASSERT(msg);
339 QTEST_ASSERT(file);
340
341 FOREACH_TEST_LOGGER
342 logger->addIncident(QAbstractTestLogger::XFail, msg, file, line);
343}
344
345void QTestLog::addXPass(const char *msg, const char *file, int line)
346{
347 QTEST_ASSERT(msg);
348 QTEST_ASSERT(file);
349
350 ++QTest::fails;
351
352 FOREACH_TEST_LOGGER
353 logger->addIncident(QAbstractTestLogger::XPass, msg, file, line);
354}
355
356void QTestLog::addBPass(const char *msg)
357{
358 QTEST_ASSERT(msg);
359
360 ++QTest::blacklists;
361
362 FOREACH_TEST_LOGGER
363 logger->addIncident(QAbstractTestLogger::BlacklistedPass, msg);
364}
365
366void QTestLog::addBFail(const char *msg, const char *file, int line)
367{
368 QTEST_ASSERT(msg);
369 QTEST_ASSERT(file);
370
371 ++QTest::blacklists;
372
373 FOREACH_TEST_LOGGER
374 logger->addIncident(QAbstractTestLogger::BlacklistedFail, msg, file, line);
375}
376
377void QTestLog::addBXPass(const char *msg, const char *file, int line)
378{
379 QTEST_ASSERT(msg);
380 QTEST_ASSERT(file);
381
382 ++QTest::blacklists;
383
384 FOREACH_TEST_LOGGER
385 logger->addIncident(QAbstractTestLogger::BlacklistedXPass, msg, file, line);
386}
387
388void QTestLog::addBXFail(const char *msg, const char *file, int line)
389{
390 QTEST_ASSERT(msg);
391 QTEST_ASSERT(file);
392
393 ++QTest::blacklists;
394
395 FOREACH_TEST_LOGGER
396 logger->addIncident(QAbstractTestLogger::BlacklistedXFail, msg, file, line);
397}
398
399void QTestLog::addSkip(const char *msg, const char *file, int line)
400{
401 QTEST_ASSERT(msg);
402 QTEST_ASSERT(file);
403
404 ++QTest::skips;
405
406 FOREACH_TEST_LOGGER
407 logger->addMessage(QAbstractTestLogger::Skip, QString::fromUtf8(msg), file, line);
408}
409
410void QTestLog::addBenchmarkResult(const QBenchmarkResult &result)
411{
412 FOREACH_TEST_LOGGER
413 logger->addBenchmarkResult(result);
414}
415
416void QTestLog::startLogging()
417{
418 elapsedTotalTime.start();
419 elapsedFunctionTime.start();
420 FOREACH_TEST_LOGGER
421 logger->startLogging();
422 QTest::oldMessageHandler = qInstallMessageHandler(QTest::messageHandler);
423}
424
425void QTestLog::stopLogging()
426{
427 qInstallMessageHandler(QTest::oldMessageHandler);
428 FOREACH_TEST_LOGGER {
429 logger->stopLogging();
430 delete logger;
431 }
432 QTest::loggers()->clear();
433 saveCoverageTool(QTestResult::currentAppName(), failCount() != 0, QTestLog::installedTestCoverage());
434}
435
436void QTestLog::addLogger(LogMode mode, const char *filename)
437{
438 if (filename && strcmp(filename, "-") == 0)
439 filename = nullptr;
440
441 QAbstractTestLogger *logger = nullptr;
442 switch (mode) {
443 case QTestLog::Plain:
444 logger = new QPlainTestLogger(filename);
445 break;
446 case QTestLog::CSV:
447 logger = new QCsvBenchmarkLogger(filename);
448 break;
449 case QTestLog::XML:
450 logger = new QXmlTestLogger(QXmlTestLogger::Complete, filename);
451 break;
452 case QTestLog::LightXML:
453 logger = new QXmlTestLogger(QXmlTestLogger::Light, filename);
454 break;
455 case QTestLog::JUnitXML:
456 logger = new QJUnitTestLogger(filename);
457 break;
458 case QTestLog::TeamCity:
459 logger = new QTeamCityLogger(filename);
460 break;
461 case QTestLog::TAP:
462 logger = new QTapTestLogger(filename);
463 break;
464#if defined(QT_USE_APPLE_UNIFIED_LOGGING)
465 case QTestLog::Apple:
466 logger = new QAppleTestLogger;
467 break;
468#endif
469#if defined(HAVE_XCTEST)
470 case QTestLog::XCTest:
471 logger = new QXcodeTestLogger;
472 break;
473#endif
474 }
475
476 QTEST_ASSERT(logger);
477 addLogger(logger);
478}
479
480/*!
481 \internal
482
483 Adds a new logger to the set of loggers that will be used
484 to report incidents and messages during testing.
485
486 The function takes ownership of the logger.
487*/
488void QTestLog::addLogger(QAbstractTestLogger *logger)
489{
490 QTEST_ASSERT(logger);
491 QTest::loggers()->append(logger);
492}
493
494int QTestLog::loggerCount()
495{
496 return QTest::loggers()->size();
497}
498
499bool QTestLog::loggerUsingStdout()
500{
501 FOREACH_TEST_LOGGER {
502 if (logger->isLoggingToStdout())
503 return true;
504 }
505
506 return false;
507}
508
509void QTestLog::warn(const char *msg, const char *file, int line)
510{
511 QTEST_ASSERT(msg);
512
513 FOREACH_TEST_LOGGER
514 logger->addMessage(QAbstractTestLogger::Warn, QString::fromUtf8(msg), file, line);
515}
516
517void QTestLog::info(const char *msg, const char *file, int line)
518{
519 QTEST_ASSERT(msg);
520
521 FOREACH_TEST_LOGGER
522 logger->addMessage(QAbstractTestLogger::Info, QString::fromUtf8(msg), file, line);
523}
524
525void QTestLog::setVerboseLevel(int level)
526{
527 QTest::verbosity = level;
528}
529
530int QTestLog::verboseLevel()
531{
532 return QTest::verbosity;
533}
534
535void QTestLog::ignoreMessage(QtMsgType type, const char *msg)
536{
537 QTEST_ASSERT(msg);
538
539 QTest::IgnoreResultList::append(QTest::ignoreResultList, type, QString::fromUtf8(msg));
540}
541
542#if QT_CONFIG(regularexpression)
543void QTestLog::ignoreMessage(QtMsgType type, const QRegularExpression &expression)
544{
545 QTEST_ASSERT(expression.isValid());
546
547 QTest::IgnoreResultList::append(QTest::ignoreResultList, type, QVariant(expression));
548}
549#endif // QT_CONFIG(regularexpression)
550
551void QTestLog::setMaxWarnings(int m)
552{
553 QTest::maxWarnings = m <= 0 ? INT_MAX : m + 2;
554}
555
556bool QTestLog::printAvailableTags = false;
557
558void QTestLog::setPrintAvailableTagsMode()
559{
560 printAvailableTags = true;
561}
562
563int QTestLog::passCount()
564{
565 return QTest::passes;
566}
567
568int QTestLog::failCount()
569{
570 return QTest::fails;
571}
572
573int QTestLog::skipCount()
574{
575 return QTest::skips;
576}
577
578int QTestLog::blacklistCount()
579{
580 return QTest::blacklists;
581}
582
583int QTestLog::totalCount()
584{
585 return passCount() + failCount() + skipCount() + blacklistCount();
586}
587
588void QTestLog::resetCounters()
589{
590 QTest::passes = 0;
591 QTest::fails = 0;
592 QTest::skips = 0;
593}
594
595void QTestLog::setInstalledTestCoverage(bool installed)
596{
597 QTest::installedTestCoverage = installed;
598}
599
600bool QTestLog::installedTestCoverage()
601{
602 return QTest::installedTestCoverage;
603}
604
605qint64 QTestLog::nsecsTotalTime()
606{
607 return elapsedTotalTime.nsecsElapsed();
608}
609
610qint64 QTestLog::nsecsFunctionTime()
611{
612 return elapsedFunctionTime.nsecsElapsed();
613}
614
615QT_END_NAMESPACE
616
617#include "moc_qtestlog_p.cpp"
618