1/****************************************************************************
2**
3** Copyright (C) 2016 The Qt Company Ltd.
4** Contact: https://www.qt.io/licensing/
5**
6** This file is part of the QtTest module of the Qt Toolkit.
7**
8** $QT_BEGIN_LICENSE:LGPL$
9** Commercial License Usage
10** Licensees holding valid commercial Qt licenses may use this file in
11** accordance with the commercial license agreement provided with the
12** Software or, alternatively, in accordance with the terms contained in
13** a written agreement between you and The Qt Company. For licensing terms
14** and conditions see https://www.qt.io/terms-conditions. For further
15** information use the contact form at https://www.qt.io/contact-us.
16**
17** GNU Lesser General Public License Usage
18** Alternatively, this file may be used under the terms of the GNU Lesser
19** General Public License version 3 as published by the Free Software
20** Foundation and appearing in the file LICENSE.LGPL3 included in the
21** packaging of this file. Please review the following information to
22** ensure the GNU Lesser General Public License version 3 requirements
23** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
24**
25** GNU General Public License Usage
26** Alternatively, this file may be used under the terms of the GNU
27** General Public License version 2.0 or (at your option) the GNU General
28** Public license version 3 or any later version approved by the KDE Free
29** Qt Foundation. The licenses are as published by the Free Software
30** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
31** included in the packaging of this file. Please review the following
32** information to ensure the GNU General Public License requirements will
33** be met: https://www.gnu.org/licenses/gpl-2.0.html and
34** https://www.gnu.org/licenses/gpl-3.0.html.
35**
36** $QT_END_LICENSE$
37**
38****************************************************************************/
39#include "qtestblacklist_p.h"
40#include "qtestresult_p.h"
41
42#include <QtTest/qtestcase.h>
43#include <QtCore/qbytearray.h>
44#include <QtCore/qfile.h>
45#include <QtCore/qset.h>
46#include <QtCore/qcoreapplication.h>
47#include <QtCore/qvariant.h>
48#include <QtCore/QSysInfo>
49#include <QtCore/QOperatingSystemVersion>
50
51#include <set>
52
53QT_BEGIN_NAMESPACE
54
55/*
56 The BLACKLIST file format is a grouped listing of keywords.
57
58 Blank lines and everything after # is simply ignored. An initial #-line
59 referring to this documentation is kind to readers. Comments can also be used
60 to indicate the reasons for ignoring particular cases.
61
62 The key "ci" applies only when run by COIN.
63 The key "cmake" applies when Qt is built using CMake. Other keys name platforms,
64 operating systems, distributions, tool-chains or architectures; a ! prefix
65 reverses what it checks. A version, joined to a key (at present, only for
66 distributions and for msvc) with a hyphen, limits the key to the specific
67 version. A keyword line matches if every key on it applies to the present
68 run. Successive lines are alternate conditions for ignoring a test.
69
70 Ungrouped lines at the beginning of a file apply to the whole testcase.
71 A group starts with a [square-bracketed] identification of a test function,
72 optionally with (after a colon, the name of) a specific data set, to ignore.
73 Subsequent lines give conditions for ignoring this test.
74
75 # See qtbase/src/testlib/qtestblacklist.cpp for format
76 # Test doesn't work on QNX at all
77 qnx
78
79 # QTBUG-12345
80 [testFunction]
81 linux
82 windows 64bit
83
84 # Flaky in COIN on macOS, not reproducible by developers
85 [testSlowly]
86 ci osx
87
88 # Needs basic C++11 support
89 [testfunction2:testData]
90 msvc-2010
91
92 QML test functions are identified using the following format:
93
94 <TestCase name>::<function name>:<data tag>
95
96 For example, to blacklist a QML test on RHEL 7.6:
97
98 # QTBUG-12345
99 [Button::test_display:TextOnly]
100 ci rhel-7.6
101
102 Keys are lower-case. Distribution name and version are supported if
103 QSysInfo's productType() and productVersion() return them.
104
105 Keys can be added via the space-separated QTEST_ENVIRONMENT
106 environment variable:
107
108 QTEST_ENVIRONMENT=ci ./tst_stuff
109
110 This can be used to "mock" a test environment. In the example above,
111 we add "ci" to the list of keys for the test environment, making it
112 possible to test BLACKLIST files that blacklist tests in a CI environment.
113
114 The other known keys are listed below:
115*/
116
117static QSet<QByteArray> keywords()
118{
119 // this list can be extended with new keywords as required
120 QSet<QByteArray> set = QSet<QByteArray>()
121 << "*"
122#ifdef Q_OS_LINUX
123 << "linux"
124#endif
125#ifdef Q_OS_MACOS
126 << "osx"
127 << "macos"
128#endif
129#if defined(Q_OS_WIN)
130 << "windows"
131#endif
132#ifdef Q_OS_IOS
133 << "ios"
134#endif
135#ifdef Q_OS_TVOS
136 << "tvos"
137#endif
138#ifdef Q_OS_WATCHOS
139 << "watchos"
140#endif
141#ifdef Q_OS_ANDROID
142 << "android"
143#endif
144#ifdef Q_OS_QNX
145 << "qnx"
146#endif
147
148#if QT_POINTER_SIZE == 8
149 << "64bit"
150#else
151 << "32bit"
152#endif
153
154#ifdef Q_CC_GNU
155 << "gcc"
156#endif
157#ifdef Q_CC_CLANG
158 << "clang"
159#endif
160#ifdef Q_CC_MSVC
161 << "msvc"
162# if _MSC_VER <= 1600
163 << "msvc-2010"
164# elif _MSC_VER <= 1700
165 << "msvc-2012"
166# elif _MSC_VER <= 1800
167 << "msvc-2013"
168# elif _MSC_VER <= 1900
169 << "msvc-2015"
170# elif _MSC_VER <= 1916
171 << "msvc-2017"
172# else
173 << "msvc-2019"
174# endif
175#endif
176
177#ifdef Q_PROCESSOR_X86
178 << "x86"
179#endif
180#ifdef Q_PROCESSOR_ARM
181 << "arm"
182#endif
183
184#ifdef QT_BUILD_INTERNAL
185 << "developer-build"
186#endif
187
188 << "cmake"
189 ;
190
191#if QT_CONFIG(properties)
192 QCoreApplication *app = QCoreApplication::instance();
193 if (app) {
194 const QVariant platformName = app->property("platformName");
195 if (platformName.isValid())
196 set << platformName.toByteArray();
197 }
198#endif
199
200 return set;
201}
202
203static QSet<QByteArray> activeConditions()
204{
205 QSet<QByteArray> result = keywords();
206
207 QByteArray distributionName = QSysInfo::productType().toLower().toUtf8();
208 QByteArray distributionRelease = QSysInfo::productVersion().toLower().toUtf8();
209 if (!distributionName.isEmpty()) {
210 if (result.find(distributionName) == result.end())
211 result.insert(distributionName);
212 // backwards compatibility with Qt 5
213 if (distributionName == "macos") {
214 if (result.find(distributionName) == result.end())
215 result.insert("osx");
216 const auto version = QOperatingSystemVersion::current();
217 if (version.majorVersion() >= 11)
218 distributionRelease = QByteArray::number(version.majorVersion());
219 }
220 if (!distributionRelease.isEmpty()) {
221 QByteArray versioned = distributionName + "-" + distributionRelease;
222 if (result.find(versioned) == result.end())
223 result.insert(versioned);
224 if (distributionName == "macos") {
225 QByteArray versioned = "osx-" + distributionRelease;
226 if (result.find(versioned) == result.end())
227 result.insert(versioned);
228 }
229 }
230 }
231
232 if (qEnvironmentVariableIsSet("QTEST_ENVIRONMENT")) {
233 for (const QByteArray &k : qgetenv("QTEST_ENVIRONMENT").split(' '))
234 result.insert(k);
235 }
236
237 return result;
238}
239
240static bool checkCondition(const QByteArray &condition)
241{
242 static const QSet<QByteArray> matchedConditions = activeConditions();
243 QList<QByteArray> conds = condition.split(' ');
244
245 for (QByteArray c : conds) {
246 bool result = c.startsWith('!');
247 if (result)
248 c.remove(0, 1);
249
250 result ^= matchedConditions.contains(c);
251 if (!result)
252 return false;
253 }
254 return true;
255}
256
257static bool ignoreAll = false;
258static std::set<QByteArray> *ignoredTests = nullptr;
259
260namespace QTestPrivate {
261
262void parseBlackList()
263{
264 QString filename = QTest::qFindTestData(QStringLiteral("BLACKLIST"));
265 if (filename.isEmpty())
266 return;
267 QFile ignored(filename);
268 if (!ignored.open(QIODevice::ReadOnly))
269 return;
270
271 QByteArray function;
272
273 while (!ignored.atEnd()) {
274 QByteArray line = ignored.readLine();
275 const int commentPosition = line.indexOf('#');
276 if (commentPosition >= 0)
277 line.truncate(commentPosition);
278 line = line.simplified();
279 if (line.isEmpty())
280 continue;
281 if (line.startsWith('[')) {
282 function = line.mid(1, line.length() - 2);
283 continue;
284 }
285 bool condition = checkCondition(line);
286 if (condition) {
287 if (!function.size()) {
288 ignoreAll = true;
289 } else {
290 if (!ignoredTests)
291 ignoredTests = new std::set<QByteArray>;
292 ignoredTests->insert(function);
293 }
294 }
295 }
296}
297
298void checkBlackLists(const char *slot, const char *data)
299{
300 bool ignore = ignoreAll;
301
302 if (!ignore && ignoredTests) {
303 QByteArray s = slot;
304 ignore = (ignoredTests->find(s) != ignoredTests->end());
305 if (!ignore && data) {
306 s += ':';
307 s += data;
308 ignore = (ignoredTests->find(s) != ignoredTests->end());
309 }
310 }
311
312 QTestResult::setBlacklistCurrentTest(ignore);
313}
314
315} // QTestPrivate
316
317QT_END_NAMESPACE
318