1//
2// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
3// See https://llvm.org/LICENSE.txt for license information.
4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5
6//
7// testfilerunner.m
8// testObjects
9//
10// Created by Blaine Garst on 9/24/08.
11//
12
13#import "testfilerunner.h"
14#import <Foundation/Foundation.h>
15#include <stdio.h>
16#include <unistd.h>
17#include <fcntl.h>
18#include <string.h>
19#include <stdlib.h>
20#include <stdbool.h>
21
22bool Everything = false; // do it also with 3 levels of optimization
23bool DoClang = false;
24
25static bool isDirectory(char *path);
26static bool isExecutable(char *path);
27static bool isYounger(char *source, char *binary);
28static bool readErrorFile(char *buffer, const char *from);
29
30__strong char *gcstrcpy2(__strong const char *arg, char *endp) {
31 unsigned size = endp - arg + 1;
32 __strong char *result = NSAllocateCollectable(size, 0);
33 strncpy(result, arg, size);
34 result[size-1] = 0;
35 return result;
36}
37__strong char *gcstrcpy1(__strong char *arg) {
38 unsigned size = strlen(arg) + 1;
39 __strong char *result = NSAllocateCollectable(size, 0);
40 strncpy(result, arg, size);
41 result[size-1] = 0;
42 return result;
43}
44
45@implementation TestFileExe
46
47@synthesize options, compileLine, shouldFail, binaryName, sourceName;
48@synthesize generator;
49@synthesize libraryPath, frameworkPath;
50
51- (NSString *)description {
52 NSMutableString *result = [NSMutableString new];
53 if (shouldFail) [result appendString:@"fail"];
54 for (id x in compileLine) {
55 [result appendString:[NSString stringWithFormat:@" %s", (char *)x]];
56 }
57 return result;
58}
59
60- (__strong char *)radar {
61 return generator.radar;
62}
63
64- (bool) compileUnlessExists:(bool)skip {
65 if (shouldFail) {
66 printf("don't use this to compile anymore!\n");
67 return false;
68 }
69 if (skip && isExecutable(binaryName) && !isYounger(sourceName, binaryName)) return true;
70 int argc = [compileLine count];
71 char *argv[argc+1];
72 for (int i = 0; i < argc; ++i)
73 argv[i] = (char *)[compileLine pointerAtIndex:i];
74 argv[argc] = NULL;
75 pid_t child = fork();
76 if (child == 0) {
77 execv(argv[0], argv);
78 exit(10); // shouldn't happen
79 }
80 if (child < 0) {
81 printf("fork failed\n");
82 return false;
83 }
84 int status = 0;
85 pid_t deadchild = wait(&status);
86 if (deadchild != child) {
87 printf("wait got %d instead of %d\n", deadchild, child);
88 exit(1);
89 }
90 if (WEXITSTATUS(status) == 0) {
91 return true;
92 }
93 printf("run failed\n");
94 return false;
95}
96
97bool lookforIn(char *lookfor, const char *format, pid_t child) {
98 char buffer[512];
99 char got[512];
100 sprintf(buffer, format, child);
101 bool gotOutput = readErrorFile(got, buffer);
102 if (!gotOutput) {
103 printf("**** didn't get an output file %s to analyze!!??\n", buffer);
104 return false;
105 }
106 char *where = strstr(got, lookfor);
107 if (!where) {
108 printf("didn't find '%s' in output file %s\n", lookfor, buffer);
109 return false;
110 }
111 unlink(buffer);
112 return true;
113}
114
115- (bool) compileWithExpectedFailure {
116 if (!shouldFail) {
117 printf("Why am I being called?\n");
118 return false;
119 }
120 int argc = [compileLine count];
121 char *argv[argc+1];
122 for (int i = 0; i < argc; ++i)
123 argv[i] = (char *)[compileLine pointerAtIndex:i];
124 argv[argc] = NULL;
125 pid_t child = fork();
126 char buffer[512];
127 if (child == 0) {
128 // in child
129 sprintf(buffer, "/tmp/errorfile_%d", getpid());
130 close(1);
131 int fd = creat(buffer, 0777);
132 if (fd != 1) {
133 fprintf(stderr, "didn't open custom error file %s as 1, got %d\n", buffer, fd);
134 exit(1);
135 }
136 close(2);
137 dup(1);
138 int result = execv(argv[0], argv);
139 exit(10);
140 }
141 if (child < 0) {
142 printf("fork failed\n");
143 return false;
144 }
145 int status = 0;
146 pid_t deadchild = wait(&status);
147 if (deadchild != child) {
148 printf("wait got %d instead of %d\n", deadchild, child);
149 exit(11);
150 }
151 if (WIFEXITED(status)) {
152 if (WEXITSTATUS(status) == 0) {
153 return false;
154 }
155 }
156 else {
157 printf("***** compiler borked/ICEd/died unexpectedly (status %x)\n", status);
158 return false;
159 }
160 char *error = generator.errorString;
161
162 if (!error) return true;
163#if 0
164 char got[512];
165 sprintf(buffer, "/tmp/errorfile_%d", child);
166 bool gotOutput = readErrorFile(got, buffer);
167 if (!gotOutput) {
168 printf("**** didn't get an error file %s to analyze!!??\n", buffer);
169 return false;
170 }
171 char *where = strstr(got, error);
172 if (!where) {
173 printf("didn't find '%s' in error file %s\n", error, buffer);
174 return false;
175 }
176 unlink(buffer);
177#else
178 if (!lookforIn(error, "/tmp/errorfile_%d", child)) return false;
179#endif
180 return true;
181}
182
183- (bool) run {
184 if (shouldFail) return true;
185 if (sizeof(long) == 4 && options & Do64) {
186 return true; // skip 64-bit tests
187 }
188 int argc = 1;
189 char *argv[argc+1];
190 argv[0] = binaryName;
191 argv[argc] = NULL;
192 pid_t child = fork();
193 if (child == 0) {
194 // set up environment
195 char lpath[1024];
196 char fpath[1024];
197 char *myenv[3];
198 int counter = 0;
199 if (libraryPath) {
200 sprintf(lpath, "DYLD_LIBRARY_PATH=%s", libraryPath);
201 myenv[counter++] = lpath;
202 }
203 if (frameworkPath) {
204 sprintf(fpath, "DYLD_FRAMEWORK_PATH=%s", frameworkPath);
205 myenv[counter++] = fpath;
206 }
207 myenv[counter] = NULL;
208 if (generator.warningString) {
209 // set up stdout/stderr
210 char outfile[1024];
211 sprintf(outfile, "/tmp/stdout_%d", getpid());
212 close(2);
213 close(1);
214 creat(outfile, 0700);
215 dup(1);
216 }
217 execve(argv[0], argv, myenv);
218 exit(10); // shouldn't happen
219 }
220 if (child < 0) {
221 printf("fork failed\n");
222 return false;
223 }
224 int status = 0;
225 pid_t deadchild = wait(&status);
226 if (deadchild != child) {
227 printf("wait got %d instead of %d\n", deadchild, child);
228 exit(1);
229 }
230 if (WIFEXITED(status) && WEXITSTATUS(status) == 0) {
231 if (generator.warningString) {
232 if (!lookforIn(generator.warningString, "/tmp/stdout_%d", child)) return false;
233 }
234 return true;
235 }
236 printf("**** run failed for %s\n", binaryName);
237 return false;
238}
239
240@end
241
242@implementation TestFileExeGenerator
243@synthesize filename, compilerPath, errorString;
244@synthesize hasObjC, hasRR, hasGC, hasCPlusPlus, wantsC99, supposedToNotCompile, open, wants32, wants64;
245@synthesize radar;
246@synthesize warningString;
247
248- (void)setFilename:(__strong char *)name {
249 filename = gcstrcpy1(name);
250}
251- (void)setCompilerPath:(__strong char *)name {
252 compilerPath = gcstrcpy1(name);
253}
254
255- (void)forMostThings:(NSMutableArray *)lines options:(int)options {
256 TestFileExe *item = nil;
257 item = [self lineForOptions:options];
258 if (item) [lines addObject:item];
259 item = [self lineForOptions:options|Do64];
260 if (item) [lines addObject:item];
261 item = [self lineForOptions:options|DoCPP];
262 if (item) [lines addObject:item];
263 item = [self lineForOptions:options|Do64|DoCPP];
264 if (item) [lines addObject:item];
265}
266
267/*
268 DoDashG = (1 << 8),
269 DoDashO = (1 << 9),
270 DoDashOs = (1 << 10),
271 DoDashO2 = (1 << 11),
272*/
273
274- (void)forAllThings:(NSMutableArray *)lines options:(int)options {
275 [self forMostThings:lines options:options];
276 if (!Everything) {
277 return;
278 }
279 // now do it with three explicit optimization flags
280 [self forMostThings:lines options:options | DoDashO];
281 [self forMostThings:lines options:options | DoDashOs];
282 [self forMostThings:lines options:options | DoDashO2];
283}
284
285- (NSArray *)allLines {
286 NSMutableArray *result = [NSMutableArray new];
287 TestFileExe *item = nil;
288
289 int options = 0;
290 [self forAllThings:result options:0];
291 [self forAllThings:result options:DoOBJC | DoRR];
292 [self forAllThings:result options:DoOBJC | DoGC];
293 [self forAllThings:result options:DoOBJC | DoGCRR];
294 //[self forAllThings:result options:DoOBJC | DoRRGC];
295
296 return result;
297}
298
299- (void)addLibrary:(const char *)dashLSomething {
300 if (!extraLibraries) {
301 extraLibraries = [NSPointerArray pointerArrayWithOptions:
302 NSPointerFunctionsStrongMemory |
303 NSPointerFunctionsCStringPersonality];
304 }
305 [extraLibraries addPointer:(void *)dashLSomething];
306}
307
308- (TestFileExe *)lineForOptions:(int)options { // nil if no can do
309 if (hasObjC && !(options & DoOBJC)) return nil;
310 if (hasCPlusPlus && !(options & DoCPP)) return nil;
311 if (hasObjC) {
312 if (!hasGC && (options & (DoGC|DoGCRR))) return nil; // not smart enough
313 if (!hasRR && (options & (DoRR|DoRRGC))) return nil;
314 }
315 NSPointerArray *pa = [NSPointerArray pointerArrayWithOptions:
316 NSPointerFunctionsStrongMemory |
317 NSPointerFunctionsCStringPersonality];
318 // construct path
319 char path[512];
320 path[0] = 0;
321 if (!compilerPath) compilerPath = "/usr/bin";
322 if (compilerPath) {
323 strcat(path, compilerPath);
324 strcat(path, "/");
325 }
326 if (options & DoCPP) {
327 strcat(path, DoClang ? "clang++" : "g++-4.2");
328 }
329 else {
330 strcat(path, DoClang ? "clang" : "gcc-4.2");
331 }
332 [pa addPointer:gcstrcpy1(path)];
333 if (options & DoOBJC) {
334 if (options & DoCPP) {
335 [pa addPointer:"-ObjC++"];
336 }
337 else {
338 [pa addPointer:"-ObjC"];
339 }
340 }
341 [pa addPointer:"-g"];
342 if (options & DoDashO) [pa addPointer:"-O"];
343 else if (options & DoDashO2) [pa addPointer:"-O2"];
344 else if (options & DoDashOs) [pa addPointer:"-Os"];
345 if (wantsC99 && (! (options & DoCPP))) {
346 [pa addPointer:"-std=c99"];
347 [pa addPointer:"-fblocks"];
348 }
349 [pa addPointer:"-arch"];
350 [pa addPointer: (options & Do64) ? "x86_64" : "i386"];
351
352 if (options & DoOBJC) {
353 switch (options & (DoRR|DoGC|DoGCRR|DoRRGC)) {
354 case DoRR:
355 break;
356 case DoGC:
357 [pa addPointer:"-fobjc-gc-only"];
358 break;
359 case DoGCRR:
360 [pa addPointer:"-fobjc-gc"];
361 break;
362 case DoRRGC:
363 printf("DoRRGC unsupported right now\n");
364 [pa addPointer:"-c"];
365 return nil;
366 }
367 [pa addPointer:"-framework"];
368 [pa addPointer:"Foundation"];
369 }
370 [pa addPointer:gcstrcpy1(filename)];
371 [pa addPointer:"-o"];
372
373 path[0] = 0;
374 strcat(path, filename);
375 strcat(path, ".");
376 strcat(path, (options & Do64) ? "64" : "32");
377 if (options & DoOBJC) {
378 switch (options & (DoRR|DoGC|DoGCRR|DoRRGC)) {
379 case DoRR: strcat(path, "-rr"); break;
380 case DoGC: strcat(path, "-gconly"); break;
381 case DoGCRR: strcat(path, "-gcrr"); break;
382 case DoRRGC: strcat(path, "-rrgc"); break;
383 }
384 }
385 if (options & DoCPP) strcat(path, "++");
386 if (options & DoDashO) strcat(path, "-O");
387 else if (options & DoDashO2) strcat(path, "-O2");
388 else if (options & DoDashOs) strcat(path, "-Os");
389 if (wantsC99) strcat(path, "-C99");
390 strcat(path, DoClang ? "-clang" : "-gcc");
391 strcat(path, "-bin");
392 TestFileExe *result = [TestFileExe new];
393 result.binaryName = gcstrcpy1(path); // could snarf copy in pa
394 [pa addPointer:result.binaryName];
395 for (id cString in extraLibraries) {
396 [pa addPointer:cString];
397 }
398
399 result.sourceName = gcstrcpy1(filename); // could snarf copy in pa
400 result.compileLine = pa;
401 result.options = options;
402 result.shouldFail = supposedToNotCompile;
403 result.generator = self;
404 return result;
405}
406
407+ (NSArray *)generatorsFromPath:(NSString *)path {
408 FILE *fp = fopen([path fileSystemRepresentation], "r");
409 if (fp == NULL) return nil;
410 NSArray *result = [self generatorsFromFILE:fp];
411 fclose(fp);
412 return result;
413}
414
415#define LOOKFOR "CON" "FIG"
416
417char *__strong parseRadar(char *line) {
418 line = strstr(line, "rdar:"); // returns beginning
419 char *endp = line + strlen("rdar:");
420 while (*endp && *endp != ' ' && *endp != '\n')
421 ++endp;
422 return gcstrcpy2(line, endp);
423}
424
425- (void)parseLibraries:(const char *)line {
426 start:
427 line = strstr(line, "-l");
428 char *endp = (char *)line + 2;
429 while (*endp && *endp != ' ' && *endp != '\n')
430 ++endp;
431 [self addLibrary:gcstrcpy2(line, endp)];
432 if (strstr(endp, "-l")) {
433 line = endp;
434 goto start;
435 }
436}
437
438+ (TestFileExeGenerator *)generatorFromLine:(char *)line filename:(char *)filename {
439 TestFileExeGenerator *item = [TestFileExeGenerator new];
440 item.filename = gcstrcpy1(filename);
441 if (strstr(line, "GC")) item.hasGC = true;
442 if (strstr(line, "RR")) item.hasRR = true;
443 if (strstr(line, "C++")) item.hasCPlusPlus = true;
444 if (strstr(line, "-C99")) {
445 item.wantsC99 = true;
446 }
447 if (strstr(line, "64")) item.wants64 = true;
448 if (strstr(line, "32")) item.wants32 = true;
449 if (strstr(line, "-l")) [item parseLibraries:line];
450 if (strstr(line, "open")) item.open = true;
451 if (strstr(line, "FAIL")) item.supposedToNotCompile = true; // old
452 // compile time error
453 if (strstr(line, "error:")) {
454 item.supposedToNotCompile = true;
455 // zap newline
456 char *error = strstr(line, "error:") + strlen("error:");
457 // make sure we have something before the newline
458 char *newline = strstr(error, "\n");
459 if (newline && ((newline-error) > 1)) {
460 *newline = 0;
461 item.errorString = gcstrcpy1(strstr(line, "error:") + strlen("error: "));
462 }
463 }
464 // run time warning
465 if (strstr(line, "runtime:")) {
466 // zap newline
467 char *error = strstr(line, "runtime:") + strlen("runtime:");
468 // make sure we have something before the newline
469 char *newline = strstr(error, "\n");
470 if (newline && ((newline-error) > 1)) {
471 *newline = 0;
472 item.warningString = gcstrcpy1(strstr(line, "runtime:") + strlen("runtime:"));
473 }
474 }
475 if (strstr(line, "rdar:")) item.radar = parseRadar(line);
476 if (item.hasGC || item.hasRR) item.hasObjC = true;
477 if (!item.wants32 && !item.wants64) { // give them both if they ask for neither
478 item.wants32 = item.wants64 = true;
479 }
480 return item;
481}
482
483+ (NSArray *)generatorsFromFILE:(FILE *)fp {
484 NSMutableArray *result = [NSMutableArray new];
485 // pretend this is a grep LOOKFOR *.[cmCM][cmCM] input
486 // look for
487 // filename: ... LOOKFOR [GC] [RR] [C++] [FAIL ...]
488 char buf[512];
489 while (fgets(buf, 512, fp)) {
490 char *config = strstr(buf, LOOKFOR);
491 if (!config) continue;
492 char *filename = buf;
493 char *end = strchr(buf, ':');
494 *end = 0;
495 [result addObject:[self generatorFromLine:config filename:filename]];
496 }
497 return result;
498}
499
500+ (TestFileExeGenerator *)generatorFromFilename:(char *)filename {
501 FILE *fp = fopen(filename, "r");
502 if (!fp) {
503 printf("didn't open %s!!\n", filename);
504 return nil;
505 }
506 char buf[512];
507 while (fgets(buf, 512, fp)) {
508 char *config = strstr(buf, LOOKFOR);
509 if (!config) continue;
510 fclose(fp);
511 return [self generatorFromLine:config filename:filename];
512 }
513 fclose(fp);
514 // guess from filename
515 char *ext = strrchr(filename, '.');
516 if (!ext) return nil;
517 TestFileExeGenerator *result = [TestFileExeGenerator new];
518 result.filename = gcstrcpy1(filename);
519 if (!strncmp(ext, ".m", 2)) {
520 result.hasObjC = true;
521 result.hasRR = true;
522 result.hasGC = true;
523 }
524 else if (!strcmp(ext, ".c")) {
525 ;
526 }
527 else if (!strcmp(ext, ".M") || !strcmp(ext, ".mm")) {
528 result.hasObjC = true;
529 result.hasRR = true;
530 result.hasGC = true;
531 result.hasCPlusPlus = true;
532 }
533 else if (!strcmp(ext, ".cc")
534 || !strcmp(ext, ".cp")
535 || !strcmp(ext, ".cxx")
536 || !strcmp(ext, ".cpp")
537 || !strcmp(ext, ".CPP")
538 || !strcmp(ext, ".c++")
539 || !strcmp(ext, ".C")) {
540 result.hasCPlusPlus = true;
541 }
542 else {
543 printf("unknown extension, file %s ignored\n", filename);
544 result = nil;
545 }
546 return result;
547
548}
549
550- (NSString *)description {
551 return [NSString stringWithFormat:@"%s: %s%s%s%s%s%s",
552 filename,
553 LOOKFOR,
554 hasGC ? " GC" : "",
555 hasRR ? " RR" : "",
556 hasCPlusPlus ? " C++" : "",
557 wantsC99 ? "C99" : "",
558 supposedToNotCompile ? " FAIL" : ""];
559}
560
561@end
562
563void printDetails(NSArray *failures, const char *whatAreThey) {
564 if ([failures count]) {
565 NSMutableString *output = [NSMutableString new];
566 printf("%s:\n", whatAreThey);
567 for (TestFileExe *line in failures) {
568 printf("%s", line.binaryName);
569 char *radar = line.generator.radar;
570 if (radar)
571 printf(" (due to %s?),", radar);
572 printf(" recompile via:\n%s\n\n", line.description.UTF8String);
573 }
574 printf("\n");
575 }
576}
577
578void help(const char *whoami) {
579 printf("Usage: %s [-fast] [-e] [-dyld librarypath] [gcc4.2dir] [-- | source1 ...]\n", whoami);
580 printf(" -fast don't recompile if binary younger than source\n");
581 printf(" -open only run tests that are thought to still be unresolved\n");
582 printf(" -clang use the clang and clang++ compilers\n");
583 printf(" -e compile all variations also with -Os, -O2, -O3\n");
584 printf(" -dyld p override DYLD_LIBRARY_PATH and DYLD_FRAMEWORK_PATH to p when running tests\n");
585 printf(" <compilerpath> directory containing gcc-4.2 (or clang) that you wish to use to compile the tests\n");
586 printf(" -- assume stdin is a grep CON" "FIG across the test sources\n");
587 printf(" otherwise treat each remaining argument as a single test file source\n");
588 printf("%s will compile and run individual test files under a variety of compilers, c, obj-c, c++, and objc++\n", whoami);
589 printf(" .c files are compiled with all four compilers\n");
590 printf(" .m files are compiled with objc and objc++ compilers\n");
591 printf(" .C files are compiled with c++ and objc++ compilers\n");
592 printf(" .M files are compiled only with the objc++ compiler\n");
593 printf("(actually all forms of extensions recognized by the compilers are honored, .cc, .c++ etc.)\n");
594 printf("\nTest files should run to completion with no output and exit (return) 0 on success.\n");
595 printf("Further they should be able to be compiled and run with GC on or off and by the C++ compilers\n");
596 printf("A line containing the string CON" "FIG within the source enables restrictions to the above assumptions\n");
597 printf("and other options.\n");
598 printf("Following CON" "FIG the string\n");
599 printf(" C++ restricts the test to only be run by c++ and objc++ compilers\n");
600 printf(" GC restricts the test to only be compiled and run with GC on\n");
601 printf(" RR (retain/release) restricts the test to only be compiled and run with GC off\n");
602 printf("Additionally,\n");
603 printf(" -C99 restricts the C versions of the test to -fstd=c99 -fblocks\n");
604 printf(" -O adds the -O optimization level\n");
605 printf(" -O2 adds the -O2 optimization level\n");
606 printf(" -Os adds the -Os optimization level\n");
607 printf("Files that are known to exhibit unresolved problems can provide the term \"open\" and this can");
608 printf("in turn allow highlighting of fixes that have regressed as well as identify that fixes are now available.\n");
609 printf("Files that exhibit known bugs may provide\n");
610 printf(" rdar://whatever such that if they fail the rdar will get cited\n");
611 printf("Files that are expected to fail to compile should provide, as their last token sequence,\n");
612 printf(" error:\n");
613 printf(" or error: substring to match.\n");
614 printf("Files that are expected to produce a runtime error message should provide, as their last token sequence,\n");
615 printf(" warning: string to match\n");
616 printf("\n%s will compile and run all configurations of the test files and report a summary at the end. Good luck.\n", whoami);
617 printf(" Blaine Garst blaine@apple.com\n");
618}
619
620int main(int argc, char *argv[]) {
621 printf("running on %s-bit architecture\n", sizeof(long) == 4 ? "32" : "64");
622 char *compilerDir = "/usr/bin";
623 NSMutableArray *generators = [NSMutableArray new];
624 bool doFast = false;
625 bool doStdin = false;
626 bool onlyOpen = false;
627 char *libraryPath = getenv("DYLD_LIBRARY_PATH");
628 char *frameworkPath = getenv("DYLD_FRAMEWORK_PATH");
629 // process options
630 while (argc > 1) {
631 if (!strcmp(argv[1], "-fast")) {
632 doFast = true;
633 --argc;
634 ++argv;
635 }
636 else if (!strcmp(argv[1], "-dyld")) {
637 doFast = true;
638 --argc;
639 ++argv;
640 frameworkPath = argv[1];
641 libraryPath = argv[1];
642 --argc;
643 ++argv;
644 }
645 else if (!strcmp(argv[1], "-open")) {
646 onlyOpen = true;
647 --argc;
648 ++argv;
649 }
650 else if (!strcmp(argv[1], "-clang")) {
651 DoClang = true;
652 --argc;
653 ++argv;
654 }
655 else if (!strcmp(argv[1], "-e")) {
656 Everything = true;
657 --argc;
658 ++argv;
659 }
660 else if (!strcmp(argv[1], "--")) {
661 doStdin = true;
662 --argc;
663 ++argv;
664 }
665 else if (!strcmp(argv[1], "-")) {
666 help(argv[0]);
667 return 1;
668 }
669 else if (argc > 1 && isDirectory(argv[1])) {
670 compilerDir = argv[1];
671 ++argv;
672 --argc;
673 }
674 else
675 break;
676 }
677 // process remaining arguments, or stdin
678 if (argc == 1) {
679 if (doStdin)
680 generators = (NSMutableArray *)[TestFileExeGenerator generatorsFromFILE:stdin];
681 else {
682 help(argv[0]);
683 return 1;
684 }
685 }
686 else while (argc > 1) {
687 TestFileExeGenerator *generator = [TestFileExeGenerator generatorFromFilename:argv[1]];
688 if (generator) [generators addObject:generator];
689 ++argv;
690 --argc;
691 }
692 // see if we can generate all possibilities
693 NSMutableArray *failureToCompile = [NSMutableArray new];
694 NSMutableArray *failureToFailToCompile = [NSMutableArray new];
695 NSMutableArray *failureToRun = [NSMutableArray new];
696 NSMutableArray *successes = [NSMutableArray new];
697 for (TestFileExeGenerator *generator in generators) {
698 //NSLog(@"got %@", generator);
699 if (onlyOpen && !generator.open) {
700 //printf("skipping resolved test %s\n", generator.filename);
701 continue; // skip closed if onlyOpen
702 }
703 if (!onlyOpen && generator.open) {
704 //printf("skipping open test %s\n", generator.filename);
705 continue; // skip open if not asked for onlyOpen
706 }
707 generator.compilerPath = compilerDir;
708 NSArray *tests = [generator allLines];
709 for (TestFileExe *line in tests) {
710 line.frameworkPath = frameworkPath; // tell generators about it instead XXX
711 line.libraryPath = libraryPath; // tell generators about it instead XXX
712 if ([line shouldFail]) {
713 if (doFast) continue; // don't recompile & don't count as success
714 if ([line compileWithExpectedFailure]) {
715 [successes addObject:line];
716 }
717 else
718 [failureToFailToCompile addObject:line];
719 }
720 else if ([line compileUnlessExists:doFast]) {
721 if ([line run]) {
722 printf("%s ran successfully\n", line.binaryName);
723 [successes addObject:line];
724 }
725 else {
726 [failureToRun addObject:line];
727 }
728 }
729 else {
730 [failureToCompile addObject:line];
731 }
732 }
733 }
734 printf("\n--- results ---\n\n%lu successes\n%lu unexpected compile failures\n%lu failure to fail to compile errors\n%lu run failures\n",
735 [successes count], [failureToCompile count], [failureToFailToCompile count], [failureToRun count]);
736 printDetails(failureToCompile, "unexpected compile failures");
737 printDetails(failureToFailToCompile, "should have failed to compile but didn't failures");
738 printDetails(failureToRun, "run failures");
739
740 if (onlyOpen && [successes count]) {
741 NSMutableSet *radars = [NSMutableSet new];
742 printf("The following tests ran successfully suggesting that they are now resolved:\n");
743 for (TestFileExe *line in successes) {
744 printf("%s\n", line.binaryName);
745 if (line.radar) [radars addObject:line.generator];
746 }
747 if ([radars count]) {
748 printf("The following radars may be resolved:\n");
749 for (TestFileExeGenerator *line in radars) {
750 printf("%s\n", line.radar);
751 }
752 }
753 }
754
755 return [failureToCompile count] + [failureToRun count];
756}
757
758#include <sys/stat.h>
759
760static bool isDirectory(char *path) {
761 struct stat statb;
762 int retval = stat(path, &statb);
763 if (retval != 0) return false;
764 if (statb.st_mode & S_IFDIR) return true;
765 return false;
766}
767
768static bool isExecutable(char *path) {
769 struct stat statb;
770 int retval = stat(path, &statb);
771 if (retval != 0) return false;
772 if (!(statb.st_mode & S_IFREG)) return false;
773 if (statb.st_mode & S_IXUSR) return true;
774 return false;
775}
776
777static bool isYounger(char *source, char *binary) {
778 struct stat statb;
779 int retval = stat(binary, &statb);
780 if (retval != 0) return true; // if doesn't exit, lie
781
782 struct stat stata;
783 retval = stat(source, &stata);
784 if (retval != 0) return true; // we're hosed
785 // the greater the timeval the younger it is
786 if (stata.st_mtimespec.tv_sec > statb.st_mtimespec.tv_sec) return true;
787 if (stata.st_mtimespec.tv_nsec > statb.st_mtimespec.tv_nsec) return true;
788 return false;
789}
790
791static bool readErrorFile(char *buffer, const char *from) {
792 int fd = open(from, 0);
793 if (fd < 0) {
794 printf("didn't open %s, (might not have been created?)\n", buffer);
795 return false;
796 }
797 int count = read(fd, buffer, 512);
798 if (count < 1) {
799 printf("read error on %s\n", buffer);
800 return false;
801 }
802 buffer[count-1] = 0; // zap newline
803 return true;
804}
805