#use strict; # usage perl harness.pl [testNum] # where testNum is an integer from 1 to 41 # testNum may also be a list of integers denoting tests to be run # ommitting testNum will lead to all tests being run # # this harness outputs the results to the file results.html as well # as reporting them to stderr. # # note that the XInclude sample application must be discoverable on # the current PATH. # # Find out the platform from 'uname -s' # open(PLATFORM, "uname -s|"); $platform = ; chomp($platform); close (PLATFORM); if ($platform =~ m/Windows/i || $platform =~ m/CYGWIN/i) { $pathsep = "\\"; } else { $pathsep = "/"; } @CORRECT_TEST_RESULTS = ( -3, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 0, 1 ); # private static boolean[] TEST_RESULTS = new boolean[] { # // one value for each test # true, true, true, true, true, true, false, true, false, true, // 10 # false, false, false, false, true, true, true, false, true, true, // 20 # true, false, true, false, false, false, true, true, false, true, // 30 # true, false, true, true, true, true, true, true, false, false, // 40 # true, }; #no need to add one to this value since there is a dummy value in the tables $NUM_TESTS = $#CORRECT_TEST_RESULTS; @ACTUAL_TEST_RESULTS = ( -3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 ); print "Running XInclude Tests\n"; $dateAndTime = localtime(); open(outfile, ">results.html"); print outfile "XInclude results generated at $dateAndTime "; print outfile "
"; print outfile ""; print outfile ""; print outfile ""; $successes = 0; $correct = 0; $numTestsToRun = $#ARGV + 1; while (<@ARGV>){ $i = $_; if ($i < 10) { $testNum = "0$i"; } else { $testNum = "$i"; } $input_file = "tests".$pathsep."xinclude".$pathsep."tests".$pathsep."test$testNum.xml"; if ($CORRECT_TEST_RESULTS[$i] == 1) { $output_file = "tests".$pathsep."xinclude".$pathsep."written".$pathsep."test$testNum.xml"; $expected_result_file = "tests".$pathsep."xinclude".$pathsep."cppoutput".$pathsep."test$testNum.xml"; } else { $output_file = "tests".$pathsep."xinclude".$pathsep."written".$pathsep."test$testNum.txt"; $expected_result_file = "tests".$pathsep."xinclude".$pathsep."cppoutput".$pathsep."test$testNum.txt"; } # run the tests and apture the output $command = "XInclude $input_file $output_file"; print "R[$command]\n"; system("$command"); $outcome = 0; if (! -e $output_file) { $ACTUAL_TEST_RESULTS[$i] = 0; # we didn't generate an output, we are correct if the test was expected to fail if($CORRECT_TEST_RESULTS[$i] eq 0) { $outcome = 1; } } else { $ACTUAL_TEST_RESULTS[$i] = 1; # we generated an output, we are correct if our result matches if($CORRECT_TEST_RESULTS[$i] eq 1) { $outcome = &compareFiles($expected_result_file, $output_file); } } print outfile ""; #test number print outfile ""; print outfile ""; # actual result $result = (($ACTUAL_TEST_RESULTS[$i])?"true":"false"); print outfile ""; # correct result $result = (($CORRECT_TEST_RESULTS[$i])?"true":"false"); print outfile ""; if ($outcome eq 1) { $successes++; $result = "Passed"; print "[test $testNum PASSED]\n"; $correct++; } else { $result = "Failed"; print "[test $testNum FAILED]\n"; } print outfile ""; } print outfile "
Results of XI Test Run
Test NumberTest Input FileGenerated Output FileReference Result FileResultExpected ResultTest Status
$testNum"; $anchor = &createHTMLAnchor("$input_file"); print outfile "$anchor"; print outfile ""; $anchor = &createHTMLAnchor("$output_file"); print outfile "$anchor"; print outfile ""; $anchor = &createHTMLAnchor("$expected_result_file"); print outfile "$anchor"; print outfile "$result$result$result
"; print "Tests Passed: $correct\n"; $percentSucceeding = $correct / $numTestsToRun * 100.0; print outfile "

$percentSucceeding% of tests that were run passed ($correct out of $numTestsToRun).

"; $shPass = 0; $shFail = 0; for ($i = 1; $i < $NUM_TESTS+1; $i++){ if ($CORRECT_TEST_RESULTS[$i]){ $shPass++; } else { $shFail++; } } print outfile "

$shPass should pass ($successes) did

"; $failures = $NUM_TESTS - $successes; print outfile "

$shFail should fail ($failures) did

"; $percentSucceeding = $correct / $NUM_TESTS * 100.0; print outfile "

$percentSucceeding% of all available tests passed ($correct out of $NUM_TESTS).

"; $percentSucceeding = $numTestsToRun / $NUM_TESTS * 100.0; print outfile "

$percentSucceeding% of all available tests were run ($numTestsToRun out of $NUM_TESTS).

"; print outfile "
"; close(outfile); # quick and dirty but functional sub compareFiles { $result = 1; print "C[$_[0]]:[$_[1]]\n"; if (! -e $_[0]) { print "No such file as $_[0] - cannot compare\n"; return 0; } if (! -e $_[1]) { print "No such file as $_[1] - cannot compare\n"; return 0; } open(expected, "<$_[0]"); open(actual, "<$_[1]"); #compare the files @expectedData = ; @actualData = ; close(expected); close(actual); for ($x = 0, $a = 0; $x < $#expectedData && $a < $#actualData; $x++, $a++){ $dataLineEx = $expectedData[$x]; $dataLineAc = $actualData[$a]; chomp($dataLineEx); chomp($dataLineAc); if ($dataLineEx ne $dataLineAc) { # check if its a warning and can be ignored at this stage if ($dataLineEx =~ /Warning/) { print "probably just a warning line:\n"; print "[$dataLineEx]\n"; $a-- } elsif ( $dataLineEx =~ m/^\s*$/ && $dataLineAc =~ m/^\s*$/ ) { $line_num = $x + 1; print "Line $line_num ws difference: \n[$dataLineEx] != \n[$dataLineAc]\n"; print "probably just white space on both lines\n"; $a-- } else { $line_num = $x + 1; print "Line $line_num difference: \n[$dataLineEx] != \n[$dataLineAc]\n"; print "From files: $_[0] $_[1]\n"; return 0; } } } return 1; } sub createHTMLAnchor { "$_[0]"; }