"""
RegressionTester.py
Author: Michael Lynch
Date: 10/2/06
COMS-4115: Prof. Edwards
DSPL Project

Performs testing on source files in a test directory and compares 
results to the correct output, displaying the result to the user.

USAGE: Python RegressionTester.py [-v] [-f FILELIST]|[-t SOURCEFILE]
    -v             - verbose mode - on failure, prints a diff of actual output to expected output
    -w             - print working - in addition to listing files that failed test at suite completion
                        print a list of programs that executed correctly
    -f FILELIST    - load suite file to execute a specific series of tests
    -t SOURCEFILE  - compile and run only a specific source file
"""

from sys import argv
import getopt
import os
import time
import sys

testDir = "testcases" + os.path.sep                  #path where we keep testing files
correctDir = testDir + "correct" + os.path.sep    #path where we keep correct output files

inputExt = ".dspl"                      #extensions to test
outputExt = ".out"                      #extension to use as standard out files
errExt = ".err"                         #extension to use as error output files
execExt = ".exe"                        #extension to use for executable files
logExt = ".log"                         #extension to use for compile log

mainProcess = r"java dspl.dsplc -c -o2e"    #process to run on input files (e.g. the compiler)
diffUtil = "diff --strip-trailing-cr"                       #command to diff two files

verboseMode = False;
inputs = None
showWorking = False;

def getArgs():
    """
    Sets program options according to command line arguments
    """
    global inputs
    global verboseMode
    global showWorking
    for i in range(len(argv)):
        if argv[i] == "-v":
            verboseMode = True
        elif argv[i] == "-t":        #program should run in single test case mode
            if (i + 1) >= len(argv):
                printUsage()
                sys.exit()
            if not os.path.exists(argv[i + 1]):
                print "Could not find file `%s'." % (argv[i + 1])
                printUsage()
                sys.exit()
            inputs = [argv[i + 1]]
        elif argv[i] == "-w":
            showWorking = True
        elif argv[i] == "-f":        #program should run in suite mode
            if (i + 1) >= len(argv):
                printUsage()
                sys.exit()
            try:
                print "opening %s" % (argv[i+1])
                f = open(argv[i + 1], "r")
                inputs = []
                for line in f:
                    if line[0] != "#":
                        filename = line.strip()
                        if os.path.sep == "\\":
                            filename = filename.replace("/", "\\")
                        elif os.path.sep == "/":
                            filename = filename.replace("\\", "/")
                        inputs.append(filename)
                f.close()
            except IOError:
                print "Error opening %s" % (argv[i + 1])
                printUsage()
                sys.exit()

def generateFilename(inputName, extension):
    """
    Creates an output name from a given input name to a ".out" filename

    @param inputName - string filename of file to derive an output name from
    """
    filenameIndex = max((inputName.rfind("\\"), inputName.rfind("/"), 0))
    inputName = inputName[filenameIndex + 1:]
                        
    #inputName = inputName.split("\\")[-1]
    
    if (inputName.endswith(inputExt)):
        baseName = inputName[:-len(inputExt)]
    else:
        baseName = inputName

    outputName = "%s%s" % (baseName, extension)
    return outputName

def printUsage():
    """
    Prints proper program usage to user.
    """
    print "USAGE: Python RegressionTester.py [-v] [-f FILELIST]|[-t SOURCEFILE]"
    print "\t-v\t\t - verbose mode - on failure, prints a diff of actual output to expected output"
    print "\t-f FILELIST\t- load suite file to execute a specific series of tests"
    print "\t-t SOURCEFILE\t- compile and run only a specific source file"

def cmpFiles(file1, file2):
    """
    Compares a program output to its expected output and displays the
    results to the user
    """
    difference = os.popen("%s %s %s" % (diffUtil, file1, file2)).readlines()
    diffOutput = []
    for line in difference:
        diffOutput.append(line)

    diffOutput = "%s%s" % ("\t", "\t".join(diffOutput))
                
    if diffOutput.strip() == "":
        print "OK!"
        return True
    else:
        print "MISMATCH - WRONG OUTPUT"
        if verboseMode:
            print "  `%s' != `%s'" % (file1, file2)
            print diffOutput
        return False


getArgs()
if not inputs:
    inputs = os.listdir(testDir)
    for i in range(len(inputs)):
        inputs[i] = testDir + inputs[i]



testName = time.strftime("%m-%d-%Y_%H-%M-%S")

if not os.path.exists("test_output"):
    os.mkdir("test_output");

#create an output path to put processed files
outputPath = "test_output" + os.path.sep + "_%s%s" % (testName, os.path.sep)
if not os.path.exists(outputPath):
    os.mkdir(outputPath)

#create an output path to put compiled files
compilerOutputDir = "test_output%s_%s%s" % (os.path.sep, testName, os.path.sep)
if not os.path.exists(compilerOutputDir):
    os.mkdir(compilerOutputDir)

successes = 0
failures = 0
failFiles = []
successFiles = []
for inFile in inputs:
    if inFile.endswith(inputExt):
        #if it's a proper input file, process it and compare output with expected
        execFile = compilerOutputDir + generateFilename(inFile, execExt)
        outputFilename = generateFilename(inFile, outputExt)
        errFilename = generateFilename(inFile, errExt)
        compileLogFilename = generateFilename(inFile, logExt)
        
        print "Running test for `%s':" % (inFile),

        if not os.path.exists(inFile):
            print "FAILED"
            print "\tCould not run compiler.\n\tSource file `%s' was not found!" % (inFile)
            failures += 1
            continue

        logFile = open("%s%s" % (outputPath, compileLogFilename), "w")
        logFile.write("Executing: %s %s %s\n\n" % (mainProcess, inFile, execFile))
        logFile.close()
        
        exitCode = os.system("%s %s %s 2>> %s%s" %
                           (mainProcess, inFile, execFile, outputPath, compileLogFilename))

        #correct for exit codes returned on Intel Mac platforms
        if (exitCode > 255):
            exitCode = exitCode >> 8

        try:
            logFile = open("%s%s" % (outputPath, compileLogFilename), "a")
            logFile.write("Process exited with code [%d]\n" % exitCode)
            logFile.close()
        except IOError:
            pass
        
        if exitCode != 0:
            #check if produced error output matches expected error output
            if os.path.exists("%s%s" % (correctDir, errFilename)):
                try:
                    expectedCode = int(open("%s%s" % (correctDir, errFilename)).readline())

                    if exitCode == expectedCode:
                        print "OK!"
                        successes += 1
                        successFiles.append(inFile)
                    else:
                        print "MISMATCH - WRONG ERROR (%d)" % (exitCode)
                        if verboseMode:
                            print "\tExpecting:\n\t< exit code=%d\n\t%s\n\tGot:\n\t> exit code=%d" % (expectedCode, "-" * 3, exitCode)
                        failures += 1
                        failFiles.append("%s" % (inFile))
                except ValueError:
                    print "MISMATCH - COULDN'T COMPILE, COULDN'T FIND EXPECTED ERROR CODE"
                    if verboseMode:
                        print "\tExpected:\n\t[NO EXPECTED INTEGER EXIT CODE IN `%s%s']" % (correctDir, errFilename)
                        print "\t%s\n\tGot:\n\texit code=%d!" % ("-" * 3, exitCode)
            #print out the difference between expected output and error output 
            elif os.path.exists("%s%s" % (correctDir, outputFilename)):
                print "MISMATCH - COULDN'T COMPILE"
                if verboseMode:
                    print "\tExpecting output:"
                    expOutput = open("%s%s" % (correctDir, outputFilename)).readlines()
                    for line in expOutput:
                        print "\t< %s" % (line),
                    print "\t%s\n\tGot:\n\t> exit code=%d" % ("-" * 3, exitCode)
                failures += 1
                failFiles.append("%s" % (inFile))
            #no expected results file exists, inform user
            else:
                print "MISMATCH - NO COMPILE, EXPECTED RESULT FILE FOR `%s' NOT FOUND!" % (inFile)
                failures += 1
                failFiles.append("%s" % (inFile))
            continue

        if not os.path.exists(execFile):
            print "MISMATCH - INCORRECT EXIT CODE (%d) - NO .exe PRODUCED" % (exitCode)
            failures += 1
            failFiles.append(inFile)
            continue

        progOut = os.popen4(execFile)[1].readlines()

        #write program's output to file
        f = open("%s%s" % (outputPath, outputFilename), "w")
        for line in progOut:
            f.write(line)
        f.close()

        #run diff on the output and expected output
        if os.path.exists("%s%s" % (correctDir, outputFilename)):
            if cmpFiles("%s%s" % (correctDir, outputFilename), "%s%s" % (outputPath, outputFilename)):
                successes += 1
                successFiles.append(inFile)
            else:
                failures += 1
                failFiles.append("%s" % (inFile))
        elif os.path.exists("%s%s" % (correctDir, errFilename)):
            print "MISMATCH - UNEXPECTED COMPILE"
            failures += 1
            failFiles.append("%s" % (inFile))
        else:
            print "COMPILED, BUT EXPECTED RESULT FILE FOR `%s' NOT FOUND!" % (inFile)
            failures += 1
            failFiles.append("%s" % (inFile))

if (successes + failures)  > 0:
    pctCorrect = 100 * successes / float(successes + failures)
else:
    pctCorrect = 0

#Print results of the tests
print "\nRESULTS: %d correct of %d (%3.2f%%)" % (successes, successes + failures, pctCorrect)
print "Failed files:"
for ff in failFiles:
    print "%s" % (ff)

if showWorking:
    print "\nSuccessful files:"
    for sf in successFiles:
        print sf
