run_unit_tests.py 5.8 KB
Newer Older
Jorn Bruggeman's avatar
Jorn Bruggeman committed
1 2 3
#!/usr/bin/env python

from __future__ import print_function
4
import sys
Jorn Bruggeman's avatar
Jorn Bruggeman committed
5 6 7 8
import os.path
import tempfile
import subprocess
import shutil
9
import argparse
10
import timeit
11
import io
Jorn Bruggeman's avatar
Jorn Bruggeman committed
12 13

script_root = os.path.abspath(os.path.dirname(__file__))
14 15 16 17
root = os.path.join(script_root, '../..')
allowed_hosts = os.listdir(os.path.join(root, 'src/drivers'))

parser = argparse.ArgumentParser()
18 19 20 21 22 23 24 25
parser.add_argument('--host', action='append', dest='hosts', choices=allowed_hosts, help='host to test (may appear multiple times)')
parser.add_argument('--cmake', default='cmake', help='path to cmake executable')
parser.add_argument('--compiler', help='Fortran compiler executable')
parser.add_argument('--performance', action='store_true', help='test performance with a specific model and environment (see --config and --env)')
parser.add_argument('--config', default='fabm.yaml', help='model configuration for performance testing, default: fabm.yaml')
parser.add_argument('--env', default='environment.yaml', help='model environment for performance testing (YAML file containing a dictionary with variable: value combinations), default: environment.yaml')
parser.add_argument('--report', default=None, help='file to write performance report to (only used with --performance), default: performance_<BRANCH>_<COMMIT>.log')
parser.add_argument('--repeat', type=int, default=5, help='number of times to run each performance test. Increase this to reduce the noise in timings')
26
parser.add_argument('-v', '--verbose', action='store_true', help='show test results even if completed successfully')
27
args, cmake_arguments = parser.parse_known_args()
28
if args.performance:
29 30 31 32 33 34
    if not os.path.isfile(args.config):
        print('Model configuration %s does not exist. Specify (or change) --config.' % args.config)
        sys.exit(2)
    if not os.path.isfile(args.env):
        print('Model environment %s does not exist. Specify (or change) --env.' % args.env)
        sys.exit(2)
35 36 37 38
    if args.report is None:
        git_branch = subprocess.check_output(['git', 'name-rev', '--name-only', 'HEAD']).decode('ascii').strip()
        git_commit = subprocess.check_output(['git', 'describe', '--always', '--dirty']).decode('ascii').strip()
        args.report = 'performance_%s_%s.log' % (git_branch, git_commit)
39
    print('Performance report will be written to %s' % args.report)
40

41 42
if args.compiler is not None:
    cmake_arguments.append('-DCMAKE_Fortran_COMPILER=%s' % args.compiler)
Jorn Bruggeman's avatar
Jorn Bruggeman committed
43 44 45 46 47

generates = {}
builds = {}
tests = {}

48 49 50 51 52
if not args.hosts:
    args.hosts = allowed_hosts
print('Selected hosts: %s' % ', '.join(args.hosts))

logs = []
53 54 55
def run(phase, args, verbose=False, **kwargs):
    proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, **kwargs)
    stdoutdata, _ = proc.communicate()
56 57
    if proc.returncode != 0:
        log_path = '%s.log' % phase
58
        with io.open(log_path, 'w') as f:
59 60
            f.write(stdoutdata)
        logs.append(log_path)
61
        print('FAILED (return code %i, log written to %s)' % (proc.returncode, log_path))
62 63
    else:
        print('SUCCESS')
64 65
    if verbose:
        print('Output:\n%s\n%s\n%s' % (80 * '-', stdoutdata, 80 * '-'))
66
    return proc.returncode
Jorn Bruggeman's avatar
Jorn Bruggeman committed
67 68 69

build_root = tempfile.mkdtemp()
try:
70
    vsconfig = 'Release' if args.performance else 'Debug'
71
    host2exe = {}
72 73 74 75
    for host in args.hosts:
        print(host)
        build_dir = os.path.join(build_root, host)
        os.mkdir(build_dir)
76
        print('  generating...', end='', flush=True)
77 78 79 80 81
        try:
            generates[host] = run('%s_generate' % host, [args.cmake, os.path.join(root, 'src'), '-DFABM_HOST=%s' % host] + cmake_arguments, cwd=build_dir)
        except FileNotFoundError:
            print('\n\ncmake executable not found. Specify its location on the command line with --cmake.')
            sys.exit(2)
Jorn Bruggeman's avatar
Jorn Bruggeman committed
82 83
        if generates[host] != 0:
            continue
84
        print('  building...', end='', flush=True)
85
        builds[host] = run('%s_build' % host, [args.cmake, '--build', build_dir, '--target', 'test_host', '--config', vsconfig])
Jorn Bruggeman's avatar
Jorn Bruggeman committed
86 87
        if builds[host] != 0:
            continue
88
        print('  testing...', end='', flush=True)
89
        for exename in ('%s/test_host.exe' % vsconfig, 'test_host'):
90 91 92
            exepath = os.path.join(build_dir, exename)
            if os.path.isfile(exepath):
                host2exe[host] = exepath
93
        tests[host] = run('%s_test' % host, [host2exe[host]], verbose=args.verbose)
94 95

    if args.performance:
96
        print('Measuring runtime')
97 98 99
        timings = {}
        shutil.copy(args.config, os.path.join(build_root, 'fabm.yaml'))
        shutil.copy(args.env, os.path.join(build_root, 'environment.yaml'))
100
        for i in range(args.repeat):
101
            print('  replicate %i' % i)
102 103 104 105
            for host in args.hosts:
                if tests.get(host, 1) != 0:
                    continue
                start = timeit.default_timer()
106
                print('    %s...' % (host,), end='')
107 108 109
                run('%s_perfrun_%i' % (host, i), [host2exe[host], '--simulate'], cwd=build_root)
                timings.setdefault(host, []).append(timeit.default_timer() - start)

Jorn Bruggeman's avatar
Jorn Bruggeman committed
110
finally:
111 112
    shutil.rmtree(build_root)

113 114
if logs:
    print('All tests complete - %i FAILED' % len(logs))
115
    print('See the following log files:\n%s' % '\n'.join(logs))
116 117
else:
    print('All tests complete - no failures')
118 119 120 121

if args.performance:
    print('Timings:')
    for host in args.hosts:
122 123
        ts = timings.get(host, ())
        timing = 'NA' if not ts else '%.3f s' % (sum(ts) / len(ts))
124
        print('  %s: %s' % (host, timing))
125 126 127 128 129 130
    with open(args.report, 'w') as f:
        f.write('host\t%s\taverage (s)\n' % '\t'.join(['run %i (s)' % i for i in range(args.repeat)]))
        for host in args.hosts:
            if host in timings:
                ts = timings[host]
                f.write('%s\t%s\t%.3f\n' % (host, '\t'.join(['%.3f' % t for t in ts]), sum(ts) / len(ts)))