#!/usr/bin/python3
"""A script to measure performance of various commits.

USAGE: ./test/time-commits REF [REF [REF [...]]]

where REF is a git commit hash, tag, or branch.

This runs the 'stashed_evaluation_long' test on the commits listed and
prints statistics about how well each did.

For example:

$ ./test/time-commits keras-one-layer

keras-one-layer
Switched to branch 'keras-one-layer'
b''
score                0.918
precision            0.921
recall               0.917
score_deviation      0.00262
accuracy             0.917
duration:           67.46

The test as defined in `test/test_webapp.py` in the latest commit is
used. That is, if you go:

[edit test/test_webapp.py]
$ git add test/test_webapp.py
$ git commit -mtest
$ ./test/time-commits old-commit older-commit

"old-commit" and "older-commit" will be tested as if they had the
latest test_webapp.py.
"""

from subprocess import run, CalledProcessError
import re
import sys
import os

SLOW_TESTS = "MOODLE_MLBACKEND_RUN_SLOW_TESTS"

# This environment variable tells test_webapp to allow the slow tests,
# which is what we want for timing purposes.
os.environ[SLOW_TESTS] = '1'

# This one makes the tests use the old style plaintext password string,
# which not all commits might have.
os.environ["MOODLE_MLBACKEND_TEST_OLD_PASSWORDS"] = '1'


TEST_WEBAPP_COMMIT = 'stashed-eval'
TEST = 'stashed_evaluation_long'
#TEST = 'stashed_evaluation_short'

PYTEST = ['python3',
          '-mpytest',
          '-Wignore::DeprecationWarning',
          '-Wignore::FutureWarning',
          '-s',
          '--durations=0']


def checkout_branch(b):
    run(['git',
         'checkout',
         b], check=True)


def checkout_test(b=TEST_WEBAPP_COMMIT):
    # Ensure the test is exactly the same in every test run.
    run(['git',
         'checkout',
         b,
         '--',
         'test/test_webapp.py'], check=True)


def run_test():
    try:
        p = run(PYTEST + ['-k', TEST],
                capture_output=True,
                check=True)
    except CalledProcessError as e:
        print(e.output.decode('utf8'))
        print(e)
        print(f'{SLOW_TESTS}=1 {" ".join(e.cmd)}')
        return

    out = p.stdout.decode('utf8')

    for metric in ['score', 'precision', 'recall', 'score_deviation', 'accuracy',
                   'balanced accuracy']:
        m = re.search(r"'%s': (\d+\.?\d+)" % (metric,), out)
        if m is None:
            print(f"{metric} is missing")
        else:
            name = f"{metric}:"
            val = float(m.group(1))
            print(f"{metric:<20} {val:.3}")

    m = re.search(r"(\d+\.?\d+)s call.+%s" % (TEST,), out)
    if m is None:
        print(f"duration is missing")
    else:
        print(f"duration:           {m.group(1)}")


def current_ref():
    """Get the branch or tag if we are on one, otherwise the commit hash"""
    try:
        p = run(['git',
                 'symbolic-ref',
                 'HEAD',
                 '--short'],
                capture_output=True,
                check=True)
    except CalledProcessError as e:
        print(e)
        p = run(['git',
                 'rev-parse',
                 'HEAD'])

    return p.stdout.decode('utf-8').strip()


def main():
    args = sys.argv[1:]
    if not args or '-h' in args or '--help' in args:
        print(__doc__)
        sys.exit()

    if args in (['-w'], ['--working-tree']):
        run_test()
        sys.exit()

    original_ref = current_ref()
    try:
        for commit in args:
            print()
            print(commit)
            checkout_branch(commit)
            checkout_test(original_ref)
            run_test()
            checkout_test(commit)
    finally:
        checkout_branch(original_ref)


main()