#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
Copyright (C) 2016 Wolfgang Rohdewald <wolfgang@rohdewald.de>

SPDX-License-Identifier: GPL-2.0

"""

import signal
import os
import sys
import subprocess
import random
import shutil
import time
import gc

from optparse import OptionParser
from locale import getdefaultlocale

from common import Debug, StrMixin, cacheDir
from util import removeIfExists, gitHead, checkMemory, popenReadlines
from kajcsv import Csv, CsvRow, CsvWriter

signal.signal(signal.SIGINT, signal.SIG_DFL)


OPTIONS = None

KNOWNCOMMITS = set()

class Clone:

    """make a temp directory for commitId"""

    def __init__(self, commitId):
        self.commitId = commitId
        if commitId != 'current':
            tmpdir = os.path.expanduser(os.path.join(cacheDir(), commitId))
            if not os.path.exists(tmpdir):
                subprocess.Popen('git clone --shared --no-checkout -q .. {temp}'.format(
                    temp=tmpdir).split()).wait()
                subprocess.Popen('git checkout -q {commitId}'.format(
                    commitId=commitId).split(), cwd=tmpdir).wait()

    def sourceDirectory(self):
        """the source directory for this git commit"""
        if self.commitId == 'current':
            tmpdir = os.path.abspath('..')
            result = os.path.join(tmpdir, 'src')
        else:
            result = os.path.join(cacheDir(), self.commitId, 'src')
        assert os.path.exists(result), '{} does not exist'.format(result)
        return result

    @classmethod
    def removeObsolete(cls):
        """remove all clones for obsolete commits"""
        for commitDir in os.listdir(cacheDir()):
            if not any(x.startswith(commitDir) for x in KNOWNCOMMITS):
                removeDir = os.path.join(cacheDir(), commitDir)
                shutil.rmtree(removeDir)

class Client:

    """a simple container, assigning process to job"""

    def __init__(self, process=None, job=None):
        self.process = process
        self.job = job


class TooManyClients(UserWarning):

    """we would surpass options.clients"""


class TooManyServers(UserWarning):

    """we would surpass options.servers"""


class Server(StrMixin):

    """represents a kajongg server instance. Called when we want to start a job."""
    servers = []
    count = 0

    def __new__(cls, job):
        """can we reuse an existing server?"""
        running = Server.allRunningJobs()
        if len(running) >= OPTIONS.clients:
            raise TooManyClients
        maxClientsPerServer = OPTIONS.clients / OPTIONS.servers
        matchingServers = [
            x for x in cls.servers
            if x.commitId == job.commitId
            and x.pythonVersion == job.pythonVersion
            and len(x.jobs) < maxClientsPerServer]
        if matchingServers:
            result = sorted(matchingServers, key=lambda x: len(x.jobs))[0]
        else:
            if len(cls.servers) >= OPTIONS.servers:
                # maybe we can kill a server without jobs?
                for server in cls.servers:
                    if not server.jobs:
                        server.stop()
                        break  # we only need to stop one server
                else:
                    # no server without jobs found
                    raise TooManyServers
            result = object.__new__(cls)
            cls.servers.append(result)
        return result

    def __init__(self, job):
        if not hasattr(self, 'jobs'):
            self.jobs = []
            self.process = None
            self.socketName = None
            self.portNumber = None
            self.commitId = job.commitId
            self.pythonVersion = job.pythonVersion
            self.clone = Clone(job.commitId)
            self.start(job)
        else:
            self.jobs.append(job)

    @classmethod
    def allRunningJobs(cls):
        """a list of all jobs on all servers"""
        return sum((x.jobs for x in cls.servers), [])

    def start(self, job):
        """start this server"""
        job.server = self
        assert self.process is None, 'Server.start already has a process'
        self.jobs.append(job)
        assert self.commitId == job.commitId, 'Server.commitId {} != Job.commitId {}'.format(
            self.commitId, job.commitId)
        cmd = [os.path.join(
            job.srcDir(),
            'kajonggserver.py')]
        cmd.insert(0, 'python{}'.format(self.pythonVersion))
        if OPTIONS.usePort:
            self.portNumber = random.randrange(1025, 65000)
            cmd.append('--port={port}'.format(port=self.portNumber))
        else:
            Server.count += 1
            self.socketName = os.path.expanduser(
                os.path.join('~', '.kajongg',
                             'sock{commit}.py{py}.{ctr}'.format(
                                 commit=self.commitId, py=self.pythonVersion, ctr=Server.count)))
            cmd.append('--socket={sock}'.format(sock=self.socketName))
        if OPTIONS.debug:
            cmd.append('--debug={dbg}'.format(dbg=','.join(OPTIONS.debug)))
        if OPTIONS.log:
            self.process = subprocess.Popen(
                cmd, cwd=job.srcDir(),
                stdout=job.logFile, stderr=job.logFile)
        else:
            # reuse this server (otherwise it stops by itself)
            cmd.append('--continue')
            self.process = subprocess.Popen(cmd, cwd=job.srcDir())
        print('{} started'.format(self))

    def stop(self, job=None):
        """maybe stop the server"""
        if self not in self.servers:
            # already stopped
            return
        if job:
            self.jobs.remove(job)
        if not self.jobs:
            self.servers.remove(self)
            if self.process:
                try:
                    self.process.terminate()
                    self.process.wait()
                except OSError:
                    pass
                print('{} killed'.format(self))
            if self.socketName:
                removeIfExists(self.socketName)

    @classmethod
    def stopAll(cls):
        """stop all servers even if clients are still there"""
        for server in cls.servers:
            for job in server.jobs[:]:
                server.stop(job)
            assert not server.jobs, 'stopAll expects no server jobs but found {}'.format(
                server.jobs)
            server.stop()

    def __str__(self):
        return 'Server {} Python{}{} {}'.format(
            self.commitId,
            self.pythonVersion,
            ' pid={}'.format(self.process.pid) if Debug.process else '',
            'port={}'.format(self.portNumber) if self.portNumber else 'socket={}'.format(self.socketName))


class Job(StrMixin):

    """a simple container"""

    def __init__(self, pythonVersion, ruleset, aiVariant, commitId, game):
        self.pythonVersion = pythonVersion
        self.ruleset = ruleset
        self.aiVariant = aiVariant
        self.commitId = commitId
        self.game = game
        self.__logFile = None
        self.logFileName = None
        self.process = None
        self.server = None
        self.started = False

    def srcDir(self):
        """the path of the directory where the particular test is running"""
        assert self.server, 'Job {} has no server'.format(self)
        assert self.server.clone, 'Job {} has no server.clone'.format(self)
        return self.server.clone.sourceDirectory()

    def __startProcess(self, cmd):
        """call Popen"""
        if OPTIONS.log:
            self.process = subprocess.Popen(
                cmd, cwd=self.srcDir(),
                stdout=self.logFile, stderr=self.logFile)
        else:
            self.process = subprocess.Popen(cmd, cwd=self.srcDir())
        print('       %s started' % (self))

    def start(self):
        """start this job"""
        # pylint: disable=too-many-branches
        self.server = Server(self)
        # never login to the same server twice at the
        # same time with the same player name
        player = self.server.jobs.index(self) + 1
        cmd = [os.path.join(self.srcDir(), 'kajongg.py'),
               '--game={game}'.format(game=self.game),
               '--player={tester} {player}'.format(
                   player=player,
                   tester='Tüster'),
               '--ruleset={ap}'.format(ap=self.ruleset)]
        if self.server.socketName:
            cmd.append('--socket={sock}'.format(sock=self.server.socketName))
        if self.server.portNumber:
            cmd.append('--port={port}'.format(port=self.server.portNumber))
        cmd.insert(0, 'python{}'.format(self.pythonVersion))
        if OPTIONS.rounds:
            cmd.append('--rounds={rounds}'.format(rounds=OPTIONS.rounds))
        if self.aiVariant != 'DefaultAI':
            cmd.append('--ai={ai}'.format(ai=self.aiVariant))
        if OPTIONS.csv:
            cmd.append('--csv={csv}'.format(csv=OPTIONS.csv))
        if OPTIONS.gui:
            cmd.append('--demo')
        else:
            cmd.append('--nogui')
        if OPTIONS.playopen:
            cmd.append('--playopen')
        if OPTIONS.debug:
            cmd.append('--debug={dbg}'.format(dbg=','.join(OPTIONS.debug)))
        self.__startProcess(cmd)
        self.started = True

    def check(self, silent=False):
        """if done, cleanup"""
        if not self.started or not self.process:
            return
        result = self.process.poll()
        if result is not None:
            self.process = None
            if not silent:
                print('       {} done{}'.format(self, 'Return code: {}'.format(result) if result else ''))
            self.server.jobs.remove(self)

    @property
    def logFile(self):
        """open if needed"""
        if self.__logFile is None:
            logDir = os.path.expanduser(
                os.path.join('~', '.kajongg', 'log', str(self.game),
                             self.ruleset, self.aiVariant, str(self.pythonVersion)))
            if not os.path.exists(logDir):
                os.makedirs(logDir)
            logFileName = self.commitId
            self.logFileName = os.path.join(logDir, logFileName)
            self.__logFile = open(self.logFileName, 'wb', buffering=0)
        return self.__logFile

    def shortRulesetName(self):
        """strip leading chars if they are identical for all rulesets"""
        names = OPTIONS.knownRulesets
        for prefix in range(100):
            if sum(x.startswith(self.ruleset[:prefix]) for x in names) == 1:
                return self.ruleset[prefix - 1:]
        return None

    def __str__(self):
        pid = 'pid={}'.format(
            self.process.pid) if self.process and Debug.process else ''
        game = 'game={}'.format(self.game)
        ruleset = self.shortRulesetName()
        aiName = 'AI={}'.format(
            self.aiVariant) if self.aiVariant != 'DefaultAI' else ''
        return ' '.join([
            self.commitId, 'Python{}'.format(self.pythonVersion), pid, game, ruleset, aiName]).replace('  ', ' ')




def cleanup_data(csv):
    """remove all data referencing obsolete commits"""
    logDir = os.path.expanduser(os.path.join('~', '.kajongg', 'log'))
    knownCommits = csv.commits()
    for dirName, _, fileNames in os.walk(logDir):
        for fileName in fileNames:
            if fileName not in knownCommits and fileName != 'current':
                os.remove(os.path.join(dirName, fileName))
        try:
            os.removedirs(dirName)
        except OSError:
            pass  # not yet empty
    Clone.removeObsolete()

def pairs(data):
    """return all consecutive pairs"""
    prev = None
    for _ in data:
        if prev:
            yield prev, _
        prev = _


class CSV(StrMixin):
    """represent kajongg.csv"""

    knownCommits = []

    def __init__(self):
        self.findKnownCommits()
        self.rows = []
        if os.path.exists(OPTIONS.csv):
            self.rows = list(sorted({CsvRow(x) for x in Csv.reader(OPTIONS.csv)}))
        self.removeInvalidCommits()

    def neutralize(self):
        """remove things we do not want to compare"""
        for row in self.rows:
            row.neutralize()

    def commits(self):
        """return set of all our commit ids"""
        # TODO: sorted by date
        return {x.commit for x in self.rows}

    def games(self):
        """return a sorted unique list of all games"""
        return sorted({x.game for x in self.rows})

    @classmethod
    def findKnownCommits(cls):
        """find known commits"""
        if not cls.knownCommits:
            cls.knownCommits = set()
            for branch in subprocess.check_output(b'git branch'.split()).decode().split('\n'):
                if 'detached' not in branch and 'no branch' not in branch:
                    cls.knownCommits |= set(subprocess.check_output(
                        'git log --max-count=400 --pretty=%H {branch}'.format(
                            branch=branch[2:]).split()).decode().split('\n'))

    @classmethod
    def onlyExistingCommits(cls, commits):
        """return a set with only  existing commits"""
        result = set()
        for commit in commits:
            if any(x.startswith(commit) for x in cls.knownCommits):
                result.add(commit)
        return result

    def removeInvalidCommits(self):
        """remove rows with invalid git commit ids"""
        csvCommits = {x.commit for x in self.rows}
        csvCommits = {
            x for x in csvCommits if set(
                x) <= set(
                    '0123456789abcdef') and len(
                        x) >= 7}
        nonExisting = csvCommits - self.onlyExistingCommits(set(x.commit for x in self.rows))
        if nonExisting:
            print(
                'removing rows from kajongg.csv for commits %s' %
                ','.join(nonExisting))
            self.rows = [x for x in self.rows if x.commit not in nonExisting]
            self.write()

    def write(self):
        """write new csv file"""
        writer = CsvWriter(OPTIONS.csv)
        for row in self.rows:
            writer.writerow(row)
        del writer

    def evaluate(self):
        """evaluate the data. Show differences as helpful as possible"""
        found_difference = False
        for ruleset, aiVariant, game in {(x.ruleset, x.aiVariant, x.game) for x in self.rows}:
            rows = list(reversed(sorted(
                x for x in self.rows
                if ruleset == x.ruleset and aiVariant == x.aiVariant and game == x.game)))
            for fixedField in (CsvRow.fields.PY_VERSION, CsvRow.fields.COMMIT):
                for fixedValue in set(x[fixedField] for x in rows):
                    checkRows = [x for x in rows if x[fixedField] == fixedValue]
                    for warned in self.compareRows(checkRows):
                        found_difference = True
                        rows.remove(warned)
            found_difference |= len(self.compareRows(rows)) > 0
            self.compareRows(rows)
        if not found_difference:
            print('found no differences in {}'.format(OPTIONS.csv))

    @staticmethod
    def compareRows(rows):
        """in absence of differences, there should be only one row.
        return a list of rows which appeared in warnings"""
        if not rows:
            return []
        msgHeader = None
        result = []
        ruleset = rows[0].ruleset
        aiVariant = rows[0].aiVariant
        game = rows[0].game
        differences = []
        for pair in pairs(rows):
            causes = pair[1].differs_for(pair[0])
            if causes:
                differences.append(tuple([pair[0], pair[1], causes]))
        for difference in sorted(differences, key=lambda x: len(x[2])):
            if not set(difference[:2]) & set(result):
                if msgHeader is None:
                    msgHeader = 'looking at game={} ruleset={} AI={}'.format(
                        game, ruleset, aiVariant)
                    print(msgHeader)
                print('   {} {}'.format(*difference[2]))
                result.extend(difference[:2])
        return result


def startingDir():
    """the path of the directory where kajonggtest has been started in"""
    return os.path.dirname(sys.argv[0])


def getJobs(jobs):
    """fill the queue"""
    try:
        while len(jobs) < OPTIONS.clients:
            jobs.append(next(OPTIONS.jobs))
    except StopIteration:
        pass
    return jobs


def doJobs():
    """now execute all jobs"""
    # pylint: disable=too-many-branches, too-many-locals, too-many-statements

    if not OPTIONS.git and OPTIONS.csv:
        if gitHead() in ('current', None):
            print(
                'Disabling CSV output: %s' %
                ('You have uncommitted changes' if gitHead() == 'current' else 'No git'))
            print()
            OPTIONS.csv = None

    try:
        jobs = []
        while getJobs(jobs):
            for checkJob in Server.allRunningJobs()[:]:
                checkJob.check()
            try:
                jobs[0].start()
                jobs = jobs[1:]
            except TooManyServers:
                time.sleep(3)
            except TooManyClients:
                time.sleep(3)
    except KeyboardInterrupt:
        Server.stopAll()
    except BaseException as exc:
        print(exc)
        raise exc
    finally:
        while True:
            running = Server.allRunningJobs()
            if not running:
                break
            for job in running:
                if not job.started:
                    if job.server:
                        job.server.jobs.remove(job)
                else:
                    job.check()
                    if job.process:
                        print('Waiting for   %s' % job)
                        job.process.wait()
            time.sleep(1)


def parse_options():
    """parse options"""
    parser = OptionParser()
    parser.add_option(
        '', '--gui', dest='gui', action='store_true',
        default=False, help='show graphical user interface')
    parser.add_option(
        '', '--ruleset', dest='rulesets', default='ALL',
        help='play like a robot using RULESET: comma separated list. If missing, test all rulesets',
        metavar='RULESET')
    parser.add_option(
        '', '--rounds', dest='rounds',
        help='play only # ROUNDS per game',
        metavar='ROUNDS')
    parser.add_option(
        '', '--ai', dest='aiVariants',
        default=None, help='use AI variants: comma separated list',
        metavar='AI')
    parser.add_option(
        '', '--python', dest='pyVersions',
        default=None, help='use python versions: comma separated list',
        metavar='PY_VERSION')
    parser.add_option(
        '', '--log', dest='log', action='store_true',
        default=False, help='write detailled debug info to ~/.kajongg/log/game/ruleset/commit.'
                            ' This starts a separate server process per job, it sets --servers to --clients.')
    parser.add_option(
        '', '--game', dest='game',
        help='start first game with GAMEID, increment for following games.' +
        ' Without this, random values are used.',
        metavar='GAMEID', type=int, default=0)
    parser.add_option(
        '', '--count', dest='count',
        help='play COUNT games. Default is unlimited',
        metavar='COUNT', type=int, default=999999999)
    parser.add_option(
        '', '--playopen', dest='playopen', action='store_true',
        help='all robots play with visible concealed tiles', default=False)
    parser.add_option(
        '', '--clients', dest='clients',
        help='start a maximum of CLIENTS kajongg instances. Default is 2',
        metavar='CLIENTS', type=int, default=1)
    parser.add_option(
        '', '--servers', dest='servers',
        help='start a maximum of SERVERS kajonggserver instances. Default is 1',
        metavar='SERVERS', type=int, default=1)
    parser.add_option(
        '', '--git', dest='git',
        help='check all commits: either a comma separated list or a range from..until')
    parser.add_option(
        '', '--debug', dest='debug',
        help=Debug.help())

    return parser.parse_args()


def improve_options():
    """add sensible defaults"""
    # pylint: disable=too-many-branches,too-many-statements
    if OPTIONS.servers < 1:
        OPTIONS.servers = 1

    cmdPath = os.path.join(startingDir(), 'kajongg.py')
    cmd = ['python3', cmdPath, '--rulesets']
    OPTIONS.knownRulesets = list(popenReadlines(cmd))
    if OPTIONS.rulesets == 'ALL':
        OPTIONS.rulesets = OPTIONS.knownRulesets
    else:
        wantedRulesets = OPTIONS.rulesets.split(',')
        usingRulesets = []
        wrong = False
        for ruleset in wantedRulesets:
            matches = [x for x in OPTIONS.knownRulesets if ruleset in x]
            if not matches:
                print('ruleset', ruleset, 'is not known', end=' ')
                wrong = True
            elif len(matches) > 1:
                exactMatch = [x for x in OPTIONS.knownRulesets if ruleset == x]
                if len(exactMatch) == 1:
                    usingRulesets.append(exactMatch[0])
                else:
                    print('ruleset', ruleset, 'is ambiguous:', matches)
                    wrong = True
            else:
                usingRulesets.append(matches[0])
        if wrong:
            sys.exit(1)
        OPTIONS.rulesets = usingRulesets
    if OPTIONS.git is not None:
        if '..' in OPTIONS.git:
            if '^' not in OPTIONS.git:
                OPTIONS.git = OPTIONS.git.replace('..', '^..')
            commits = subprocess.check_output(
                'git log --grep _SILENT --invert-grep --pretty=%h {range}'.format(
                    range=OPTIONS.git).split()).decode()
            _ = list(x.strip() for x in commits.split('\n') if x.strip())
            OPTIONS.git = list(reversed(_))
        else:
            OPTIONS.git = CSV.onlyExistingCommits(OPTIONS.git.split(','))
            if not OPTIONS.git:
                sys.exit(1)
    if OPTIONS.debug is None:
        OPTIONS.debug = []
    else:
        OPTIONS.debug = [OPTIONS.debug]
    if OPTIONS.log:
        OPTIONS.servers = OPTIONS.clients
        OPTIONS.debug.extend(
            'neutral,dangerousGame,explain,originalCall,robbingKong,robotAI,scores,traffic,hand'.split(','))
    if gitHead() not in ('current', None) and not OPTIONS.log:
        OPTIONS.debug.append('git')
    if not OPTIONS.aiVariants:
        OPTIONS.aiVariants = 'DefaultAI'
    if OPTIONS.pyVersions:
        OPTIONS.pyVersions = OPTIONS.pyVersions.split(',')
    else:
        OPTIONS.pyVersions = ['3']
    OPTIONS.allAis = OPTIONS.aiVariants.split(',')
    if OPTIONS.count:
        print('rulesets:', ', '.join(OPTIONS.rulesets))
        _ = ' '.join(OPTIONS.allAis)
        if _ != 'DefaultAI':
            print('AIs:', _)
    if OPTIONS.git:
        print('commits:', ' '.join(OPTIONS.git))
        # since we order jobs by game, commit we want one permanent server per
        # commit
    OPTIONS.jobs = allJobs()
    OPTIONS.games = allGames()
    OPTIONS.jobCount = 0
    OPTIONS.usePort = os.name == 'nt'


def allGames():
    """a generator returning game ids"""
    while True:
        if OPTIONS.game:
            result = OPTIONS.game
            OPTIONS.game += 1
        else:
            result = int(random.random() * 10 ** 9)
        yield result


def allJobs():
    """a generator returning Job instances"""
    # pylint: disable=too-many-nested-blocks
    for game in OPTIONS.games:
        for commitId in OPTIONS.git or ['current']:
            for ruleset in OPTIONS.rulesets:
                for aiVariant in OPTIONS.allAis:
                    for pyVersion in OPTIONS.pyVersions:
                        OPTIONS.jobCount += 1
                        if OPTIONS.jobCount > OPTIONS.count:
                            return
                        yield Job(pyVersion, ruleset, aiVariant, commitId, game)

def main():
    """parse options, play, evaluate results"""
    global OPTIONS  # pylint: disable=global-statement

    locale_encoding = getdefaultlocale()[1]
    if locale_encoding != 'UTF-8':
        print('we need default encoding UTF-8 but have {}'.format(locale_encoding))
        sys.exit(2)

    # we want only english in the logs because i18n and friends
    # behave differently in kde and kde
    os.environ['LANG'] = 'en_US.UTF-8'
    (OPTIONS, args) = parse_options()
    OPTIONS.csv = os.path.expanduser(
        os.path.join('~', '.kajongg', 'kajongg.csv'))
    if not os.path.exists(os.path.dirname(OPTIONS.csv)):
        os.makedirs(os.path.dirname(OPTIONS.csv))

    csv = CSV()

    improve_options()

    csv.evaluate()

    errorMessage = Debug.setOptions(','.join(OPTIONS.debug))
    if errorMessage:
        print(errorMessage)
        sys.exit(2)

    if args and ''.join(args):
        print('unrecognized arguments:', ' '.join(args))
        sys.exit(2)

    print()

    if OPTIONS.count:
        doJobs()
        if OPTIONS.csv:
            CSV().evaluate()

def cleanup(sig, unusedFrame):
    """at program end"""
    Server.stopAll()
    sys.exit(sig)

# is one server for two clients.
if __name__ == '__main__':
    signal.signal(signal.SIGABRT, cleanup)
    signal.signal(signal.SIGTERM, cleanup)
    signal.signal(signal.SIGINT, cleanup)
    if os.name != 'nt':
        signal.signal(signal.SIGHUP, cleanup)
        signal.signal(signal.SIGQUIT, cleanup)

    main()
    gc.collect()
    checkMemory()
