# -*- coding: utf-8 -*-
# vim:set shiftwidth=4 tabstop=4:
#
# Copyright 2013-2016 Univention GmbH
#
# http://www.univention.de/
#
# All rights reserved.
#
# The source code of this program is made available
# under the terms of the GNU Affero General Public License version 3
# (GNU AGPL V3) as published by the Free Software Foundation.
#
# Binary versions of this program provided by Univention to you as
# well as other copyrighted, protected or trademarked materials like
# Logos, graphics, fonts, specific documentations and configurations,
# cryptographic keys etc. are subject to a license agreement between
# you and Univention and not subject to the GNU AGPL V3.
#
# In the case you use this program under the terms of the GNU AGPL V3,
# the program is provided in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License with the Debian GNU/Linux or Univention distribution in file
# /usr/share/common-licenses/AGPL-3; if not, see
# <http://www.gnu.org/licenses/>.

from __future__ import print_function
from ConfigParser import NoOptionError, NoSectionError
import re
import ipaddr
assert re and ipaddr, "Required for paramiko"
import os
import paramiko
import subprocess
import sys
import getpass
import glob
import time
import traceback
import json
import M2Crypto
import datetime
from threading import Thread
from socket import gethostbyname, error as socket_error, timeout as socket_timeout
from select import select
from ssl import SSLError
from base64 import b64encode
from lxml import etree
import nmap
try:
    from shlex import quote
except ImportError:
    from pipes import quote

import boto
from boto.ec2 import regions, blockdevicemapping

BOTO_VERSION = tuple(int(v) for v in boto.Version.split('.'))
PATH_UCS_KT_GET = '/usr/bin/ucs-kt-get'
PATH_UCS_KT_INSTANCE_CREATE = '/usr/bin/ucs-kt-instance-create'
EC2_WINDOWS_DEFAULT_PASSWORD = 'Univention@99#+?=$'
EC2_WINDOWS_USER_DATA = '''
<powershell>
    netsh advfirewall firewall add rule name="WinRM HTTP" dir=in localport=5985 protocol=TCP action=allow
    netsh advfirewall firewall add rule name=winexe1 dir=in action=allow protocol=TCP localport=139
    netsh advfirewall firewall add rule name=winexe2 dir=in action=allow protocol=TCP localport=445
    winrm quickconfig -q
    Enable-PSRemoting -Force
    Enable-WSManCredSSP -Role Server -Force
    Enable-WSManCredSSP -Role Client -DelegateComputer * -Force
    Set-Item -Path "wsman:\localhost\service\auth\credSSP" -Value $True -Force
    New-Item -Path HKLM:\SOFTWARE\Policies\Microsoft\Windows\CredentialsDelegation -Name AllowFreshCredentialsWhenNTLMOnly -Force
    New-ItemProperty -Path HKLM:\SOFTWARE\Policies\Microsoft\Windows\CredentialsDelegation\AllowFreshCredentialsWhenNTLMOnly -Name 1 -Value * -PropertyType String
    winrm set winrm/config '@{MaxTimeoutms="7200000"}'
    winrm set winrm/config/winrs '@{MaxMemoryPerShellMB="0"}'
    winrm set winrm/config/winrs '@{MaxProcessesPerShell="0"}'
    winrm set winrm/config/winrs '@{MaxShellsPerUser="0"}'
    winrm set winrm/config/service '@{AllowUnencrypted="true"}'
    winrm set winrm/config/service/auth '@{Basic="true"}'
    winrm set winrm/config/client/auth '@{Basic="true"}'
    winrm set winrm/config/client '@{TrustedHosts="*"}'
    Restart-Service winrm

    # the default random password is not good enough for dc_promo
    # as workaround we set a valid password
    $a=Get-WmiObject win32_useraccount -Filter "LocalAccount=TRUE AND SID LIKE 'S-1-5-%%-500'" | select Name
    net users $a.Name %s
</powershell>
''' % EC2_WINDOWS_DEFAULT_PASSWORD

KVM_NETWORKS = {
    'appliance-net': '''
        <network ipv6='yes'>
            <name>appliance-net</name>
            <forward mode='nat'/>
            <ip address='192.168.150.1' netmask='255.255.255.0'>
                <dhcp>
                    <range start='192.168.150.10' end='192.168.150.254'/>
                </dhcp>
            </ip>
        </network>''',
    'appliance-net-no-nat': '''
        <network ipv6='yes'>
            <name>appliance-net-no-nat</name>
            <ip address='192.168.151.1' netmask='255.255.255.0'>
                <dhcp>
                    <range start='192.168.151.10' end='192.168.151.254'/>
                </dhcp>
            </ip>
        </network>''',
    'install-net': '''
        <network ipv6='yes'>
            <name>install-net</name>
            <forward mode='nat'/>
            <ip address='192.168.152.1' netmask='255.255.255.0'>
                <dhcp>
                    <range start='192.168.152.10' end='192.168.152.254'/>
                </dhcp>
            </ip>
        </network>''',
    'install-net-nodhcp': '''
        <network ipv6='yes'>
            <name>install-net-nodhcp</name>
            <forward mode='nat'/>
            <ip address='192.168.153.1' netmask='255.255.255.0'></ip>
        </network>''',
}


def mac2IPv6linklocal(mac):
    """
    Converta mac address into a IPv6 link local address.
    >>> mac2IPv6linklocal('0:1:2:3:4:5')
    'fe80::0201:02ff:fe03:0405'
    >>> mac2IPv6linklocal('52:54:00:eb:7c:79')
    'fe80::5054:00ff:feeb:7c79'
    """
    octets = [int(_, 16) for _ in mac.split(':')]
    octets[0] &= ~1  # clear broadcast bit
    octets[0] ^= 2  # flip universal/local bit
    octets = octets[0:3] + [0xff, 0xfe] + octets[3:6]
    groups = ['%02x%02x' % _ for _ in zip(octets[0::2], octets[1::2])]
    return 'fe80::' + ':'.join(groups)


def _split_config(lines):
    """
    Iterate over lines of config option.
    """
    for line in lines.split('\n'):
        line = line.strip()
        if not line:
            continue
        if line.startswith('#'):
            continue
        yield line


class TimeoutError(Exception):
    pass


class VM(object):

    """
    Generic instance.
    """
    failed_commands = 0

    PARAMS_REQUIRED = []
    PARAMS_OPTIONAL = []

    def __init__(self, section, config, virtualisation):
        ''' Initialize a VM instance '''
        self.section = section
        self.config = config
        self.virtualisation = virtualisation
        self.do_init_connect = True
        self.do_init_copy_files = True

        # Merge global DEFAULTS
        for key in self.PARAMS_REQUIRED + self.PARAMS_OPTIONAL:
            if not config.has_option(section, key):
                if config.has_option('Global', key):
                    config.set(section, key, config.get('Global', key))

        self.private_key = None
        self.ssh_pub_key = None
        if config.has_option(section, 'ec2_keypair_file'):
            self.private_key = os.path.expanduser(config.get(section, 'ec2_keypair_file'))
        if config.has_option(section, 'kvm_keypair_file'):
            self.private_key = os.path.expanduser(config.get(section, 'kvm_keypair_file'))
            self.key = M2Crypto.RSA.load_key(self.private_key)
            if config.has_option(section, 'kvm_keypair'):
                self.ssh_pub_key = 'ssh-rsa ' + b64encode('\x00\x00\x00\x07ssh-rsa%s%s' % (self.key.pub()[0], self.key.pub()[1])) + ' ' + config.get(section, 'kvm_keypair')

        # Save the profile, it will be later written to the VM
        try:
            self.profile = config.get(section, 'profile')
        except NoOptionError:
            self.profile = None

        # Save environment
        try:
            self.environment = config.get(section, 'environment')
        except NoOptionError:
            self.environment = None

        # see if its a windows vm or not
        try:
            self._is_windows = config.getboolean(section, 'windows')
        except NoOptionError:
            self._is_windows = None

        # Read and save the file lines and ignore comments
        try:
            lines = config.get(section, 'files')
            self.files = list(_split_config(lines))
        except NoOptionError:
            self.files = []

        # list of commands
        self.commands = {}
        for key, value in config.items(self.section):
            if not key.startswith('command'):
                continue
            i = int(key[len('command'):], 10)
            self.commands[i] = list(_split_config(value))

        # logfile
        for sname in (section, 'Global'):
            if config.has_option(sname, 'logfile'):
                self.logfile = os.path.expanduser(config.get(sname, 'logfile'))
                break
        else:
            self.logfile = None

        # Create the logfile
        if self.logfile:
            log = open(self.logfile, 'a+')
            log.write('Created instance %s at %s\n' % (section, datetime.datetime.now().isoformat()))
            log.close()
        self.logfile_fd = None

        for sname in (section, 'Global'):
            if config.has_option(sname, 'recover'):
                self.recover = config.getint(sname, 'recover')
                break
        else:
            self.recover = 0

        self.instance = None
        self.client = None
        self.client_sftp = None
        self.predefined_ip = None

    def valid(self):
        return bool(self._is_windows or self.commands) and all(
            self.config.has_option(self.section, key) for key in self.PARAMS_REQUIRED
        )

    def _connect_vm(self):
        """Connect via ssh to VM."""
        raise NotImplementedError()

    def _waiting_for_open_ports(self, ports, timeout=3600):
        """ Probe given ports until one of them is reported 'open' (returns True) or timeout is reached (returns False). Ports must be a List ['a', 'b']. """
        scanner = nmap.PortScanner()
        ports_string = ','.join(ports)
        ip = self.get_ip()
        start = now = time.time()

        # scan until timout is reachedo or there is an open port
        while now - start < timeout:
            scan_result = scanner.scan(ip, ports_string, '-PN')
            now = time.time()
            self._log('Pending %d...' % (timeout - now + start))
            if not scan_result['scan']:
                self._log('Error: No scan results were returned')
                continue
            port_status = scan_result['scan'].get(ip, {}).get('tcp')
            if port_status is None:
                continue
            for port in ports:
                if not int(port) in port_status:
                    continue
                if port_status[int(port)]['state'] == u'open':
                    return True

        self._log("Port(s) %s could not be reached" % (', '.join(ports), ))
        return False

    def connect(self):
        ''' Wait until the connection is ready '''
        if self.do_init_connect is not True:
            self.do_init_connect = True
            return
        self.client = paramiko.SSHClient()
        self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        start = now = time.time()
        # TODO: make the timeout configurable
        timeout = 3600
        # connect via ssh if it's a linux host
        if not self._is_windows:
            while now - start < timeout:
                try:
                    self._connect_vm()
                    break
                except socket_error as ex:
                    self._log('[%d] Network error: %s' %
                              (timeout - now + start, ex))
                except paramiko.AuthenticationException as ex:
                    self._log('[%d]: Authentication failed... [%s]' %
                              (timeout - now + start, ex))
                except paramiko.SSHException as ex:
                    self._log('[%d]: SSH failed... [%s]' %
                              (timeout - now + start, ex))
                except Exception as ex:
                    self._log('[%d]: Unknown error "%s"...' %
                              (timeout - now + start, ex))
                    raise
                time.sleep(5)
                now = time.time()
            else:
                raise TimeoutError(timeout)
        # port probe if it's a windows host
        else:
            # if port 139 or 445 is open we should be able to connect to the
            # windows host
            if not self._waiting_for_open_ports(['139', '445']):
                self._log('Not able to reach %s' % (self.get_ip(), ))
                raise TimeoutError(timeout)
        if not self._is_windows:
            if self.ssh_pub_key:
                self._ssh_exec('mkdir -p /root/.ssh')
                self._ssh_exec('echo "\n%s" >> /root/.ssh/authorized_keys' % self.ssh_pub_key)
                self.ssh_pub_key = None

    def create_environment(self):
        ''' Write environment file '''
        if self._is_windows:
            return
        if not self.environment:
            return
        self._open_client_sftp_connection()
        remote_env = self.client_sftp.file('/root/.ssh/environment', 'w')
        for env in self.environment.split('\n'):
            if env:
                print('%s' % env, file=remote_env)
        remote_env.close()
        self._ssh_exec('ucr set sshd/PermitUserEnvironment=yes')
        self._ssh_exec('ucr set sshd/config/PermitUserEnvironment=yes')
        self._ssh_exec('echo "PermitUserEnvironment yes" >> /etc/ssh/sshd_config')
        self._ssh_exec('echo "sleep 1;service ssh restart" | at now')
        self._close_client_sftp_connection()

    def create_profiles(self):
        ''' Write the given profile to the instance '''
        if self._is_windows:
            return
        if not self.profile:
            return
        self._open_client_sftp_connection()
        remote_profile = self.client_sftp.file(
            '/var/cache/univention-system-setup/profile', 'w')
        print(self.profile, file=remote_profile)
        remote_profile.close()
        self._close_client_sftp_connection()

    def list_files(self):
        '''Iterate files to copy'''
        if not self.files:
            return

        for line in self.files:
            localfiles, remotedir = line.rsplit(' ', 1)
            expanded_files = glob.glob(os.path.expanduser(localfiles))
            if expanded_files:
                for localfile in expanded_files:
                    yield localfile, remotedir
            else:
                yield localfiles, None

    def copy_files(self):
        ''' Copy the given files to the instance '''
        if self._is_windows:
            return
        if self.do_init_copy_files is not True:
            self.do_init_copy_files = True
            return
        self._open_client_sftp_connection()
        for localfile, remotedir in self.list_files():
            if not remotedir:
                continue
            fname = os.path.basename(localfile)
            remfile = os.path.join(remotedir, fname)
            self._remote_mkdir(remotedir)
            self.client_sftp.put(localfile, remfile)
            self._log('{2} copy {0} -> {1}'.format(localfile, remfile, self.get_name()), newline=False)
            self.client_sftp.chmod(remfile, os.stat(localfile).st_mode & 0o777)
        # copy shared utils
        for utils in ['.', './ucs-ec2-tools', '../ucs-ec2-tools', './tools', './bin']:
            local = os.path.join(utils, 'shared-utils')
            if os.path.isdir(local):
                self._copy_recursive(local, '/root/shared-utils')
                break
        # copy utils
        for utils in ['.', './ucs-ec2-tools', '../ucs-ec2-tools', './tools', './bin']:
            local = os.path.join(utils, 'utils')
            if os.path.isdir(local):
                self._copy_recursive(local, '/root')
                break
        # produc tests
        for utils in ['.', './ucs-ec2-tools', '../ucs-ec2-tools', './tools', './bin']:
            local = os.path.join(utils, 'product-tests')
            if os.path.isdir(local):
                self._copy_recursive(local, '/root/product-tests')
                break
        self._close_client_sftp_connection()

    def _copy_recursive(self, local, remote):
        for dirpath, dirnames, filenames in os.walk(local):
            remote_path = os.path.join(remote, dirpath[len(local) + 1:])
            self._remote_mkdir(remote_path)
            for filename in filenames:
                local_file = os.path.join(dirpath, filename)
                remote_file = os.path.join(remote_path, filename)
                self.client_sftp.put(local_file, remote_file)
                self._log('{2} copy {0} -> {1}'.format(local_file, remote_file, self.get_name()), newline=False)
                self.client_sftp.chmod(remote_file, os.stat(local_file).st_mode & 0o777)

    def _exec_local(self, cmdline):
        """
        Execute local command redirecting output to local logfile.
        """
        self._log('')
        self._log('=> Execute local command: %s at %s' % (cmdline, datetime.datetime.now().isoformat()))
        self._open_logfile()
        ret = subprocess.call(
            cmdline, shell=True, stdout=self.logfile_fd, stderr=self.logfile_fd)
        self._close_logfile()
        return ret

    def _open_logfile(self):
        """
        Open internal self.logfile.
        """
        if self.logfile:
            self.logfile_fd = open(self.logfile, 'a+')
        else:
            self.logfile_fd = sys.stdout

    def _close_logfile(self):
        """
        Close internal self.logfile.
        """
        if self.logfile_fd and self.logfile_fd != sys.stdout:
            self.logfile_fd.close()
        self.logfile_fd = None

    def run_commands(self, phase, shared_data=None):
        ''' Run all commands for a given phase e.g. for command1 '''
        if not self.commands:
            return
        try:
            commands = self.commands[phase]
        except LookupError:
            _print_done('fail: phase %s does not exists' % (phase,))
            return
        if VM.failed_commands and phase < self.recover:
            _print_done('skip: phase %d due to previous errors' % (phase,))
            return
        connection_required = True
        if not shared_data.get(self.section):
            shared_data[self.section] = dict()
        if hasattr(self, 'get_network_mac'):
            shared_data[self.section]['network_mac'] = self.get_network_mac()
        for cmdline in commands:
            if VM.failed_commands and phase < self.recover:
                _print_done(
                    'skip: cmd "%s" due to previous errors' % (cmdline,))
                continue
            try:
                for host in shared_data:
                    if shared_data[host].get('network_dhcp_ip'):
                        _r = '[{0}_NETWORK_DHCP_IP]'.format(host)
                        if _r in cmdline:
                            cmdline = cmdline.replace(_r, shared_data[host]['network_dhcp_ip'])
                    if shared_data[host].get('network_mac'):
                        _r = '[{0}_NETWORK_MAC]'.format(host)
                        if _r in cmdline:
                            cmdline = cmdline.replace(_r, shared_data[host]['network_mac'])
                _print_process('  %s' % cmdline)
                abort_phase = False
                if cmdline.startswith('ABORT_PHASE_TEST '):
                    _tmp, cmdline = cmdline.split('ABORT_PHASE_TEST ', 1)
                    abort_phase = True
                if cmdline == 'RESET':
                    self.reset()
                    continue
                if cmdline == 'COPY_FILES':
                    self.connect()
                    self.copy_files()
                    _print_done()
                    continue
                if cmdline == 'DEACTIVATE_NETWORK':
                    self.deactivate_interface_on_network()
                    continue
                if cmdline == 'ACTIVATE_NETWORK':
                    self.activate_interface_on_network()
                    continue
                if cmdline == 'GET_NETWORK_DHCP_IP':
                    if hasattr(self, 'get_network_dhcp_ip'):
                            shared_data[self.section]['network_dhcp_ip'] = self.get_network_dhcp_ip()
                    _print_done()
                    continue
                if cmdline.startswith('ASSIGN_PRIVATE_IP_ADDRESSE'):
                    tmp = cmdline.split()
                    self.ec2_assign_private_ip_addresses(tmp[1])
                    continue
                if cmdline.startswith('LOCAL'):
                    cmdline = cmdline[len('LOCAL'):]
                    ret = self._exec_local(cmdline)
                else:
                    try:
                        if connection_required:
                            self.connect()
                            connection_required = False
                        ret = self._ssh_exec(cmdline)
                    except paramiko.ssh_exception.SSHException:
                        self.connect()
                        connection_required = False
                        ret = self._ssh_exec(cmdline)
                # since 4.2 the paramiko->ssh->reboot fails and there
                # seems to be not way of catch this error
                # so this ugly hack just ignores the return code for
                # 'do'reboot' commands ...
                if 'do_reboot' in cmdline:
                    _print_done(msg='done (ignoring do_reboot command failure)')
                elif 'wait_for_reboot' in cmdline:
                    _print_done(msg='done (ignoring wait_for_reboot command failure)')
                else:
                    if ret != 0:
                        if abort_phase:
                            _print_done('skip commands in phase %s due to ABORT_PHASE_TEST "%s" failed with %s' % (phase, cmdline, ret))
                            return
                        else:
                            _print_done('fail: return code %s' % ret)
                            VM.failed_commands += 1
                    else:
                        _print_done()
            except Exception:
                self._print_exception_to_file()
                _print_done('fail')

    def _print_exception_to_file(self):
        """
        Write current exception to self.logfile.
        """
        self._open_logfile()
        traceback.print_exc(file=self.logfile_fd)
        self._close_logfile()

    def command_count(self):
        ''' Retrun the  IP address of the started VM '''
        return max(self.commands or (0,))

    def get_ip(self):
        ''' Retrun the IP address of the started VM '''
        if self.predefined_ip:
            return self.predefined_ip
        if self.instance.vpc_id:
            return self.instance.private_ip_address
        else:
            return self.instance.ip_address

    def get_name(self):
        '''    Return the configured name for the VM, this is the section for
                this host in the config file '''
        return '[%s]' % self.section

    # Helper functions
    def _ssh_exec(self, command, sshconnection=None, stdout=None, stderr=None):
        '''    Execute command using ssh and writes output to logfile.
                sshconnection defines, which ssh connection shall be used - default is self.client.
        '''
        transport = (sshconnection or self.client).get_transport()
        transport.set_keepalive(15)
        session = transport.open_session()
        session.setblocking(False)
        try:
            self._log('')
            self._log('=> Execute: %s on %s at %s' % (command, self.get_name(), datetime.datetime.now().isoformat()))
            session.exec_command(command)
            # Close STDIN for remote command
            session.shutdown_write()
            while True:
                data_available = False
                r_list, _w_list, _e_list = select([session], [], [], 10)
                try:
                    while session.recv_ready():
                        data_available = True
                        data = session.recv(4096)
                        if hasattr(stdout, 'extend'):
                            stdout.extend(data)
                        elif hasattr(stdout, 'write'):
                            stdout.write(data)
                        else:
                            prefix = '{0} {1}'.format(self.get_name(), datetime.datetime.now().isoformat())
                            self._log(data, newline=False, indent=True, prefix=prefix)
                except socket_timeout:
                    self._log('*** Timeout stdout')

                try:
                    while session.recv_stderr_ready():
                        data_available = True
                        data = session.recv_stderr(4096)
                        if hasattr(stderr, 'extend'):
                            stderr.extend(data)
                        elif hasattr(stderr, 'write'):
                            stderr.write(data)
                        else:
                            prefix = '{0} {1}'.format(self.get_name(), datetime.datetime.now().isoformat())
                            self._log(data, newline=False, indent=True, prefix=prefix)
                except socket_timeout:
                    self._log('*** Timeout stderr')

                if not data_available and session.exit_status_ready():
                    break
            if session.exit_status != 0:
                self._log('*** Failed %d: %s' % (session.exit_status, command))
        finally:
            session.close()
        return session.exit_status

    def _open_client_sftp_connection(self):
        '''    Open the SFTP connection and save the connection as self.client_sftp '''
        if not self.client_sftp:
            self.client_sftp = self.client.open_sftp()

    def _close_client_sftp_connection(self):
        '''    Close the SFTP connection '''
        if self.client_sftp:
            self.client_sftp.close()
        self.client_sftp = None

    def _remote_mkdir(self, directory):
        '''    Helpder function to create the given directory structure through
                the SFTP connection '''
        if not directory.startswith('/'):
            return

        try:
            if os.path.exists(directory):
                mode = os.stat(directory).st_mode & 0o777
            else:
                mode = 0o777
            self.client_sftp.mkdir(directory, mode=mode)
        except IOError:
            self._remote_mkdir(directory.rsplit("/", 1)[0])

    # Write a message to the log file
    def _log(self, msg, newline=True, indent=False, prefix=None):
        '''    Write a message to self.logfile '''
        if self.logfile:
            lfile = open(self.logfile, 'a+')
            for line in msg.strip('\n').split('\n'):
                line = line.strip('\n')
                if line:
                    if indent:
                        line = '\t%s' % line
                    if prefix:
                        line = '%s%s' % (prefix, line)
                    lfile.write(str(line))
                    lfile.write('\n')
            if newline:
                lfile.write('\n')
            lfile.close()
        else:
            print(msg)


class VM_KVM(VM):

    """
    Local KVM instance.
    """

    PARAMS_REQUIRED = [
        'kvm_server',
        'kvm_user',
        'kvm_ucsversion',
        'kvm_architecture',
        'kvm_template',
        'kvm_interface',
    ]
    PARAMS_OPTIONAL = [
        'kvm_external_disk',
        'kvm_isofile',
        'kvm_memory',
        'kvm_operating_system',
        'kvm_extra_label',
        'kvm_network',
        'kvm_network_disabled',
        'kvm_keypair',
        'kvm_keypair_file',
        'kvm_connect',
        'kvm_copy_files',
        'kvm_iso',
    ]

    def __init__(self, section, config):
        ''' Initialize a VM instance in local KVM environment '''
        super(VM_KVM, self).__init__(section, config, 'kvm')

        self.kvm_name_full = '%s_%s' % (os.getenv('USER'), self.section,)
        self.reuse = None
        self.kvm_vnc_display = None
        self.xml = None
        self.kvm_interface = self.config.get(self.section, 'kvm_interface')
        if self.config.has_option(self.section, 'kvm_connect'):
            if self.config.get(self.section, 'kvm_connect') == 'false':
                self.do_init_connect = False
        if self.config.has_option(self.section, 'kvm_copy_files'):
            if self.config.get(self.section, 'kvm_copy_files') == 'false':
                self.do_init_copy_files = False

    def _connect_vm(self):
        """Connect to KVM."""
        if self.private_key:
            self.client.connect(
                self.get_ip(),
                port=22,
                username='root',
                key_filename=self.private_key)
        else:
            self.client.connect(
                self.get_ip(),
                port=22,
                username='root',
                password='univention',
                look_for_keys=False,
                allow_agent=False,
            )

    def _connect_kvm_host(self):
        '''Connect to KVM host.'''
        server = paramiko.SSHClient()
        server.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        kvm_server = self.config.get(self.section, 'kvm_server')
        try:
            kvm_user = self.config.get(self.section, 'kvm_user')
        except NoOptionError:
            kvm_user = getpass.getuser()
        try:
            server.connect(
                kvm_server,
                username=kvm_user,
                port=22,
            )
        except (paramiko.ssh_exception.AuthenticationException, socket_error) as ex:
            self._log('Failed to connect to %s as %s...' %
                      (kvm_server, kvm_user))
            _print_done('fail (%s)' % (ex,))
            sys.exit(1)

        transport = server.get_transport()
        transport.set_keepalive(15)

        return server

    def _create_kvm_network(self, server):
        name = self.config.get(self.section, 'kvm_network')
        stdout, stderr = bytearray(), bytearray()
        if name not in KVM_NETWORKS:
            raise Exception('Given network name "%s" unknown in KVM_NETWORKS' % name)
        cmdline = 'virsh net-dumpxml {0}'.format(name)
        if self._ssh_exec(cmdline, server, stdout, stderr):
            virsh_net = KVM_NETWORKS[name]
            cmdline = 'mktemp'
            stdout, stderr = bytearray(), bytearray()
            if self._ssh_exec(cmdline, server, stdout, stderr):
                raise Exception('Failed to create KVM net: %s, %s' % (stderr, stdout))
            tmp_file = stdout
            stdout, stderr = bytearray(), bytearray()
            cmdline = 'echo "%s" > %s' % (virsh_net, tmp_file)
            if self._ssh_exec(cmdline, server, stdout, stderr):
                raise Exception('Failed to create KVM net: %s, %s' % (stderr, stdout))
            stdout, stderr = bytearray(), bytearray()
            cmdline = 'virsh net-create %s' % tmp_file
            if self._ssh_exec(cmdline, server, stdout, stderr):
                raise Exception('Failed to create KVM net: %s, %s' % (stderr, stdout))

    def deactivate_interface_on_network(self):
        mac = self.get_network_mac()
        if mac is not None:
            server = self._connect_kvm_host()
            cmd = 'virsh domif-setlink {0} {1} down'.format(self.kvm_name_full, mac)
            self.run_on_kvm_server(server, cmd)

    def activate_interface_on_network(self):
        mac = self.get_network_mac()
        if mac is not None:
            server = self._connect_kvm_host()
            cmd = 'virsh domif-setlink {0} {1} up'.format(self.kvm_name_full, mac)
            self.run_on_kvm_server(server, cmd)

    def get_network_mac(self):
        self._log('get_network_mac')
        if not self.config.has_option(self.section, 'kvm_network'):
            self._log('mac None')
            return None
        network = self.config.get(self.section, 'kvm_network')
        mac = None
        for i in etree.fromstring(self.xml).findall('.//interface'):
            if i.find('source[@network="{0}"]'.format(network)) is not None:
                m = i.find('mac')
                if m is not None and m.attrib.get('address'):
                    mac = m.attrib['address']
        self._log('mac {0}'.format(mac))
        return mac

    def get_kvm_interface_mac(self):
        ''' get mac of interface with <source bridge='kvm_interface'/> '''
        self._log('get_kvm_interface_mac')
        mac = None
        for i in etree.fromstring(self.xml).findall('.//interface'):
            if i.find('source[@bridge="{0}"]'.format(self.kvm_interface)) is not None:
                m = i.find('mac')
                if m is not None and m.attrib.get('address'):
                    mac = m.attrib['address']
        self._log('mac {0}'.format(mac))
        return mac

    def get_network_dhcp_ip(self):
        self._log('get_network_dhcp_ip')
        if not self.config.has_option(self.section, 'kvm_network'):
            self._log('ip None')
            return None
        #network = self.config.get(self.section, 'kvm_network')
        ip = None
        mac = self.get_network_mac()
        if mac is not None:
            server = self._connect_kvm_host()
            stdout, stderr = bytearray(), bytearray()
            # this does not work, the lease expires after ca 30min
            #ret = self._ssh_exec('virsh net-dhcp-leases {0} | grep {1} | grep ipv4'.format(network, mac), server, stdout, stderr)
            #ret = self._ssh_exec('cat /var/lib/libvirt/dnsmasq/{0}.leases | grep {1} | awk \'{{print $3}}\''.format(network, mac), server, stdout, stderr)
            ret = self._ssh_exec("/usr/sbin/arp -n | grep {0} | cut -d' ' -f 1".format(mac), server, stdout, stderr)
            self._log('stdout {0}'.format(stdout))
            self._log('stderr {0}'.format(stderr))
            if ret == 0:
                #ip = str(stdout).split()[4].split('/')[0]
                ip = str(stdout).strip()
            server.close()
        self._log('ip {0}'.format(ip))
        return ip

    def add_kvm_interface(self, server):
        mac = self.get_kvm_interface_mac()
        if mac is None:
            self._log('need to create bridged interface for {0}'.format(self.kvm_interface))
            out, err = bytearray(), bytearray()
            cmdline = 'virsh attach-interface {0} bridge {1} --persistent --live --model virtio'.format(self.kvm_name_full, self.kvm_interface)
            ret = self._ssh_exec(cmdline, server, out, err)
            if ret:
                _print_done('fail:\ncmd=%s\nret=%d\nstdout=%s\nstderr=%s' % (cmdline, ret, out, err))
                sys.exit(1)
            self.dumpxml(server)
            mac = self.get_kvm_interface_mac()
        return mac

    def add_kvm_network(self, server):
        mac = None
        if self.config.has_option(self.section, 'kvm_network'):
            network = self.config.get(self.section, 'kvm_network')
            mac = self.get_network_mac()
            if mac is None:
                self._log('need to create interface in {0} for {1}'.format(network, self.kvm_name_full))
                cmdline = 'virsh attach-interface {0} network {1} --persistent --live --model virtio'.format(self.kvm_name_full, network)
                out, err = bytearray(), bytearray()
                ret = self._ssh_exec(cmdline, server, out, err)
                if ret:
                    _print_done('fail:\ncmd=%s\nret=%d\nstdout=%s\nstderr=%s' % (cmdline, ret, out, err))
                    sys.exit(1)
                self.dumpxml(server)
                mac = self.get_network_mac()
        return mac

    def start(self, keep_cdrom=False, exact_match=False):
        ''' Start the VM '''
        server = self._connect_kvm_host()
        kvm_iso = None
        if self.config.has_option(self.section, 'kvm_iso'):
            kvm_iso = self.config.get(self.section, 'kvm_iso')
        if self.reuse:
            self.kvm_name_full = self.reuse
        else:
            # create network
            if self.config.has_option(self.section, 'kvm_network'):
                self._create_kvm_network(server)
            # create temporary result file for ucs-kt-get
            cmdline = 'mktemp'
            stdout, stderr = self.run_on_kvm_server(server, cmdline)
            fn_kvm_results = str(stdout.strip())
            # label
            label = self.section
            if self.config.has_option(self.section, 'kvm_extra_label'):
                label += '-' + self.config.get(self.section, 'kvm_extra_label')
            # ucs-kt-get
            if not [disk for disk in self.config.items(self.section) if 'kvm_external_disk' in disk[0]]:
                cmdline = ' '.join(quote(arg) for arg in [
                    PATH_UCS_KT_GET,
                    '--yes',
                    '--version', self.config.get(self.section, 'kvm_ucsversion'),
                    '--architecture', self.config.get(self.section, 'kvm_architecture'),
                    '--label', label,
                    '--onlyone',
                    '--resultfile', fn_kvm_results,
                    self.config.get(self.section, 'kvm_template'),
                ])
                if exact_match:
                    cmdline += ' -e'
                if keep_cdrom:
                    cmdline += " --keep-cdrom"
                if self.config.has_option(self.section, 'kvm_operating_system'):
                    cmdline += " -O %s" % self.config.get(self.section, 'kvm_operating_system')
            else:
                isofile = self.config.get(self.section, 'kvm_isofile')
                if not isofile:
                    # dummy isofile
                    isofile = '/mnt/omar/vmwares/iso-images/others/virtio-win-1.1.16.iso'
                cmdline = ' '.join(quote(arg) for arg in [
                    PATH_UCS_KT_INSTANCE_CREATE,
                    '--name', self.section,
                    '-P',
                    '--cpus', '2',
                    '--memory', self.config.get(self.section, 'kvm_memory'),
                    '--architecture', self.config.get(self.section, 'kvm_architecture'),
                    '--iso-file', self.config.get(self.section, 'kvm_isofile'),
                    '--resultfile', fn_kvm_results,
                    '--external-disk', self.config.get(self.section, 'kvm_external_disk'),
                ])

            # iso support, i dont want to mess around with the kvm_external_disk and kvm_isofile
            # so add a new options kvm_iso, if set the instance is not started by ucs-kt-get
            # but created, the iso is added and then the instance is started
            if kvm_iso:
                cmdline += ' -n'

            # start
            self.run_on_kvm_server(server, cmdline)
            _print_done()

            # get and remove temporary file
            _print_process('Determine instance name')
            server_sftp = server.open_sftp()
            kvm_results = server_sftp.file(fn_kvm_results).read()
            server_sftp.unlink(fn_kvm_results)
            server_sftp.close()
            try:
                # convert JSON kvm_results from ucs-kt-get into python structure
                kvm_results = json.loads(kvm_results)
                if not kvm_results:
                    raise ValueError('kvm_results is empty')
                vm_info, = kvm_results
                if vm_info['status'] != 'ok':
                    raise ValueError('Failed to start VM: %r' % (vm_info,))
                try:
                    self.kvm_name_full = vm_info['name']
                except KeyError:
                    raise ValueError(
                        'name is not set in kvm_results: %r' % (vm_info,))
            except (EnvironmentError, TypeError, ValueError) as ex:
                self._log('Failed to load result file %s of %s: %s' %
                          (fn_kvm_results, PATH_UCS_KT_GET, ex,))
                _print_done('error (unreadable results)')
                sys.exit(1)

            # now add the iso and start the instance
            if kvm_iso:
                cmdline = 'virsh attach-disk {0} {1} hda --type cdrom --mode readonly --config'.format(self.kvm_name_full, kvm_iso)
                self.run_on_kvm_server(server, cmdline)
                cmdline = 'virsh start {0}'.format(self.kvm_name_full)
                self.run_on_kvm_server(server, cmdline)

        _print_done('done (VM: %s)' % self.kvm_name_full)

        # vnc display
        _print_process('Detecting VNC display')
        cmd = 'virsh vncdisplay %s' % self.kvm_name_full
        stdout, stderr = self.run_on_kvm_server(server, cmd)
        if stdout:
            stdout = stdout.strip()
            self.kvm_vnc_display = '{0}{1}'.format(self.config.get(self.section, 'kvm_server'), stdout)
        _print_done('done (VNC: %s)' % self.kvm_vnc_display)

        # network
        _print_process('Detecting IPv6 address')
        self.dumpxml(server)
        self.add_kvm_network(server)
        mac = self.add_kvm_interface(server)
        # disable kvm network if required
        if self.config.has_option(self.section, 'kvm_network_disabled'):
            self.deactivate_interface_on_network()
        self.instance = {
            'section': self.section,
            'mac': mac,
            'ipv6': '%s%%%s' % (
                mac2IPv6linklocal(mac),
                self.get_used_interface_for_host(self.config.get(self.section, 'kvm_server'))
            )
        }
        self._log('Instance %(section)s: MAC=%(mac)s  IPv6=%(ipv6)s' % self.instance)
        server.close()

    def run_on_kvm_server(self, server, cmd):
        stdout, stderr = bytearray(), bytearray()
        ret = self._ssh_exec(cmd, server, stdout, stderr)
        if ret:
            _print_done('fail:\ncmd=%s\nret=%d\nstdout=%s\nstderr=%s' % (cmd, ret, stdout, stderr))
            sys.exit(1)
        return stdout, stderr

    def get_used_interface_for_host(self, host):
        ip = gethostbyname(host)
        route = subprocess.check_output(['ip', 'route', 'get', ip])
        for line in route.splitlines():
            if 'dev' in line:
                return line.split('dev')[1].split()[0]

    def dumpxml(self, server):
        stdout, stderr = bytearray(), bytearray()
        cmdline = '/usr/bin/virsh dumpxml %s' % (quote(self.kvm_name_full))
        ret = self._ssh_exec(cmdline, server, stdout, stderr)
        if ret:
            _print_done('fail:\ncmd=%s\nret=%d\nstdout=%s\nstderr=%s' % (cmdline, ret, stdout, stderr))
            sys.exit(1)
        self.xml = str(stdout)

    def get_ip(self):
        ''' Return the IP address of the started VM '''
        return self.instance['ipv6']

    def reset(self):
        _print_process('Reset VM')
        server = self._connect_kvm_host()
        stdout, stderr = bytearray(), bytearray()
        # This should basically be 'virsh reset ...', but with a reset changes
        # in the hardware (e.g. a new NIC was added) are not registered.
        cmdline = 'virsh destroy %s' % (quote(self.kvm_name_full),)
        ret = self._ssh_exec(cmdline, server, stdout, stderr)
        if not ret:
            cmdline = 'virsh start %s' % (quote(self.kvm_name_full),)
            ret = self._ssh_exec(cmdline, server, stdout, stderr)
        if ret:
            _print_done('fail:\ncmd=%s\nret=%d\nstdout=%s\nstderr=%s' % (cmdline, ret, stdout, stderr))
            sys.exit(1)
        return

    def terminate(self):
        _print_process('Terminating VM')
        cmdline = ' '.join(quote(arg) for arg in [
            'ucs-kt-remove',
            '-t',
            self.kvm_name_full,
        ])
        server = self._connect_kvm_host()
        ret = self._ssh_exec(cmdline, server)
        if ret != 0:
            _print_done('fail:' % (ret,))
            sys.exit(1)
        else:
            _print_done()
        server.close()


class VM_EC2(VM):

    """
    Amazon EC2 instance.
    """

    PARAMS_REQUIRED = [
        'ec2_ami',
        'ec2_instance_type',
        'ec2_region',
    ]
    PARAMS_OPTIONAL = [
        'ec2_ami_name',
        'ec2_subnet_id',
        'ec2_security_group',
        'ec2_instance_ebsOptimized',
        'ec2_partition_size',
        'ec2_instance_store',
        'ec2_volume_type',
        'ec2_volume_iops',
        'ec2_keypair',
        'ec2_keypair_file',
    ]

    def __init__(self, section, config):
        ''' Initialize a VM instance '''
        super(VM_EC2, self).__init__(section, config, 'ec2')

        self.aws_cfg = {}
        for key in self.PARAMS_REQUIRED + self.PARAMS_OPTIONAL + ['ec2_reuse']:
            if config.has_option(section, key):
                self.aws_cfg[key] = config.get(section, key)

        try:
            self.tags = dict(config.items('EC2Tags'))
        except NoSectionError:
            self.tags = {}

        self.ec2 = None

    def _create_blockdevicemapping(self, ami):
        """
        Create explicit block device with given size.
        Also make additional ephemeral disk available.
        """
        size = self.aws_cfg.get('ec2_partition_size')
        max_ephemeral = int(self.aws_cfg.get('ec2_instance_store', 4))
        bdm = blockdevicemapping.BlockDeviceMapping()

        last_used = 0
        for dev, desc in ami.block_device_mapping.iteritems():
            bdm[dev] = blockdevicemapping.BlockDeviceType(
                ephemeral_name=desc.ephemeral_name,
                volume_id=desc.volume_id,
                snapshot_id=desc.snapshot_id,
                size=int(size) if size else desc.size,
                delete_on_termination=True,
            )
            if BOTO_VERSION >= (2, 6):
                iops = self.aws_cfg.get('ec2_volume_iops')
                if iops:
                    bdm[dev].iops = int(iops)
                bdm[dev].volume_type = self.aws_cfg.get('ec2_volume_type')
            if desc.ephemeral_name:
                max_ephemeral = 0
            last_used = max(last_used, ord(dev.rstrip("0123456789")[-1]))

        for ephemeral in range(max_ephemeral):
            dev = u'/dev/xvd%c' % (chr(last_used + 1 + ephemeral),)
            bdm[dev] = blockdevicemapping.BlockDeviceType(
                ephemeral_name=u"ephemeral%d" % (ephemeral,),
            )

        return bdm

    def _connect_vm(self):
        """Connect to EC2 VM."""
        self.client.connect(
            self.get_ip(),
            port=22,
            username='root',
            key_filename=self.private_key,
        )

    def start(self):
        ''' Start the VM '''
        # self.ec2 = boto.connect_ec2(**self.aws_cfg)
        aws_cfg = {}
        for region in regions(**aws_cfg):
            if region.name == self.aws_cfg['ec2_region']:
                aws_cfg['region'] = region
                break

        env_vars = ('JOB_NAME', 'BUILD_NUMBER', 'BUILD_URL')
        user_data = '\n'.join(
            ['%s=%s' % (v, os.getenv(v, '')) for v in env_vars])
        if self._is_windows:
            user_data += EC2_WINDOWS_USER_DATA
        self.ec2 = boto.connect_ec2(**aws_cfg)
        reuse = self.aws_cfg.get('ec2_reuse')
        if reuse:
            reservation = self.ec2.get_all_instances(instance_ids=[reuse])[0]
        else:
            if 'ec2_ami_name' in self.aws_cfg:
                ami = None
                epoch = None
                try:
                    amis = self.ec2.get_all_images(owners=['self', 'amazon', 'aws-marketplace'], filters=dict(name=self.aws_cfg['ec2_ami_name'], state='available'))
                except SSLError:
                    self._log('SSLError exception while executing self.ec2.get_all_images. Wait for one minute and try again.')
                    time.sleep(60)
                    amis = self.ec2.get_all_images(owners=['self', 'amazon', 'aws-marketplace'], filters=dict(name=self.aws_cfg['ec2_ami_name'], state='available'))
                for candidate in amis:
                    d = datetime.datetime.strptime(candidate.creationDate.split('.')[0], '%Y-%m-%dT%H:%M:%S')
                    if epoch is None or epoch < time.mktime(d.timetuple()):
                        ami = candidate
                        epoch = time.mktime(d.timetuple())
                if ami is None:
                    raise Exception('No ami found for %s!' % self.aws_cfg['ec2_ami_name'])
            else:
                ami = self.ec2.get_image(self.aws_cfg['ec2_ami'])

            param = dict(
                image_id=ami.id,
                max_count=1,
                key_name=self.aws_cfg['ec2_keypair'],
                user_data=user_data,
                instance_type=self.aws_cfg['ec2_instance_type'],
                instance_initiated_shutdown_behavior='terminate',  # 'save'
                block_device_map=self._create_blockdevicemapping(ami),
            )
            if self.aws_cfg.get('ec2_subnet_id'):
                param['subnet_id'] = self.aws_cfg['ec2_subnet_id']
                param['security_group_ids'] = [
                    self.aws_cfg['ec2_security_group']]
            else:
                param['security_groups'] = [self.aws_cfg['ec2_security_group']]
            if BOTO_VERSION >= (2, 9, 3):
                if self.aws_cfg.get('ec2_instance_ebsOptimized', '').lower() in ('1', 'true', 'yes'):
                    param['ebs_optimized'] = True

            reservation = self.ec2.run_instances(**param)

        self.instance = reservation.instances[0]

        self._wait_instance()

        user = os.getenv('USER')
        tags = {
            'Name': 'Test-%s-%s' % (user, self.section),
            'class': 'ucs-test',
            'usecase': os.getenv('USECASE', 'user-%s' % (user,)),
        }
        tags.update(self.tags)
        tags.update(dict((var.lower(), os.getenv(var, '')) for var in env_vars))
        try:
            self.instance.add_tags(tags)
        except AttributeError:
            for name, value in tags.iteritems():
                self.instance.add_tag(name, value)

    def reset(self):
        _print_process('Reset VM')
        self.instance.reboot()

    def terminate(self):
        _print_process('Terminating VM')
        self.instance.terminate()

    def _wait_instance(self, timeout=600):
        """
        Wait until instance is created.
        """
        start = now = time.time()
        while now - start < timeout:
            self._log('DEBUG: now: %d; start: %d; timeout: %d' % (now, start, timeout))
            if self.instance.state == 'running':
                break
            if self.instance.state == 'pending':
                self._log('Pending %d...' % (timeout - now + start))
            time.sleep(10)

            try:
                self.instance.update()
            except boto.exception.EC2ResponseError as ex:
                for error in ex.errors:
                    self._log('Error code: %r', error.error_code)
                    if error.error_code == 'InvalidInstanceID.NotFound':
                        break
                else:
                    self._log('Unexcpected error waiting for instance: %s', ex)
                    raise
            now = time.time()
        else:
            self._log('Timeout waiting for instance')
            self._log('DEBUG: now: %d; start: %d; timeout: %d' % (now, start, timeout))
            raise TimeoutError(timeout)

    def get_windows_password(self):
        key = M2Crypto.RSA.load_key(self.private_key)
        # wait max 10min for password
        for i in range(0, 60):
            data = self.ec2.get_password_data(self.instance.id)
            if data:
                return EC2_WINDOWS_DEFAULT_PASSWORD
                #return key.private_decrypt(data.decode('base64'), M2Crypto.RSA.pkcs1_padding)
            time.sleep(10)
        return None

    @property
    def ami_name(self):
        ami = self.ec2.get_image(self.instance.image_id)
        return ami.name

    def ec2_assign_private_ip_addresses(self, new_ip):
        if self.instance.interfaces:
            self._log('assign private ip {0} to network {1}'.format(new_ip, self.instance.interfaces[0].id))
            ret = self.ec2.assign_private_ip_addresses(network_interface_id=self.instance.interfaces[0].id, private_ip_addresses=new_ip)
            self._log('=> {0}'.format(ret))
            if ret is not True:
                raise Exception('Could not assign ip {0} to interface {1}'.format(new_ip, self.instance.interfaces[0].id))


def load_vms(config, clazz):
    vms = []
    for host in config.sections():
        if host in ['Global', 'EC2Tags']:
            continue
        _print_process('Creating VM [%s]' % host)
        cfg = clazz(host, config)
        if cfg.valid():
            vms.append(cfg)
            _print_done()
        else:
            _print_done('invalid')
    return vms


def _print_process(msg):
    '''Print s status line'''
    if not os.isatty(sys.stdout.fileno()):
        print(msg)
    elif len(msg) > 64:
        print('%s..' % msg[:63], end='')
    else:
        print('%-65s' % msg, end='')
    sys.stdout.flush()


def _print_done(msg='done'):
    '''Close the status line opend with _print_process'''
    print(msg)
    sys.stdout.flush()


def check_missing_files(vms):
    missing = [
        localfile
        for vm in vms
        for localfile, remotedir in vm.list_files()
        if not os.path.exists(localfile) or not remotedir
    ]
    if missing:
        print('fail: missing local files:\n%s' % ('\n'.join(missing),), file=sys.stderr)
        if os.isatty(sys.stdout.fileno()):
            sys.exit(1)


def each_vm(vms, parallel, run, *args, **kwargs):
    if parallel:
        threads = []
        for vm in vms:
            thread = Thread(
                target=run, name=vm.get_name(), args=(vm,) + args, kwargs=kwargs)
            thread.start()
            threads.append(thread)
        for thread in threads:
            thread.join()
    else:
        for vm in vms:
            run(vm, *args, **kwargs)


if __name__ == '__main__':
    import doctest
    print(doctest.testmod())
