#!/usr/bin/env python
"""
Script to manage ALP environment setup.
Copyright: Dmitry Zamaruev (avenger@zoral.com.ua)

"""
__version__ = "3.0"

import os
import re
import pwd
import sys
import time
import urllib
import socket
import shutil
import logging
import zipfile
import tempfile
from glob import glob
from fnmatch import fnmatch
from subprocess import call, Popen, PIPE, STDOUT
from logging import error, info, warning, debug
from getopt import getopt, GetoptError
from xml.dom.minidom import parse, parseString

from ConfigParser import SafeConfigParser

CONFIG_NAME = 'environment.conf'
USER = pwd.getpwuid(os.getuid())[0]
HOME = pwd.getpwuid(os.getuid())[5]
HOSTNAME = socket.gethostname().split('.')[0]
FQDN = socket.gethostname()

common_variables = {
                    'home': HOME,
                    'user': USER,
                    'currtime': time.strftime("%F.%H%M%S"),
                    'hostname': HOSTNAME,
                    'host.ip' : socket.gethostbyname(FQDN),
                   }

def usage():#{{{
    print("Usage: alpsetup [options] <action> <service>")
    print("\nWhere options are:")
    print("\t-h           - this help")
    print("\t-d           - enable debug output")
    print("\t-u, --update - update downloaded files")
    print("\nParams:")
    print("\t<action>     - action to launch")
    print("\t<service>    - service to operate")#}}}

def unpack(archive, dest_dir, skip=0, files=None):#{{{
    """ Unpacks archive to given destination

    Arguments:
      archive  -- source archive file
      dest_dir -- destination directory
      skip     -- how many top level dirs to skip (default 0)
      files    -- list of filenames to extract (default None)

    Return: 
      None (raises Exception on error)

    Notes:
      If 'dest_dir' does not exists it will be created.

    """    
    if not os.path.exists(archive):
        raise RuntimeError("File not found '%s'!"%archive)

    if not os.path.exists(dest_dir):
        os.makedirs(dest_dir)

    if   archive.endswith('.zip') \
      or archive.endswith('.war') \
      or archive.endswith('.jar') \
      or archive.endswith('.ear'):
        zipobj = zipfile.ZipFile(archive)
        for name in zipobj.namelist():
            _name = os.path.join(*name.split('/')[skip:])
            # If we have filelist to unpack
            # check first if we need to unpack current path
            if files:
                _unpack = False
                for f in files:
                    if   f.startswith(name) \
                      or fnmatch(name, f):
                        _unpack = True
                if not _unpack:
                    continue

            if name.endswith('/'):
                _dir = os.path.join(dest_dir, _name)
                if os.path.exists(_dir):
                    warning("Directory exists: %s"%_dir)
                else:
                    os.makedirs(os.path.join(dest_dir, _name))
            else:
                outfile = open(os.path.join(dest_dir, _name), 'wb')
                outfile.write(zipobj.read(name))
                outfile.close()
    else:
        raise RuntimeError("Archive format not supported")#}}}

def unpack_temp(filelist, tempdir):#{{{
    """ Unpack specific file from a nested archives

    Arguments:
      filelist -- list of nested archives and file to extract
      tempdir  -- temporary directory for operations

    Return:
      path to unpacked file

    Notes:
      'tempdir' should exist before calling this function

    Example:
      filelist = ['/home/user/application.ear','content.jar','some.properties']
      Unpack 'some.properties' file from 'content.jar' 
      which in turn packed into 'application.ear'

    """
    for f in filelist[:-1]:
        i = filelist.index(f)
        to_extract = filelist[i+1]
        if i == 0:
            unpack(f, tempdir, files=[to_extract])
        else:
            _f = os.path.join(tempdir, f)
            unpack(glob(_f)[0], tempdir, files=[to_extract])
    return os.path.join(tempdir, filelist[-1])#}}}

def remove_files(archive, *files):#{{{
    """ Remove files from archive

    Arguments:
      archive -- source archive
      ...     -- variable number of files to remove

    Return:
      None

    """
    tempdir = tempfile.mkdtemp()
    try:
        tempname = os.path.join(tempdir, 'new.zip')
        zipread = zipfile.ZipFile(archive, 'r')
        zipwrite = zipfile.ZipFile(tempname, 'w')
        for item in zipread.infolist():
            if item.filename not in files:
                data = zipread.read(item.filename)
                zipwrite.writestr(item, data)
        zipwrite.close()
        zipread.close()
        shutil.move(tempname, archive)
    finally:
        shutil.rmtree(tempdir)#}}}

def pack(archive, source, name):#{{{
    """ Pack file into archive

    Arguments:
      archive -- destination archive
      source  -- file to pack
      name    -- name of file in archive

    Return:
      None

    """
    if not os.path.exists(archive):
        raise RuntimeError("File not found '%s'!"%archive)

    if   archive.endswith('.zip') \
      or archive.endswith('.war') \
      or archive.endswith('.jar') \
      or archive.endswith('.ear'):
        remove_files(archive, name)
        zipobj = zipfile.ZipFile(archive, 'a')
        zipobj.write(source, name)
        zipobj.close()
    else:
        raise RuntimeError("Archive format not supported")#}}}

def pack_temp(filelist, tempdir):#{{{
    """ Pack file into nested archives

    Arguments:
      filelist -- list of nested archives and source file
      tempdir  -- temporary dir populated from unpack_temp()

    Return:
      None

    Notes:
      'filelist' should be in the same order as feed to unpack_temp() call

    Example:
      filelist = ['/home/user/application.ear','content.jar','some.properties']
      unpacked_file = unpack_temp(filelist, '/tmp/temp_dir')
      change_file(unpacked_file)
      pack_temp(filelist, '/tmp/temp_dir')

    """
    reverse = [i for i in reversed(filelist)]
    for f in reverse[:-1]:
        i = reverse.index(f)
        archive = reverse[i+1]
        if i < len(reverse)-1:
            pack_file = glob(os.path.join(tempdir, f))[0]
            pack_name = pack_file[len(tempdir)+1:]
            archive = glob(os.path.join(tempdir, archive))[0]
            pack(archive, pack_file, pack_name)
        else:
            pack_file = glob(os.path.join(tempdir, f))[0]
            pack_name = pack_file[len(tempdir)+1:]
            pack(archive, pack_file, pack_name)#}}}

def copy(source, destination):#{{{
    """ Copies file from source to destination
    (currently just shortcut to shutil.copy() method)

    Arguments:
      source -- source file
      destination -- destination file or directory

    Returns:
      None

    Notes:
      If 'destination' is directory, then file with source name created
      in that directory. If file exists - it will be overwritten

    """
    # TODO: Should we add directory creation if does not exists?
    shutil.copy(source, destination)#}}}

def items_reorder(items):#{{{
    """ Reorder numbered items read from ConfigParser objects

    Arguments:
      items -- list of tuples (param, value) 

    Returns:
      ordered list of dictionaries {param: value} 
      with order information removed

    Note:
      All input items must have order information, i.e:
      10.param = value
      20.param = value

    Example:
      >>> items = [('30.action','echo'), ('10.cmd','some'), ('30.param','me')]
      >>> items_reorder(items)
      [{'cmd': 'some'}, {'action': 'echo', 'param': 'me'}]
      >>>

    """
    i = sorted(items, key=lambda x: int(x[0].split('.')[0]))
    # Convert order to array
    _i = {}
    for param, value in i:
        order, key = param.split('.', 1)
        if order not in _i.keys():
            _i[order] = {}
        _i[order][key] = value
    itms = []
    for i in sorted(_i.keys()):
        itms.append(_i[i])

    return itms#}}}

def items_to_dict(items):#{{{
    """ Convert list of tuples (param, value) to dict {param: value}

    Arguments:
      items -- list of tuples (param, value)

    Returns:
      dictionary {param: value}

    Example:
      >>> items = [('param1','value1'), ('param2','value2'), ('param3', 'value3')]
      >>> items_to_dict(items)
      {'param3': 'value3', 'param2': 'value2', 'param1': 'value1'}
      >>> 

    """
    _i = {}
    for key, value in items:
        _i[key] = value
    return _i#}}}

def xml_nodes(path, dom):#{{{
    """ Search XML minidom document for nodes of the given path.

    Arguments:
      path -- search path
      dom  -- Document object to search in

    Returns:
      Array of found nodes

    Notes:
      'path' is an XPath-like expression that could contain
      tag name and attribute. Currently only one attribute is supported.
      'path' could be absolute (starting from /) or relative. In later case
      search will begin from all tags matching root element name
      Syntax of 'path' is:
      '(/)?<tagname>([<attribute>="<value>"])?(/<tagname>([<attribute>="<value>"])?)*

    Example:
    >>> from xml.dom.minidom import parseString
    >>>
    >>> xml = \"\"\"
    ... <xml>
    ...   <Node name="node1">
    ...     <Element attr="el1"><SubElement name="sub1" /></Element>
    ...     <Element attr="el2"><SubElement name="sub2" /></Element>
    ...   </Node>
    ...   <Node name="node2">
    ...     <SubElement name="sub3" />
    ...     <Element attr="el4"><SubElement name="sub4" /></Element>
    ...     <Element attr="el5"><SubElement name="sub5" /></Element>
    ...   </Node>
    ... </xml>
    ... \"\"\"
    >>>
    >>> dom = parseString(xml)
    >>>
    >>> # Return all <Element> tags
    >>> print xml_nodes("Element", dom) # doctest: +ELLIPSIS
    [<DOM Element: Element at 0x...>, <DOM Element: Element at 0x...>, <DOM Element: Element at 0x...>, <DOM Element: Element at 0x...>]
    >>> # Return only <Element> with attribute 'attr' equal to 'el4'
    >>> print xml_nodes('Element[attr="el4"]', dom) # doctest: +ELLIPSIS
    [<DOM Element: Element at 0x...>]
    >>> # Return all <SubElement> tags within <Node> tag
    >>> print xml_nodes('Node/SubElement', dom) # doctest: +ELLIPSIS
    [<DOM Element: SubElement at 0x...>]
    >>> # Return all <SubElement> tags within <Element attr="el5"> tag
    >>> print xml_nodes('Element[attr="el5"]/SubElement', dom) # doctest: +ELLIPSIS
    [<DOM Element: SubElement at 0x...>]

    """
    path = path.strip()
    found_nodes = [list(i) for i in re.findall('(?P<node>([\w:\-]+)(\[.+?\])?)', path)]
    for node in found_nodes:
        if node[2] != '':
            # Parse attribute
            attr, val = node[2][1:-1].split('=', 1)
            val = val.strip('"')
            node[2] = (attr, val)

    elements = []
    if path[0] == '/':
        elements.append(dom)
    else:
        _elements = dom.getElementsByTagName(found_nodes[0][1])
        # If we have attribute info - check it
        if found_nodes[0][2] != '':
            attr = found_nodes[0][2]
            for e in _elements:
                try:
                    if e.attributes[attr[0]].value == attr[1]:
                        elements.append(e)
                except KeyError:
                    pass
        else:
            elements = _elements
        found_nodes = found_nodes[1:]

    for n in found_nodes:
        tmp_ = []
        for e in elements:
            for c in e.childNodes:
                if c.nodeName.lower() != n[1].lower():
                    continue
                if n[2] != '':
                    try:
                        if c.attributes[n[2][0]].value != n[2][1]:
                            continue
                    except KeyError:
                        continue
                tmp_.append(c)
        elements = tmp_
        if not elements:
            break

    return elements#}}}

def xml_strip_nl(node):#{{{
    """ Strips newlines from XML nodes
    FIXME: Should be more intellect in detecting text nodes
    """
    if node.nodeType == node.TEXT_NODE:
        node.nodeValue = node.nodeValue.strip(" \r\n")
        if node.nodeValue == "":
            parent = node.parentNode
            parent.removeChild(node)
    else:
        for n in node.childNodes[:]:
            xml_strip_nl(n)#}}}

def xml_write(filename, dom):#{{{
    """ Writes given DOM to file

    Arguments:
      filename -- file to write to (file is created or overwriten)
      dom      -- XML Document to write

    """
    # FIXME: Alter xml_strip_nl behavoiur to produce human-readable xml
    #xml_strip_nl(dom)
    f = file(filename, 'w')
    dom.writexml(f)#, indent="  ", addindent="  ", newl="\n")
    f.flush()
    f.close()#}}}

class URLopener(urllib.FancyURLopener):
    """ Override default to raise error on wrong server response """
    def http_error_default(self, url, fp, errcode, errmsg, headers):
        urllib.URLopener.http_error_default(self, url, fp, errcode, errmsg, headers)
urllib._urlopener = URLopener()

def download_file(url, filename=None, update=False):#{{{
    """ Downloads file from given URL, saving to given location

    Arguments:
      url -- URL to download
      filename -- location to save file
      update -- whether we should update file if it is already exists

    Notes:
      if 'filename' is None - download location will be created using mktemp

    Returns:
      path to downloaded file

    """
    # Ensure that directories exists
    # before we extract files there 
    if filename is not None:
        _dir = os.path.dirname(filename)
        if _dir:
            os.makedirs(_dir)
        # Check if file already exists
        # Do not download if not required
        if os.path.exists(filename) and not update:
            return filename

    # Try to download file
    # raise exception on any error
    try:
        res = urllib.urlretrieve(url, filename)
    except IOError, e:
        if len(e.args) != 4:
            raise
        raise RuntimeError('Could not retrieve: %s (%d %s)'%(url,e[1],e[2]))

    return res[0]#}}}

class UpdateFile (object):
    """ Class that is responsible for updating file contents

    Should be initialized with parent service object and filename to alter.
    File section in config should have following format

    [file:<filename>]
    file = <real file path>
    <order>.<action> = <value>
    ...

    Where:
    <filename> is unique identifier for section, could be different from
        real filename or location.
    <real file path> should be absolute path to file.
        Could point to file in archives. In such case pathes inside archives
        are relative to archive. Archive contents separated by #.
        Nested archives also supported. Example:
        /some/path/archive.zip#file.txt
        /some/path/archive.zip#dir/nested.zip#somedir/file.txt
    <order>.<action> numeric <order> represent order of execution - lower numbers
        have higher priority.
        <action> should be implemented as do_<action> method, with one positional
        argument that will be passed <value>
    <value> will be passed as the first positional param to action function

    """
    def __init__(self, service, filename_section):#{{{
        self.srv = service
        self.section = 'file:'+filename_section

        mgr = service.mgr
        cfg = service.config

        # Load all defenitions using variables from all services
        items = cfg.items(self.section, raw=False,
                          vars=mgr.variables())

        # Filter out substitution variables (ConfigParser specific behavior)  
        for name in mgr.services():
            items = [i for i in items if not i[0].startswith("%s."%name)]
        items = [i for i in items if i[0] not in cfg.defaults().keys()]

        # Filter out specific items (i.e. for create_file)
        items = [i for i in items if i[0] not in ('content',)]

        # Get filename
        filename = ''
        for i in items:
            if i[0] == 'file':
                filename = i[1]
                items.remove(i)

        if not filename:
            raise RuntimeError("filename not specified")
        self._filename = filename

        items = items_reorder(items)

        for i in items:
            a = i.keys()[0]
            action = a.split('.', 1)
            params = []
            if len(action) > 1:
                params.append(action[1])
            action = action[0]
            params.append(i[a])
            # Get implementation 
            try:
                action_func = self.__getattribute__('do_%s'%action)
            except:
                raise RuntimeError("UpdateFile: '%s' not implemented"%action)

            i.clear()
            i['params'] = params
            i['action'] = action
            i['func'] = action_func

        self.items = items#}}}

    def execute(self):#{{{
        """ Executes given set of actions on file.
        In case if file is packed inside archives - file is unpacked
        before actions, and packed back after all chanches occur.
        """
        tempdir = tempfile.mkdtemp()
        try:
            files = self._filename.split('#')
            if len(files) > 1:
                self.filename = unpack_temp(files, tempdir)
            else:
                self.filename = files[0]

            for item in self.items:
                self.srv.debug('update_file(): executing %s()'%item['action'])
                item['func'](*item['params'])

            if len(files) > 1:
                pack_temp(files, tempdir)

        finally:
            shutil.rmtree(tempdir)#}}}

    def do_xml_delete(self, nodes_list):#{{{
        """ Action 'xml_delete'
        Delete all nodes described by path expression
        (see xml_nodes for path description).
        Value could contain list of different nodes - path expressions
        separated by comma.

        Example:
        10.xml_delete = Server/Connector[protocol="ajp"]
        20.xml_delete = Server/Connector, Server/Host

        """
        xml = parse(self.filename)
        paths = nodes_list.split(',')
        for path in paths:
            nodes = xml_nodes(path, xml)
            for node in nodes:
                parent = node.parentNode
                parent.removeChild(node)
        xml_write(self.filename, xml)#}}}

    def do_xml_append(self, value):#{{{
        """ Action 'xml_append'
        Append given XML to each found node described by path expression
        (see xml_nodes for path description).
        Value should contain path expression on first string
        and XML to append as next lines

        Example:
        10.xml_append = /Server
            <Connector proto="HTTP/1.0" />

        """
        xml = parse(self.filename)
        path, code = value.split('\n', 1)
        xml_code = parseString("<xml>"+code+"</xml>")
        for node in xml_nodes(path, xml):
            for c in xml_code.childNodes[0].childNodes:
                node.appendChild(c.cloneNode(deep=True))
        xml_write(self.filename, xml)#}}}

    def do_xml_insert(self, value):#{{{
        """ Action 'xml_insert'
        Insert given XML to each found node described by path expression
        before all node childs (see xml_nodes for path description).
        Value should contain path expression on first string
        and XML to insert as next lines

        Example:
        10.xml_insert = /Server
            <Connector proto="HTTP/1.0" />

        """
        xml = parse(self.filename)
        path, code = value.split('\n', 1)
        xml_code = parseString("<xml>"+code+"</xml>")
        for node in xml_nodes(path, xml):
            for c in xml_code.childNodes[0].childNodes:
                if node.hasChildNodes():
                    # Clone node with deep copy
                    node.insertBefore(c.cloneNode(True), node.childNodes[0])
                else:
                    node.appendChild(c.cloneNode(deep=True))
        xml_write(self.filename, xml)#}}}

    def do_xml_property(self, paths):#{{{
        """ Action 'xml_property'
        Update property for each found node described by path expression
        (see xml_nodes for path description).
        Value should contain path expression follwed by [<property>=<value>]
        that describe what value to set for given property.

        Example:
        # Set port="9005" for /Server node
        10.xml_property = /Server[port="9005"]
        # Set value="true" for node
        # /log4j:configuration/appender[name="FILE"]/param[name="Append"]
        10.xml_property = /log4j:configuration/appender[name="FILE"]/param[name="Append"][value="true"]

        """
        xml = parse(self.filename)
        params = re.findall('^(.*)(\[.+?\])$', paths.strip())
        if not params:
            raise RuntimeError("Wrong xml_property setting: %s"%paths)
        params = params[0]
        path = params[0]
        attr, value = params[1].strip('[]').split('=', 1)
        value = value.strip('"')
        for node in xml_nodes(path, xml):
            node.setAttribute(attr, value)
        xml_write(self.filename, xml)#}}}

    def do_regexp(self, regexp):#{{{
        """ Action 'regexp'
        Replaces line using provided regexp.
        Value could contain several lines of replacement regexps in format:
        'search pattern' > 'replace pattern'
        You could use groups reference in 'replace pattern' in form 
        \g<1> for first group, \g<2> - for second, etc.

        Example:
        # following regexp will replace value of 'shotoku.web.localpath' parameter
        # in some file to '/some/path/web' preserving spaces (formating) and newlines
        10.regexp = '(shotoku\.web\.localpath\s*=\s*).*?(\n)' > '\g<1>/some/path/web\g<2>'
        # following example is equivalent
        10.regexp = '(shotoku\.web\.localpath\s*=\s*).*?(\n)' > '\g<1>/some/path/web\g<2>'
        20.regexp = '(shotoku\.app\.localpath\s*=\s*).*?(\n)' > '\g<1>/some/path/app\g<2>'
        # to that
        10.regexp = '(shotoku\.web\.localpath\s*=\s*).*?(\n)' > '\g<1>/some/path/web\g<2>'
            '(shotoku\.app\.localpath\s*=\s*).*?(\n)' > '\g<1>/some/path/app\g<2>'

        """
        newlines = []
        f = file(self.filename, 'r').readlines()
        for line in f:
            for rline in regexp.split('\n'):
                (regex, subs) = rline.split('>', 1)
                regex = regex.strip().strip("'")
                subs = subs.strip().strip("'")
                newline = re.sub(regex, subs, line)
                if newline != line:
                    break
            newlines.append(newline)
        file(self.filename, 'w').writelines(newlines)#}}}

    def do_regexp_mline(self, regexp):#{{{
        """ Action 'regexp_mline'
        Same action as 'regexp' (see above) but dot will match newlines,
        so set of lines could be replaced.

        """
        newlines = []
        content = ''.join(file(self.filename, 'r').readlines())
        for rline in regexp.split('\n'):
            (regex, subs) = rline.split('>', 1)
            regex = regex.strip().strip("'")
            subs = subs.strip().strip("'")
            _re = re.compile(regex, flags=re.DOTALL)
            new = _re.sub(subs, content)
        file(self.filename, 'w').write(new)#}}}

    def do_line_append(self, line):#{{{
        """ Action 'line_append'
        Appends line to the end of file, but only if such line
        does already not exist in file.

        """
        lines = file(self.filename, 'r').readlines()
        # Fix last line - should have \n
        if lines[-1][-1] != '\n':
            lines[-1] = lines[-1] + '\n'
        matched = False
        for l in lines:
            if re.search("^%s$"%line, l):
                matched = True
        if not matched:
            lines.append(line + '\n')
        self.srv.debug_pp(lines)
        file(self.filename, 'w').writelines(lines)#}}}

    def do_line_delete(self, line):#{{{
        """ Action 'line_delete'
        Searches and deletes given line.
        Search is done via re.search() so line could contain regexp.

        """
        lines = file(self.filename, 'r').readlines()
        for l in lines:
            if re.search("^%s"%line, l):
                lines.remove(l)
                break
        file(self.filename, 'w').writelines(lines)#}}}

class Action (object):
    """ Class that is responsible for executing service actions.

    Should be initialized with parent service object and name of action.
    Action section in config should have following format

    [action:<service>:<action_name>]
    <order>.<param> = <value>
    ...

    A set of <param>s with same <order> describes one sub-action.
    Sub-action type is described with 'action' param and should be implemented
    as do_<subaction> method of this class. 'action' param is mandatory.
    All other <param> values will be passed as 'param' dictionary to method.

    There are also special cases, where sub-action could be described in one line
    Such sub-actions are 'cmd', 'update_file', 'create_file', example:
      10.cmd = rm -rf /
    is equivalent to:
      10.action = shell
      10.cmd = rm -rf /
    later syntax allows you to add more params to sub-action, not just command
    such as 'fail' and 'allowed_fail':
      10.fail = False     # Optional param
      10.allowed_fail = 0 # Optional param

    Param 'fail' tells system that this sub-action is subject to fail.
    In case of successfull completetion of sub-action - no further sub-action runs!
    But if this sub-action fails - further sub-actions (for example to kill hanged
    process that failed to stop in previous sub-action)

    Param 'allowed_fail' - shows which return code not to treat as failed result
    (for example kill will return 1 if it cannot find PID, so we could count this
     as successfull 'kill -9' operation)

    """
    def __init__(self, service, name):
        self.srv = service
        self.config = service.config
        self.section = ':'.join([service.action_section, name])
        self.env = {}

        # Load all action defenitions
        items = self.config.items(self.section,
                                  raw=False,
                                  vars=service.variables())
        # Get environment overrides
        env = [i for i in items if i[0].startswith("env.")]
        for var, value in env:
            self.env[var[4:].upper()] = value
        # Filter out substitution variables (ConfigParser specific behavior)
        items = [i for i in items if not i[0].startswith("%s."%self.srv.name)]
        items = [i for i in items if not i[0].startswith("env.")]
        items = [i for i in items if i[0] not in self.config.defaults().keys()]
        items = items_reorder(items)

        # Sort out action names and params
        for i in items:
            # Check that there only one action in required order
            action = None
            params = {}
            can_fail = False
            allowed_fail = 0
            for a in i.keys():
                if a == 'cmd' and len(i) == 1:
                    action = 'shell'
                    params['cmd'] = i[a]
                elif a == 'update_file' and len(i) == 1:
                    action = 'update_file'
                    params['file'] = i[a]
                elif a == 'create_file' and len(i) == 1:
                    action = 'create_file'
                    params['file'] = i[a]
                elif a == 'action':
                    action = i[a]
                elif a == 'fail':
                    can_fail = bool(i[a])
                elif a == 'allowed_fail':
                    allowed_fail = int(i[a])
                else:
                    params[a] = i[a]
            # Get implementation for action
            try:
                action_func = self.__getattribute__('do_%s'%action)
            except:
                self.srv.error("Action '%s' not implemented"%action)
                raise RuntimeError

            i.clear()
            i['params'] = params
            i['action'] = action
            i['func'] = action_func
            i['can_fail'] = can_fail
            i['allowed_fail'] = allowed_fail

        self.items = items


    def do_shell(self, params={}):
        """ Sub-action 'shell'
        Executes arbitary shell command

        Defined as:
        10.action = shell
        10.cmd = /some/command --param 1
        10.output = False   # Optional param

        Param 'output' tells not to hide output from command.
        With set to true command is run with '>stdout 2>&1'

        Additional optional params allowed:
        10.fail = False
        10.allowed_fail = 0

        Allowed short form:
        10.cmd = /some/command --param 1

        In short form command will be run with defaults:
        output=False, fail=False

        """
        cmd = params.get('cmd', None)
        output = bool(params.get('output', False))

        self.srv.info('Running shell(%s)'%cmd)

        if not cmd:
            raise RuntimeError("Shell command not specified!")

        env = self.srv.environment()
        env.update(self.env)
        try:
            stdout = None
            if not output:
                stdout = file('/dev/null','w')
            rc = call('(%s)'%cmd, shell=True, env=env, stdout=stdout, stderr=STDOUT)
            if rc < 0:
                self.srv.error("Child was terminated by signal(%d)", -rc)
            elif rc != 0:
                raise RuntimeError("Process exited with (%d)"%rc, rc)
        except OSError, e:
            self.srv.error("Execution failed:", e)


    def do_artifact(self, params={}):
        """ Sub-action 'artifact'
        Deploys artifact to given dir/file, unpacking it if needed

        Defined as:
        10.action = artifact
        10.file = /absolute/path/to/file
        10.destination_file = /destination/file # Could be optional, see below
        10.destination_dir = /destination/dir   # Could be optional, see below

        Optional params:
        10.extract = False
        10.skip_dirs = 0
        10.url = http://site/download

        Param 'file' - path to source file
        Param 'destination_file' - where to copy source file (with name change)
        Param 'destination_dir' - where to copy source file (preserving name)
          You must specify one of destination_dir/file parameters,
          if both specified - destination_dir will take precedence
        Param 'extract' - shows that artifact is archive and should be extracted
          When 'extract' specified - you must specify 'destination_dir' parameter.
          If specified 'destination_dir' does not exist - it will be created
          (as part of extraction process)
        Param 'skip_dirs' - skip given number of leading directories
          from file names before extraction
        Param 'url' - sets URL used to download artifact. Artifact downloaded
          to location set by 'file' parameter. If file exists at 'file' location
          nothing is done. To force download (file renew) you should pass '-u'
          command-line param to 'alpsetup'

        Additional optional params allowed:
        10.fail = False

        Allowed short form:
        None

        """
        artifact = params.get('file', None)
        url = params.get('url', None)
        destination_file = params.get('destination_file', None)
        destination_dir = params.get('destination_dir', None)
        extract = bool(params.get('extract', False))
        skip_dirs = int(params.get('skip_dirs', 0))

        if not url:
            self.srv.info('Running artifact(%s)'%artifact)
        else:
            self.srv.info('Running artifact(%s)'%url)

        if not artifact and not url:
            raise RuntimeError("No source file for artifact!")

        if url:
            update_file = self.config.getboolean('config', 'update')
            artifact = download_file(url, artifact, update=update_file)

        if extract:
            if not destination_dir:
                raise RuntimeError("'destination_dir' param empty!")
            unpack(artifact, destination_dir, skip=skip_dirs)
        else:
            if destination_dir:
                # Copy file inside dir
                copy(artifact, destination_dir)
            elif destination_file:
                # Copy to newfilename
                copy(artifact, destination_file)
            else:
                raise RuntimeError("'destination_file' or 'destination_dir' param empty!")


    def do_update_file(self, params={}):
        file_section = params.get('file', None)

        self.srv.info('Running update_file(%s)'%file_section)

        if not file_section:
            raise RuntimeError("'file' param empty!")

        f = UpdateFile(self.srv, file_section)
        f.execute()


    def do_create_file(self, params={}):
        file_section = params.get('file', None)

        self.srv.info('Running create_file(%s)'%file_section)

        if not file_section:
            raise RuntimeError("'file' param empty!")

        mgr = self.srv.mgr
        cfg = self.config

        # Load all defenitions using variables from all services
        items = cfg.items("file:%s"%file_section, raw=False,
                          vars=mgr.variables())
        # Filter out substitution variables (ConfigParser specific behavior)  
        for name in mgr.services():
            items = [i for i in items if not i[0].startswith("%s."%name)]
        items = [i for i in items if i[0] not in cfg.defaults().keys()]
        items = items_to_dict(items)

        filename = items.get('file', None)
        content = items.get('content', None)
        f = file(filename, 'w')
        f.write(content)
        f.close()

        # If there are other directives - threat them as update_file
        items.pop('file')
        items.pop('content')
        if len(items.keys()) > 0:
            f = UpdateFile(self.srv, file_section)
            f.execute()

    def do_wait_log_line(self, params={}):
        logfile = params.get('file', None)
        line = params.get('line', None)
        timeout = int(params.get('timeout', 600))

        self.srv.info('Running wait_log_line(%s)'%logfile)

        if not logfile:
            raise RuntimeError("'file' param empty!")

        if not line:
            raise RuntimeError("'line' param empty!")

        _re = re.compile(line)
        t1 = time.time()
        t2 = time.time()
        found = False
        file_timeout = 10
        while True:
            try:
                f = file(logfile, 'r')
            except IOError:
                pass
            else:
                break
            if t2 - t1 > file_timeout:
                raise RuntimeError("wait_log_line() file not acquired in %d seconds"%file_timeout)
        lines = f.readlines()
        self.srv.debug("Read lines so far:")
        self.srv.debug_pp(lines)
        while t2 - t1 < timeout:
            pos = f.tell()
            line = f.readline()
            self.srv.debug("[%d]: %s"%(pos,line.strip()))
            if not line:
                time.sleep(5)
                f.seek(pos)
            else:
                if _re.search(line):
                    found = True
                    break
            t2 = time.time()
        f.close()
        if not found:
            raise RuntimeError("wait_log_line() timeout!")

        self.srv.info("Condition success!")

    def do_wait_cmd_line(self, params={}):
        cmd = params.get('cmd', None)
        line = params.get('line', None)
        timeout = int(params.get('timeout', 600))

        self.srv.info('Running wait_cmd_line(%s)'%cmd)

        if not cmd:
            raise RuntimeError("'cmd' param empty!")

        if not line:
            raise RuntimeError("'line' param empty!")

        env = self.srv.environment()
        env.update(self.env)

        _re = re.compile(line)
        t1 = time.time()
        t2 = time.time()
        found = False
        while t2 - t1 < timeout:
            output = Popen(cmd, shell=True, env=env, stdout=PIPE, stderr=STDOUT).communicate()[0]
            self.srv.debug("Got output:")
            self.srv.debug_pp(output)
            if not output:
                time.sleep(1)
            else:
                for line in output.split('\n'):
                    if _re.search(line):
                        found = True
                if found:
                    break
            t2 = time.time()
        if not found:
            raise RuntimeError("wait_cmd_line() timeout!")

        self.srv.info("Condition success!")

    def execute(self):
        for i in self.items:
            try:
                rc = i['func'](i['params'])
                if i['can_fail']:
                    break
            except RuntimeError, e:
                if len(e.args) > 1:
                    rc = e.args[1]
                    if rc != i['allowed_fail']:
                        raise
                    break
                elif not i['can_fail']:
                    raise
                pass


class Service (object):

    def __init__(self, manager, name):
        self.mgr = manager
        self.config = manager.config
        self.name = name
        self._vars = {}   # Variables to be used in actions
        self._env = {}    # Environment for shell actions
        self._actions = []

        self.service_section = "service:%s"%self.name
        self.action_section = "action:%s"%self.name

        # Load all variables if exist
        if self.config.has_section(self.service_section):
            self.debug("loading variables from %s"%self.service_section)
            for var, value in self.config.items(self.service_section):
                if var.startswith('env.'):
                    # Put into environment list
                    self._env[var[4:].upper()] = value
                elif var not in self.config.defaults().keys():
                    self._vars["%s.%s"%(self.name,var)] = value
        self.debug('Variables:')
        self.debug_pp(self._vars)
        self.debug('Environment:')
        self.debug_pp(self._env)

        # Load available actions
        actions = [a for a in self.config.sections() \
                        if a.startswith(self.action_section)]
        self._actions = [a[len(self.action_section)+1:] for a in actions]
        self.debug('Actions:')
        self.debug_pp(self._actions)


    # Data access

    def variables(self):
        return self._vars

    def environment(self):
        return self._env


    # Action functions

    def do_action(self, name):
        # If we have no action - just return
        if name not in self._actions:
            self.warning("action '%s' not defined"%name)
            return

        action = Action(self, name)
        action.execute()


    # Logging functions

    def info(self, msg, *args, **kwargs):
        info("[%s]: %s"%(self.name, msg), *args, **kwargs)

    def warning(self, msg, *args, **kwargs):
        warning("[%s]: %s"%(self.name, msg), *args, **kwargs)

    def error(self, msg, *args, **kwargs):
        error("[%s]: %s"%(self.name, msg), *args, **kwargs)

    def debug(self, msg, *args, **kwargs):
        debug("[%s]: %s"%(self.name, msg), *args, **kwargs)

    def debug_pp(self, obj):
        import pprint
        msg = pprint.pformat(obj)
        for line in msg.split('\n'):
            self.debug(line)

    def __repr__(self):
        return unicode('<Service: %s>'%self.name)


class ServiceManager (object):

    def __init__(self, config):
        self._services = {}
        self.config = config
        self.service_bundle = {}
        self.action_bundle = {}

        # Check if we have main section
        if 'main' not in config.sections():
            error("No 'main' configuration section")
            raise RuntimeError
        # Load all declared services
        for sect in config.sections():
            if sect.startswith('service:'):
                s = sect[8:]
                self._services[s] = Service(self, s)
        # Load all service bundles
        for opt in [o for o in config.options('main') if o.startswith('services.')]:
            bundle = opt[9:]
            services = config.get('main', opt)
            services = services.split(',')
            services = [s.strip() for s in services]
            _bundle_services = []
            for s in services:
                if s not in self._services.keys():
                    raise RuntimeError("Service '%s' not defined!"%s)
                _bundle_services.append(s)
            self.service_bundle[bundle] = _bundle_services
            debug('Loaded service bundle [%s]: %s'%(bundle, ','.join(_bundle_services)))
        # Load all action bundles
        # action could be unimplemented - so we not checking its existense
        for opt in [o for o in config.options('main') if o.startswith('actions.')]:
            bundle = opt[8:]
            actions = config.get('main', opt)
            actions = actions.split(',')
            actions = [s.strip() for s in actions]
            self.action_bundle[bundle] = actions
            debug('Loaded action bundle [%s]: %s'%(bundle, ','.join(actions)))


    # Data access

    def variables(self):
        _vars = {}
        for s in self._services.keys():
            _vars.update(self._services[s].variables())

        return _vars

    def services(self):
        return self._services.keys()

    # Action

    def do_action(self, service, action):
        """ Expands services and actions, 
            and starts all given actions on all services
            Expansion is done only once with no recursion,
            so it should be safe to have same name of bundle and service:

              services.deploy = configure, deploy

        """
        _srv = [service]
        if service in self.service_bundle.keys():
            _srv = self.service_bundle[service]
        else:
            # Check if service exists
            if service not in self._services.keys():
                raise RuntimeError("Service '%s' not defined!"%s)

        _actions = [action]
        if action in self.action_bundle.keys():
            _actions = self.action_bundle[action]

        for a in _actions:
            # Check if this action should be applied in reverse order
            reverse = False
            if self.config.has_option('main', 'action.%s.reverse'%a):
                reverse = self.config.getboolean('main', 'action.%s.reverse'%a)
            # Reverse processing order if any
            services = _srv
            if reverse:
                services = reversed(_srv)
            # Process each service
            for s in services:
                self._services[s].do_action(a)

    # Logging

    def __repr__(self):
        srvs = ','.join(self._services.keys())
        return unicode('<ServiceManager: %s>'%srvs)


class OverrideParser(SafeConfigParser):
    def __init__(self, *args, **kwargs):
        SafeConfigParser.__init__(self, *args, **kwargs)

    def read(self, filenames):
        """
        Overrided method that provides section overriding
        rather than key/value addition if same sections
        are in different files
        """
        if isinstance(filenames, basestring):
            filenames = [filenames]
        read_ok = []
        sections = []
        for filename in filenames:
            try:
                fp = open(filename)
            except IOError:
                continue
            if hasattr(self, '_dict'):
                curr_sections = self._dict()
            else:
                curr_sections = {}
            self._sections = curr_sections
            self._read(fp, filename)
            fp.close()
            sections.append(self._sections)
            read_ok.append(filename)
        if len(sections) > 0:
            self._sections = sections[0]
            for sect in sections[1:]:
                for s in sect.keys():
                    if 'override' in sect[s].keys():
                        self._sections[s] = sect[s]
                    elif s in self._sections.keys():
                        self._sections[s].update(sect[s])
                    else:
                        self._sections[s] = sect[s]
        return read_ok


def main():
    # Check that we are not running as root
    uid = os.geteuid()
    if uid == 0:
        print("Cannot run as root!")
        sys.exit(1)
    # Get command line arguments
    try:
        opts, args = getopt(sys.argv[1:], 'hdu', ['update'])
    except GetoptError, e:
        usage()
        sys.exit(2)

    log_level = logging.INFO
    update_files = False
    for o, a in opts:
        if o == '-d':
            log_level = logging.DEBUG
        elif o in ('-u', '--update'):
            update_files = True
        elif o in ('-h',):
            usage()
            sys.exit(0)

    if len(args) < 2:
        usage()
        sys.exit(2)

    # Setup logging
    logging.basicConfig(level=log_level,
                        format='%(asctime)s %(levelname)s %(message)s',
                        stream=sys.stdout)
    # Look for config in homedir and script directory
    config_files = []
    config_files.append(os.path.join(sys.path[0], CONFIG_NAME))
    config_files.append(os.path.join(sys.path[0], "environment.%s.conf"%HOSTNAME))
    config_files.append(os.path.join("/etc/alp", CONFIG_NAME))
    config_files.append(os.path.join(sys.path[0], "environment.%s.%s.conf"%(HOSTNAME, USER)))
    config_files.append(os.path.join(HOME, CONFIG_NAME))

    config = OverrideParser(common_variables)
    read_files = config.read(config_files)
    if len(read_files) == 0:
        error("Can't find any configuration files: %s", str(config_files))
        sys.exit(1)

    info("Using configuration from: %s", str(read_files))

    command = args[0]
    target = args[1]

    # Update configs with options
    config.add_section('config')
    config.set('config', 'update', str(update_files))

    try:
        # Load known services
        services = ServiceManager(config)

        services.do_action(target, command)

    except RuntimeError, e:
        if str(e) != '':
            error(str(e))
        sys.exit(1)


if __name__ == '__main__':
    main()