#coding:utf-8
import re
import os
import sys
import pwd
import xml
import yaml
import time
import fcntl
import psutil
import sunhpc
import shutil
import socket
import syslog
import struct
import sqlite3
import termios
import argparse
import textwrap
import datetime
import subprocess
import prettytable
import configparser
import sunhpc.invoke
import sqlalchemy.engine.result

from struct  import pack
from collections import Counter
from xml.sax import saxutils
from xml.sax import handler
from xml.sax import make_parser
from xml.sax._exceptions import SAXParseException
from prettytable import DEFAULT,MSWORD_FRIENDLY,PLAIN_COLUMNS,RANDOM

LEFT_PADDING    = 8
RIGHT_PADDING   = 8
DEFAULT_HELP_WIDTH = 8

def get_help_width():
    try:
        data = fcntl.ioctl(sys.stdout, termios.TIOCGWINSZ, '1234')
        columns = int(struct.unpack('hh', data)[1])
    except (IOError, ValueError) as e:
        print ("terminal size detection failed, using default width.")
        return DEFAULT_HELP_WIDTH

    columns = columns - RIGHT_PADDING
    if columns > 0:
        width = columns
    else:
        width = DEFAULT_HELP_WIDTH

    return width

class DatabaseConnection(object):

    def __init__(self, db):
        self.database = db

    def search(self, command):
        return self.database.search(command)

    def execute(self, command):
        return self.database.execute(command)

    def fetchone(self):
        return self.database.fetchone()

    def fetchall(self):
        return self.database.fetchall()

    def getSession(self):
        """helper function to get the session"""
        return self.database.getSession()

    def checkHostnameValidity(self, hostname):
        return self.database.checkHostnameValidity(hostname)

    def getHostname(self, hostname=None):
        return self.database.getHostname(hostname)

    def getHostIp(self, hostname=None):
        return self.database.getHostIp(hostname)

    def getHostAttr(self, host, key):
        return self.getHostAttrs(host).get(key)

    def getHostAttrs(self, host):
        hostname = self.getHostname(host)
        return self.database.getHostAttrs(hostname)

    def getFrontendName(self):
        return self.database.getFrontendName()

    def setNewHostAttr(self, node, attr, value):
        return self.database.setNewHostAttr(node, attr, value)

    def commit(self):
        return self.database.commit()

    @property
    def getDBFile(self):
        return self.database.getDBFile()

    @property
    def getEngine(self):
        return self.database.getEngine()

class HostArgumentProcessor(object):

    def getHostnames(self, names=None, managed_only=0):

        list = []
        if not names:
            query = 'select name from nodes'
            self.db.execute(query)
            for host, in self.db.fetchall():
                list.append(host)

            if managed_only:
                managed_list = []
                for hostname in list:
                    if self.db.getHostAttr(hostname, 
                        'managed') == 'true':
                        managed_list.append(hostname)
                return managed_list
            return list

        groups = {}
        self.db.execute('select min(rack), max(rack) from nodes')
        min,max = self.db.fetchone()
        for i in range(min, max+1): # racks
            self.db.execute("""select n.name from nodes n where n.rack=%d""" % i)
            l = []
            for node, in self.db.fetchall():
                l.append(node)
            groups['rack%d' % i] = l

        dict = {}
        for name in names:
            if name.find('select') == 0:    # SQL select
                self.db.execute(name)
                for host, in self.db.fetchall():
                    dict[host] = 1
            elif name.find('%') >= 0:   # SQL % pattern
                self.db.execute("""select name from nodes where
                    name like '%s'""" % name)
                for h, in self.db.fetchall():
                    dict[h] = 1
            elif name in groups:  # group name
                for host in groups[name]:
                    dict[host] = 1
            else:               # host name
                dict[self.db.getHostname(name)] = 1

        list = sorted(dict.keys())
        return list

class NetworkFunc(object):

    def netmask_to_cidr(self, netmask):
        return sum([bin(int(i)).count('1') for i in str(netmask).split('.')])

    def cidr_to_netmask(self, cidr):
        return socket.inet_ntoa(pack('>I', 0xffffffff ^ (1 << 32 - int(cidr)) - 1))

    def getNetwork(self, addr, mask):
        address = sunhpc.core.ip.IPAddr(addr)
        netmask = sunhpc.core.ip.IPAddr(mask)
        return sunhpc.core.ip.IPAddr(address & netmask)
        
class FirewallConnection(object):

    def __init__(self, cmd):
        self.cmd = cmd

    def addports(self, ports=[], proto=[]):
        """prots=['80', '10-20'], proto=['tcp', 'udp']"""

        if not self.isRunning('firewalld', active='is-active'):
            return False

        if not ports: return False
        pr = ' --permanent >/dev/null 1>&2'
        fw = '/usr/bin/firewall-cmd --add-port='

        pplist = []
        if proto:
            for c in proto:
                for p in ports:
                    pplist.append('%s/%s' % (p, c))
        else:
            for p in ports:
                pplist.append('%s/tcp' % p)

        cmds = []
        for pp in pplist:
            if not self.query('--query-port=', pp):
                cmds.append(fw + pp + pr)

        for cmd in cmds:
            ret = subprocess.run(cmd, shell=True,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE, encoding="utf-8", timeout=3)

        self.reload()

        for pp in pplist:
            if self.query('--query-port=', pp):
                self.cmd.msg('Adding port \t%s successfull.' % pp)
            else:
                self.cmd.msg('Adding port \t%s Failed.' % pp, 'w')

    def addservice(self, name):
        """firewall-cmd --add-service=tftp --permanent"""
        if not name: return
        pr = ' --permanent >/dev/null 1>&2'
        fw = '/usr/bin/firewall-cmd --add-service='

        srvlist = []
        if isinstance(name, type([])):
            srvlist.extend(name)
        else:
            srvlist.append(name)

        cmds = []
        for s in srvlist:
            if not self.query('--query-service=', s):
                cmds.append(fw + s + pr)

        for cmd in cmds:
            ret = subprocess.run(cmd, shell=True,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE, encoding="utf-8", timeout=3)

        self.reload()
        for s in srvlist:
            if self.query('--query-service=', s):
                self.cmd.msg('Adding service \t%s successfull.' % s)
            else:
                self.cmd.msg('Adding service \t%s Failed.'   % s, 'w')
        
    def reload(self):
        self.cmd.shcmd('/usr/bin/firewall-cmd --reload')

    def isRunning(self, name, active='status'):
        cmd = '/usr/bin/systemctl %s %s >/dev/null 1>&2' % (active, name)
        ret = subprocess.run(cmd, shell=True,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE, encoding="utf-8", timeout=3)

        # shell success code 0
        if ret.returncode:
            return False
        return True

    def query(self, active, name):
        cmd = '/usr/bin/firewall-cmd %s%s >/dev/null 1>&2' % (active, name)
        ret = subprocess.run(cmd, shell=True,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE, encoding="utf-8", timeout=3)

        # shell success code 0
        if ret.returncode:
            return False
        return True

class KickstartProcess(object):
        
    def __init__(self, command):
        self.command        = command
        self.ks             = {}
        self.ks['main']     = []
        self.ks['pre']      = []
        self.ks['post']     = []
        self.ks['packages'] = []
        self.ks['addons']   = []
        self.ks['anaconda'] = []

        self.services_list  = []
        self.services_list  = []

        self.ksprelog       = '/mnt/sysimage/root/ks-sunhpc-pre.log'
        self.kspostlog      = '/root/ks-sunhpc-post.log'

        self.isFrontend     = False

    def getKickstart(self, phases):
        return self.ks[phases]

    def addMain(self, txt):
        if isinstance(txt, type([])):
            self.ks['main'].append(txt)
        else:
            self.ks['main'].append([txt])

    def addPre(self, txt, i='', arg='', log=None):
        """arg: --nochroot"""
        
        if i in ['sh', 'bash', 'python']:
            interpreter = '%%pre --interpreter=/usr/bin/%s' % i
        else:
            interpreter = '%pre'

        if log:
            log = '--log=%s' % log
        else:
            log = '--log=%s' % self.ksprelog
            
        list = []
        if self.isFrontend:
            list.extend(self.getContent(txt))
        else:
            list.append(' '.join([interpreter, arg, log]))
            list.extend(self.getContent(txt))
            list.append('%end\n')
        self.ks['pre'].append(list)

    def addPost(self, txt, i='', arg='', log=None):
        """arg: --nochroot"""
        
        if i in ['sh', 'bash', 'python']:
            interpreter = '%%post --interpreter=/usr/bin/%s' % i
        else:
            interpreter = '%post'

        if log:
            log = '--log=%s' % log
        else:
            log = '--log=%s' % self.kspostlog
            
        list = []
        if self.isFrontend:
            list.extend(self.getContent(txt))
        else:
            list.append(' '.join([interpreter, arg, log]))
            list.extend(self.getContent(txt))
            list.append('%end\n')
        self.ks['post'].append(list)

    def addPackages(self, txt, arg=''):

        if self.isFrontend:
            self.ks['packages'].append(self.getContent(txt))
        else:
            self.ks['packages'].append(['%%packages %s' % arg])
            self.ks['packages'].append(self.getContent(txt))
            self.ks['packages'].append(['%end\n'])

    def addAddons(self, txt, arg=''):

        if not self.isFrontend:
            self.ks['addons'].append(self.getContent(txt))
            self.ks['addons'].append(['%end\n'])

    def addAnaconda(self, txt, arg=''):

        if not self.isFrontend:
            self.ks['anaconda'].append(['%%anaconda %s' % arg])
            self.ks['anaconda'].append(self.getContent(txt))
            self.ks['anaconda'].append(['%end\n'])

    def getContent(self, txt):
        content = []
        if isinstance(txt, type([])) or \
                    isinstance(txt, type(())):
            content.extend(txt)
        else:
            content.append(txt)
        return content

    def makefile(self, text='', name='', mode='',
                    owner='', perms='', expr='', quot=''):
        """
        text:   list content
        name:   file path
        mode:   append or pipe key 
        owner:  root.apache
        perms:  755
        src:    relation mode args. cat src to text.
        expr:   exec command > or >> filename
        """

        if isinstance(text, type([])) or \
                isinstance(text, type(())):
            fileText = ''.join(text)
        else:
            fileText = textwrap.dedent(text)

        fileName    = name
        fileMode    = mode
        fileOwner   = owner
        filePerms   = perms
        fileCommand = expr
        fileQuoting = quot

        s = ''
        if fileName:
            # split path
            paths, fname = os.path.split(fileName)
            if paths:
                mkdirs = "[ ! -e %s ] && mkdir -p %s\n" % (paths, paths)
            else:
                mkdirs = ""

            if fileMode == 'append':
                gt = '>>'
            else:
                gt = '>'

            if fileCommand:
                s += "%s %s %s\n" % (fileCommand, gt, fileName)
            else:
                if not fileText:
                    s += 'touch %s\n' % fileName
                else:
                    if fileQuoting == 'expand':
                        eof = "EOF"
                    else:
                        eof = "'EOF'"

                    s += mkdirs
                    s += "cat %s %s << %s" % (gt, fileName, eof)
                    if fileText[0] != '\n':
                        s += '\n'
                    s += fileText
                    if fileText[-1] != '\n':
                        s += '\n'
                    s += "EOF\n"

            if filePerms:
                s += 'chmod %s %s\n' % (filePerms, fileName)

            if fileOwner:
                s += 'chown %s %s\n' % (fileOwner, fileName)
        return s

class ModulesArgument(object):
    """
    Subclass of ArgumentParser that also examines boot arguments.
        formatter_class=argparse.RawTextHelpFormatter,
        formatter_class=lambda prog: argparse.ArgumentDefaultsHelpFormatter(
                prog, max_help_position=LEFT_PADDING, width=get_help_width()),
    """

    def __init__(self, *args, **kwargs):
        self.parser     = None
        self.subparser  = None

        self.autoGenerate(*args, **kwargs)

    def autoGenerate(self, *args, **kwargs):
        self.parser = argparse.ArgumentParser(
            epilog='Sunhpc online help: <https://www.sunhpc.com/software/coreutils/>',
            formatter_class=lambda prog: argparse.ArgumentDefaultsHelpFormatter(
                prog, max_help_position=LEFT_PADDING, width=get_help_width()),
            *args, **kwargs
        )

        # version
        self.parser.add_argument('-v', '--version', action='version', version='Sunhpc ' + sunhpc.version)

    def handler(self):
        return self.parser

    def add_mutux(self):
        """
        互斥参数
        mp = self.ap.add_mutux()
        mp.add_argument('--aaa', action='store_true')
        mp.add_argument('--bbb', action='store_true')
        mp.add_argument('--ccc', action='store_true')
        同时使用aaa,bbb,ccc只能使用其中一个,否则会报错.
        """
        return self.parser.add_mutually_exclusive_group()

    def add_group(self, gname):
        """
        自定义参数组
        op = self.ap.add_group('Configure')
        op.add_argument('-c', dest='config', metavar='config',
                default='/opt/sunhpc/etc/sunhpc.conf',
                help='use config file'
                )
        """
        return self.parser.add_argument_group('%s' % gname)

    def add_sub(self, command, helps=''):
        """
        嵌套解析器
        sub = self.ap.add_sub('create', 'Create a directory')
        sub.add_argument('--dirname', action='store', help='New directory to create')

        sub = self.ap.add_sub('delete', 'Remove a directory')
        sub.add_argument('--dirname', action='store', help='The directory to remove')
        """
        if not self.subparser:
            self.subparser = self.parser.add_subparsers()
        cmd = self.subparser.add_parser('%s' % command, help=helps)
        return cmd


class Command(object):
    """Base class for all Sunhpc commands the general command line form
    is as follows:

        sunhpc ACTION COMPONENT OBJECT [ <ARGNAME ARGS> ... ]
        
        ACTION(s):
            add
            create
            list
            load
            sync
    """
    MustBeRoot = 1

    def __init__(self, database):

        self.db         = DatabaseConnection(database)
        self.fw         = FirewallConnection(self)
        self.ks         = KickstartProcess(self)
        self.ap         = ModulesArgument()
        self.newdb      = self.db.database

        self.os         = os.uname()[0].lower()
        self.arch       = os.uname()[4]
        self.net        = NetworkFunc()
        self.text       = ''
        self.output     = []
        self.fmtOutput  = []

        self._args      = None
        self._params    = None

        self.major      = sunhpc.version_major
        self.minor      = sunhpc.version_minor
        self.micro      = sunhpc.version_micro

        self.prefix     = os.environ.get('SUNHPC_HOME') if os.environ.get('SUNHPC_HOME') else '/opt/sunhpc'
        self._debug     = True if os.environ.get('SUNHPC_DEBUG') else False

        self.modules    = sunhpc.core.utils.index_modules()
        self.modules_count = Counter()
        self.modules_count.update([module.split('.')[0] for module in self.modules])

    @property
    def debug(self):
        return self._debug
    def setdebug(self, status=None):
        self._debug = True if status else False

    def getVersions(self, key='', suffix='', dirs=''):
        """Gets the version number in all of the file names."""
        versions = []
        for name in os.listdir(dirs):
            #pattern = r'\d+\.\d*'
            pattern = r'\d+\.(?:\d+\.)*\d+'
            numbers = re.findall(pattern, name)
            if name.lower().find(key) != -1 and name.endswith(suffix):
                versions.append('.'.join(numbers))
        return versions

    def getNetcard(self):
        netcard_info = {}
        info = psutil.net_if_addrs()
        for k,v in info.items():
            for item in v:
                if item[0] == 2 and not item[1] == '127.0.0.1':
                    netcard_info[k] = item.address

        return netcard_info

    def writeConfig(self, filename, content):
        text = ''
        if type(content) in [type([]), type(())]:
            for c in content:
                text += c
        else:
            text += content

        with open(filename, 'w') as f:
            f.write(text)

    def create_user(self, username, passwd='', uid=None, gid=None, home=None, nohome=False, shell='/bin/bash'):

        with open('/etc/passwd', 'r') as fd:
            for line in fd:
                matchusername = re.search(r'%s' % username, line, re.I)
        if not matchusername:
            # useradd -M -s /sbin/nologin apache
            cmd = 'useradd '
            if nohome: cmd += '-M '
            if uid:    cmd += '-u %d ' % uid
            if gid:
                self.shcmd('groupadd -g %d %s' % (gid, username))
                cmd += '-g %d ' % gid
            if home:   cmd += '-d %s ' % home
            if shell:  cmd += '-s %s ' % shell
            if username:
                cmd += '%s ' % username
            if passwd:
                cmd += '%s' % passwd

            os.system(cmd)
            self.msg('"%s" is create successfully.' % username)
        else:
            self.msg('"%s" is already exists.' % username, 'w')

    def msg(self, msg, level='', r=None, q=False):
        """ .e,g.. self.msg(messages, 'e') """

        if q: return
        r = str(r)

        if level in ['e', 'err', 'error']:
            if not r:
                print("\033[91m[errs] %s\033[0m" % (msg))
            else:
                r = "\033[92m%s\033[0m\033[31m" % r
                print("\033[31m[errs] %s \033[0m" % msg.replace('*', r, 1))

        elif level in ['w', 'war', 'warn', 'warning']:
            if not r:
                print("\033[95m[warn] %s\033[0m" % (msg))
            else:
                r = "\033[92m%s\033[0m\033[95m" % r
                print("\033[95m[warn] %s \033[0m" % msg.replace('*', r, 1))

        elif level in ['i', 'inf', 'info', 'infomation']:
            if not r:
                print("\033[92m[info] %s\033[0m" % (msg))
            else:
                r = "\033[92m%s\033[0m\033[93m" % r
                print("\033[93m[info] %s \033[0m" % msg.replace('*', r, 1))

        elif level in ['o', 'ok']:
            if not r:
                print("\033[92m[+]\033[0m \033[96m%s\033[0m" % (msg))
            else:
                r = "\033[96m%s\033[0m\033[92m" % r
                print("\033[92m[+] %s \033[0m" % msg.replace('*', r, 1))

        elif level in ['a', 'abrt', 'abort']:
            if not r:
                #print("\033[31m[abrt] %s\033[0m" % (msg))
                msg = "\033[31m[abrt] %s\033[0m" % (msg)
            else:
                r = "\033[93m%s\033[0m\033[31m" % r
                msg = "\033[31m[abrt] %s \033[0m" % msg.replace('*', r, 1)
            self.abort(msg)
        else:
            #print("\033[92m[+]\033[0m \033[96m%s\033[0m" % (msg))
            print("\033[92m[+]\033[0m %s" % (msg))

    def message(self, msg, tag='[+]', end=''):
        header = '\033[92m%s\033[0m' % tag
        ender  = '\033[95m%s\033[0m' % end
        msger  = '\033[31m%s\033[0m' % msg
        result = '%s %s %s' % (header, msger, ender)
        return result


    def abort(self, msg):
        syslog.syslog(syslog.LOG_ERR, msg.split('\n')[0])
        raise sunhpc.core.utils.CommandError(msg)

    def matchText(self, name, txt):
        """name is file path, txt: match string"""
        # 如果文件不存在，直接返回真，可以直接写入这个文件.
        if not os.path.exists(name):
            return 1

        # 如果找到 txt 在文件中返回 False,不在附加到文件.
        cmd = 'grep -q "%s" %s' % (txt, name)
        ret = subprocess.run(cmd, shell=True)
        if ret.returncode:
            return 1
        return 0

    def isRootUser(self):
        return True if os.geteuid() == 0 else False

    def isApacheUser(self):
        try:
            if os.geteuid() == pwd.getpwnam('apache')[3]:
                return 1
        except:
            pass
        return 0

    def str2bool(self, s):
        if s and s.upper() in [ 'ON', 'YES', 'Y', 'TRUE', '1' ]:
            return 1
        return 0

    def bool2str(self, b):
        if b:
            return 'yes'
        return 'no'

    def sbin(self, name, path="sbin"):
        """Return sunhpc bin/sbin command path"""
        return os.path.join(self.prefix, path, name)

    def netup(self, addr, port=80):
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.settimeout(5.0)
        try:
            sock.connect((addr, port))
            sock.close()
        except:
            return 0
        return 1

    def confinfo(self, filename, keys):

        status = True
        if not os.path.exists(filename):
            with open(filename, 'w') as f:
                f.write('#\n# Generated by sunhpc report yumrepos commands.\n#\n\n')

        with open(filename, 'r') as f:
            config = yaml.safe_load(f)

        if config is None:
            config = {
                keys: {'date': datetime.datetime.now()}
            }
            status = False

        elif isinstance(config, dict):
            if keys not in config:
                config[keys] = {}
                config[keys]['date'] = datetime.datetime.now()
                status = False

        with open(filename, 'w') as f:
            yaml.dump(config, f, sort_keys=True, indent=4)

        return status

    def tranpath(self, filename):

        if '/' in filename:
            # 转换成需要的格式文件名.
            fullpath = os.path.join(os.sep, filename)
            dotPath  = '..'.join(fullpath.split('.'))
            dotPath  = '.'.join(dotPath.split('/'))[1:]
        else:
            # 反转成正常路径文件名.
            dotFullPath = '/'.join(filename.split('.'))
            dotPath = '.'.join(dotFullPath.split('//'))
            dotPath = os.path.join(os.sep, dotPath)
        return dotPath

    def replace(self, files, srckey, dstkey):
        """替换指定文件中的关键字"""
        if not os.path.exists(files):
            self.abort('The %s no exists.' % files)

        line = []
        with open(files, 'r') as f:
            line.extend(f.readlines())

        newline = []
        for i in line:
            if srckey in i:
                rep = i.replace(srckey, dstkey)
                newline.append(rep)
            else:
                newline.append(i)

        return newline

    def shrun(self, cmd):
        return subprocess.run(cmd, shell=True, check=True)

    def shcmd(self, cmd, ret='str', flag=None, code=False, env=None, cwd=None, preexec_fn=None):
        info = {}
        cmd  = ' '.join(cmd.split())
        p = subprocess.Popen(cmd, shell=True, bufsize = 0,
                cwd=cwd, env=env, preexec_fn=preexec_fn,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE)

        if ret == 'str':
            info['o'] = p.stdout.read().decode('UTF-8')
            info['e'] = p.stderr.read().decode('UTF-8')
        if ret == 'list':
            info['o'] = [ l.decode('UTF-8') for l in p.stdout.readlines() ]
            info['e'] = [ l.decode('UTF-8') for l in p.stderr.readlines() ]
        if code:
            #info['code'] = p.poll()

            # 使用p.wait() 容易造成死锁. ulimit -a 查看pipe size 缓存大小
            # 如果pipe size 太小,输出超出了界限,p.wait会一直等待.
            # info['c'] = p.wait()

            # 所以我们使用p.communicate() 函数替代, 此函数是将输出直接放入内存
            # 所以这样基本不会出问题了.
            # info['o'], info['e'] = p.communicate() # now wait
            p.communicate() # now wait

            # None, 表示没有执行结束
            # -N,   表示进程被N号信号终止.
            info['c'] = p.returncode # get exe return value
        return info

    def copyfiles(self, files, dstdirs):

        if not files:
            self.msg('supply files list is empty.', 'e')
            return

        if not os.path.exists(dstdirs):
            os.makedirs(dstdirs)

        sys.stdout.write("\33[?25l")
        for i in range(len(files)):
            self.shcmd("/usr/bin/cp --parents %s %s" % (files[i], dstdirs))
            self.precent(i+1, len(files), files[i])
        sys.stdout.write("\33[?25h")
        print ('', end='\n')

    def precent(self, step, total, name=''):

        rate = step / total
        percent = int(rate * 100) 
        txt  = "\033[92m[+]\033[0m\033[95m Running " 
        txt += "%02d%% [%s/%s] finished.\033[0m\r" % (percent, step, total)
        print (txt, end="")

    def systemctl(self, name, active=None):

        cmd = '/usr/bin/systemctl is-active %s' % name
        ret = subprocess.run(cmd, shell=True, 
                stdout=subprocess.PIPE, 
                stderr=subprocess.PIPE, encoding="utf-8", timeout=3)

        # shell success code 0
        if ret.returncode:
            return False
        return True

    def system(self, cmd, std=1):

        sys.stdout.write("\33[?25l")
        if std:
            subprocess.call(cmd, shell=True)
        else:
            p = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE,
                    stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                        close_fds=True)

            w, r ,e = (p.stdin, p.stdout, p.stderr)
            currLength  = 0
            prevLength  = 0
            spinChars   = '-\|/'
            spinIndex   = 0
            while 1:
                line = e.readline().decode()
                if not line:
                    break
                if len(line) > 79:
                    data = line[0:78]
                else:
                    data = line[:-1]

                currLength = len(data)
                pad = ''
                # 清理输出数据后面的字符无法消除
                for i in range(0, prevLength - currLength):
                    pad = pad + ' '
                spin  = spinChars[spinIndex % len(spinChars)]
                spinIndex = spinIndex + 1
                print (spin + data + pad + '\r', end='')
                prevLength = currLength
                sys.stdout.flush()
            r.close()
            w.close()
            e.close()

            pad = ''
            for i in range(0,78):
                pad = pad + ' '
            print ('\r%s\r' % pad,)
        sys.stdout.write("\33[?25h")

    def loadModules(self, name, args=''):
        """Parser Kickstart modules sunhpc.modules.*"""

        module_list = []
        for mod in self.modules:
            try:
                mod_name = mod.split('.')[0]
            except KeyError:
                mod_name = None

            if name != mod_name: continue

            module = 'sunhpc.modules.%s' % mod
            module = sunhpc.core.utils.import_centrals(module)(self)
            module_list.append(module)

        if not module_list: return

        text = []
        for mod in module_list:
            modname =  mod.__module__.split('.')
            if modname[2] == 'kickstart':
                text.append('# Include kickstart module %s' % modname[-1])
            else:
                text.append('# Include kickstart module %s' % modname[-1])

            txt = mod.run(args)
            if not txt: continue

            if isinstance(txt, type([])):
                text.extend(txt)
            elif isinstance(txt, type(())):
                for e in txt:
                    text.append(e)
            else:
                text.append(txt)

        return text

    def loadPlugins(self):

        plugins = []
        #
        # 每个命令下面都可以添加plugin文件,所以这里的模块
        # 路径需要根据每个命令,自行获取.
        # 例如: /commands/sync/users 命令等.
        dirs = eval('%s.__path__[0]' % self.__module__)
        for fe in os.listdir(dirs):
            # 所有命令目录下面的扩展模块名称必须以,
            # "plugin_00_*.py" 开头方能够被识别.
            if fe.split('_')[0] != 'plugin':
                continue
            if os.path.splitext(fe)[1] != '.py':
                continue

            # 获取命令目录下面的文件模块的名称然后进行连接.
            # e.g., sunhpc.commands.sync.users.plugin_fixusers
            module = '%s.%s' % (self.__module__, os.path.splitext(fe)[0])

            # 导入模块
            __import__(module)
            module = eval(module)
            try:
                # 调用模块中Plugin类.
                o = getattr(module, 'Plugin')(self)
            except AttributeError:
                continue

            plugins.append(o)

        return plugins

    def runPlugins(self, args='', plugins=None):
        if not plugins:
            plugins = self.loadPlugins()

        if not plugins: 
            return

        for plugin in plugins:
            syslog.syslog(syslog.LOG_INFO, 'Run %s' % plugin)
            plugin.run(args)

    def clearText(self):
        self.text = ''
    def addText(self, s):
        if s: self.text += s
    def getText(self):
        return self.text

    def dictOutput(self, strDict, sep='\t'):
        maxlen = max(map(len, strDict.keys()))
        for k in strDict:
            strings = str(strDict[k])
            if strings.strip().lower() in ['ok', 'on', 'yes']:
                value = '|\033[1;32m %s \033[0m' % strings.upper()
            elif strings.strip().lower() in ['fail', 'off', 'no', 'error', 'failed']:
                value = '|\033[1;31m %s \033[0m' % strings.upper()
            else:
                value = '| %s' % strings
            print ('\t\033[1;95m%s\033[0m %s \033[1;96m%s\033[0m' % (k.ljust(maxlen), sep, value))

    def beginFmtOutput(self, header=[]):
        self.fmtOutput = prettytable.PrettyTable(header)
    def addFmtOutput(self, list_string):
        """use prettytable fotmatting output buffer."""

        # 如果不是列表,转换成列表进行添加.
        if not isinstance(list_string, type([])):
            list_string = [ list_string ]
        if isinstance(list_string, type(())):
            list_string = list(list_string)

        # 如果是[[]]列表,使用rows一次添加多行.否则使用单行添加row.
        if list_string and isinstance(list_string[0], type([])):
            self.fmtOutput.add_rows(list_string)
        else:
            self.fmtOutput.add_row(list_string)
    def endFmtOutput(self, name='', sort='', rever=False, style='def', align='l'):
        """
        :types name: string
        :param name: 需要匹配对其的名称.

        :types sort: string
        :param sort: 需要匹配排序的名称.

        :types rever: bool
        :param rever: 反向排序.

        :types style: string
        :param style: 设置边框样式.

        :types align: string
        :param align: 对其方式, l左对齐,r右对齐,c居中.
        """
        style_id = {'def':DEFAULT, 'ms':MSWORD_FRIENDLY, 'plain': PLAIN_COLUMNS, 'random': RANDOM}
        if style in style_id:
            self.fmtOutput.set_style(style_id[style])

        if align in ['l', 'r', 'c']:
            names = name.split(',')
            if names[0] != '':
                for n in names:
                    self.fmtOutput.align[n] = align
            else:
                self.fmtOutput.align = align

        if sort != '':
            print (self.fmtOutput.get_string(sortby=sort, reversesort=rever))
        else:
            print (self.fmtOutput)

    def beginOutput(self):
        self.output = []

    def addOutput(self, owner, vals):

        plist = [ '%s:' % owner ]
        if isinstance(vals, type([])):
            plist.extend(vals)
        elif isinstance(vals, type(())) or \
                isinstance(vals, sqlalchemy.engine.result.Row):
            for e in vals:
                plist.append(e)
        else:
            plist.append(vals)
        self.output.append(plist)

    def endOutput(self, header=[], padChar='-', trimOwner=1,linesep='\n'):

        if not self.output: return

        showHeader = True
        if 'output-header' in self._params:
            showHeader = self.str2bool(self._params['output-header'])
        
        self.outputCols = []
        if 'output-col' in self._params:
            showCols = self._params['output-col'].split(',')
            for i in header:
                if i.lower() in showCols: 
                    self.outputCols.append(True)
                else:
                    self.outputCols.append(False)

        if trimOwner:
            owner = ''
            self.startOfLine = 1
            for line in self.output:
                if not owner:
                    owner = line[0]
                if not owner == line[0]:
                    self.startOfLine = 0
        else:
            self.startOfLine = 0

        if header and showHeader:
            plist = []
            for field in header:
                plist.append(field.upper())
            output = [ plist ]
            output.extend(self.output)
        else:
            output = self.output
            
        colwidth = []
        for line in output:
            for i in range(0, len(line)):
                if len(colwidth) <= i:
                    colwidth.append(0)
                if type(line[i]) != type(str):
                    if line[i] == None:
                        itemlen = 0
                    else:
                        itemlen = len(repr(line[i]))
                else:
                    itemlen = len(line[i])

                if itemlen > colwidth[i]:
                    colwidth[i] = itemlen
                
        o = ''
        for line in output:
            plist = []
            for i in range(self.startOfLine, len(line)):
                if line[i] == None:
                    s = ''
                else:
                    s = str(line[i])
                if padChar != '':
                    if s:
                        o = s.ljust(colwidth[i])
                    else:
                        o = ''.ljust(colwidth[i], padChar)
                else:
                    o = s
                plist.append(o)

            self.addText('%s%s' % (self.outputRow(plist),linesep))

    def outputRow(self, plist):
        if self.outputCols:
            l = []
            for i in range(0, len(plist)):
                if self.outputCols[i + self.startOfLine]:
                    l.append(plist[i])
            return ' '.join(l)
        else:
            return ' '.join(plist)

    def usage(self):
        if self.__doc__:
            handler = DocStringHandler()
            parser = make_parser()
            parser.setContentHandler(handler)
            try:
                parser.feed('<docstring>%s</docstring>' % self.__doc__)
            except:
                return '-- invalid doc string --'
            return handler.getUsageText()
        else:
            return '-- missing doc string --'

    def help(self, command, flags={}):

        if not self.__doc__: return

        if self.MustBeRoot:
            users = [ 'root', 'apache' ]
        else:
            users = []
            
        if 'format' in flags:
            formats = flags['format'].lower()
        else:
            formats = 'plain'
        
        if formats == 'raw':
            i = 1
            for line in self.__doc__.split('\n'):
                self.addText('%d:%s\n' % (i, line))
                i += 1
        else:
            handler = DocStringHandler(command, users)
            parser  = make_parser()
            parser.setContentHandler(handler)
            parser.feed('<docstring>%s</docstring>' % self.__doc__)
            if formats == 'docbook':
                self.addText(handler.getDocbookText())
            elif formats == 'parsed':
                self.addText(handler.getParsedText())
            elif formats == 'sphinx':
                self.addText(handler.getSphinxText())
            else:
                self.addText(handler.getPlainText())

    def fillPositionalArgs(self, names, params=None, args=None):

        if not type(names) in [ type([]), type(()) ]:
            names = [ names ]

        if not params:
            params = self._params
        if not args:
            args = self._args

        plist = []
        for name in names:
            if name in params:
                plist.append(params[name])
            else:
                plist.append(None)

        variate = []
        trimmed = args
        plist.reverse()
        for e in plist:
            if not e and len(trimmed):
                variate.append(trimmed[-1])
                trimmed = trimmed[:-1]
            else:
                variate.append(e)
        variate.reverse()

        rlist = []
        rlist.append(trimmed)
        rlist.extend(variate)
        return rlist

    def fillParams(self, names, params=None):

        if not type(names) in [ type([]), type(()) ]:
            names = [ names ]

        pdlist = []
        for e in names:
            if type(e) in [ type([]), type(()) ] and len(e) == 2:
                tuples = ( e[0], e[1] )
            else:
                tuples = ( e[0], None )
            pdlist.append(tuples)

        if not params:
            params = self._params

        plist = []
        for (key, default) in pdlist:
            if key in params:
                plist.append(params[key])
            else:
                plist.append(default)
        return plist

    def command(self, command, args=[]):

        modpath = 'sunhpc.commands.%s' % command
        __import__(modpath)
        mod = eval(modpath)

        try:
            o = getattr(mod, 'Command')(self.db)
            n = ' '.join(command.split('.'))
        except AttributeError:
            return ''

        o.runWrapper(n, args)
        return o.getText()

    def runWrapper(self, name, args):

        username = pwd.getpwuid(os.geteuid())[0]
        if args:
            command = '%s %s' % (name, ' '.join(args))
        else:
            command = name

        syslog.syslog(syslog.LOG_INFO,
            'user %s called "%s"' % (username, command))
            
        pdict   = {}
        plist   = []
        nparams = 0
        flagpattern=re.compile("^[a-zA-z0-9\-_+]+=")
        for arg in args:
            tokens = arg.split()
            if tokens[0] == 'select':
                plist.append(arg)
            elif flagpattern.match(arg):
                (key, val) = arg.split('=', 1)
                pdict[key] = val
                if nparams == 0:
                    pdict['@SUNHPCPARAM0'] = arg
                nparams += 1
            else:
                plist.append(arg)

        if plist and plist[0] == 'help':
            self.help(name, pdict)
        else:
            if self.MustBeRoot and not \
                (self.isRootUser() or self.isApacheUser()):
                self.abort('command "%s" requires root' % name)
            else:
                self._args   = plist
                self._params = pdict
                try:
                    self.run(self._params, self._args)
                except sunhpc.core.utils.HostnotfoundException as e:
                    if self.debug:
                        traceback.print_exc()
                    self.abort(str(e))
                except sqlite3.OperationalError as e:
                    if self.debug:
                        traceback.print_exc()
                    self.abort("Dabase error: " + str(e))

    def getFunction(self, name):
        import inspect
        path = inspect.getfile(name)
        self.msg(path)

    def makeEULA(self, dst):
        s  = 'SunHPC Linux %s EULA\n\n' % sunhpc.version_major
        s += 'This version was created using SunHPC %s\n\n' % sunhpc.version
        s += 'The Distribution is released as CentOS. Individual packages in the\n'
        s += 'distribution come with their own licences.\n'
        with open(dst, 'w') as f:
            f.write(s)

    def makeBuildTag(self, dst):
        timefmt = time.strftime('%Y%m%d-%H%M', time.localtime())
        with open(dst, 'w') as f:
            f.write('%s\n' % timefmt)

    def makeDiscInfo(self, dst):
        with open(dst, 'w') as f:
            f.write('%s\n' % time.time())
            f.write('%s\n' % sunhpc.version)
            f.write('%s\n' % self.arch)

    def makeTreeInfo(self, dst):
        config = configparser.ConfigParser()
        config.add_section('general')
        config.set('general', 'name', 'Sunhpc-%s' % sunhpc.version_major)
        config.set('general', 'family', 'Sunhpc')
        config.set('general', 'timestamp', '%.2f' % time.time())
        config.set('general', 'variant', '')
        config.set('general', 'version', sunhpc.version_major)
        config.set('general', 'packagedir', '')
        config.set('general', 'arch', self.arch)

        config.add_section('stage2')
        config.set('stage2', 'mainimage', 'images/install.img')

        config.add_section('images-%s' % self.arch)

        vmlinuz = 'images/pxeboot/vmlinuz-%s-%s' % (sunhpc.version, self.arch)
        initrds = 'images/pxeboot/initrd-%s-%s' % (sunhpc.version, self.arch)
        updates = 'images/pxeboot/updates.img'
        config.set('images-%s'% self.arch, 'kernel', vmlinuz)
        config.set('images-%s'% self.arch, 'initrd', initrds)
        #config.set('images-%s'% self.arch, 'upgrade', updates)
        config.set('images-%s'% self.arch, 'boot.iso', 'images/boot.iso')

        with open(dst, 'w') as f:
            config.write(f)

    def buildstamp(self, dst_stamp):
        uuidTime = time.strftime('%Y%m%d%H%M%S', time.localtime())
        with open(dst_stamp, 'w') as f:
            f.write('[Main]\n')
            f.write('Product=Sunhpc\n')
            f.write('Version=%s\n' % sunhpc.version_major)
            f.write('BugURL=your distribution provided bug reporting tool\n')
            f.write('IsFinal=True\n')
            f.write('UUID=%s.%s\n' % (uuidTime, self.arch))
            f.write('[Compose]\n')
            f.write('Lorax=19.7.19-1\n')

    def run(self, flags, args):
        pass

class DocStringHandler(
            handler.ContentHandler, handler.ErrorHandler,
            handler.EntityResolver, handler.DTDHandler ):
    
    def __init__(self, name='', users=[]):
        handler.ContentHandler.__init__(self)
        self.text   = ''
        self.name   = name
        self.users  = users
        self.section= {}
        self.section['arg']         = []
        self.section['param']       = []
        self.section['example']     = []
        self.section['related']     = []
        self.section['description'] = ''
        self.parser = make_parser()
        self.parser.setContentHandler(self)

    def getDocbookText(self):
        s  = ''
        s += '<section id="sunhpc-%s" xreflabel="%s">\n' % \
            (string.join(self.name.split(' '), '-'), self.name)
        s += '<title>%s</title>\n' % self.name
        s += '<cmdsynopsis>\n'
        s += '\t<command>sunhpc %s</command>\n' % self.name
        for ((name, type, opt, rep), txt) in self.section['arg']:
            if opt:
                choice = 'opt'
            else:
                choice = 'req'
            if rep:
                repeat = 'repeat'
            else:
                repeat = 'norepeat'
            s += '\t<arg rep="%s" choice="%s">%s</arg>\n' % \
                (repeat, choice, name)
        for ((name, type, opt, rep), txt) in self.section['param']:
            if opt:
                choice = 'opt'
            else:
                choice = 'req'
            if rep:
                repeat = 'repeat'
            else:
                repeat = 'norepeat'
            s += '\t<arg rep="%s" choice="%s">' % (repeat, choice)
            s += '%s=<replaceable>%s</replaceable>' % (name, type)
            s += '</arg>\n'
        s += '</cmdsynopsis>\n'
        s += '<para>\n'
        s += saxutils.escape(self.section['description'])
        s += '\n</para>\n'
        if self.section['arg']:
            s += '<variablelist><title>arguments</title>\n'
            for ((name, type, opt, rep), txt) in \
                self.section['arg']:
                s += '\t<varlistentry>\n'
                if opt:
                    term = '<optional>%s</optional>' % name
                else:
                    term = name
                s += '\t<term>%s</term>\n' % term
                s += '\t<listitem>\n'
                s += '\t<para>\n'
                s += saxutils.escape(txt)
                s += '\n\t</para>\n'
                s += '\t</listitem>\n'
                s += '\t</varlistentry>\n'
            s += '</variablelist>\n'
        if self.section['param']:
            s += '<variablelist><title>parameters</title>\n'
            for ((name, type, opt, rep), txt) in \
                self.section['param']:
                s += '\t<varlistentry>\n'
                if opt:
                    optStart = '<optional>'
                    optEnd   = '</optional>'
                else:
                    optStart = ''
                    optEnd   = ''
                key = '%s=' % name
                val = '<replaceable>%s</replaceable>' % type
                s += '\t<term>%s%s%s%s</term>\n' % \
                    (optStart, key, val, optEnd)
                s += '\t<listitem>\n'
                s += '\t<para>\n'
                s += saxutils.escape(txt)
                s += '\n\t</para>\n'
                s += '\t</listitem>\n'
                s += '\t</varlistentry>\n'
            s += '</variablelist>\n'
        if self.section['example']:
            s += '<variablelist><title>examples</title>\n'
            for (cmd, txt) in self.section['example']:
                s += '\t<varlistentry>\n'
                s += '\t<term>\n'
                if 'root' in self.users:
                    s += '# '
                else:
                    s += '$ '
                s += 'sunhpc %s' % cmd
                s += '\n\t</term>\n'
                s += '\t<listitem>\n'
                s += '\t<para>\n'
                s += saxutils.escape(txt)
                s += '\n\t</para>\n'
                s += '\t</listitem>\n'
                s += '\t</varlistentry>\n'
            s += '</variablelist>\n'
        if self.section['related']:
            s += '<variablelist><title>related commands</title>\n'
            for related in self.section['related']:
                s += '\t<varlistentry>\n'
                s += '\t<term>'
                s += '<xref linkend="sunhpc-%s">' % \
                    string.join(related.split(' '), '-')
                s += '</term>\n'
                s += '\t<listitem>\n'
                s += '\t<para>\n'
                s += '\n\t</para>\n'
                s += '\t</listitem>\n'
                s += '\t</varlistentry>\n'
            s += '</variablelist>\n'
        s += '</section>'
        return s

    def getUsageText(self):
        s = ''
        for ((name, type, opt, rep), txt) in self.section['arg']:
            if opt:
                s += '[%s]' % name
            else:
                s += '{%s}' % name
            if rep:
                s += '...'
            s += ' '
        for ((name, type, opt, rep), txt) in self.section['param']:
            if opt:
                s += '[%s=%s]' % (name, type)
            else:
                s += '{%s=%s}' % (name, type)
            if rep:
                s += '...'
            s += ' '
        if s and s[-1] == ' ':
            return s[:-1]
        else:
            return s

    def getSphinxText(self):
        if 'root' in self.users:
            prompt = '#'
        else:
            prompt = '$'

        s  = ':orphan:\n\n'
        s += '%s\n' % self.name
        s += '%s\n\n' % ("-" * len(self.name))
        s += '.. role:: defn\n\n' 
        utxt = self.getUsageText()
        if len(utxt): 
            s += ':defn:`sunhpc %s` *%s*\n' % (self.name, utxt)
        else:         
            s += ':defn:`sunhpc %s` %s\n' % (self.name, utxt)
        s += '\n\n**Description:**\n'
        s += self.section['description'].replace('\t','   ')
        if self.section['arg']:
            s += '\n**Arguments:**\n\n'
            for ((name, type, opt, rep), txt) in \
                self.section['arg']:
                if opt:
                    s += '*[%s]*' % name
                else:
                    s += '*{%s}*' % name
                txt = txt.replace('*', '\*')
                s += '\n%s\n' % txt.replace('\t', '   ')
        if self.section['param']:
            s += '\n**Parameters:**\n\n'
            for ((name, type, opt, rep), txt) in \
                self.section['param']:
                if opt:
                    s += '*[%s=%s]*' % (name, type)
                else:
                    s += '*{%s=%s}*' % (name, type)
                txt = txt.replace('*', '\*')
                s += '\n%s\n' % txt.replace('\t', '   ')
        if self.section['example']:
            s += '\n**Examples:**\n'
            for (cmd, txt) in self.section['example']:
                txt = txt.replace('*', '\*')
                s += '%s::\n\n' % txt.replace('\t','   ')
                s += '        %s sunhpc %s\n' % (prompt, cmd)
        if self.section['related']:
            s += '\n**Related Commands:**\n\n'
            for related in self.section['related']:
                s += '   * :ref:`sunhpc-%s`\n' % related.replace(' ','-')

        word = self.name.split()[0]
        s += '\n:ref:`%s commands <%s-ref>`\n' % (word, word)

        return s

    def getPlainText(self):
        if 'root' in self.users:
            prompt = '\033[91m#\033[0m'
        else:
            prompt = '\033[96m$\033[0m'
        s  = ''
        s += 'sunhpc %s %s' % (self.name, self.getUsageText())
        s += '\n\n\033[95m Description: \033[0m\n'
        s += '\033[96m%s\033[0m' % self.section['description']
        if self.section['arg']:
            s += '\n\033[95m Arguments: \033[0m\n\n'
            for ((name, type, opt, rep), txt) in \
                self.section['arg']:
                if opt:
                    s += '\t\033[92m[%s]\033[0m' % name
                else:
                    s += '\t\033[92m{%s}\033[0m' % name
                s += '\033[96m%s\033[0m' % txt
        if self.section['param']:
            s += '\n\033[95m Parameters: \033[0m\n\n'
            for ((name, type, opt, rep), txt) in \
                self.section['param']:
                if opt:
                    s += '\t\033[92m[%s=%s]\033[0m' % (name, type)
                else:
                    s += '\t\033[92m{%s=%s}\033[0m' % (name, type)
                #s += '\n%s\n' % txt
                s += '\033[96m%s\033[0m' % txt
        if self.section['example']:
            s += '\n\033[95m Examples: \033[0m\n\n'
            for (cmd, txt) in self.section['example']:
                s += '\t%s sunhpc %s' % (prompt, cmd)
                #s += '%s\n' % txt
                s += '\033[96m%s\033[0m' % txt
        if self.section['related']:
            s += '\n\033[95m Related Commands: \033[0m\n'
            for related in self.section['related']:
                s += '\tsunhpc %s\n' % related
        return s
        
    def getParsedText(self):
        return '%s' % self.section
        
    def startElement(self, name, attrs):
        if not self.section['description']:
            self.section['description'] = self.text
        self.key  = None
        self.text = ''
        if name in [ 'arg', 'param' ]:
            try:
                type = attrs.get('type')
            except:
                type = 'string'
            try:
                optional = int(attrs.get('optional'))
            except:
                if name == 'arg':
                    optional = 0
                if name == 'param':
                    optional = 1
            try:
                repeat = int(attrs.get('repeat'))
            except:
                repeat = 0
            name = attrs.get('name')
            self.key = (name, type, optional, repeat)
        elif name == 'example':
            self.key = attrs.get('cmd')
        
    def endElement(self, name):
        if name == 'docstring':
            self.section['param'].sort()
            self.section['related'].sort()
        elif name in [ 'arg', 'param', 'example' ]:
            self.section[name].append((self.key, self.text))
        else:
            if name in self.section:
                self.section[name].append(self.text)
        
    def characters(self, s):
        self.text += s

class RollArgumentProcessor(object):
    """An Interface class to add the ability to process roll arguments."""
    
    def getRollNames(self, args, params):
        if 'version' in params:
            version = params['version']
        else:
            version = '%' # SQL wildcard
    
        plist = []
        if not args: args = [ '%' ] # find all roll names
        for arg in args:
            rows = self.db.search("""select distinct name,version
                from rolls where name like '%s' and 
                version like '%s'""" % (arg, version))
            if rows == 0 and arg == '%': # empty table is OK
                continue
            if rows < 1:
                self.abort('unknown roll name "%s"' % arg)
            for (name, ver) in self.db.fetchall():
                plist.append((name, ver))
                
        return plist

class Plugin(object):
    """Base class for all Sunhpc command plug-ins."""
    def __init__(self, command):
        self.cmd = command
        self.db  = command.db
        
    def run(self, args):
        """All derived classes should override this method. This
        is the entry point into the Plugin object."""
        pass
