<commit_before>'''
The module used to execute states in salt. A state is unlike a module execution
in that instead of just executing a command it ensure that a certain state is
present on the system.

The data sent to the state calls is as follows:
    { 'state': '<state module name>',
      'fun': '<state function name>',
      'name': '<the name argument passed to all states>'
      'argn': '<arbitrairy argument, can have many of these>'
      }
'''
# Import python modules
import sys
import os
import copy
import inspect
import tempfile
import logging
# Import Salt modules
import salt.loader
import salt.minion

log = logging.getLogger(__name__)

def format_log(ret):
    '''
    Format the state into a log message
    '''
    msg = ''
    if type(ret) == type(dict()):
        # Looks like the ret may be a valid state return
        if ret.has_key('changes'):
            # Yep, looks like a valid state return
            chg = ret['changes']
            if not chg:
                msg = 'No changes made for {0[name]}'.format(ret)
            elif type(chg) == type(dict()):
                if chg.has_key('diff'):
                    if type(chg['diff']) == type(str()):
                        msg = 'File changed:\n{0}'.format(
                                chg['diff'])
                if type(chg[chg.keys()[0]]) == type(dict()):
                    if chg[chg.keys()[0]].has_key('new'):
                        # This is the return data from a package install
                        msg = 'Installed Packages:\n'
                        for pkg in chg:
                            old = 'absent'
                            if chg[pkg]['old']:
                                old = chg[pkg]['old']
                            msg += '{0} changed from {1} to {2}\n'.format(
                                    pkg, old, chg[pkg]['new'])
            if not msg:
                msg = str(ret['changes'])
            if ret['result']:
                log.info(msg)
            else:
                log.error(msg)
    else:
        # catch unhandled data
        log.info(str(ret))

class StateError(Exception): pass

class State(object):
    '''
    Class used to execute salt states
    '''
    def __init__(self, opts):
        if not opts.has_key('grains'):
            opts['grains'] = salt.loader.grains(opts)
        self.opts = opts
        self.functions = salt.loader.minion_mods(self.opts)
        self.states = salt.loader.states(self.opts, self.functions)
        self.rend = salt.loader.render(self.opts, self.functions)

    def verify_data(self, data):
        '''
        Verify the data, return an error statement if something is wrong
        '''
        errors = []
        if not data.has_key('state'):
            errors.append('Missing "state" data')
        if not data.has_key('fun'):
            errors.append('Missing "fun" data')
        if not data.has_key('name'):
            errors.append('Missing "name" data')
        if errors:
            return errors
        full = data['state'] + '.' + data['fun']
        if not self.states.has_key(full):
            errors.append('Specified state ' + full + ' is unavailable.')
        else:
            aspec = inspect.getargspec(self.states[full])
            arglen = 0
            deflen = 0
            if type(aspec[0]) == type(list()):
                arglen = len(aspec[0])
            if type(aspec[3]) == type(tuple()):
                deflen = len(aspec[3])
            for ind in range(arglen - deflen):
                if not data.has_key(aspec[0][ind]):
                    errors.append('Missing paramater ' + aspec[0][ind]\
                                + ' for state ' + full)
        return errors

    def verify_chunks(self, chunks):
        '''
        Verify the chunks in a list of low data structures
        '''
        err = []
        for chunk in chunks:
            err += self.verify_data(chunk)
        return err

    def format_call(self, data):
        '''
        Formats low data into a list of dict's used to actually call the state,
        returns:
        {
        'full': 'module.function',
        'args': [arg[0], arg[1], ...]
        }
        used to call the function like this:
        self.states[ret['full']](*ret['args'])

        It is assumed that the passed data has already been verified with
        verify_data
        '''
        ret = {}
        ret['full'] = data['state'] + '.' + data['fun']
        ret['args'] = []
        aspec = inspect.getargspec(self.states[ret['full']])
        arglen = 0
        deflen = 0
        if type(aspec[0]) == type(list()):
            arglen = len(aspec[0])
        if type(aspec[3]) == type(tuple()):
            deflen = len(aspec[3])
        kwargs = {}
        for ind in range(arglen - 1, 0, -1):
            minus = arglen - ind
            if deflen - minus > -1:
                kwargs[aspec[0][ind]] = aspec[3][-minus]
        for arg in kwargs:
            if data.has_key(arg):
                kwargs[arg] = data[arg]
        for arg in aspec[0]:
            if kwargs.has_key(arg):
                ret['args'].append(kwargs[arg])
            else:
                ret['args'].append(data[arg])
        return ret

    def compile_high_data(self, high):
        '''
        "Compile" the high data as it is retrieved from the cli or yaml into
        the individual state executor structures
        '''
        chunks = []
        for name, body in high.items():
            for state, run in body.items():
                chunk = {'state': state,
                         'name': name}
                funcs = set()
                names = set()
                for arg in run:
                    if type(arg) == type(str()):
                        funcs.add(arg)
                        continue
                    if type(arg) == type(dict()):
                        for key, val in arg.items():
                            if key == 'names':
                                names.update(val)
                                continue
                            else:
                                chunk.update(arg)
                if names:
                    for name in names:
                        live  = copy.deepcopy(chunk)
                        live['name'] = name
                        for fun in funcs:
                            live['fun'] = fun
                            chunks.append(live)
                else:
                    live  = copy.deepcopy(chunk)
                    for fun in funcs:
                        live['fun'] = fun
                        chunks.append(live)

        return sorted(chunks, key=lambda k: k['state'] + k['name'] + k['fun'])

    def compile_template(self, template):
        '''
        Take the path to a template and return the high data structure derived
        from the template.
        '''
        if not isinstance(template, str):
            return {}
        if not os.path.isfile(template):
            return {}
        return self.rend[self.opts['renderer']](template)

    def compile_template_str(self, template):
        '''
        Take the path to a template and return the high data structure derived
        from the template.
        '''
        fn_ = tempfile.mkstemp()[1]
        open(fn_, 'w+').write(template)
        high = self.rend[self.opts['renderer']](fn_)
        os.remove(fn_)
        return high

    def call(self, data):
        '''
        Call a state directly with the low data structure, verify data before
        processing.
        '''
        log.info(
                'Executing state {0[state]}.{0[fun]} for {0[name]}'.format(data)
                )
        cdata = self.format_call(data)
        ret = self.states[cdata['full']](*cdata['args'])
        format_log(ret)
        return ret

    def call_chunks(self, chunks):
        '''
        Iterate over a list of chunks and call them, checking for requires.
        '''
        running = {}
        for low in chunks:
            running = self.call_chunk(low, running, chunks)
        return running

    def check_requires(self, low, running, chunks):
        '''
        Look into the running data to see if the requirement has been met
        '''
        if not low.has_key('require'):
            return 'met'
        reqs = []
        status = 'unmet'
        for req in low['require']:
            for chunk in chunks:
                if chunk['name'] == req[req.keys()[0]]:
                    if chunk['state'] == req.keys()[0]:
                        reqs.append(chunk)
        fun_stats = []
        for req in reqs:
            tag = req['state'] + '.' + req['name'] + '.' + req['fun']
            if not running.has_key(tag):
                fun_stats.append('unmet')
            else:
                fun_stats.append('met' if running[tag]['result'] else 'fail')
        for stat in fun_stats:
            if stat == 'unmet':
                return stat
            elif stat == 'fail':
                return stat
        return 'met'

    def check_watchers(self, low, running, chunks):
        '''
        Look into the running data to see if the watched states have been run
        '''
        if not low.has_key('watch'):
            return 'nochange'
        reqs = []
        status = 'unmet'
        for req in low['watch']:
            for chunk in chunks:
                if chunk['name'] == req[req.keys()[0]]:
                    if chunk['state'] == req.keys()[0]:
                        reqs.append(chunk)
        fun_stats = []
        for req in reqs:
            tag = req['state'] + '.' + req['name'] + '.' + req['fun']
            if not running.has_key(tag):
                fun_stats.append('unmet')
            else:
                fun_stats.append('change' if running[tag]['changes'] else 'nochange')
        for stat in fun_stats:
            if stat == 'change':
                return stat
            elif stat == 'unmet':
                return stat
        return 'nochange'

    def call_chunk(self, low, running, chunks):
        '''
        Check if a chunk has any requires, execute the requires and then the
        chunk
        '''
        tag = low['state'] + '.' + low['name'] + '.' + low['fun']
        if low.has_key('require'):
            status = self.check_requires(low, running, chunks)
            if status == 'unmet':
                reqs = []
                for req in low['require']:
                    for chunk in chunks:
                        if chunk['name'] == req[req.keys()[0]]:
                            if chunk['state'] == req.keys()[0]:
                                reqs.append(chunk)
                for chunk in reqs:
                    running = self.call_chunk(chunk, running, chunks)
                running = self.call_chunk(low, running, chunks)
            elif status == 'met':
                running[tag] = self.call(low)
            elif status == 'fail':
                running[tag] = {'changes': {},
                                'result': False,
                                'comment': 'One or more require failed'}
        elif low.has_key('watch'):
            status = self.check_watchers(low, running, chunks)
            if status == 'unmet':
                reqs = []
                for req in low['watch']:
                    for chunk in chunks:
                        if chunk['name'] == req[req.keys()[0]]:
                            if chunk['state'] == req.keys()[0]:
                                reqs.append(chunk)
                for chunk in reqs:
                    running = self.call_chunk(chunk, running, chunks)
                running = self.call_chunk(low, running, chunks)
            elif status == 'nochange':
                running[tag] = self.call(low)
            elif status == 'change':
                ret = self.call(low)
                if not ret['changes']:
                    low['fun'] = 'watcher'
                    ret = self.call(low)
                running[tag] = ret
        else:
            running[tag] = self.call(low)
        return running

    def call_high(self, high):
        '''
        Process a high data call and ensure the defined states.
        '''
        err = []
        rets = []
        chunks = self.compile_high_data(high)
        errors = self.verify_chunks(chunks)
        if errors:
            return errors
        return self.call_chunks(chunks)

    def call_template(self, template):
        '''
        Enforce the states in a template
        '''
        high = self.compile_template(template)
        if high:
            return self.call_high(high)
        return high

    def call_template_str(self, template):
        '''
        Enforce the states in a template, pass the template as a string
        '''
        high = self.compile_template_str(template)
        if high:
            return self.call_high(high)
        return high

class HighState(object):
    '''
    Generate and execute the salt "High State". The High State is the compound
    state derived from a group of template files stored on the salt master or
    in a the local cache.
    '''
    def __init__(self, opts):
        self.client = salt.minion.FileClient(opts)
        self.opts = self.__gen_opts(opts)
        self.state = State(self.opts)
        self.matcher = salt.minion.Matcher(self.opts)

    def __gen_opts(self, opts):
        '''
        The options used by the High State object are derived from options on
        the minion and the master, or just the minion if the high state call is
        entirely local.
        '''
        # If the state is intended to be applied locally, then the local opts
        # should have all of the needed data, otherwise overwrite the local
        # data items with data from the master
        if opts.has_key('local_state'):
            if opts['local_state']:
                return opts
        mopts = self.client.master_opts()
        opts['renderer'] = mopts['renderer']
        if mopts['state_top'].startswith('salt://'):
            opts['state_top'] = mopts['state_top']
        elif mopts['state_top'].startswith('/'):
            opts['state_top'] = os.path.join('salt://', mopts['state_top'][1:])
        else:
            opts['state_top'] = os.path.join('salt://', mopts['state_top'])
        return opts

    def get_top(self):
        '''
        Returns the high data derived from the top file
        '''
        top = self.client.cache_file(self.opts['state_top'], 'base')
        return self.state.compile_template(top)

    def top_matches(self, top):
        '''
        Search through the top high data for matches and return the states that
        this minion needs to execute.

        Returns:
        {'env': ['state1', 'state2', ...]}
        '''
        matches = {}
        for env, body in top.items():
            for match, data in body.items():
                if self.matcher.confirm_top(match, data):
                    if not matches.has_key(env):
                        matches[env] = []
                    for item in data:
                        if type(item) == type(str()):
                            matches[env].append(item)
        return matches

    def gather_states(self, matches):
        '''
        Gather the template files from the master
        '''
        group = []
        for env, states in matches.items():
            for sls in states:
                state = self.client.get_state(sls, env)
                if state:
                    group.append(state)
        return group

    def render_state(self, sls, env, mods):
        '''
        Render a state file and retrive all of the include states
        '''
        fn_ = self.client.get_state(sls, env)
        state = self.state.compile_template(fn_)
        mods.add(sls)
        nstate = None
        if state:
            if state.has_key('include'):
                for sub_sls in state.pop('include'):
                    if not list(mods).count(sub_sls):
                        nstate, mods = self.render_state(sub_sls, env, mods)
                    if nstate:
                        state.update(nstate)
        return state, mods

    def render_highstate(self, matches):
        '''
        Gather the state files and render them into a single unified salt high
        data structure.
        '''
        highstate = {}
        for env, states in matches.items():
            mods = set()
            for sls in states:
                state, mods = self.render_state(sls, env, mods)
                if state:
                    highstate.update(state)
        return highstate

    def call_highstate(self):
        '''
        Run the sequence to execute the salt highstate for this minion
        '''
        top = self.get_top()
        matches = self.top_matches(top)
        high = self.render_highstate(matches)
        return self.state.call_high(high)
<commit_msg>Pass the sls data through to the low data structure.<commit_after>'''
The module used to execute states in salt. A state is unlike a module execution
in that instead of just executing a command it ensure that a certain state is
present on the system.

The data sent to the state calls is as follows:
    { 'state': '<state module name>',
      'fun': '<state function name>',
      'name': '<the name argument passed to all states>'
      'argn': '<arbitrairy argument, can have many of these>'
      }
'''
# Import python modules
import sys
import os
import copy
import inspect
import tempfile
import logging
# Import Salt modules
import salt.loader
import salt.minion

log = logging.getLogger(__name__)

def format_log(ret):
    '''
    Format the state into a log message
    '''
    msg = ''
    if type(ret) == type(dict()):
        # Looks like the ret may be a valid state return
        if ret.has_key('changes'):
            # Yep, looks like a valid state return
            chg = ret['changes']
            if not chg:
                msg = 'No changes made for {0[name]}'.format(ret)
            elif type(chg) == type(dict()):
                if chg.has_key('diff'):
                    if type(chg['diff']) == type(str()):
                        msg = 'File changed:\n{0}'.format(
                                chg['diff'])
                if type(chg[chg.keys()[0]]) == type(dict()):
                    if chg[chg.keys()[0]].has_key('new'):
                        # This is the return data from a package install
                        msg = 'Installed Packages:\n'
                        for pkg in chg:
                            old = 'absent'
                            if chg[pkg]['old']:
                                old = chg[pkg]['old']
                            msg += '{0} changed from {1} to {2}\n'.format(
                                    pkg, old, chg[pkg]['new'])
            if not msg:
                msg = str(ret['changes'])
            if ret['result']:
                log.info(msg)
            else:
                log.error(msg)
    else:
        # catch unhandled data
        log.info(str(ret))

class StateError(Exception): pass

class State(object):
    '''
    Class used to execute salt states
    '''
    def __init__(self, opts):
        if not opts.has_key('grains'):
            opts['grains'] = salt.loader.grains(opts)
        self.opts = opts
        self.functions = salt.loader.minion_mods(self.opts)
        self.states = salt.loader.states(self.opts, self.functions)
        self.rend = salt.loader.render(self.opts, self.functions)

    def verify_data(self, data):
        '''
        Verify the data, return an error statement if something is wrong
        '''
        errors = []
        if not data.has_key('state'):
            errors.append('Missing "state" data')
        if not data.has_key('fun'):
            errors.append('Missing "fun" data')
        if not data.has_key('name'):
            errors.append('Missing "name" data')
        if errors:
            return errors
        full = data['state'] + '.' + data['fun']
        if not self.states.has_key(full):
            errors.append('Specified state ' + full + ' is unavailable.')
        else:
            aspec = inspect.getargspec(self.states[full])
            arglen = 0
            deflen = 0
            if type(aspec[0]) == type(list()):
                arglen = len(aspec[0])
            if type(aspec[3]) == type(tuple()):
                deflen = len(aspec[3])
            for ind in range(arglen - deflen):
                if not data.has_key(aspec[0][ind]):
                    errors.append('Missing paramater ' + aspec[0][ind]\
                                + ' for state ' + full)
        return errors

    def verify_chunks(self, chunks):
        '''
        Verify the chunks in a list of low data structures
        '''
        err = []
        for chunk in chunks:
            err += self.verify_data(chunk)
        return err

    def format_call(self, data):
        '''
        Formats low data into a list of dict's used to actually call the state,
        returns:
        {
        'full': 'module.function',
        'args': [arg[0], arg[1], ...]
        }
        used to call the function like this:
        self.states[ret['full']](*ret['args'])

        It is assumed that the passed data has already been verified with
        verify_data
        '''
        ret = {}
        ret['full'] = data['state'] + '.' + data['fun']
        ret['args'] = []
        aspec = inspect.getargspec(self.states[ret['full']])
        arglen = 0
        deflen = 0
        if type(aspec[0]) == type(list()):
            arglen = len(aspec[0])
        if type(aspec[3]) == type(tuple()):
            deflen = len(aspec[3])
        kwargs = {}
        for ind in range(arglen - 1, 0, -1):
            minus = arglen - ind
            if deflen - minus > -1:
                kwargs[aspec[0][ind]] = aspec[3][-minus]
        for arg in kwargs:
            if data.has_key(arg):
                kwargs[arg] = data[arg]
        for arg in aspec[0]:
            if kwargs.has_key(arg):
                ret['args'].append(kwargs[arg])
            else:
                ret['args'].append(data[arg])
        return ret

    def compile_high_data(self, high):
        '''
        "Compile" the high data as it is retrieved from the cli or yaml into
        the individual state executor structures
        '''
        chunks = []
        for name, body in high.items():
            for state, run in body.items():
                if state.startswith('__'):
                    continue
                chunk = {'state': state,
                         'name': name}
                if body.has_key('__sls__'):
                    chunk['__sls__'] = body['__sls__']
                funcs = set()
                names = set()
                for arg in run:
                    if type(arg) == type(str()):
                        funcs.add(arg)
                        continue
                    if type(arg) == type(dict()):
                        for key, val in arg.items():
                            if key == 'names':
                                names.update(val)
                                continue
                            else:
                                chunk.update(arg)
                if names:
                    for name in names:
                        live  = copy.deepcopy(chunk)
                        live['name'] = name
                        for fun in funcs:
                            live['fun'] = fun
                            chunks.append(live)
                else:
                    live  = copy.deepcopy(chunk)
                    for fun in funcs:
                        live['fun'] = fun
                        chunks.append(live)

        return sorted(chunks, key=lambda k: k['state'] + k['name'] + k['fun'])

    def compile_template(self, template):
        '''
        Take the path to a template and return the high data structure derived
        from the template.
        '''
        if not isinstance(template, str):
            return {}
        if not os.path.isfile(template):
            return {}
        return self.rend[self.opts['renderer']](template)

    def compile_template_str(self, template):
        '''
        Take the path to a template and return the high data structure derived
        from the template.
        '''
        fn_ = tempfile.mkstemp()[1]
        open(fn_, 'w+').write(template)
        high = self.rend[self.opts['renderer']](fn_)
        os.remove(fn_)
        return high

    def call(self, data):
        '''
        Call a state directly with the low data structure, verify data before
        processing.
        '''
        log.info(
                'Executing state {0[state]}.{0[fun]} for {0[name]}'.format(data)
                )
        cdata = self.format_call(data)
        ret = self.states[cdata['full']](*cdata['args'])
        format_log(ret)
        return ret

    def call_chunks(self, chunks):
        '''
        Iterate over a list of chunks and call them, checking for requires.
        '''
        running = {}
        for low in chunks:
            running = self.call_chunk(low, running, chunks)
        return running

    def check_requires(self, low, running, chunks):
        '''
        Look into the running data to see if the requirement has been met
        '''
        if not low.has_key('require'):
            return 'met'
        reqs = []
        status = 'unmet'
        for req in low['require']:
            for chunk in chunks:
                if chunk['name'] == req[req.keys()[0]]:
                    if chunk['state'] == req.keys()[0]:
                        reqs.append(chunk)
        fun_stats = []
        for req in reqs:
            tag = req['state'] + '.' + req['name'] + '.' + req['fun']
            if not running.has_key(tag):
                fun_stats.append('unmet')
            else:
                fun_stats.append('met' if running[tag]['result'] else 'fail')
        for stat in fun_stats:
            if stat == 'unmet':
                return stat
            elif stat == 'fail':
                return stat
        return 'met'

    def check_watchers(self, low, running, chunks):
        '''
        Look into the running data to see if the watched states have been run
        '''
        if not low.has_key('watch'):
            return 'nochange'
        reqs = []
        status = 'unmet'
        for req in low['watch']:
            for chunk in chunks:
                if chunk['name'] == req[req.keys()[0]]:
                    if chunk['state'] == req.keys()[0]:
                        reqs.append(chunk)
        fun_stats = []
        for req in reqs:
            tag = req['state'] + '.' + req['name'] + '.' + req['fun']
            if not running.has_key(tag):
                fun_stats.append('unmet')
            else:
                fun_stats.append('change' if running[tag]['changes'] else 'nochange')
        for stat in fun_stats:
            if stat == 'change':
                return stat
            elif stat == 'unmet':
                return stat
        return 'nochange'

    def call_chunk(self, low, running, chunks):
        '''
        Check if a chunk has any requires, execute the requires and then the
        chunk
        '''
        tag = low['state'] + '.' + low['name'] + '.' + low['fun']
        if low.has_key('require'):
            status = self.check_requires(low, running, chunks)
            if status == 'unmet':
                reqs = []
                for req in low['require']:
                    for chunk in chunks:
                        if chunk['name'] == req[req.keys()[0]]:
                            if chunk['state'] == req.keys()[0]:
                                reqs.append(chunk)
                for chunk in reqs:
                    running = self.call_chunk(chunk, running, chunks)
                running = self.call_chunk(low, running, chunks)
            elif status == 'met':
                running[tag] = self.call(low)
            elif status == 'fail':
                running[tag] = {'changes': {},
                                'result': False,
                                'comment': 'One or more require failed'}
        elif low.has_key('watch'):
            status = self.check_watchers(low, running, chunks)
            if status == 'unmet':
                reqs = []
                for req in low['watch']:
                    for chunk in chunks:
                        if chunk['name'] == req[req.keys()[0]]:
                            if chunk['state'] == req.keys()[0]:
                                reqs.append(chunk)
                for chunk in reqs:
                    running = self.call_chunk(chunk, running, chunks)
                running = self.call_chunk(low, running, chunks)
            elif status == 'nochange':
                running[tag] = self.call(low)
            elif status == 'change':
                ret = self.call(low)
                if not ret['changes']:
                    low['fun'] = 'watcher'
                    ret = self.call(low)
                running[tag] = ret
        else:
            running[tag] = self.call(low)
        return running

    def call_high(self, high):
        '''
        Process a high data call and ensure the defined states.
        '''
        err = []
        rets = []
        chunks = self.compile_high_data(high)
        errors = self.verify_chunks(chunks)
        if errors:
            return errors
        return self.call_chunks(chunks)

    def call_template(self, template):
        '''
        Enforce the states in a template
        '''
        high = self.compile_template(template)
        if high:
            return self.call_high(high)
        return high

    def call_template_str(self, template):
        '''
        Enforce the states in a template, pass the template as a string
        '''
        high = self.compile_template_str(template)
        if high:
            return self.call_high(high)
        return high

class HighState(object):
    '''
    Generate and execute the salt "High State". The High State is the compound
    state derived from a group of template files stored on the salt master or
    in a the local cache.
    '''
    def __init__(self, opts):
        self.client = salt.minion.FileClient(opts)
        self.opts = self.__gen_opts(opts)
        self.state = State(self.opts)
        self.matcher = salt.minion.Matcher(self.opts)

    def __gen_opts(self, opts):
        '''
        The options used by the High State object are derived from options on
        the minion and the master, or just the minion if the high state call is
        entirely local.
        '''
        # If the state is intended to be applied locally, then the local opts
        # should have all of the needed data, otherwise overwrite the local
        # data items with data from the master
        if opts.has_key('local_state'):
            if opts['local_state']:
                return opts
        mopts = self.client.master_opts()
        opts['renderer'] = mopts['renderer']
        if mopts['state_top'].startswith('salt://'):
            opts['state_top'] = mopts['state_top']
        elif mopts['state_top'].startswith('/'):
            opts['state_top'] = os.path.join('salt://', mopts['state_top'][1:])
        else:
            opts['state_top'] = os.path.join('salt://', mopts['state_top'])
        return opts

    def get_top(self):
        '''
        Returns the high data derived from the top file
        '''
        top = self.client.cache_file(self.opts['state_top'], 'base')
        return self.state.compile_template(top)

    def top_matches(self, top):
        '''
        Search through the top high data for matches and return the states that
        this minion needs to execute.

        Returns:
        {'env': ['state1', 'state2', ...]}
        '''
        matches = {}
        for env, body in top.items():
            for match, data in body.items():
                if self.matcher.confirm_top(match, data):
                    if not matches.has_key(env):
                        matches[env] = []
                    for item in data:
                        if type(item) == type(str()):
                            matches[env].append(item)
        return matches

    def gather_states(self, matches):
        '''
        Gather the template files from the master
        '''
        group = []
        for env, states in matches.items():
            for sls in states:
                state = self.client.get_state(sls, env)
                if state:
                    group.append(state)
        return group

    def render_state(self, sls, env, mods):
        '''
        Render a state file and retrive all of the include states
        '''
        fn_ = self.client.get_state(sls, env)
        state = self.state.compile_template(fn_)
        mods.add(sls)
        nstate = None
        if state:
            if state.has_key('include'):
                for sub_sls in state.pop('include'):
                    if not list(mods).count(sub_sls):
                        nstate, mods = self.render_state(sub_sls, env, mods)
                    if nstate:
                        state.update(nstate)
            for name in state:
                if not state[name].has_key('__sls__'):
                    state[name]['__sls__'] = sls
        return state, mods

    def render_highstate(self, matches):
        '''
        Gather the state files and render them into a single unified salt high
        data structure.
        '''
        highstate = {}
        for env, states in matches.items():
            mods = set()
            for sls in states:
                state, mods = self.render_state(sls, env, mods)
                if state:
                    highstate.update(state)
        return highstate

    def call_highstate(self):
        '''
        Run the sequence to execute the salt highstate for this minion
        '''
        top = self.get_top()
        matches = self.top_matches(top)
        high = self.render_highstate(matches)
        return self.state.call_high(high)
<|endoftext|>


<commit_before>"""
Copyright (c) 2014 Brian Muller
Copyright (c) 2015 OpenBazaar
"""

import abc
import random
import nacl.signing
import nacl.encoding
import nacl.hash
from base64 import b64encode
from binascii import hexlify
from constants import PROTOCOL_VERSION
from dht.node import Node
from dht.utils import digest
from hashlib import sha1
from log import Logger
from protos.message import Message, Command, NOT_FOUND, HOLE_PUNCH
from protos.objects import FULL_CONE, RESTRICTED, SYMMETRIC
from twisted.internet import defer, reactor
from txrudp.connection import State

class RPCProtocol:
    """
    This is an abstract class for processing and sending rpc messages.
    A class that implements the `MessageProcessor` interface probably should
    extend this as it does most of the work of keeping track of messages.
    """
    __metaclass__ = abc.ABCMeta

    def __init__(self, sourceNode, router, waitTimeout=2.5):
        """
        Args:
            proto: A protobuf `Node` object containing info about this node.
            router: A `RoutingTable` object from dht.routing. Implies a `network.Server` object
                    must be started first.
            waitTimeout: Consider it a connetion failure if no response
                    within this time window.
            noisy: Whether or not to log the output for this class.
            testnet: The network parameters to use.

        """
        self.sourceNode = sourceNode
        self.router = router
        self._waitTimeout = waitTimeout
        self._outstanding = {}
        self.log = Logger(system=self)

    def receive_message(self, datagram, connection, ban_score):
        m = Message()
        try:
            m.ParseFromString(datagram)
            sender = Node(m.sender.guid,
                          m.sender.nodeAddress.ip,
                          m.sender.nodeAddress.port,
                          m.sender.signedPublicKey,
                          None if not m.sender.HasField("relayAddress") else
                          (m.sender.relayAddress.ip, m.sender.relayAddress.port),
                          m.sender.natType,
                          m.sender.vendor)
        except Exception:
            # If message isn't formatted property then ignore
            self.log.warning("received unknown message from %s, ignoring" % str(connection.dest_addr))
            return False

        if m.testnet != self.multiplexer.testnet:
            self.log.warning("received message from %s with incorrect network parameters." %
                             str(connection.dest_addr))
            connection.shutdown()
            return False

        if m.protoVer < PROTOCOL_VERSION:
            self.log.warning("received message from %s with incompatible protocol version." %
                             str(connection.dest_addr))
            connection.shutdown()
            return False

        # Check that the GUID is valid. If not, ignore
        if self.router.isNewNode(sender):
            try:
                pubkey = m.sender.signedPublicKey[len(m.sender.signedPublicKey) - 32:]
                verify_key = nacl.signing.VerifyKey(pubkey)
                verify_key.verify(m.sender.signedPublicKey)
                h = nacl.hash.sha512(m.sender.signedPublicKey)
                pow_hash = h[64:128]
                if int(pow_hash[:6], 16) >= 50 or hexlify(m.sender.guid) != h[:40]:
                    raise Exception('Invalid GUID')

            except Exception:
                self.log.warning("received message from sender with invalid GUID, ignoring")
                connection.shutdown()
                return False

        if m.sender.vendor:
            self.db.VendorStore().save_vendor(m.sender.guid.encode("hex"), m.sender.SerializeToString())

        msgID = m.messageID
        if m.command == NOT_FOUND:
            data = None
        else:
            data = tuple(m.arguments)
        if msgID in self._outstanding:
            self._acceptResponse(msgID, data, sender)
        elif m.command != NOT_FOUND:
            #ban_score.process_message(m)
            self._acceptRequest(msgID, str(Command.Name(m.command)).lower(), data, sender, connection)

    def _acceptResponse(self, msgID, data, sender):
        if data is not None:
            msgargs = (b64encode(msgID), sender)
            self.log.debug("received response for message id %s from %s" % msgargs)
        else:
            self.log.warning("received 404 error response from %s" % sender)
        d = self._outstanding[msgID][0]
        if self._outstanding[msgID][2].active():
            self._outstanding[msgID][2].cancel()
        d.callback((True, data))
        del self._outstanding[msgID]

    def _acceptRequest(self, msgID, funcname, args, sender, connection):
        self.log.debug("received request from %s, command %s" % (sender, funcname.upper()))
        f = getattr(self, "rpc_%s" % funcname, None)
        if f is None or not callable(f):
            msgargs = (self.__class__.__name__, funcname)
            self.log.error("%s has no callable method rpc_%s; ignoring request" % msgargs)
            return False
        if funcname == "hole_punch":
            f(sender, *args)
        else:
            d = defer.maybeDeferred(f, sender, *args)
            d.addCallback(self._sendResponse, funcname, msgID, sender, connection)
            d.addErrback(self._sendResponse, "bad_request", msgID, sender, connection)

    def _sendResponse(self, response, funcname, msgID, sender, connection):
        self.log.debug("sending response for msg id %s to %s" % (b64encode(msgID), sender))
        m = Message()
        m.messageID = msgID
        m.sender.MergeFrom(self.sourceNode.getProto())
        m.protoVer = PROTOCOL_VERSION
        m.testnet = self.multiplexer.testnet
        if response is None:
            m.command = NOT_FOUND
        else:
            m.command = Command.Value(funcname.upper())
            if not isinstance(response, list):
                response = [response]
            for arg in response:
                m.arguments.append(str(arg))
        connection.send_message(m.SerializeToString())

    def timeout(self, node):
        """
        This timeout is called by the txrudp connection handler. We will run through the
        outstanding messages and callback false on any waiting on this IP address.
        """
        address = (node.ip, node.port)
        for msgID, val in self._outstanding.items():
            if address == val[1]:
                val[0].callback((False, None))
                del self._outstanding[msgID]

        if node.id is not None:
            self.router.removeContact(node)
        try:
            self.multiplexer[address].shutdown()
        except Exception:
            pass

    def rpc_hole_punch(self, sender, ip, port, relay="False"):
        """
        A method for handling an incoming HOLE_PUNCH message. Relay the message
        to the correct node if it's not for us. Otherwise send a datagram to allow
        the other node to punch through our NAT.
        """
        if relay == "True":
            self.hole_punch(Node(None, ip, int(port), nat_type=FULL_CONE), sender.ip, sender.port)
        else:
            self.log.debug("punching through NAT for %s:%s" % (ip, port))
            # pylint: disable=W0612
            for i in range(20):
                self.multiplexer.send_datagram("", (ip, int(port)))

    def __getattr__(self, name):
        if name.startswith("_") or name.startswith("rpc_"):
            return object.__getattr__(self, name)

        try:
            return object.__getattr__(self, name)
        except AttributeError:
            pass

        def func(node, *args):
            msgID = sha1(str(random.getrandbits(255))).digest()

            m = Message()
            m.messageID = msgID
            m.sender.MergeFrom(self.sourceNode.getProto())
            m.command = Command.Value(name.upper())
            m.protoVer = PROTOCOL_VERSION
            for arg in args:
                m.arguments.append(str(arg))
            m.testnet = self.multiplexer.testnet
            data = m.SerializeToString()

            address = (node.ip, node.port)
            relay_addr = None
            if node.nat_type == SYMMETRIC:
                relay_addr = node.relay_node

            d = defer.Deferred()
            if m.command != HOLE_PUNCH:
                timeout = timeout = reactor.callLater(self._waitTimeout, self.timeout, node)
                self._outstanding[msgID] = [d, address, timeout]

            self.multiplexer.send_message(data, address, relay_addr)
            self.log.debug("calling remote function %s on %s (msgid %s)" % (name, address, b64encode(msgID)))

            if self.multiplexer[address].state != State.CONNECTED and node.nat_type == RESTRICTED:
                self.hole_punch(Node(digest("null"), node.relay_node[0], node.relay_node[1], nat_type=FULL_CONE),
                                address[0], address[1], "True")
                self.log.debug("sending hole punch message to %s" % address[0] + ":" + str(address[1]))

            return d

        return func
<commit_msg>Increase wait timeout<commit_after>"""
Copyright (c) 2014 Brian Muller
Copyright (c) 2015 OpenBazaar
"""

import abc
import random
import nacl.signing
import nacl.encoding
import nacl.hash
from base64 import b64encode
from binascii import hexlify
from constants import PROTOCOL_VERSION
from dht.node import Node
from dht.utils import digest
from hashlib import sha1
from log import Logger
from protos.message import Message, Command, NOT_FOUND, HOLE_PUNCH
from protos.objects import FULL_CONE, RESTRICTED, SYMMETRIC
from twisted.internet import defer, reactor
from txrudp.connection import State

class RPCProtocol:
    """
    This is an abstract class for processing and sending rpc messages.
    A class that implements the `MessageProcessor` interface probably should
    extend this as it does most of the work of keeping track of messages.
    """
    __metaclass__ = abc.ABCMeta

    def __init__(self, sourceNode, router, waitTimeout=4):
        """
        Args:
            proto: A protobuf `Node` object containing info about this node.
            router: A `RoutingTable` object from dht.routing. Implies a `network.Server` object
                    must be started first.
            waitTimeout: Consider it a connetion failure if no response
                    within this time window.
            noisy: Whether or not to log the output for this class.
            testnet: The network parameters to use.

        """
        self.sourceNode = sourceNode
        self.router = router
        self._waitTimeout = waitTimeout
        self._outstanding = {}
        self.log = Logger(system=self)

    def receive_message(self, datagram, connection, ban_score):
        m = Message()
        try:
            m.ParseFromString(datagram)
            sender = Node(m.sender.guid,
                          m.sender.nodeAddress.ip,
                          m.sender.nodeAddress.port,
                          m.sender.signedPublicKey,
                          None if not m.sender.HasField("relayAddress") else
                          (m.sender.relayAddress.ip, m.sender.relayAddress.port),
                          m.sender.natType,
                          m.sender.vendor)
        except Exception:
            # If message isn't formatted property then ignore
            self.log.warning("received unknown message from %s, ignoring" % str(connection.dest_addr))
            return False

        if m.testnet != self.multiplexer.testnet:
            self.log.warning("received message from %s with incorrect network parameters." %
                             str(connection.dest_addr))
            connection.shutdown()
            return False

        if m.protoVer < PROTOCOL_VERSION:
            self.log.warning("received message from %s with incompatible protocol version." %
                             str(connection.dest_addr))
            connection.shutdown()
            return False

        # Check that the GUID is valid. If not, ignore
        if self.router.isNewNode(sender):
            try:
                pubkey = m.sender.signedPublicKey[len(m.sender.signedPublicKey) - 32:]
                verify_key = nacl.signing.VerifyKey(pubkey)
                verify_key.verify(m.sender.signedPublicKey)
                h = nacl.hash.sha512(m.sender.signedPublicKey)
                pow_hash = h[64:128]
                if int(pow_hash[:6], 16) >= 50 or hexlify(m.sender.guid) != h[:40]:
                    raise Exception('Invalid GUID')

            except Exception:
                self.log.warning("received message from sender with invalid GUID, ignoring")
                connection.shutdown()
                return False

        if m.sender.vendor:
            self.db.VendorStore().save_vendor(m.sender.guid.encode("hex"), m.sender.SerializeToString())

        msgID = m.messageID
        if m.command == NOT_FOUND:
            data = None
        else:
            data = tuple(m.arguments)
        if msgID in self._outstanding:
            self._acceptResponse(msgID, data, sender)
        elif m.command != NOT_FOUND:
            #ban_score.process_message(m)
            self._acceptRequest(msgID, str(Command.Name(m.command)).lower(), data, sender, connection)

    def _acceptResponse(self, msgID, data, sender):
        if data is not None:
            msgargs = (b64encode(msgID), sender)
            self.log.debug("received response for message id %s from %s" % msgargs)
        else:
            self.log.warning("received 404 error response from %s" % sender)
        d = self._outstanding[msgID][0]
        if self._outstanding[msgID][2].active():
            self._outstanding[msgID][2].cancel()
        d.callback((True, data))
        del self._outstanding[msgID]

    def _acceptRequest(self, msgID, funcname, args, sender, connection):
        self.log.debug("received request from %s, command %s" % (sender, funcname.upper()))
        f = getattr(self, "rpc_%s" % funcname, None)
        if f is None or not callable(f):
            msgargs = (self.__class__.__name__, funcname)
            self.log.error("%s has no callable method rpc_%s; ignoring request" % msgargs)
            return False
        if funcname == "hole_punch":
            f(sender, *args)
        else:
            d = defer.maybeDeferred(f, sender, *args)
            d.addCallback(self._sendResponse, funcname, msgID, sender, connection)
            d.addErrback(self._sendResponse, "bad_request", msgID, sender, connection)

    def _sendResponse(self, response, funcname, msgID, sender, connection):
        self.log.debug("sending response for msg id %s to %s" % (b64encode(msgID), sender))
        m = Message()
        m.messageID = msgID
        m.sender.MergeFrom(self.sourceNode.getProto())
        m.protoVer = PROTOCOL_VERSION
        m.testnet = self.multiplexer.testnet
        if response is None:
            m.command = NOT_FOUND
        else:
            m.command = Command.Value(funcname.upper())
            if not isinstance(response, list):
                response = [response]
            for arg in response:
                m.arguments.append(str(arg))
        connection.send_message(m.SerializeToString())

    def timeout(self, node):
        """
        This timeout is called by the txrudp connection handler. We will run through the
        outstanding messages and callback false on any waiting on this IP address.
        """
        address = (node.ip, node.port)
        for msgID, val in self._outstanding.items():
            if address == val[1]:
                val[0].callback((False, None))
                del self._outstanding[msgID]

        if node.id is not None:
            self.router.removeContact(node)
        try:
            self.multiplexer[address].shutdown()
        except Exception:
            pass

    def rpc_hole_punch(self, sender, ip, port, relay="False"):
        """
        A method for handling an incoming HOLE_PUNCH message. Relay the message
        to the correct node if it's not for us. Otherwise send a datagram to allow
        the other node to punch through our NAT.
        """
        if relay == "True":
            self.hole_punch(Node(None, ip, int(port), nat_type=FULL_CONE), sender.ip, sender.port)
        else:
            self.log.debug("punching through NAT for %s:%s" % (ip, port))
            # pylint: disable=W0612
            for i in range(20):
                self.multiplexer.send_datagram("", (ip, int(port)))

    def __getattr__(self, name):
        if name.startswith("_") or name.startswith("rpc_"):
            return object.__getattr__(self, name)

        try:
            return object.__getattr__(self, name)
        except AttributeError:
            pass

        def func(node, *args):
            msgID = sha1(str(random.getrandbits(255))).digest()

            m = Message()
            m.messageID = msgID
            m.sender.MergeFrom(self.sourceNode.getProto())
            m.command = Command.Value(name.upper())
            m.protoVer = PROTOCOL_VERSION
            for arg in args:
                m.arguments.append(str(arg))
            m.testnet = self.multiplexer.testnet
            data = m.SerializeToString()

            address = (node.ip, node.port)
            relay_addr = None
            if node.nat_type == SYMMETRIC:
                relay_addr = node.relay_node

            d = defer.Deferred()
            if m.command != HOLE_PUNCH:
                timeout = timeout = reactor.callLater(self._waitTimeout, self.timeout, node)
                self._outstanding[msgID] = [d, address, timeout]

            self.multiplexer.send_message(data, address, relay_addr)
            self.log.debug("calling remote function %s on %s (msgid %s)" % (name, address, b64encode(msgID)))

            if self.multiplexer[address].state != State.CONNECTED and node.nat_type == RESTRICTED:
                self.hole_punch(Node(digest("null"), node.relay_node[0], node.relay_node[1], nat_type=FULL_CONE),
                                address[0], address[1], "True")
                self.log.debug("sending hole punch message to %s" % address[0] + ":" + str(address[1]))

            return d

        return func
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright 2015 Criteo. All rights reserved.
#
# The contents of this file are licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with the
# License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.

import argparse
from ciscoconfparse import CiscoConfParse
import sys
import re
import os

def cli_parser():
    parser = argparse.ArgumentParser(
        description=('Generating configuration commands by finding differences'
                     ' between two Cisco IOS style configuration files'))
    parser.add_argument('origin', metavar='origin',
                        type=str, help='Origin configuration file')
    parser.add_argument('target', metavar='target',
                        type=str, help='Target configuration file')
    parser.add_argument('no_command', metavar='no_command',
                        type=str, help='Negation command keyword')
    return parser.parse_args()

def clean_file(file, no_command):
    with open(file) as file_opened:
        list = file_opened.readlines()

    list_clean = []

    if no_command == 'undo':
        line_parent_sharp = False
        for line in list:
            line_clean = line.rstrip(' \t\r\n\0')
            if line_parent_sharp is False:
                if line_clean == '#' or line_clean == '!':
                    line_parent_sharp = True
                elif re.match('^!.*$', line_clean) or re.match(
                  '^#.*$', line_clean):
                    pass
                elif line_clean == 'return' or line_clean == 'exit':
                    pass
                elif line_clean == '':
                    pass
                else:
                    list_clean.append(line_clean)
            else:
                if line_clean == '#' or line_clean == '!':
                    line_parent_sharp = True
                elif re.match('^!.*$', line_clean) or re.match(
                  '^#.*$', line_clean):
                    pass
                elif line_clean == 'return' or line_clean == 'exit':
                    pass
                elif re.match('^\s.*$', line_clean):
                    line_without_space = re.match('^\s(.*)$', line_clean)
                    list_clean.append(line_without_space.group(1))
                elif line_clean == '':
                    pass
                else:
                    list_clean.append(line_clean)
                    line_parent_sharp = False
    else:
        for line in list:
            if 'Current Configuration ...' in line:
                pass
            else:
                list_clean.append(line.rstrip(' \t\r\n\0'))

    return list_clean

def compare_children_prefix_no(obj_origin, obj_target, no_command):
    command_list = [obj_origin.text]
    for child in obj_origin.children:
        match = obj_target.find_objects_w_parents(
            obj_origin.text,
            '^'+re.escape(child.text)+'$')
        if len(match) == 0 and not re.match(
          "\s*" + no_command + "\s.*", child.text):
            line_text_without_indent = re.match("(\s*)(.*)", child.text)
            command_list.append(
                line_text_without_indent.group(1) +
                no_command +
                " " +
                line_text_without_indent.group(2))
        elif len(match) == 0 and re.match(
          "\s*" + no_command + "\s.*", child.text):
            line_text_without_no = re.match(
              "(\s*)" + no_command + "\s(.*)", child.text)
            command_list.append(
                line_text_without_no.group(1) +
                line_text_without_no.group(2))
        elif len(child.children) > 0:
            for result_recursive in compare_children_prefix_no(
                    child, obj_target, no_command):
                command_list.append(result_recursive)
    if len(command_list) > 1:
        return command_list
    else:
        return []

def compare_children(obj_origin, obj_target):
    command_list = [obj_origin.text]
    for child in obj_origin.children:
        match = obj_target.find_objects_w_parents(
            obj_origin.text,
            '^'+re.escape(child.text)+'$')
        if len(match) == 0:
            command_list.append(child.text)
        elif len(child.children) > 0:
            for result_recursive in compare_children(child, obj_target):
                command_list.append(result_recursive)
    if len(command_list) > 1:
        return command_list
    else:
        return []

def merge_commands(list1, list2):
    command_list = []
    for line_list1 in list1:
        if isinstance(line_list1, list) is True:
            for line_children_list1 in line_list1:
                command_list.append(line_children_list1)
            for line_list2 in list2:
                if isinstance(line_list2, list) is True:
                    if line_list1[0] == line_list2[0]:
                        for line_children_list2 in line_list2[1:]:
                            command_list.append(line_children_list2)
                        list2.remove(line_list2)
        else:
            command_list.append(line_list1)
    for line_list2 in list2:
        if isinstance(line_list2, list) is True:
            for line_children_list2 in line_list2:
                command_list.append(line_children_list2)
        else:
            command_list.append(line_list2)
    return command_list

def netcompare(origin, target, no_command):
    origin_file = CiscoConfParse(origin, syntax='ios', factory=False)
    target_file = CiscoConfParse(target, syntax='ios', factory=False)
    diff_add_no_commands = []
    for line in origin_file.objs:
        if line.parent.linenum == line.linenum and line.is_config_line is True:
            parent_match = target_file.find_objects(
                '^'+re.escape(line.text)+'$',
                exactmatch=True, ignore_ws=False)
            if len(parent_match) == 0 and not re.match(
              no_command + "\s.*", line.text):
                diff_add_no_commands.append(no_command + ' ' + line.text)
            elif len(parent_match) == 0 and re.match(
              no_command + "\s.*", line.text):
                line_text_without_no = re.match(
                  no_command + "\s(.*)", line.text)
                diff_add_no_commands.append(line_text_without_no.group(1))
            if len(line.children) > 0 and len(parent_match) != 0:
                result_comparison = compare_children_prefix_no(
                    line, target_file, no_command)
                if len(result_comparison) > 0:
                    diff_add_no_commands.append(result_comparison)
    diff_add_commands = []
    for line in target_file.objs:
        if line.parent.linenum == line.linenum and line.is_config_line is True:
            parent_match = origin_file.find_objects(
                '^'+re.escape(line.text)+'$',
                exactmatch=True, ignore_ws=False)
            if len(parent_match) == 0:
                parent_with_children = target_file.find_all_children(
                    '^'+re.escape(line.text)+'$')
                for line_in_parent_with_children in parent_with_children:
                    diff_add_commands.append(line_in_parent_with_children)
            if len(line.children) > 0 and len(parent_match) != 0:
                result_comparison = compare_children(line, origin_file)
                if len(result_comparison) > 0:
                    diff_add_commands.append(
                        compare_children(line, origin_file))
    return merge_commands(diff_add_no_commands, diff_add_commands)

def main():
    args = cli_parser()

    origin_list = clean_file(args.origin, args.no_command)
    target_list = clean_file(args.target, args.no_command)

    display_commands = netcompare(origin_list, target_list, args.no_command)
    for line in display_commands:
        print line

if __name__ == '__main__':
    main()
<commit_msg>BugFix when add a children with children<commit_after>#!/usr/bin/env python
# Copyright 2015 Criteo. All rights reserved.
#
# The contents of this file are licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with the
# License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.

import argparse
from ciscoconfparse import CiscoConfParse
import sys
import re
import os

def cli_parser():
    parser = argparse.ArgumentParser(
        description=('Generating configuration commands by finding differences'
                     ' between two Cisco IOS style configuration files'))
    parser.add_argument('origin', metavar='origin',
                        type=str, help='Origin configuration file')
    parser.add_argument('target', metavar='target',
                        type=str, help='Target configuration file')
    parser.add_argument('no_command', metavar='no_command',
                        type=str, help='Negation command keyword')
    return parser.parse_args()

def clean_file(file, no_command):
    with open(file) as file_opened:
        list = file_opened.readlines()

    list_clean = []

    if no_command == 'undo':
        line_parent_sharp = False
        for line in list:
            line_clean = line.rstrip(' \t\r\n\0')
            if line_parent_sharp is False:
                if line_clean == '#' or line_clean == '!':
                    line_parent_sharp = True
                elif re.match('^!.*$', line_clean) or re.match(
                  '^#.*$', line_clean):
                    pass
                elif line_clean == 'return' or line_clean == 'exit':
                    pass
                elif line_clean == '':
                    pass
                else:
                    list_clean.append(line_clean)
            else:
                if line_clean == '#' or line_clean == '!':
                    line_parent_sharp = True
                elif re.match('^!.*$', line_clean) or re.match(
                  '^#.*$', line_clean):
                    pass
                elif line_clean == 'return' or line_clean == 'exit':
                    pass
                elif re.match('^\s.*$', line_clean):
                    line_without_space = re.match('^\s(.*)$', line_clean)
                    list_clean.append(line_without_space.group(1))
                elif line_clean == '':
                    pass
                else:
                    list_clean.append(line_clean)
                    line_parent_sharp = False
    else:
        for line in list:
            if 'Current Configuration ...' in line:
                pass
            else:
                list_clean.append(line.rstrip(' \t\r\n\0'))

    return list_clean

def compare_children_prefix_no(obj_origin, obj_target, no_command):
    command_list = [obj_origin.text]
    for child in obj_origin.children:
        match = obj_target.find_objects_w_parents(
            obj_origin.text,
            '^'+re.escape(child.text)+'$')
        if len(match) == 0 and not re.match(
          "\s*" + no_command + "\s.*", child.text):
            line_text_without_indent = re.match("(\s*)(.*)", child.text)
            command_list.append(
                line_text_without_indent.group(1) +
                no_command +
                " " +
                line_text_without_indent.group(2))
        elif len(match) == 0 and re.match(
          "\s*" + no_command + "\s.*", child.text):
            line_text_without_no = re.match(
              "(\s*)" + no_command + "\s(.*)", child.text)
            command_list.append(
                line_text_without_no.group(1) +
                line_text_without_no.group(2))
        elif len(child.children) > 0:
            for result_recursive in compare_children_prefix_no(
                    child, obj_target, no_command):
                command_list.append(result_recursive)
    if len(command_list) > 1:
        return command_list
    else:
        return []

def compare_children(obj_origin, obj_target):
    command_list = [obj_origin.text]
    for child in obj_origin.children:
        match = obj_target.find_objects_w_parents(
            obj_origin.text,
            '^'+re.escape(child.text)+'$')
        if len(child.children) > 0:
            for result_recursive in compare_children(child, obj_target):
                command_list.append(result_recursive)
        elif len(match) == 0:
            command_list.append(child.text)
    if len(command_list) > 1:
        return command_list
    else:
        return []

def merge_commands(list1, list2):
    command_list = []
    for line_list1 in list1:
        if isinstance(line_list1, list) is True:
            for line_children_list1 in line_list1:
                command_list.append(line_children_list1)
            for line_list2 in list2:
                if isinstance(line_list2, list) is True:
                    if line_list1[0] == line_list2[0]:
                        for line_children_list2 in line_list2[1:]:
                            command_list.append(line_children_list2)
                        list2.remove(line_list2)
        else:
            command_list.append(line_list1)
    for line_list2 in list2:
        if isinstance(line_list2, list) is True:
            for line_children_list2 in line_list2:
                command_list.append(line_children_list2)
        else:
            command_list.append(line_list2)
    return command_list

def netcompare(origin, target, no_command):
    origin_file = CiscoConfParse(origin, syntax='ios', factory=False)
    target_file = CiscoConfParse(target, syntax='ios', factory=False)
    diff_add_no_commands = []
    for line in origin_file.objs:
        if line.parent.linenum == line.linenum and line.is_config_line is True:
            parent_match = target_file.find_objects(
                '^'+re.escape(line.text)+'$',
                exactmatch=True, ignore_ws=False)
            if len(parent_match) == 0 and not re.match(
              no_command + "\s.*", line.text):
                diff_add_no_commands.append(no_command + ' ' + line.text)
            elif len(parent_match) == 0 and re.match(
              no_command + "\s.*", line.text):
                line_text_without_no = re.match(
                  no_command + "\s(.*)", line.text)
                diff_add_no_commands.append(line_text_without_no.group(1))
            if len(line.children) > 0 and len(parent_match) != 0:
                result_comparison = compare_children_prefix_no(
                    line, target_file, no_command)
                if len(result_comparison) > 0:
                    diff_add_no_commands.append(result_comparison)
    diff_add_commands = []
    for line in target_file.objs:
        if line.parent.linenum == line.linenum and line.is_config_line is True:
            parent_match = origin_file.find_objects(
                '^'+re.escape(line.text)+'$',
                exactmatch=True, ignore_ws=False)
            if len(parent_match) == 0:
                parent_with_children = target_file.find_all_children(
                    '^'+re.escape(line.text)+'$')
                for line_in_parent_with_children in parent_with_children:
                    diff_add_commands.append(line_in_parent_with_children)
            if len(line.children) > 0 and len(parent_match) != 0:
                result_comparison = compare_children(line, origin_file)
                if len(result_comparison) > 0:
                    diff_add_commands.append( result_comparison )
    return merge_commands(diff_add_no_commands, diff_add_commands)

def main():
    args = cli_parser()

    origin_list = clean_file(args.origin, args.no_command)
    target_list = clean_file(args.target, args.no_command)

    display_commands = netcompare(origin_list, target_list, args.no_command)
    for line in display_commands:
        print line

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
Topological sorting for Directed Acyclic Graph (DAG) is a linear ordering of vertices such that for every directed edge uv, vertex u comes before v in the ordering. Topological Sorting for a graph is not possible if the graph is not a DAG.

Time Complexity: The algorithm is simply DFS with an extra stack. So time complexity is same as DFS which is O(V+E).
"""

from collections import defaultdict

class Graph:
	def __init__(self, vertices):
		self.graph = defaultdict(list)
		self.V = vertices

	def add_edge(self, u, v):
		self.graph[u].append(v)

	def _topological_sort_util(self, vertex, visited, stack):
		visited[vertex] = True

		for i in self.graph[vertex]:
			if visited[i] is False:
				self._topological_sort_util(i, visited, stack)

		stack.insert(0, vertex)

	def topological_sort(self):
		visited = [False] * self.V
		stack = []

		print self.graph.items(), self.V
		for each in self.graph.keys():
			if visited[each] is False:
				self._topological_sort_util(each, visited, stack)

		return stack

g = Graph(6)
g.add_edge(5, 2)
g.add_edge(5, 0)
g.add_edge(4, 0)
g.add_edge(4, 1)
g.add_edge(2, 3)
g.add_edge(3, 1)

print g.topological_sort()<commit_msg>topological sort<commit_after>"""
Topological sorting for Directed Acyclic Graph (DAG) is a linear ordering of vertices such that for every directed edge uv, vertex u comes before v in the ordering. Topological Sorting for a graph is not possible if the graph is not a DAG.

Time Complexity: The algorithm is simply DFS with an extra stack. So time complexity is same as DFS which is O(V+E).
"""

from collections import defaultdict

class Graph:
	def __init__(self, vertices):
		self.graph = defaultdict(set)
		self.V = vertices

	def add_edge(self, u, v):
		self.graph[u].add(v)

	def _topological_sort_util(self, vertex, visited, stack):
		visited[vertex] = True
		print vertex
		print '\n'
		for i in self.graph[vertex]:
			if visited[i] is False:
				self._topological_sort_util(i, visited, stack)

		print 'here' + str(vertex)
		stack.insert(0, vertex)

	def topological_sort(self):
		visited = [False] * self.V
		stack = []

		print self.graph.items(), self.V
		for each in self.graph.keys():
			if visited[each] is False:
				self._topological_sort_util(each, visited, stack)

		return stack

g = Graph(6)
g.add_edge(5, 2)
g.add_edge(5, 0)
g.add_edge(4, 0)
g.add_edge(4, 1)
g.add_edge(2, 3)
g.add_edge(3, 1)

print g.topological_sort()<|endoftext|>


<commit_before># Time:  O(n * h)
# Space: O(n * h)

# Given a binary tree, return all duplicate subtrees.
# For each kind of duplicate subtrees, you only need to return the root node of any one of them.
#
# Two trees are duplicate if they have the same structure with same node values.
#
# Example 1:
#         1
#        / \
#       2   3
#      /   / \
#     4   2   4
#        /
#       4
# The following are two duplicate subtrees:
#       2
#      /
#     4
# and
#     4
# Therefore, you need to return above trees' root in the form of a list.

# Definition for a binary tree node.
# class TreeNode(object):
#     def __init__(self, x):
#         self.val = x
#         self.left = None
#         self.right = None

class Solution(object):
    def findDuplicateSubtrees(self, root):
        """
        :type root: TreeNode
        :rtype: List[TreeNode]
        """
        def postOrderTraversal(node, lookup, result):
            if not node:
                return ""
            s = "(" + postOrderTraversal(node.left, lookup, result) + \
                str(node.val) + \
                postOrderTraversal(node.right, lookup, result) + \
                ")"
            if lookup[s] == 1:
                result.append(node)
            lookup[s] += 1
            return s

        lookup = collections.defaultdict(int)
        result = []
        postOrderTraversal(root, lookup, result)
        return result
<commit_msg>Update find-duplicate-subtrees.py<commit_after># Time:  O(n)
# Space: O(n)

# Given a binary tree, return all duplicate subtrees.
# For each kind of duplicate subtrees, you only need to return the root node of any one of them.
#
# Two trees are duplicate if they have the same structure with same node values.
#
# Example 1:
#         1
#        / \
#       2   3
#      /   / \
#     4   2   4
#        /
#       4
# The following are two duplicate subtrees:
#       2
#      /
#     4
# and
#     4
# Therefore, you need to return above trees' root in the form of a list.

# Definition for a binary tree node.
# class TreeNode(object):
#     def __init__(self, x):
#         self.val = x
#         self.left = None
#         self.right = None

class Solution(object):
    def findDuplicateSubtrees(self, root):
        """
        :type root: TreeNode
        :rtype: List[TreeNode]
        """
        def getid(root, result):
            if root:
                id = lookup[root.val, getid(root.left, result), getid(root.right, result)]
                result[id].append(root)
                return id
        result = collections.defaultdict(list)
        lookup = collections.defaultdict()
        lookup.default_factory = lookup.__len__
        getid(root, result)
        return [roots[0] for roots in result.values() if len(roots) > 1]

# Time:  O(n * h)
# Space: O(n * h)
class Solution2(object):
    def findDuplicateSubtrees(self, root):
        """
        :type root: TreeNode
        :rtype: List[TreeNode]
        """
        def postOrderTraversal(node, lookup, result):
            if not node:
                return ""
            s = "(" + postOrderTraversal(node.left, lookup, result) + \
                str(node.val) + \
                postOrderTraversal(node.right, lookup, result) + \
                ")"
            if lookup[s] == 1:
                result.append(node)
            lookup[s] += 1
            return s

        lookup = collections.defaultdict(int)
        result = []
        postOrderTraversal(root, lookup, result)
        return result
<|endoftext|>


<commit_before>import os
import sys
import json

from PySide import QtGui, QtCore

from kraken.ui import images_rc
from kraken.ui.kraken_menu import KrakenMenu
from kraken.ui.kraken_ui import KrakenUI
from kraken.ui.preferences import Preferences
from kraken.plugins.logger import OutputLog

class KrakenWindow(QtGui.QMainWindow):
    """Main Kraken Window that loads the UI."""

    def __init__(self, parent=None):
        super(KrakenWindow, self).__init__(parent)
        self.setObjectName('KrakenMainWindow')
        self.setWindowTitle('Kraken Editor')
        self.setWindowIcon(QtGui.QIcon(':/images/Kraken_Icon.png'))
        self.setAttribute(QtCore.Qt.WA_DeleteOnClose)

        # Set system output to write to output log object
        self.outputLog = OutputLog()
        sys.stdout = self.outputLog

        QtCore.QCoreApplication.setOrganizationName("Kraken")
        QtCore.QCoreApplication.setApplicationName("Kraken Editor")
        self.settings = QtCore.QSettings("Kraken", "Kraken Editor")
        self.preferences = Preferences()

        cssPath = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                               'kraken_ui.css')

        styleData = ''
        with open(cssPath) as cssFile:
            styleData = cssFile.read()

        self.setStyleSheet(styleData)

        self.createLayout()
        self.createConnections()

    def createLayout(self):

        # Setup Status Bar
        statusBar = self.statusBar()
        self.outputLogButton = QtGui.QPushButton('Log', self)
        self.outputLogButton.setObjectName('outputLog_button')
        statusBar.insertPermanentWidget(0, self.outputLogButton)

        mainWidget = QtGui.QWidget()

        # Main Layout
        self.mainLayout = QtGui.QVBoxLayout()
        self.mainLayout.setContentsMargins(0, 0, 0, 0)

        self.krakenUI = KrakenUI(self)
        self.krakenMenu = KrakenMenu(self)
        self.krakenUI.graphViewWidget.setGuideRigName('MyRig')
        self.krakenMenu.updateRigNameLabel()

        self.mainLayout.addWidget(self.krakenMenu)
        self.mainLayout.addWidget(self.krakenUI, 1)

        mainWidget.setLayout(self.mainLayout)
        self.setCentralWidget(mainWidget)

        self.statusBar().showMessage('Ready', 2000)

        self.setGeometry(250, 150, 800, 475)
        self.center()

        self.readSettings()

    def createConnections(self):
        self.outputLogButton.clicked.connect(self.showOutputLog)
        self.krakenMenu.newAction.triggered.connect(self.krakenUI.graphViewWidget.newRigPreset)
        self.krakenUI.graphViewWidget.rigLoaded.connect(self.krakenMenu.buildRecentFilesMenu)
        self.krakenUI.graphViewWidget.rigNameChanged.connect(self.krakenMenu.updateRigNameLabel)

    def getKrakenUI(self):
        return self.krakenUI

    def center(self):
        qr = self.frameGeometry()
        cp = QtGui.QDesktopWidget().availableGeometry().center()
        qr.moveCenter(cp)
        self.move(qr.topLeft())

    # =========
    # Settings
    # =========
    def getSettings(self):
        return self.settings

    def writeSettings(self):
        self.settings.beginGroup('MainWindow')
        self.settings.setValue('size', self.size())
        self.settings.setValue('pos', self.pos())
        self.settings.setValue('preferences', json.dumps(self.preferences.getPreferences()))
        self.settings.endGroup()
        self.krakenMenu.writeSettings(self.settings)
        self.krakenUI.writeSettings(self.settings)

    def readSettings(self):
        self.settings.beginGroup('MainWindow')
        self.resize(self.settings.value('size', self.size()))
        self.move(self.settings.value('pos', self.pos()))
        self.preferences.loadPreferences(json.loads(self.settings.value('preferences', '{}')))
        self.settings.endGroup()

        self.krakenMenu.readSettings(self.settings)
        self.krakenUI.readSettings(self.settings)

    # =======
    # Events
    # =======
    def closeEvent(self, event):

        msgBox = QtGui.QMessageBox(self)
        msgBox.setObjectName('SaveMessageBox')
        msgBox.setWindowTitle("Kraken Editor")
        msgBox.setText("You are closing Kraken.")
        msgBox.setInformativeText("Do you want to save your changes?")
        msgBox.setStandardButtons(QtGui.QMessageBox.Save | QtGui.QMessageBox.Discard | QtGui.QMessageBox.Cancel)
        msgBox.setDefaultButton(QtGui.QMessageBox.Save)

        ret = msgBox.exec_()

        if ret == QtGui.QMessageBox.Cancel:
            event.ignore()
            return

        elif ret == QtGui.QMessageBox.Save:
            self.kraken_ui.graphViewWidget.saveRigPreset()

            self.statusBar().showMessage('Closing')

        self.writeSettings()

    def showOutputLog(self):
        outputDialog = OutputLogDialog(self)
        outputDialog.show()
        outputDialog.setText(self.outputLog.getLog())
        outputDialog.textWidget.moveCursor(QtGui.QTextCursor.End)

class OutputLogDialog(QtGui.QDialog):
    """Output Dialog"""

    def __init__(self, parent=None):
        super(OutputLogDialog, self).__init__(parent)
        self.setObjectName('outputLog')
        self.resize(700, 300)
        self.setWindowTitle('Kraken Output Log')

        self.createLayout()
        self.createConnections()

    def createLayout(self):
        """Sets up the layout for the dialog."""

        self.textWidget = QtGui.QTextEdit()
        self.textWidget.setLineWrapMode(QtGui.QTextEdit.NoWrap)
        self.textWidget.setReadOnly(True)
        self.textWidget.setContextMenuPolicy(QtCore.Qt.CustomContextMenu)

        self.outputLogLayout = QtGui.QVBoxLayout(self)
        self.outputLogLayout.addWidget(self.textWidget)

        self.setLayout(self.outputLogLayout)

    def createConnections(self):
        """Connects widgets to methods or other signals."""

        self.textWidget.customContextMenuRequested.connect(self.createContextMenu)

    def setText(self, text):
        """Sets the text of the text widget."""

        self.textWidget.setText(text)

    # =============
    # Context Menu
    # =============
    def createContextMenu(self):
        self.contextMenu = QtGui.QMenu(self)
        selectAllAction = self.contextMenu.addAction("Select All")
        copyAction = self.contextMenu.addAction("Copy")
        self.contextMenu.addSeparator()
        clearAction = self.contextMenu.addAction("Clear")

        selectAllAction.triggered.connect(self.contextSelectAll)
        copyAction.triggered.connect(self.contextCopy)
        clearAction.triggered.connect(self.contextClear)

        self.contextMenu.exec_(QtGui.QCursor.pos())

    def contextSelectAll(self):
        self.textWidget.selectAll()

    def contextCopy(self):
        self.textWidget.copy()

    def contextClear(self):
        outputLog = self.parent().outputLog
        outputLog.clear()
        self.textWidget.clear()

def createSplash(app):
    """Creates a splash screen object to show while the Window is loading.

    Return:
    SplashScreen object.

    """

    splashPixmap = QtGui.QPixmap(':/images/KrakenUI_Splash.png')

    splash = QtGui.QSplashScreen(splashPixmap)
    splash.setMask(splashPixmap.mask())
    splash.showMessage("Loading Extensions...",
                       QtCore.Qt.AlignBottom | QtCore.Qt.AlignLeft,
                       QtCore.Qt.white)

    splash.show()

    app.processEvents()

    return splash

if __name__ == "__main__":
    app = QtGui.QApplication(sys.argv)

    splash = createSplash(app)

    window = KrakenWindow()
    window.show()

    splash.finish(window)

    sys.exit(app.exec_())
<commit_msg>Only 1 output dialog is now created.<commit_after>import os
import sys
import json

from PySide import QtGui, QtCore

from kraken.ui import images_rc
from kraken.ui.kraken_menu import KrakenMenu
from kraken.ui.kraken_ui import KrakenUI
from kraken.ui.preferences import Preferences
from kraken.plugins.logger import OutputLog

class KrakenWindow(QtGui.QMainWindow):
    """Main Kraken Window that loads the UI."""

    def __init__(self, parent=None):
        super(KrakenWindow, self).__init__(parent)
        self.setObjectName('KrakenMainWindow')
        self.setWindowTitle('Kraken Editor')
        self.setWindowIcon(QtGui.QIcon(':/images/Kraken_Icon.png'))
        self.setAttribute(QtCore.Qt.WA_DeleteOnClose)

        # Set system output to write to output log object
        self.outputLog = OutputLog()
        sys.stdout = self.outputLog

        QtCore.QCoreApplication.setOrganizationName("Kraken")
        QtCore.QCoreApplication.setApplicationName("Kraken Editor")
        self.settings = QtCore.QSettings("Kraken", "Kraken Editor")
        self.preferences = Preferences()

        cssPath = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                               'kraken_ui.css')

        styleData = ''
        with open(cssPath) as cssFile:
            styleData = cssFile.read()

        self.setStyleSheet(styleData)

        self.createLayout()
        self.createConnections()

    def createLayout(self):

        # Setup Status Bar
        statusBar = self.statusBar()
        self.outputLogButton = QtGui.QPushButton('Log', self)
        self.outputLogButton.setObjectName('outputLog_button')
        statusBar.insertPermanentWidget(0, self.outputLogButton)

        mainWidget = QtGui.QWidget()

        # Main Layout
        self.mainLayout = QtGui.QVBoxLayout()
        self.mainLayout.setContentsMargins(0, 0, 0, 0)

        self.krakenUI = KrakenUI(self)
        self.krakenMenu = KrakenMenu(self)
        self.krakenUI.graphViewWidget.setGuideRigName('MyRig')
        self.krakenMenu.updateRigNameLabel()

        self.mainLayout.addWidget(self.krakenMenu)
        self.mainLayout.addWidget(self.krakenUI, 1)

        mainWidget.setLayout(self.mainLayout)
        self.setCentralWidget(mainWidget)

        self.statusBar().showMessage('Ready', 2000)

        self.setGeometry(250, 150, 800, 475)
        self.center()

        self.readSettings()

    def createConnections(self):
        self.outputLogButton.clicked.connect(self.showOutputLog)
        self.krakenMenu.newAction.triggered.connect(self.krakenUI.graphViewWidget.newRigPreset)
        self.krakenUI.graphViewWidget.rigLoaded.connect(self.krakenMenu.buildRecentFilesMenu)
        self.krakenUI.graphViewWidget.rigNameChanged.connect(self.krakenMenu.updateRigNameLabel)

    def getKrakenUI(self):
        return self.krakenUI

    def center(self):
        qr = self.frameGeometry()
        cp = QtGui.QDesktopWidget().availableGeometry().center()
        qr.moveCenter(cp)
        self.move(qr.topLeft())

    # =========
    # Settings
    # =========
    def getSettings(self):
        return self.settings

    def writeSettings(self):
        self.settings.beginGroup('MainWindow')
        self.settings.setValue('size', self.size())
        self.settings.setValue('pos', self.pos())
        self.settings.setValue('preferences', json.dumps(self.preferences.getPreferences()))
        self.settings.endGroup()
        self.krakenMenu.writeSettings(self.settings)
        self.krakenUI.writeSettings(self.settings)

    def readSettings(self):
        self.settings.beginGroup('MainWindow')
        self.resize(self.settings.value('size', self.size()))
        self.move(self.settings.value('pos', self.pos()))
        self.preferences.loadPreferences(json.loads(self.settings.value('preferences', '{}')))
        self.settings.endGroup()

        self.krakenMenu.readSettings(self.settings)
        self.krakenUI.readSettings(self.settings)

    # =======
    # Events
    # =======
    def closeEvent(self, event):

        msgBox = QtGui.QMessageBox(self)
        msgBox.setObjectName('SaveMessageBox')
        msgBox.setWindowTitle("Kraken Editor")
        msgBox.setText("You are closing Kraken.")
        msgBox.setInformativeText("Do you want to save your changes?")
        msgBox.setStandardButtons(QtGui.QMessageBox.Save | QtGui.QMessageBox.Discard | QtGui.QMessageBox.Cancel)
        msgBox.setDefaultButton(QtGui.QMessageBox.Save)

        ret = msgBox.exec_()

        if ret == QtGui.QMessageBox.Cancel:
            event.ignore()
            return

        elif ret == QtGui.QMessageBox.Save:
            self.kraken_ui.graphViewWidget.saveRigPreset()

            self.statusBar().showMessage('Closing')

        self.writeSettings()

    def showOutputLog(self):
        outputDialog = self.findChild(QtGui.QDialog, 'outputLog')
        if outputDialog is None:
            outputDialog = OutputLogDialog(self)

        outputDialog.show()
        outputDialog.setText(self.outputLog.getLog())
        outputDialog.textWidget.moveCursor(QtGui.QTextCursor.End)

class OutputLogDialog(QtGui.QDialog):
    """Output Dialog"""

    def __init__(self, parent=None):
        super(OutputLogDialog, self).__init__(parent)
        self.setObjectName('outputLog')
        self.resize(700, 300)
        self.setWindowTitle('Kraken Output Log')

        self.createLayout()
        self.createConnections()

    def createLayout(self):
        """Sets up the layout for the dialog."""

        self.textWidget = QtGui.QTextEdit()
        self.textWidget.setLineWrapMode(QtGui.QTextEdit.NoWrap)
        self.textWidget.setReadOnly(True)
        self.textWidget.setContextMenuPolicy(QtCore.Qt.CustomContextMenu)

        self.outputLogLayout = QtGui.QVBoxLayout(self)
        self.outputLogLayout.addWidget(self.textWidget)

        self.setLayout(self.outputLogLayout)

    def createConnections(self):
        """Connects widgets to methods or other signals."""

        self.textWidget.customContextMenuRequested.connect(self.createContextMenu)

    def setText(self, text):
        """Sets the text of the text widget."""

        self.textWidget.setText(text)

    # =============
    # Context Menu
    # =============
    def createContextMenu(self):
        self.contextMenu = QtGui.QMenu(self)
        selectAllAction = self.contextMenu.addAction("Select All")
        copyAction = self.contextMenu.addAction("Copy")
        self.contextMenu.addSeparator()
        clearAction = self.contextMenu.addAction("Clear")

        selectAllAction.triggered.connect(self.contextSelectAll)
        copyAction.triggered.connect(self.contextCopy)
        clearAction.triggered.connect(self.contextClear)

        self.contextMenu.exec_(QtGui.QCursor.pos())

    def contextSelectAll(self):
        self.textWidget.selectAll()

    def contextCopy(self):
        self.textWidget.copy()

    def contextClear(self):
        outputLog = self.parent().outputLog
        outputLog.clear()
        self.textWidget.clear()

def createSplash(app):
    """Creates a splash screen object to show while the Window is loading.

    Return:
    SplashScreen object.

    """

    splashPixmap = QtGui.QPixmap(':/images/KrakenUI_Splash.png')

    splash = QtGui.QSplashScreen(splashPixmap)
    splash.setMask(splashPixmap.mask())
    splash.showMessage("Loading Extensions...",
                       QtCore.Qt.AlignBottom | QtCore.Qt.AlignLeft,
                       QtCore.Qt.white)

    splash.show()

    app.processEvents()

    return splash

if __name__ == "__main__":
    app = QtGui.QApplication(sys.argv)

    splash = createSplash(app)

    window = KrakenWindow()
    window.show()

    splash.finish(window)

    sys.exit(app.exec_())
<|endoftext|>


<commit_before># Creating a csv file from nytimes article on all Trump lies
# "Text to data"
# Thanks to Kevin Markham

import urllib.request, urllib.parse, urllib.error
from bs4 import BeautifulSoup
import ssl

#ignore ssl errors:
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE

#actual url and bs4 necessities:
url = 'https://www.nytimes.com/interactive/2017/06/23/opinion/trumps-lies.html'
html = urllib.request.urlopen(url, context=ctx).read()
soup = BeautifulSoup(html, 'html.parser')

#the magic line:
results = soup.find_all('span', attrs={'class':'short-desc'})
#testing:
#print(len(results))
#first_result = results[0]
#print(first_result.contents[1])

#creating data set:
records = []
for result in results:
    date = result.find('strong').text[0:-1] + ', 2017'
    lie = result.contents[1][1:-2]
    check = result.find('a').text[1:-1]
    link = result.find('a')['href']
    records.append((date, lie, factcheck, link))
print(len(records))
print(records[0:1])

import pandas as pd
df = pd.DataFrame(records, columns=['date', 'lie', 'factcheck', 'link'])

#make date format more universal (for analysis):
df['date'] = pd.to_datetime(df['date'])

#turn to csv:
df.to_csv('trump_lies.csv', index=False, encoding='utf-8')

#reading it back to pandas from csv:
#df = pd.read_csv('trump_lies.csv', parse_dates=['date'], encoding='utf-8')
<commit_msg>Delete TrumpLiesNYtimes.py<commit_after><|endoftext|>


<commit_before>#! /usr/bin/env python3

import sys, time
from time import sleep
from quick2wire.parts.pcf8591 import *
from quick2wire.i2c import I2CMaster
#from vis import volt_plot
from writefile import *
from tkinter import *

class adc_reader():

    def __init__():

        self.address = int(sys.argv[1]) if len(sys.argv) > 1 else BASE_ADDRESS
        self.pin_index1 = int(sys.argv[2]) if len(sys.argv) > 2 else 0
        self.pin_index2 = int(sys.argv[3]) if len(sys.argv) > 3 else 1

    def run(filename):
        with I2CMaster() as i2c:
            adc = PCF8591(i2c, THREE_DIFFERENTIAL)
            pin1 = adc.differential_input(1)

            #pin2 = adc.single_ended_input(pin_index2)
            # root = Tk()
            # def key(event):
            #     print(event.char)
            #     if event.char == 'a':
            #         subvocal = 'Y'
            #         voltage = pin1.value * 3.3
            #         print("read: {} : {}".format(count, voltage))
            #         current = time.clock() - start
            #         write_file('test.csv', current, voltage, subvocal)
            #     else:
            #         subvocal = "N"
            #         voltage = pin1.value * 3.3
            #         print("read: {} : {}".format(count, voltage))
            #         current = time.clock() - start
            #         write_file('test.csv', current, voltage, subvocal)

            # frame = Frame(root, width=300, height=300)
            # frame.bind("<Key>", key)
            # frame.pack()
            # root.mainloop()
            #data = {}
            count = 0
            start = time.clock()

            with open(filename, 'w') as csvfile:

                fieldnames = ['time', 'count', 'voltage']
                writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
                writer.writeheader()
                count = 1
                while self.record:
                    voltage = pin1.value * 3.3
                    strength = ['-' for i in range(int(pin1.value//16))]
                    reading += [bar for bar in strength]
                    print (reading)
                    #sleep(0.1)
                    count += 1
                    current = time.clock() - start
                    #data[count] = pin1.raw_value
                    #volt_plot(count, data)
                    writer.writerow({'time': current, 'count': count, 'voltage': voltage})
            current += 1
<commit_msg>fix: class syntax error<commit_after>#! /usr/bin/env python3

import sys, time
from time import sleep
from quick2wire.parts.pcf8591 import *
from quick2wire.i2c import I2CMaster
#from vis import volt_plot
from writefile import *
from tkinter import *

class adc_reader():

    def __init__(self):

        self.address = int(sys.argv[1]) if len(sys.argv) > 1 else BASE_ADDRESS
        self.pin_index1 = int(sys.argv[2]) if len(sys.argv) > 2 else 0
        self.pin_index2 = int(sys.argv[3]) if len(sys.argv) > 3 else 1

    def run(filename):
        with I2CMaster() as i2c:
            adc = PCF8591(i2c, THREE_DIFFERENTIAL)
            pin1 = adc.differential_input(1)

            #pin2 = adc.single_ended_input(pin_index2)
            # root = Tk()
            # def key(event):
            #     print(event.char)
            #     if event.char == 'a':
            #         subvocal = 'Y'
            #         voltage = pin1.value * 3.3
            #         print("read: {} : {}".format(count, voltage))
            #         current = time.clock() - start
            #         write_file('test.csv', current, voltage, subvocal)
            #     else:
            #         subvocal = "N"
            #         voltage = pin1.value * 3.3
            #         print("read: {} : {}".format(count, voltage))
            #         current = time.clock() - start
            #         write_file('test.csv', current, voltage, subvocal)

            # frame = Frame(root, width=300, height=300)
            # frame.bind("<Key>", key)
            # frame.pack()
            # root.mainloop()
            #data = {}
            count = 0
            start = time.clock()

            with open(filename, 'w') as csvfile:

                fieldnames = ['time', 'count', 'voltage']
                writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
                writer.writeheader()
                count = 1
                while self.record:
                    voltage = pin1.value * 3.3
                    strength = ['-' for i in range(int(pin1.value//16))]
                    reading += [bar for bar in strength]
                    print (reading)
                    #sleep(0.1)
                    count += 1
                    current = time.clock() - start
                    #data[count] = pin1.raw_value
                    #volt_plot(count, data)
                    writer.writerow({'time': current, 'count': count, 'voltage': voltage})
            current += 1
<|endoftext|>