<commit_before>
__author__    = "Andre Merzky, Ole Weidner"
__copyright__ = "Copyright 2012-2013, The SAGA Project"
__license__   = "MIT"

import saga.utils.singleton
import saga.utils.logger
import saga.engine.engine
import saga.context
import saga.base

import saga.utils.signatures as sus

# ------------------------------------------------------------------------------
#
class _ContextList (list) :
    """
    The `saga.Session` class has a 'contexts' member which is a mutable list of
    `saga.Context` instances.  Adding contexts to that list is semantically
    equivalent to calling `saga.Session.add_context (ctx)`, which (a) creates
    a deep copy of the context, and (b) initializes the context (i.e. calls
    `ctx._initialize (self)`, with `self` being the session instance).  We thus
    create our own provate `_ContextList` class which inherits from the native
    Python `list` class, and overload the `append()` call with said semantics.
    """

    # --------------------------------------------------------------------------
    #
    def __init__ (self, session, *args, **kwargs) :

        self.session = session

        l = super  (_ContextList, self)
        l.__init__ (*args, **kwargs)

    # --------------------------------------------------------------------------
    #
    def append (self, ctx) :

        if  not isinstance (ctx, saga.Context) :
            raise TypeError, "appended item is not a saga.Context instance"

        # create a deep copy of the context (this keeps _adaptor etc)
        ctx_clone = saga.Context  (ctx.type)
        ctx._attributes_deep_copy (ctx_clone)

        # from pudb import set_trace; set_trace()

        # try to initialize that context, i.e. evaluate its attributes and
        # infer additional runtime information as needed
        self.session._logger.debug ("adding  context : %s" \
                                 % (ctx_clone))
        try :
            ctx_clone._initialize (self.session)
        except saga.exceptions.SagaException as e :
            msg = "Cannot add context, initialization failed (%s)"  %  str(e)
            raise saga.exceptions.BadParameter (msg)

        # context initialized ok, add it to the list of known contexts
        super (_ContextList, self).append (ctx_clone)

# ------------------------------------------------------------------------------
#
class _DefaultSession (object) :

    __metaclass__ = saga.utils.singleton.Singleton

    # --------------------------------------------------------------------------
    #
    @sus.takes   ('_DefaultSession')
    @sus.returns (sus.nothing)
    def __init__ (self) :

        # the default session picks up default contexts, from all context
        # adaptors.  To implemented, we have to do some legwork: get the engine,
        # dig through the registered context adaptors, and ask each of them for
        # default contexts.

        self.contexts  = _ContextList (session=self)
        self._logger   = saga.utils.logger.getLogger ('saga.DefaultSession')

        _engine = saga.engine.engine.Engine ()

        if not 'saga.Context' in _engine._adaptor_registry :
            self._logger.warn ("no context adaptors found")
            return

        for schema in   _engine._adaptor_registry['saga.Context'] :
            for info in _engine._adaptor_registry['saga.Context'][schema] :

                default_ctxs = info['adaptor_instance']._get_default_contexts ()
                # FIXME:
              # default_ctxs = []
                for default_ctx in default_ctxs :

                    self.contexts.append (default_ctx)
                    self._logger.debug ("default context [%-20s] : %s" \
                                     % (info['adaptor_name'], default_ctx))

# ------------------------------------------------------------------------------
#
class Session (saga.base.SimpleBase) :
    """A SAGA Session object as defined in GFD.90.

    A SAGA session has the purpose of scoping the use of security credentials
    for remote operations.  In other words, a session instance acts as
    a container for security Context instances -- SAGA objects (such as
    job.Service or filesystem.File) created in that session will then use
    exactly the security contexts from that session (and no others).

    That way, the session serves two purposes:  (1) it helps SAGA to decide
    which security mechanism should be used for what interaction, and (2) it
    helps SAGA to find security credentials which would be difficult to pick up
    automatically.

    The use of a session is as follows:

    Example::

        # define an ssh context
        c = saga.Context()
        c.context_type = 'ssh'
        c.user_cert = '$HOME/.ssh/special_id_rsa'
        c.user_key  = '$HOME/.ssh/special_id_rsa.pub'

        # add it to a session
        s = saga.Session
        s.add_context(c)

        # create a job service in this session -- that job service can now
        # *only* use that ssh context.
        j = saga.job.Service('ssh://remote.host.net/', s)

    The session argument to the L{job.Service} constructor is fully optional --
    if left out, SAGA will use default session, which picks up some default
    contexts as described above -- that will suffice for the majority of use
    cases.

    A session instance exposes a `context` property, which is a list of
    authentication contexts managed by this session.  As the contexts and the
    session are stateless, it is safe to modify this list as needed.
    """

    # FIXME: session deep copy not implemented

    # --------------------------------------------------------------------------
    #
    @sus.takes   ('Session',
                  sus.optional(bool))
    @sus.returns (sus.nothing)
    def __init__ (self, default=True) :
        """
        default: bool
        ret:     None
        """

        saga.base.SimpleBase.__init__ (self)

        # if the default session is expected, we point our context list to the
        # shared list of the default session singleton.  Otherwise, we create
        # a private list which is not populated.

        if default :
            default_session  = _DefaultSession ()
            self.contexts    = default_session.contexts
        else :
            self.contexts    = _ContextList (session = self)

    # ----------------------------------------------------------------
    #
    @sus.takes   ('Session')
    @sus.returns (basestring)
    def __str__  (self):
        """String represenation."""

        import traceback
        traceback.print_stack ()
        return "Registered contexts: %s" % (str(self.contexts))

    # ----------------------------------------------------------------
    #
    @sus.takes      ('Session',
                     saga.context.Context)
    @sus.returns    (sus.nothing)
    def add_context (self, ctx) :
        """
        ctx:     saga.Context
        ret:     None

        Add a security L{Context} to the session.
        It is encouraged to use the L{contexts} property instead.
        """

        return self.contexts.append (ctx)

    # ----------------------------------------------------------------
    #
    @sus.takes   ('Session',
                  saga.context.Context)
    @sus.returns (sus.nothing)
    def remove_context (self, ctx) :
        """
        ctx:     saga.Context
        ret:     None

        Remove a security L{Context} from the session.
        It is encouraged to use the L{contexts} property instead.
        """

        if ctx in self.contexts :
            self.contexts.remove (ctx)

    # ----------------------------------------------------------------
    #
    @sus.takes   ('Session')
    @sus.returns (sus.list_of (saga.context.Context))
    def list_contexts  (self) :
        """
        ret:     list[saga.Context]

        Retrieve all L{Context} objects attached to the session.
        It is encouraged to use the L{contexts} property instead.
        """

        return self.contexts

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

<commit_msg>fix 224: make sure contexts are initialized with a valid session<commit_after>
__author__    = "Andre Merzky, Ole Weidner"
__copyright__ = "Copyright 2012-2013, The SAGA Project"
__license__   = "MIT"

import saga.utils.singleton
import saga.utils.logger
import saga.engine.engine
import saga.context
import saga.base

import saga.utils.signatures as sus

# ------------------------------------------------------------------------------
#
class _ContextList (list) :
    """
    The `saga.Session` class has a 'contexts' member which is a mutable list of
    `saga.Context` instances.  Adding contexts to that list is semantically
    equivalent to calling `saga.Session.add_context (ctx)`, which (a) creates
    a deep copy of the context, and (b) initializes the context (i.e. calls
    `ctx._initialize (self)`, with `self` being the session instance).  We thus
    create our own provate `_ContextList` class which inherits from the native
    Python `list` class, and overload the `append()` call with said semantics.
    """

    # --------------------------------------------------------------------------
    #
    def __init__ (self, *args, **kwargs) :

        base_list = super  (_ContextList, self)
        base_list.__init__ (*args, **kwargs)

    # --------------------------------------------------------------------------
    #
    def append (self, session, ctx) :

        if  not isinstance (ctx, saga.Context) :
            raise TypeError, "appended item is not a saga.Context instance"

        # create a deep copy of the context (this keeps _adaptor etc)
        ctx_clone = saga.Context  (ctx.type)
        ctx._attributes_deep_copy (ctx_clone)

        # from pudb import set_trace; set_trace()

        # try to initialize that context, i.e. evaluate its attributes and
        # infer additional runtime information as needed
        session._logger.debug ("adding  context : %s" \
                            % (ctx_clone))
        try :
            ctx_clone._initialize (session)
        except saga.exceptions.SagaException as e :
            msg = "Cannot add context, initialization failed (%s)"  %  str(e)
            raise saga.exceptions.BadParameter (msg)

        # context initialized ok, add it to the list of known contexts
        super (_ContextList, self).append (ctx_clone)

# ------------------------------------------------------------------------------
#
class _DefaultSession (object) :

    __metaclass__ = saga.utils.singleton.Singleton

    # --------------------------------------------------------------------------
    #
    @sus.takes   ('_DefaultSession')
    @sus.returns (sus.nothing)
    def __init__ (self) :

        # the default session picks up default contexts, from all context
        # adaptors.  To implemented, we have to do some legwork: get the engine,
        # dig through the registered context adaptors, and ask each of them for
        # default contexts.

        self.contexts  = _ContextList ()
        self._logger   = saga.utils.logger.getLogger ('saga.DefaultSession')

        _engine = saga.engine.engine.Engine ()

        if not 'saga.Context' in _engine._adaptor_registry :
            self._logger.warn ("no context adaptors found")
            return

        for schema in   _engine._adaptor_registry['saga.Context'] :
            for info in _engine._adaptor_registry['saga.Context'][schema] :

                default_ctxs = info['adaptor_instance']._get_default_contexts ()

                for default_ctx in default_ctxs :

                    self.contexts.append (session=self, ctx=default_ctx)
                    self._logger.debug   ("default context [%-20s] : %s" \
                                     %   (info['adaptor_name'], default_ctx))

# ------------------------------------------------------------------------------
#
class Session (saga.base.SimpleBase) :
    """A SAGA Session object as defined in GFD.90.

    A SAGA session has the purpose of scoping the use of security credentials
    for remote operations.  In other words, a session instance acts as
    a container for security Context instances -- SAGA objects (such as
    job.Service or filesystem.File) created in that session will then use
    exactly the security contexts from that session (and no others).

    That way, the session serves two purposes:  (1) it helps SAGA to decide
    which security mechanism should be used for what interaction, and (2) it
    helps SAGA to find security credentials which would be difficult to pick up
    automatically.

    The use of a session is as follows:

    Example::

        # define an ssh context
        c = saga.Context()
        c.context_type = 'ssh'
        c.user_cert = '$HOME/.ssh/special_id_rsa'
        c.user_key  = '$HOME/.ssh/special_id_rsa.pub'

        # add it to a session
        s = saga.Session
        s.add_context(c)

        # create a job service in this session -- that job service can now
        # *only* use that ssh context.
        j = saga.job.Service('ssh://remote.host.net/', s)

    The session argument to the L{job.Service} constructor is fully optional --
    if left out, SAGA will use default session, which picks up some default
    contexts as described above -- that will suffice for the majority of use
    cases.

    A session instance exposes a `context` property, which is a list of
    authentication contexts managed by this session.  As the contexts and the
    session are stateless, it is safe to modify this list as needed.
    """

    # FIXME: session deep copy not implemented

    # --------------------------------------------------------------------------
    #
    @sus.takes   ('Session',
                  sus.optional(bool))
    @sus.returns (sus.nothing)
    def __init__ (self, default=True) :
        """
        default: bool
        ret:     None
        """

        simple_base = super  (Session, self)
        simple_base.__init__ ()

        # if the default session is expected, we point our context list to the
        # shared list of the default session singleton.  Otherwise, we create
        # a private list which is not populated.

        if  default :
            default_session  = _DefaultSession ()
            self.contexts    = default_session.contexts
        else :
            self.contexts    = _ContextList (session = self)

    # ----------------------------------------------------------------
    #
    @sus.takes   ('Session')
    @sus.returns (basestring)
    def __str__  (self):
        """String represenation."""

        return "Registered contexts: %s" % (str(self.contexts))

    # ----------------------------------------------------------------
    #
    @sus.takes      ('Session',
                     saga.context.Context)
    @sus.returns    (sus.nothing)
    def add_context (self, ctx) :
        """
        ctx:     saga.Context
        ret:     None

        Add a security L{Context} to the session.
        It is encouraged to use the L{contexts} property instead.
        """

        return self.contexts.append (session=self, ctx=ctx)

    # ----------------------------------------------------------------
    #
    @sus.takes   ('Session',
                  saga.context.Context)
    @sus.returns (sus.nothing)
    def remove_context (self, ctx) :
        """
        ctx:     saga.Context
        ret:     None

        Remove a security L{Context} from the session.
        It is encouraged to use the L{contexts} property instead.
        """

        if ctx in self.contexts :
            self.contexts.remove (ctx)

    # ----------------------------------------------------------------
    #
    @sus.takes   ('Session')
    @sus.returns (sus.list_of (saga.context.Context))
    def list_contexts  (self) :
        """
        ret:     list[saga.Context]

        Retrieve all L{Context} objects attached to the session.
        It is encouraged to use the L{contexts} property instead.
        """

        return self.contexts

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

<|endoftext|>


<commit_before>#!/usr/bin/env python

###########################################################
##                                                       ##
##   acts.py                                             ##
##                                                       ##
##                Author: Tony Fischetti                 ##
##                        tony.fischetti@gmail.com       ##
##                                                       ##
###########################################################
#
##############################################################################
#                                                                            #
# Copyright (c) 2013, 2014, Tony Fischetti                                   #
#                                                                            #
# MIT License, http://www.opensource.org/licenses/mit-license.php            #
#                                                                            #
# Permission is hereby granted, free of charge, to any person obtaining a    #
# copy of this software and associated documentation files (the "Software"), #
# to deal in the Software without restriction, including without limitation  #
# the rights to use, copy, modify, merge, publish, distribute, sublicense,   #
# and/or sell copies of the Software, and to permit persons to whom the      #
# Software is furnished to do so, subject to the following conditions:       #
#                                                                            #
# The above copyright notice and this permission notice shall be included in #
# all copies or substantial portions of the Software.                        #
#                                                                            #
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR #
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,   #
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL    #
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER #
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING    #
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER        #
# DEALINGS IN THE SOFTWARE.                                                  #
#                                                                            #
##############################################################################

"""
Various actions that the main entry delegates to
"""

from __future__ import unicode_literals
from __future__ import print_function
from subprocess import Popen
import networkx as nx
import codecs
import os
import re
import fnmatch
import string
import glob
import sys
import yaml

if sys.version_info[0] < 3:
    import codecs
    open = codecs.open

def parse(file, text, includes):
    try:
        sakefile = yaml.load(text) or {}
    except yaml.YAMLError as exc:
        sys.stderr.write("Error: {} failed to parse as valid YAML\n".format(file))
        if hasattr(exc, 'problem_mark'):
            mark = exc.problem_mark
            sys.stderr.write("Error near line {}\n".format(str(mark.line+1)))
        sys.exit(1)
    for filename, (subdata, subincludes) in includes.items():
        sakefile.update(parse(filename, subdata, subincludes))
    return sakefile

def clean_path(a_path, force_os=None, force_start=None):
    """
    This function is used to normalize the path (of an output or
    dependency) and also provide the path in relative form. It is
    relative to the current working directory
    """
    if not force_start:
        force_start = os.curdir
    if force_os == "windows":
        import ntpath
        return ntpath.relpath(ntpath.normpath(a_path),
                             start=force_start)
    if force_os == "posix":
        import posixpath
        return posixpath.relpath(posixpath.normpath(a_path),
                                start=force_start)
    return os.path.relpath(os.path.normpath(a_path),
                          start=force_start)

def escp(target_name):
    """
    This function is used by sake help. Since sakefiles allow
    for targets with spaces in them, sake help needs to quote
    all targets with spaces. This takes a target name and
    quotes it if necessary
    """
    if ' ' in target_name:
        return '"{}"'.format(target_name)
    return target_name

def get_help(sakefile):
    """
    Returns the prettily formatted help strings (for printing)

    Args:
        A dictionary that is the parsed Sakefile (from sake.py)

    NOTE:
        the list sorting in this function is required for this
        function to be deterministic
    """
    full_string = "You can 'sake' one of the following...\n\n"
    errmes = "target '{}' is not allowed to not have help message\n"
    outerlines = []
    for target in sakefile:
        if target == "all":
            # this doesn't have a help message
            continue
        middle_lines = []
        if "formula" not in sakefile[target]:
            # this means it's a meta-target
            inner = "{}:\n  - {}\n\n".format(escp(target),
                                             sakefile[target]["help"])
            for atom_target in sakefile[target]:
                if atom_target == "help":
                    continue
                inner += "    {}:\n      -  {}\n\n".format(escp(atom_target),
                                                         sakefile[target][atom_target]["help"])
            middle_lines.append(inner)
        else:
            middle_lines.append("{}:\n  - {}\n\n".format(escp(target),
                                                         sakefile[target]["help"]))
        if middle_lines:
            outerlines.append('\n'.join(sorted(middle_lines)))

    if outerlines:
        full_string += '\n'.join(sorted(outerlines))
    what_clean_does = "remove all targets' outputs and start from scratch"
    full_string += "\nclean:\n  -  {}\n\n".format(what_clean_does)
    what_visual_does = "output visual representation of project's dependencies"
    full_string += "visual:\n  -  {}\n".format(what_visual_does)
    full_string = re.sub("\n{3,}", "\n\n", full_string)
    return full_string

def expand_macros(raw_text, macros={}):
    """
    this gets called before the sakefile is parsed. it looks for
    macros defined anywhere in the sakefile (the start of the line
    is '#!') and then replaces all occurences of '$variable' with the
    value defined in the macro. it then returns the contents of the
    file with the macros expanded.
    """
    includes = {}
    result = []
    pattern = re.compile("#!\s*(\w+)\s*=\s*(.+$)", re.UNICODE)
    ipattern = re.compile("#<\s*(\S+)\s+(optional|or\s+(.+))?$")
    for line in raw_text.split("\n"):
        line = string.Template(line).safe_substitute(macros)
        # note that the line is appended to result before it is checked for macros
        # this prevents macros expanding into themselves
        result.append(line)
        if line.startswith("#!"):
            match = pattern.match(line)
            try:
                var, val = match.group(1, 2)
            except:
                raise InvalidMacroError("Failed to parse macro {}\n".format(line))
            macros[var] = val
        elif line.startswith("#<"):
            match = ipattern.match(line)
            try:
                filename = match.group(1)
            except:
                raise IncludeError("Failed to parse include {}\n".format(line))
            try:
                with open(filename, 'r') as f:
                    includes[filename] = expand_macros(f.read(), macros)
            except IOError:
                if match.group(2):
                    if match.group(2).startswith('or '):
                        print(match.group(3))
                else:
                    raise IncludeError("Nonexistent include {}\n".format(filename))
    return "\n".join(result), includes

def check_for_dep_in_outputs(dep, verbose, G):
    """
    Function to help construct_graph() identify dependencies

    Args:
        A dependency
        A flag indication verbosity
        A (populated) NetworkX DiGraph

    Returns:
        A list of targets that build given dependency

    """
    if verbose:
        print("checking dep {}".format(dep))
    ret_list = []
    for node in G.nodes(data=True):
        if "output" not in node[1]:
            continue
        for out in node[1]['output']:
            if fnmatch.fnmatch(out, dep):
                ret_list.append(node[0])
                break
    return ret_list

def construct_graph(sakefile, verbose):
    """
    Takes the sakefile dictionary and builds a NetworkX graph

    Args:
        A dictionary that is the parsed Sakefile (from sake.py)
        A flag indication verbosity

    Returns:
        A NetworkX graph
    """
    G = nx.DiGraph()
    if verbose:
        print("Going to construct Graph")
    for target in sakefile:
        if target == "all":
            # we don't want this node
            continue
        if "formula" not in sakefile[target]:
            # that means this is a meta target
            for atomtarget in sakefile[target]:
                if atomtarget == "help":
                    continue
                if verbose:
                    print("Adding '{}'".format(atomtarget))
                data_dict = sakefile[target][atomtarget]
                data_dict["parent"] = target
                G.add_node(atomtarget, data_dict)
        else:
            if verbose:
                print("Adding '{}'".format(target))
            G.add_node(target, sakefile[target])
    if verbose:
        print("Nodes are built\nBuilding connections")
    for node in G.nodes(data=True):
        if verbose:
            print("checking node {} for dependencies".format(node[0]))
        # normalize all paths in output
        for k, v in node[1].items():
            if v is None: node[1][k] = []
        if "output" in node[1]:
            for index, out in enumerate(node[1]['output']):
                node[1]['output'][index] = clean_path(node[1]['output'][index])
        if "dependencies" not in node[1]:
            continue
        if verbose:
            print("it has dependencies")
        connects = []
        # normalize all paths in dependencies
        for index, dep in enumerate(node[1]['dependencies']):
            dep = os.path.normpath(dep)
            shrt = "dependencies"
            node[1]['dependencies'][index] = clean_path(node[1][shrt][index])
    for node in G.nodes(data=True):
        connects = []
        if "dependencies" not in node[1]:
            continue
        for dep in node[1]['dependencies']:
            matches = check_for_dep_in_outputs(dep, verbose, G)
            if not matches:
                continue
            for match in matches:
                if verbose:
                    print("Appending {} to matches".format(match))
                connects.append(match)
        if connects:
            for connect in connects:
                G.add_edge(connect, node[0])
    return G

def get_all_outputs(node_dict):
    """
    This function takes a node dictionary and returns a list of
    the node's output files. Some of the entries in the 'output'
    attribute may be globs, and without this function, sake won't
    know how to handle that. This will unglob all globs and return
    the true list of *all* outputs.
    """
    outlist = []
    for item in node_dict['output']:
        glist = glob.glob(item)
        if glist:
            for oneglob in glist:
                outlist.append(oneglob)
        else:
            outlist.append(item)
    return outlist

def clean_all(G, verbose, quiet, recon):
    """
    Removes all the output files from all targets. Takes
    the graph as the only argument

    Args:
        The networkx graph object
        A flag indicating verbosity
        A flag indicating quiet mode

    Returns:
        0 if successful
        1 if removing even one file failed
    """
    all_outputs = []
    for node in G.nodes(data=True):
        if "output" in node[1]:
            for item in get_all_outputs(node[1]):
                all_outputs.append(item)
    all_outputs.append(".shastore")
    retcode = 0
    for item in all_outputs:
        if os.path.isfile(item):
            if recon:
                print("Would remove file: {}".format(item))
                continue
            if verbose:
                mesg = "Attempting to remove file '{}'"
                print(mesg.format(item))
            try:
                os.remove(item)
                if verbose:
                    print("Removed file")
            except:
                errmeg = "Error: file '{}' failed to be removed\n"
                sys.stderr.write(errmeg.format(item))
                retcode = 1
    if not retcode and not recon:
        print("All clean")
    return retcode

def write_dot_file(G, filename):
    """
    Writes the graph G in dot file format for graphviz visualization.

    Args:
        a Networkx graph
        A filename to name the dot files
    """
    with open(filename, "w", encoding="utf-8") as fh:
        fh.write("strict digraph DependencyDiagram {\n")
        edge_list = G.edges()
        node_list = set(G.nodes())
        if edge_list:
            for edge in G.edges():
                source, targ = edge
                node_list = node_list - set(source)
                node_list = node_list - set(targ)
                line = '"{}" -> "{}";\n'
                fh.write(line.format(source, targ))
        # draw nodes with no links
        if node_list:
            for node in node_list:
                line = '"{}"\n'.format(node)
                fh.write(line)
        fh.write("}")

def visualize(G, filename="dependencies", no_graphviz=False):
    """
    Uses networkX to draw a graphviz dot file either (a) calls the
    graphviz command "dot" to turn it into a SVG and remove the
    dotfile (default), or (b) if no_graphviz is True, just output
    the graphviz dot file

    Args:
        a NetworkX DiGraph
        a filename (a default is provided
        a flag indicating whether graphviz should *not* be called

    Returns:
        0 if everything worked
        will cause fatal error on failure
    """
    if no_graphviz:
        write_dot_file(G, filename)
        return 0
    write_dot_file(G, "tempdot")
    renderer = "svg"
    if re.search("\.jpg$", filename, re.IGNORECASE):
        renderer = "jpg"
    elif re.search("\.jpeg$", filename, re.IGNORECASE):
        renderer = "jpg"
    elif re.search("\.svg$", filename, re.IGNORECASE):
        renderer = "svg"
    elif re.search("\.png$", filename, re.IGNORECASE):
        renderer = "png"
    elif re.search("\.gif$", filename, re.IGNORECASE):
        renderer = "gif"
    elif re.search("\.ps$", filename, re.IGNORECASE):
        renderer = "ps"
    elif re.search("\.pdf$", filename, re.IGNORECASE):
        renderer = "pdf"
    else:
        renderer = "svg"
        filename += ".svg"
    command = "dot -T{} tempdot -o {}".format(renderer, filename)
    p = Popen(command, shell=True)
    p.communicate()
    if p.returncode:
        errmes = "Either graphviz is not installed, or its not on PATH"
        os.remove("tempdot")
        sys.stderr.write(errmes)
        sys.exit(1)
    os.remove("tempdot")
    return 0

#####################
##  CUSTOM ERRORS  ##
#####################

class Error(Exception):
    """Base class for exceptions in this module."""
    pass

class InvalidMacroError(Error):
    def __init__(self, message):
        self.message = message

class IncludeError(Error):
    def __init__(self, message):
        self.message = message

<commit_msg>Slight logic error in regex<commit_after>#!/usr/bin/env python

###########################################################
##                                                       ##
##   acts.py                                             ##
##                                                       ##
##                Author: Tony Fischetti                 ##
##                        tony.fischetti@gmail.com       ##
##                                                       ##
###########################################################
#
##############################################################################
#                                                                            #
# Copyright (c) 2013, 2014, Tony Fischetti                                   #
#                                                                            #
# MIT License, http://www.opensource.org/licenses/mit-license.php            #
#                                                                            #
# Permission is hereby granted, free of charge, to any person obtaining a    #
# copy of this software and associated documentation files (the "Software"), #
# to deal in the Software without restriction, including without limitation  #
# the rights to use, copy, modify, merge, publish, distribute, sublicense,   #
# and/or sell copies of the Software, and to permit persons to whom the      #
# Software is furnished to do so, subject to the following conditions:       #
#                                                                            #
# The above copyright notice and this permission notice shall be included in #
# all copies or substantial portions of the Software.                        #
#                                                                            #
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR #
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,   #
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL    #
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER #
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING    #
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER        #
# DEALINGS IN THE SOFTWARE.                                                  #
#                                                                            #
##############################################################################

"""
Various actions that the main entry delegates to
"""

from __future__ import unicode_literals
from __future__ import print_function
from subprocess import Popen
import networkx as nx
import codecs
import os
import re
import fnmatch
import string
import glob
import sys
import yaml

if sys.version_info[0] < 3:
    import codecs
    open = codecs.open

def parse(file, text, includes):
    try:
        sakefile = yaml.load(text) or {}
    except yaml.YAMLError as exc:
        sys.stderr.write("Error: {} failed to parse as valid YAML\n".format(file))
        if hasattr(exc, 'problem_mark'):
            mark = exc.problem_mark
            sys.stderr.write("Error near line {}\n".format(str(mark.line+1)))
        sys.exit(1)
    for filename, (subdata, subincludes) in includes.items():
        sakefile.update(parse(filename, subdata, subincludes))
    return sakefile

def clean_path(a_path, force_os=None, force_start=None):
    """
    This function is used to normalize the path (of an output or
    dependency) and also provide the path in relative form. It is
    relative to the current working directory
    """
    if not force_start:
        force_start = os.curdir
    if force_os == "windows":
        import ntpath
        return ntpath.relpath(ntpath.normpath(a_path),
                             start=force_start)
    if force_os == "posix":
        import posixpath
        return posixpath.relpath(posixpath.normpath(a_path),
                                start=force_start)
    return os.path.relpath(os.path.normpath(a_path),
                          start=force_start)

def escp(target_name):
    """
    This function is used by sake help. Since sakefiles allow
    for targets with spaces in them, sake help needs to quote
    all targets with spaces. This takes a target name and
    quotes it if necessary
    """
    if ' ' in target_name:
        return '"{}"'.format(target_name)
    return target_name

def get_help(sakefile):
    """
    Returns the prettily formatted help strings (for printing)

    Args:
        A dictionary that is the parsed Sakefile (from sake.py)

    NOTE:
        the list sorting in this function is required for this
        function to be deterministic
    """
    full_string = "You can 'sake' one of the following...\n\n"
    errmes = "target '{}' is not allowed to not have help message\n"
    outerlines = []
    for target in sakefile:
        if target == "all":
            # this doesn't have a help message
            continue
        middle_lines = []
        if "formula" not in sakefile[target]:
            # this means it's a meta-target
            inner = "{}:\n  - {}\n\n".format(escp(target),
                                             sakefile[target]["help"])
            for atom_target in sakefile[target]:
                if atom_target == "help":
                    continue
                inner += "    {}:\n      -  {}\n\n".format(escp(atom_target),
                                                         sakefile[target][atom_target]["help"])
            middle_lines.append(inner)
        else:
            middle_lines.append("{}:\n  - {}\n\n".format(escp(target),
                                                         sakefile[target]["help"]))
        if middle_lines:
            outerlines.append('\n'.join(sorted(middle_lines)))

    if outerlines:
        full_string += '\n'.join(sorted(outerlines))
    what_clean_does = "remove all targets' outputs and start from scratch"
    full_string += "\nclean:\n  -  {}\n\n".format(what_clean_does)
    what_visual_does = "output visual representation of project's dependencies"
    full_string += "visual:\n  -  {}\n".format(what_visual_does)
    full_string = re.sub("\n{3,}", "\n\n", full_string)
    return full_string

def expand_macros(raw_text, macros={}):
    """
    this gets called before the sakefile is parsed. it looks for
    macros defined anywhere in the sakefile (the start of the line
    is '#!') and then replaces all occurences of '$variable' with the
    value defined in the macro. it then returns the contents of the
    file with the macros expanded.
    """
    includes = {}
    result = []
    pattern = re.compile("#!\s*(\w+)\s*=\s*(.+$)", re.UNICODE)
    ipattern = re.compile("#<\s*(\S+)\s*(optional|or\s+(.+))?$")
    for line in raw_text.split("\n"):
        line = string.Template(line).safe_substitute(macros)
        # note that the line is appended to result before it is checked for macros
        # this prevents macros expanding into themselves
        result.append(line)
        if line.startswith("#!"):
            match = pattern.match(line)
            try:
                var, val = match.group(1, 2)
            except:
                raise InvalidMacroError("Failed to parse macro {}\n".format(line))
            macros[var] = val
        elif line.startswith("#<"):
            match = ipattern.match(line)
            try:
                filename = match.group(1)
            except:
                raise IncludeError("Failed to parse include {}\n".format(line))
            try:
                with open(filename, 'r') as f:
                    includes[filename] = expand_macros(f.read(), macros)
            except IOError:
                if match.group(2):
                    if match.group(2).startswith('or '):
                        print(match.group(3))
                else:
                    raise IncludeError("Nonexistent include {}\n".format(filename))
    return "\n".join(result), includes

def check_for_dep_in_outputs(dep, verbose, G):
    """
    Function to help construct_graph() identify dependencies

    Args:
        A dependency
        A flag indication verbosity
        A (populated) NetworkX DiGraph

    Returns:
        A list of targets that build given dependency

    """
    if verbose:
        print("checking dep {}".format(dep))
    ret_list = []
    for node in G.nodes(data=True):
        if "output" not in node[1]:
            continue
        for out in node[1]['output']:
            if fnmatch.fnmatch(out, dep):
                ret_list.append(node[0])
                break
    return ret_list

def construct_graph(sakefile, verbose):
    """
    Takes the sakefile dictionary and builds a NetworkX graph

    Args:
        A dictionary that is the parsed Sakefile (from sake.py)
        A flag indication verbosity

    Returns:
        A NetworkX graph
    """
    G = nx.DiGraph()
    if verbose:
        print("Going to construct Graph")
    for target in sakefile:
        if target == "all":
            # we don't want this node
            continue
        if "formula" not in sakefile[target]:
            # that means this is a meta target
            for atomtarget in sakefile[target]:
                if atomtarget == "help":
                    continue
                if verbose:
                    print("Adding '{}'".format(atomtarget))
                data_dict = sakefile[target][atomtarget]
                data_dict["parent"] = target
                G.add_node(atomtarget, data_dict)
        else:
            if verbose:
                print("Adding '{}'".format(target))
            G.add_node(target, sakefile[target])
    if verbose:
        print("Nodes are built\nBuilding connections")
    for node in G.nodes(data=True):
        if verbose:
            print("checking node {} for dependencies".format(node[0]))
        # normalize all paths in output
        for k, v in node[1].items():
            if v is None: node[1][k] = []
        if "output" in node[1]:
            for index, out in enumerate(node[1]['output']):
                node[1]['output'][index] = clean_path(node[1]['output'][index])
        if "dependencies" not in node[1]:
            continue
        if verbose:
            print("it has dependencies")
        connects = []
        # normalize all paths in dependencies
        for index, dep in enumerate(node[1]['dependencies']):
            dep = os.path.normpath(dep)
            shrt = "dependencies"
            node[1]['dependencies'][index] = clean_path(node[1][shrt][index])
    for node in G.nodes(data=True):
        connects = []
        if "dependencies" not in node[1]:
            continue
        for dep in node[1]['dependencies']:
            matches = check_for_dep_in_outputs(dep, verbose, G)
            if not matches:
                continue
            for match in matches:
                if verbose:
                    print("Appending {} to matches".format(match))
                connects.append(match)
        if connects:
            for connect in connects:
                G.add_edge(connect, node[0])
    return G

def get_all_outputs(node_dict):
    """
    This function takes a node dictionary and returns a list of
    the node's output files. Some of the entries in the 'output'
    attribute may be globs, and without this function, sake won't
    know how to handle that. This will unglob all globs and return
    the true list of *all* outputs.
    """
    outlist = []
    for item in node_dict['output']:
        glist = glob.glob(item)
        if glist:
            for oneglob in glist:
                outlist.append(oneglob)
        else:
            outlist.append(item)
    return outlist

def clean_all(G, verbose, quiet, recon):
    """
    Removes all the output files from all targets. Takes
    the graph as the only argument

    Args:
        The networkx graph object
        A flag indicating verbosity
        A flag indicating quiet mode

    Returns:
        0 if successful
        1 if removing even one file failed
    """
    all_outputs = []
    for node in G.nodes(data=True):
        if "output" in node[1]:
            for item in get_all_outputs(node[1]):
                all_outputs.append(item)
    all_outputs.append(".shastore")
    retcode = 0
    for item in all_outputs:
        if os.path.isfile(item):
            if recon:
                print("Would remove file: {}".format(item))
                continue
            if verbose:
                mesg = "Attempting to remove file '{}'"
                print(mesg.format(item))
            try:
                os.remove(item)
                if verbose:
                    print("Removed file")
            except:
                errmeg = "Error: file '{}' failed to be removed\n"
                sys.stderr.write(errmeg.format(item))
                retcode = 1
    if not retcode and not recon:
        print("All clean")
    return retcode

def write_dot_file(G, filename):
    """
    Writes the graph G in dot file format for graphviz visualization.

    Args:
        a Networkx graph
        A filename to name the dot files
    """
    with open(filename, "w", encoding="utf-8") as fh:
        fh.write("strict digraph DependencyDiagram {\n")
        edge_list = G.edges()
        node_list = set(G.nodes())
        if edge_list:
            for edge in G.edges():
                source, targ = edge
                node_list = node_list - set(source)
                node_list = node_list - set(targ)
                line = '"{}" -> "{}";\n'
                fh.write(line.format(source, targ))
        # draw nodes with no links
        if node_list:
            for node in node_list:
                line = '"{}"\n'.format(node)
                fh.write(line)
        fh.write("}")

def visualize(G, filename="dependencies", no_graphviz=False):
    """
    Uses networkX to draw a graphviz dot file either (a) calls the
    graphviz command "dot" to turn it into a SVG and remove the
    dotfile (default), or (b) if no_graphviz is True, just output
    the graphviz dot file

    Args:
        a NetworkX DiGraph
        a filename (a default is provided
        a flag indicating whether graphviz should *not* be called

    Returns:
        0 if everything worked
        will cause fatal error on failure
    """
    if no_graphviz:
        write_dot_file(G, filename)
        return 0
    write_dot_file(G, "tempdot")
    renderer = "svg"
    if re.search("\.jpg$", filename, re.IGNORECASE):
        renderer = "jpg"
    elif re.search("\.jpeg$", filename, re.IGNORECASE):
        renderer = "jpg"
    elif re.search("\.svg$", filename, re.IGNORECASE):
        renderer = "svg"
    elif re.search("\.png$", filename, re.IGNORECASE):
        renderer = "png"
    elif re.search("\.gif$", filename, re.IGNORECASE):
        renderer = "gif"
    elif re.search("\.ps$", filename, re.IGNORECASE):
        renderer = "ps"
    elif re.search("\.pdf$", filename, re.IGNORECASE):
        renderer = "pdf"
    else:
        renderer = "svg"
        filename += ".svg"
    command = "dot -T{} tempdot -o {}".format(renderer, filename)
    p = Popen(command, shell=True)
    p.communicate()
    if p.returncode:
        errmes = "Either graphviz is not installed, or its not on PATH"
        os.remove("tempdot")
        sys.stderr.write(errmes)
        sys.exit(1)
    os.remove("tempdot")
    return 0

#####################
##  CUSTOM ERRORS  ##
#####################

class Error(Exception):
    """Base class for exceptions in this module."""
    pass

class InvalidMacroError(Error):
    def __init__(self, message):
        self.message = message

class IncludeError(Error):
    def __init__(self, message):
        self.message = message

<|endoftext|>


<commit_before>'''
The actual saltkey functional code
'''

# Import python modules
import os
import shutil
import sys
import logging
# Import salt modules
import salt.crypt
import salt.utils

log = logging.getLogger(__name__)

class Key(object):
    '''
    The object that encapsulates saltkey actions
    '''
    def __init__(self, opts):
        self.opts = opts
        self.colors = salt.utils.get_colors(not bool(self.opts['no_color']))

    def _keys(self, key_type, full_path=False):
        '''
        Safely return the names of the unaccepted keys, pass True to return
        the full key paths. Returns a set.
        '''
        ret = set()
        subdir = ''
        if key_type == 'pre':
            subdir = 'minions_pre'
        elif key_type == 'rej':
            subdir = 'minions_rejected'
        elif key_type == 'acc':
            subdir = 'minions'
        dir_ = os.path.join(self.opts['pki_dir'], subdir)
        if not os.path.isdir(dir_):
            err = ('The {0} directory is not present, ensure that '
                   'the master server has been started').format(subdir)
            self._log(err, level='error')
            sys.exit(42)
        keys = os.listdir(dir_)
        if full_path:
            for key in keys:
                ret.add(os.path.join(dir_, key))
        else:
            ret = set(keys)
        return ret

    def _log(self, message, level=''):
        if hasattr(log, level):
            log_msg = getattr(log, level)
            log_msg(message)
        if not self.opts['quiet']:
            print(message)

    def _list_pre(self, header=True, printer=None):
        '''
        List the unaccepted keys
        '''
        if header == True:
            self._log('{0}Unaccepted Keys:{1}'.format(
                self.colors['LIGHT_RED'], self.colors['ENDC']
                ))
        keys = self._keys('pre')
        if printer is None:
            for key in sorted(keys):
                output = '{0}{1}{2}'.format(
                        self.colors['RED'],
                        key,
                        self.colors['ENDC']
                        )
                self._log(output)
        else:
            printer(list(keys))

    def _list_accepted(self, header=True, printer=None):
        '''
        List the accepted public keys
        '''
        if header == True:
            self._log('{0}Accepted Keys:{1}'.format(
                self.colors['LIGHT_GREEN'], self.colors['ENDC']
                ))
        keys = self._keys('acc')
        if printer is None:
            for key in sorted(keys):
                self._log('{0}{1}{2}'.format(
                    self.colors['GREEN'], key, self.colors['ENDC']
                    ))
        else:
            printer(list(keys))

    def _list_rejected(self, header=True, printer=None):
        '''
        List the unaccepted keys
        '''
        if header == True:
            self._log('{0}Rejected:{1}'.format(
                self.colors['LIGHT_BLUE'], self.colors['ENDC']
                ))
        keys = self._keys('rej')
        if printer is None:
            for key in sorted(keys):
                self._log('{0}{1}{2}'.format(
                    self.colors['BLUE'], key, self.colors['ENDC']
                    ))
        else:
            printer(list(keys))

    def _list(self, name):
        '''
        List keys
        '''
        printout = self._get_outputter()
        if 'json_out' in self.opts and self.opts['json_out']:
            printout.indent = 2
        if name in ('pre', 'un', 'unaccept', 'unaccepted'):
            self._list_pre(header=False, printer=printout)
        elif name in ('acc', 'accept', 'accepted'):
            self._list_accepted(header=False, printer=printout)
        elif name in ('rej', 'reject', 'rejected'):
            self._list_rejected(header=False, printer=printout)
        elif name in ('all',):
            if printout is not None:
                keys = {
                    'rejected': list(self._keys('rej')),
                    'accepted': list(self._keys('acc')),
                    'unaccepted': list(self._keys('pre')),
                }
                printout(keys)
            else:
                self._list_pre(printer=printout)
                self._list_accepted(printer=printout)
                self._list_rejected(printer=printout)
        else:
            err = ('Unrecognized key type "{0}".  Run with -h for options.'
                    ).format(name)
            self._log(err, level='error')

    def _get_outputter(self):
        get_outputter = salt.output.get_outputter
        if self.opts['raw_out']:
            printout = get_outputter('raw')
        elif self.opts['json_out']:
            printout = get_outputter('json')
        elif self.opts['yaml_out']:
            printout = get_outputter('yaml')
        else:
            printout = None # use default color output
        return printout

    def _print_key(self, name):
        '''
        Print out the specified public key
        '''
        keys = self._keys('pre', True).union(self._keys('acc', True))
        for key in sorted(keys):
            if key.endswith(name):
                with open(key, 'r') as kfn:
                    self._log(kfn.read())

    def _print_all(self):
        '''
        Print out the public keys, all of em'
        '''
        self._log('{0}Unaccepted keys:{1}'.format(
            self.colors['LIGHT_RED'], self.colors['ENDC']
            ))
        for key in sorted(self._keys('pre', True)):
            self._log('  {0}{1}{2}'.format(
                self.colors['RED'],
                os.path.basename(key),
                self.colors['ENDC']
                ))
            with open(key, 'r') as kfn:
                self._log(kfn.read())
        self._log('{0}Accepted keys:{1}'.format(
            self.colors['LIGHT_GREEN'], self.colors['ENDC']
            ))
        for key in sorted(self._keys('acc', True)):
            self._log('  {0}{1}{2}'.format(
                self.colors['GREEN'],
                os.path.basename(key),
                self.colors['ENDC']
                ))
            with open(key, 'r') as kfn:
                self._log(kfn.read())
        self._log('{0}Rejected keys:{1}'.format(
            self.colors['LIGHT_BLUE'], self.colors['ENDC']
            ))
        for key in sorted(self._keys('pre', True)):
            self._log('  {0}{1}{2}'.format(
                self.colors['BLUE'],
                os.path.basename(key),
                self.colors['ENDC']))
            with open(key, 'r') as kfn:
                self._log(kfn.read())

    def _accept(self, key):
        '''
        Accept a specified host's public key
        '''
        (minions_accepted,
         minions_pre,
         minions_rejected) = self._check_minions_directories()
        pre = os.listdir(minions_pre)
        if key not in pre:
            err = ('The key named {0} does not exist, please accept an '
                   'available key').format(key)
            #log.error(err)
            self._log(err, level='error')
            sys.exit(43)
        shutil.move(os.path.join(minions_pre, key),
                    os.path.join(minions_accepted, key))
        self._log(
                'Key for {0} accepted.'.format(key),
                level='info'
                )

    def _accept_all(self):
        '''
        Accept all keys in pre
        '''
        (minions_accepted,
         minions_pre,
         minions_rejected) = self._check_minions_directories()
        for key in os.listdir(minions_pre):
            self._accept(key)

    def _delete_key(self, delete=None):
        '''
        Delete a key
        '''
        (minions_accepted,
         minions_pre,
         minions_rejected) = self._check_minions_directories()
        if delete is None:
            delete = self.opts['delete']
        pre = os.path.join(minions_pre, delete)
        acc = os.path.join(minions_accepted, delete)
        rej = os.path.join(minions_rejected, delete)
        if os.path.exists(pre):
            os.remove(pre)
            self._log('Removed pending key {0}'.format(delete),
                         level='info')
        if os.path.exists(acc):
            os.remove(acc)
            self._log('Removed accepted key {0}'.format(delete),
                         level='info')
        if os.path.exists(rej):
            os.remove(rej)
            self._log('Removed rejected key {0}'.format(delete),
                         level='info')

    def _delete_all(self):
        '''
        Delete all keys
        '''
        for dir in ("acc", "rej", "pre"):
            for key in self._keys(dir):
                self._delete_key(key)

    def _reject(self, key):
        '''
        Reject a specified host's public key
        '''
        (minions_accepted,
         minions_pre,
         minions_rejected) = self._check_minions_directories()
        pre = os.listdir(minions_pre)
        if key not in pre:
            err = ('The host named {0} is unavailable, please accept an '
                   'available key').format(key)
            self._log(err, level='error')
            sys.exit(43)
        shutil.move(os.path.join(minions_pre, key),
                    os.path.join(minions_rejected, key))
        self._log('{0} key rejected.'.format(key), level='info')

    def _reject_all(self):
        '''
        Reject all keys in pre
        '''
        (minions_accepted,
         minions_pre,
         minions_rejected) = self._check_minions_directories()
        for key in os.listdir(minions_pre):
            self._reject(key)

    def _check_minions_directories(self):
        minions_accepted = os.path.join(self.opts['pki_dir'], 'minions')
        minions_pre = os.path.join(self.opts['pki_dir'], 'minions_pre')
        minions_rejected = os.path.join(self.opts['pki_dir'],
                                        'minions_rejected')
        for dir_ in [minions_accepted, minions_pre, minions_rejected]:
            if not os.path.isdir(dir_):
                err = ('The minions directory {0} is not present, ensure '
                       'that the master server has been started'.format(dir_))
                self._log(err, level='error')
                sys.exit(42)
        return minions_accepted, minions_pre, minions_rejected

    def run(self):
        '''
        Run the logic for saltkey
        '''
        if self.opts['gen_keys']:
            salt.crypt.gen_keys(
                    self.opts['gen_keys_dir'],
                    self.opts['gen_keys'],
                    self.opts['keysize'])
            return
        if self.opts['list']:
            self._list(self.opts['list'])
        elif self.opts['list_all']:
            self._list('all')
        elif self.opts['print']:
            self._print_key(self.opts['print'])
        elif self.opts['print_all']:
            self._print_all()
        elif self.opts['accept']:
            self._accept(self.opts['accept'])
        elif self.opts['accept_all']:
            self._accept_all()
        elif self.opts['reject']:
            self._reject(self.opts['reject'])
        elif self.opts['reject_all']:
            self._reject_all()
        elif self.opts['delete']:
            self._delete_key()
        elif self.opts['delete_all']:
            self._delete_all()
        else:
            self._list('all')
<commit_msg>Fix color loading for saltkey api calls<commit_after>'''
The actual saltkey functional code
'''

# Import python modules
import os
import shutil
import sys
import logging
# Import salt modules
import salt.crypt
import salt.utils

log = logging.getLogger(__name__)

class Key(object):
    '''
    The object that encapsulates saltkey actions
    '''
    def __init__(self, opts):
        self.opts = opts
        self.colors = salt.utils.get_colors(
                not bool(self.opts.get('no_color', False))
                )

    def _keys(self, key_type, full_path=False):
        '''
        Safely return the names of the unaccepted keys, pass True to return
        the full key paths. Returns a set.
        '''
        ret = set()
        subdir = ''
        if key_type == 'pre':
            subdir = 'minions_pre'
        elif key_type == 'rej':
            subdir = 'minions_rejected'
        elif key_type == 'acc':
            subdir = 'minions'
        dir_ = os.path.join(self.opts['pki_dir'], subdir)
        if not os.path.isdir(dir_):
            err = ('The {0} directory is not present, ensure that '
                   'the master server has been started').format(subdir)
            self._log(err, level='error')
            sys.exit(42)
        keys = os.listdir(dir_)
        if full_path:
            for key in keys:
                ret.add(os.path.join(dir_, key))
        else:
            ret = set(keys)
        return ret

    def _log(self, message, level=''):
        if hasattr(log, level):
            log_msg = getattr(log, level)
            log_msg(message)
        if not self.opts['quiet']:
            print(message)

    def _list_pre(self, header=True, printer=None):
        '''
        List the unaccepted keys
        '''
        if header == True:
            self._log('{0}Unaccepted Keys:{1}'.format(
                self.colors['LIGHT_RED'], self.colors['ENDC']
                ))
        keys = self._keys('pre')
        if printer is None:
            for key in sorted(keys):
                output = '{0}{1}{2}'.format(
                        self.colors['RED'],
                        key,
                        self.colors['ENDC']
                        )
                self._log(output)
        else:
            printer(list(keys))

    def _list_accepted(self, header=True, printer=None):
        '''
        List the accepted public keys
        '''
        if header == True:
            self._log('{0}Accepted Keys:{1}'.format(
                self.colors['LIGHT_GREEN'], self.colors['ENDC']
                ))
        keys = self._keys('acc')
        if printer is None:
            for key in sorted(keys):
                self._log('{0}{1}{2}'.format(
                    self.colors['GREEN'], key, self.colors['ENDC']
                    ))
        else:
            printer(list(keys))

    def _list_rejected(self, header=True, printer=None):
        '''
        List the unaccepted keys
        '''
        if header == True:
            self._log('{0}Rejected:{1}'.format(
                self.colors['LIGHT_BLUE'], self.colors['ENDC']
                ))
        keys = self._keys('rej')
        if printer is None:
            for key in sorted(keys):
                self._log('{0}{1}{2}'.format(
                    self.colors['BLUE'], key, self.colors['ENDC']
                    ))
        else:
            printer(list(keys))

    def _list(self, name):
        '''
        List keys
        '''
        printout = self._get_outputter()
        if 'json_out' in self.opts and self.opts['json_out']:
            printout.indent = 2
        if name in ('pre', 'un', 'unaccept', 'unaccepted'):
            self._list_pre(header=False, printer=printout)
        elif name in ('acc', 'accept', 'accepted'):
            self._list_accepted(header=False, printer=printout)
        elif name in ('rej', 'reject', 'rejected'):
            self._list_rejected(header=False, printer=printout)
        elif name in ('all',):
            if printout is not None:
                keys = {
                    'rejected': list(self._keys('rej')),
                    'accepted': list(self._keys('acc')),
                    'unaccepted': list(self._keys('pre')),
                }
                printout(keys)
            else:
                self._list_pre(printer=printout)
                self._list_accepted(printer=printout)
                self._list_rejected(printer=printout)
        else:
            err = ('Unrecognized key type "{0}".  Run with -h for options.'
                    ).format(name)
            self._log(err, level='error')

    def _get_outputter(self):
        get_outputter = salt.output.get_outputter
        if self.opts['raw_out']:
            printout = get_outputter('raw')
        elif self.opts['json_out']:
            printout = get_outputter('json')
        elif self.opts['yaml_out']:
            printout = get_outputter('yaml')
        else:
            printout = None # use default color output
        return printout

    def _print_key(self, name):
        '''
        Print out the specified public key
        '''
        keys = self._keys('pre', True).union(self._keys('acc', True))
        for key in sorted(keys):
            if key.endswith(name):
                with open(key, 'r') as kfn:
                    self._log(kfn.read())

    def _print_all(self):
        '''
        Print out the public keys, all of em'
        '''
        self._log('{0}Unaccepted keys:{1}'.format(
            self.colors['LIGHT_RED'], self.colors['ENDC']
            ))
        for key in sorted(self._keys('pre', True)):
            self._log('  {0}{1}{2}'.format(
                self.colors['RED'],
                os.path.basename(key),
                self.colors['ENDC']
                ))
            with open(key, 'r') as kfn:
                self._log(kfn.read())
        self._log('{0}Accepted keys:{1}'.format(
            self.colors['LIGHT_GREEN'], self.colors['ENDC']
            ))
        for key in sorted(self._keys('acc', True)):
            self._log('  {0}{1}{2}'.format(
                self.colors['GREEN'],
                os.path.basename(key),
                self.colors['ENDC']
                ))
            with open(key, 'r') as kfn:
                self._log(kfn.read())
        self._log('{0}Rejected keys:{1}'.format(
            self.colors['LIGHT_BLUE'], self.colors['ENDC']
            ))
        for key in sorted(self._keys('pre', True)):
            self._log('  {0}{1}{2}'.format(
                self.colors['BLUE'],
                os.path.basename(key),
                self.colors['ENDC']))
            with open(key, 'r') as kfn:
                self._log(kfn.read())

    def _accept(self, key):
        '''
        Accept a specified host's public key
        '''
        (minions_accepted,
         minions_pre,
         minions_rejected) = self._check_minions_directories()
        pre = os.listdir(minions_pre)
        if key not in pre:
            err = ('The key named {0} does not exist, please accept an '
                   'available key').format(key)
            #log.error(err)
            self._log(err, level='error')
            sys.exit(43)
        shutil.move(os.path.join(minions_pre, key),
                    os.path.join(minions_accepted, key))
        self._log(
                'Key for {0} accepted.'.format(key),
                level='info'
                )

    def _accept_all(self):
        '''
        Accept all keys in pre
        '''
        (minions_accepted,
         minions_pre,
         minions_rejected) = self._check_minions_directories()
        for key in os.listdir(minions_pre):
            self._accept(key)

    def _delete_key(self, delete=None):
        '''
        Delete a key
        '''
        (minions_accepted,
         minions_pre,
         minions_rejected) = self._check_minions_directories()
        if delete is None:
            delete = self.opts['delete']
        pre = os.path.join(minions_pre, delete)
        acc = os.path.join(minions_accepted, delete)
        rej = os.path.join(minions_rejected, delete)
        if os.path.exists(pre):
            os.remove(pre)
            self._log('Removed pending key {0}'.format(delete),
                         level='info')
        if os.path.exists(acc):
            os.remove(acc)
            self._log('Removed accepted key {0}'.format(delete),
                         level='info')
        if os.path.exists(rej):
            os.remove(rej)
            self._log('Removed rejected key {0}'.format(delete),
                         level='info')

    def _delete_all(self):
        '''
        Delete all keys
        '''
        for dir in ("acc", "rej", "pre"):
            for key in self._keys(dir):
                self._delete_key(key)

    def _reject(self, key):
        '''
        Reject a specified host's public key
        '''
        (minions_accepted,
         minions_pre,
         minions_rejected) = self._check_minions_directories()
        pre = os.listdir(minions_pre)
        if key not in pre:
            err = ('The host named {0} is unavailable, please accept an '
                   'available key').format(key)
            self._log(err, level='error')
            sys.exit(43)
        shutil.move(os.path.join(minions_pre, key),
                    os.path.join(minions_rejected, key))
        self._log('{0} key rejected.'.format(key), level='info')

    def _reject_all(self):
        '''
        Reject all keys in pre
        '''
        (minions_accepted,
         minions_pre,
         minions_rejected) = self._check_minions_directories()
        for key in os.listdir(minions_pre):
            self._reject(key)

    def _check_minions_directories(self):
        minions_accepted = os.path.join(self.opts['pki_dir'], 'minions')
        minions_pre = os.path.join(self.opts['pki_dir'], 'minions_pre')
        minions_rejected = os.path.join(self.opts['pki_dir'],
                                        'minions_rejected')
        for dir_ in [minions_accepted, minions_pre, minions_rejected]:
            if not os.path.isdir(dir_):
                err = ('The minions directory {0} is not present, ensure '
                       'that the master server has been started'.format(dir_))
                self._log(err, level='error')
                sys.exit(42)
        return minions_accepted, minions_pre, minions_rejected

    def run(self):
        '''
        Run the logic for saltkey
        '''
        if self.opts['gen_keys']:
            salt.crypt.gen_keys(
                    self.opts['gen_keys_dir'],
                    self.opts['gen_keys'],
                    self.opts['keysize'])
            return
        if self.opts['list']:
            self._list(self.opts['list'])
        elif self.opts['list_all']:
            self._list('all')
        elif self.opts['print']:
            self._print_key(self.opts['print'])
        elif self.opts['print_all']:
            self._print_all()
        elif self.opts['accept']:
            self._accept(self.opts['accept'])
        elif self.opts['accept_all']:
            self._accept_all()
        elif self.opts['reject']:
            self._reject(self.opts['reject'])
        elif self.opts['reject_all']:
            self._reject_all()
        elif self.opts['delete']:
            self._delete_key()
        elif self.opts['delete_all']:
            self._delete_all()
        else:
            self._list('all')
<|endoftext|>


<commit_before>import sys

__version_info__ = (0, 10, 4)
__version__ = '.'.join(map(str, __version_info__))

# If we can get a version from Git use that instead, otherwise carry on
try:
    import subprocess
    from salt.utils import which

    git = which('git')
    if git:
        p = subprocess.Popen([git, 'describe'],
                stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
        out, err = p.communicate()
        if out:
            __version__ = '{0}'.format(out.strip().lstrip('v'))
            __version_info__ = tuple(__version__.split('-', 1)[0].split('.'))
except Exception:
    pass

def versions_report():
    libs = (
        ("Jinja2", "jinja2", "__version__"),
        ("M2Crypto", "M2Crypto", "version"),
        ("msgpack-python", "msgpack", "version"),
        ("msgpack-pure", "msgpack_pure", "version"),
        ("pycrypto", "Crypto", "__version__"),
        ("PyYAML", "yaml", "__version__"),
        ("PyZMQ", "zmq", "__version__"),
    )

    padding = len(max([lib[0] for lib in libs], key=len)) + 1

    fmt = '{0:>{pad}}: {1}'

    yield fmt.format("Salt", __version__, pad=padding)

    yield fmt.format(
        "Python", sys.version.rsplit('\n')[0].strip(), pad=padding
    )

    for name, imp, attr in libs:
        try:
            imp = __import__(imp)
            version = getattr(imp, attr)
            if not isinstance(version, basestring):
                version = '.'.join(map(str, version))
            yield fmt.format(name, version, pad=padding)
        except ImportError:
            yield fmt.format(name, "not installed", pad=padding)

if __name__ == '__main__':
    print(__version__)
<commit_msg>Cast `__version_info__` elements to `int`'s.<commit_after>import sys

__version_info__ = (0, 10, 4)
__version__ = '.'.join(map(str, __version_info__))

# If we can get a version from Git use that instead, otherwise carry on
try:
    import subprocess
    from salt.utils import which

    git = which('git')
    if git:
        p = subprocess.Popen([git, 'describe'],
                stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
        out, err = p.communicate()
        if out:
            __version__ = '{0}'.format(out.strip().lstrip('v'))
            __version_info__ = tuple(
                [int(i) for i in __version__.split('-', 1)[0].split('.')]
            )
except Exception:
    pass

def versions_report():
    libs = (
        ("Jinja2", "jinja2", "__version__"),
        ("M2Crypto", "M2Crypto", "version"),
        ("msgpack-python", "msgpack", "version"),
        ("msgpack-pure", "msgpack_pure", "version"),
        ("pycrypto", "Crypto", "__version__"),
        ("PyYAML", "yaml", "__version__"),
        ("PyZMQ", "zmq", "__version__"),
    )

    padding = len(max([lib[0] for lib in libs], key=len)) + 1

    fmt = '{0:>{pad}}: {1}'

    yield fmt.format("Salt", __version__, pad=padding)

    yield fmt.format(
        "Python", sys.version.rsplit('\n')[0].strip(), pad=padding
    )

    for name, imp, attr in libs:
        try:
            imp = __import__(imp)
            version = getattr(imp, attr)
            if not isinstance(version, basestring):
                version = '.'.join(map(str, version))
            yield fmt.format(name, version, pad=padding)
        except ImportError:
            yield fmt.format(name, "not installed", pad=padding)

if __name__ == '__main__':
    print(__version__)
<|endoftext|>


<commit_before>'''
Set up the version of Salt
'''

# Import python libs
import os
import re
import sys
import warnings
import subprocess

__version_info__ = (0, 12, 0)
__version__ = '.'.join(map(str, __version_info__))

GIT_DESCRIBE_RE = re.compile(
    r'(?P<major>[\d]{1,2}).(?P<minor>[\d]{1,2}).(?P<bugfix>[\d]{1,2})'
    r'(?:(?:.*)-(?P<noc>[\d]+)-(?P<sha>[a-z0-9]{8}))?'
)

def __get_version_info_from_git(version, version_info):
    '''
    If we can get a version from Git use that instead, otherwise we carry on
    '''
    try:
        kwargs = dict(
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            cwd=os.path.abspath(os.path.dirname(__file__))
        )
        if not sys.platform.startswith('win'):
            # Let's not import `salt.utils` for the above check
            kwargs['close_fds'] = True

        process = subprocess.Popen(['git', 'describe', '--tags'], **kwargs)
        out, err = process.communicate()
        if not out.strip() or err.strip():
            return version, version_info

        match = GIT_DESCRIBE_RE.search(out.strip())
        if not match:
            return version, version_info

        parsed_version = '{0}.{1}.{2}-{3}-{4}'.format(
            match.group('major'),
            match.group('minor'),
            match.group('bugfix'),
            match.group('noc'),
            match.group('sha')
        )
        parsed_version_info = tuple([
            int(g) for g in match.groups()[:3] if g.isdigit()
        ])
        if parsed_version_info != version_info:
            warnings.warn(
                'In order to get the proper salt version with the '
                'git hash you need to update salt\'s local git '
                'tags. Something like: \'git fetch --tags\' or '
                '\'git fetch --tags upstream\' if you followed '
                'salt\'s contribute documentation. The version '
                'string WILL NOT include the git hash.',
                UserWarning,
                stacklevel=2
            )
            return version, version_info
        return parsed_version, parsed_version_info
    except OSError, err:
        if not hasattr(err, 'child_traceback'):
            # This is not an exception thrown within the Popen created child.
            # Let's raise it so it can be catch by the developers
            raise
        # Popen child exceptions are not raised
    return version, version_info

# Get version information from git if available
__version__, __version_info__ = \
    __get_version_info_from_git(__version__, __version_info__)
# This function has executed once, we're done with it. Delete it!
del __get_version_info_from_git

def versions_report():
    '''
    Report on all of the versions for dependant software
    '''
    libs = (
        ('Jinja2', 'jinja2', '__version__'),
        ('M2Crypto', 'M2Crypto', 'version'),
        ('msgpack-python', 'msgpack', 'version'),
        ('msgpack-pure', 'msgpack_pure', 'version'),
        ('pycrypto', 'Crypto', '__version__'),
        ('PyYAML', 'yaml', '__version__'),
        ('PyZMQ', 'zmq', '__version__'),
    )

    padding = len(max([lib[0] for lib in libs], key=len)) + 1

    fmt = '{0:>{pad}}: {1}'

    yield fmt.format('Salt', __version__, pad=padding)

    yield fmt.format(
        'Python', sys.version.rsplit('\n')[0].strip(), pad=padding
    )

    for name, imp, attr in libs:
        try:
            imp = __import__(imp)
            version = getattr(imp, attr)
            if not isinstance(version, basestring):
                version = '.'.join(map(str, version))
            yield fmt.format(name, version, pad=padding)
        except ImportError:
            yield fmt.format(name, 'not installed', pad=padding)

if __name__ == '__main__':
    print(__version__)
<commit_msg>Don't fail on Windows if git is not installed or not on `PATH`.<commit_after>'''
Set up the version of Salt
'''

# Import python libs
import os
import re
import sys
import warnings
import subprocess

__version_info__ = (0, 12, 0)
__version__ = '.'.join(map(str, __version_info__))

GIT_DESCRIBE_RE = re.compile(
    r'(?P<major>[\d]{1,2}).(?P<minor>[\d]{1,2}).(?P<bugfix>[\d]{1,2})'
    r'(?:(?:.*)-(?P<noc>[\d]+)-(?P<sha>[a-z0-9]{8}))?'
)

def __get_version_info_from_git(version, version_info):
    '''
    If we can get a version from Git use that instead, otherwise we carry on
    '''
    try:
        kwargs = dict(
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            cwd=os.path.abspath(os.path.dirname(__file__))
        )
        if not sys.platform.startswith('win'):
            # Let's not import `salt.utils` for the above check
            kwargs['close_fds'] = True

        try:
            process = subprocess.Popen(['git', 'describe', '--tags'], **kwargs)
        except WindowsError, err:
            if err.errno != 2:
                # Raise any other WindowsError exception so we know what's
                # going on.
                raise
            # The system cannot find the file specified
            return version, version_info

        out, err = process.communicate()
        if not out.strip() or err.strip():
            return version, version_info

        match = GIT_DESCRIBE_RE.search(out.strip())
        if not match:
            return version, version_info

        parsed_version = '{0}.{1}.{2}-{3}-{4}'.format(
            match.group('major'),
            match.group('minor'),
            match.group('bugfix'),
            match.group('noc'),
            match.group('sha')
        )
        parsed_version_info = tuple([
            int(g) for g in match.groups()[:3] if g.isdigit()
        ])
        if parsed_version_info != version_info:
            warnings.warn(
                'In order to get the proper salt version with the '
                'git hash you need to update salt\'s local git '
                'tags. Something like: \'git fetch --tags\' or '
                '\'git fetch --tags upstream\' if you followed '
                'salt\'s contribute documentation. The version '
                'string WILL NOT include the git hash.',
                UserWarning,
                stacklevel=2
            )
            return version, version_info
        return parsed_version, parsed_version_info
    except OSError, err:
        if not hasattr(err, 'child_traceback'):
            # This is not an exception thrown within the Popen created child.
            # Let's raise it so it can be catch by the developers
            raise
        # Popen child exceptions are not raised
    return version, version_info

# Get version information from git if available
__version__, __version_info__ = \
    __get_version_info_from_git(__version__, __version_info__)
# This function has executed once, we're done with it. Delete it!
del __get_version_info_from_git

def versions_report():
    '''
    Report on all of the versions for dependant software
    '''
    libs = (
        ('Jinja2', 'jinja2', '__version__'),
        ('M2Crypto', 'M2Crypto', 'version'),
        ('msgpack-python', 'msgpack', 'version'),
        ('msgpack-pure', 'msgpack_pure', 'version'),
        ('pycrypto', 'Crypto', '__version__'),
        ('PyYAML', 'yaml', '__version__'),
        ('PyZMQ', 'zmq', '__version__'),
    )

    padding = len(max([lib[0] for lib in libs], key=len)) + 1

    fmt = '{0:>{pad}}: {1}'

    yield fmt.format('Salt', __version__, pad=padding)

    yield fmt.format(
        'Python', sys.version.rsplit('\n')[0].strip(), pad=padding
    )

    for name, imp, attr in libs:
        try:
            imp = __import__(imp)
            version = getattr(imp, attr)
            if not isinstance(version, basestring):
                version = '.'.join(map(str, version))
            yield fmt.format(name, version, pad=padding)
        except ImportError:
            yield fmt.format(name, 'not installed', pad=padding)

if __name__ == '__main__':
    print(__version__)
<|endoftext|>


<commit_before># Copyright 2014 Google Inc. All rights reserved.

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import, unicode_literals

from caniusepython3.test import unittest

ALL_GOOD = True
try:
    import io
    import sys
    import tokenize

    from astroid import test_utils
    from pylint import testutils
    from pylint.testutils import CheckerTestCase

    from caniusepython3 import pylint_checker as checker
except (ImportError, SyntaxError):
    ALL_GOOD = False
    CheckerTestCase = unittest.TestCase

def python2_only(test):
    """Decorator for any tests that will fail under Python 3."""
    return unittest.skipIf(sys.version_info[0] > 2, 'Python 2 only')(test)

unittest.skipIf(not ALL_GOOD, 'Pylint requires Python 2.7/3.3 or newer')
class StrictPython3CheckerTest(CheckerTestCase):

    CHECKER_CLASS = checker.StrictPython3Checker

    def check_not_builtin(self, builtin_name, message):
        node = test_utils.extract_node(builtin_name + '  #@')
        with self.assertAddsMessages(testutils.Message(message, node=node)):
            self.checker.visit_name(node)

    @python2_only
    def test_map_builtin(self):
        self.check_not_builtin('map', 'map-builtin')

    @python2_only
    def test_range_builtin(self):
        self.check_not_builtin('range', 'range-builtin')

    @python2_only
    def test_zip_builtin(self):
        self.check_not_builtin('zip', 'zip-builtin')

    @python2_only
    def test_round_builtin(self):
        self.check_not_builtin('round', 'round-builtin')

    @python2_only
    def test_open_builtin(self):
        self.check_not_builtin('open', 'open-builtin')

class UnicodeCheckerTest(CheckerTestCase):

    CHECKER_CLASS = checker.UnicodeChecker

    def tokenize(self, source):
        return tokenize.generate_tokens(io.StringIO(source).readline)

    def test_bytes_okay(self):
        tokens = self.tokenize("b'abc'")
        with self.assertNoMessages():
            self.checker.process_tokens(tokens)

    def test_unicode_okay(self):
        tokens = self.tokenize("u'abc'")
        with self.assertNoMessages():
            self.checker.process_tokens(tokens)

    def test_native_string(self):
        arg = "val = 'abc'"
        tokens = self.tokenize(arg)
        with self.assertAddsMessages(testutils.Message('native-string', line=1)):
            self.checker.process_tokens(tokens)

    def test_future_unicode(self):
        arg = "from __future__ import unicode_literals; val = 'abc'"
        tokens = self.tokenize(arg)
        with self.assertNoMessages():
            self.checker.process_tokens(tokens)

    def test_future_unicode_after_module_docstring(self):
        module = '"""Module docstring"""\nfrom __future__ import unicode_literals'
        tokens = self.tokenize(module)
        with self.assertNoMessages():
            self.checker.process_tokens(tokens)

    def test_future_unicode_after_shebang_and_module_docstring(self):
        module = '#! /usr/bin/python2.7\n"""Module docstring"""\nfrom __future__ import unicode_literals'
        tokens = self.tokenize(module)
        with self.assertNoMessages():
            self.checker.process_tokens(tokens)

if __name__ == '__main__':
    import unittest
    unittest.main()
<commit_msg>Another NameError fix to make 2.6/3.2 happy<commit_after># Copyright 2014 Google Inc. All rights reserved.

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import, unicode_literals

from caniusepython3.test import unittest

ALL_GOOD = True
try:
    import io
    import sys
    import tokenize

    from astroid import test_utils
    from pylint import testutils
    from pylint.testutils import CheckerTestCase

    from caniusepython3.pylint_checker import StrictPython3Checker, UnicodeChecker
except (ImportError, SyntaxError):
    ALL_GOOD = False
    CheckerTestCase = unittest.TestCase
    StrictlyPython3Checker = None
    UnicodeChecker = None

def python2_only(test):
    """Decorator for any tests that will fail under Python 3."""
    return unittest.skipIf(sys.version_info[0] > 2, 'Python 2 only')(test)

unittest.skipIf(not ALL_GOOD, 'Pylint requires Python 2.7/3.3 or newer')
class StrictPython3CheckerTest(CheckerTestCase):

    CHECKER_CLASS = StrictPython3Checker

    def check_not_builtin(self, builtin_name, message):
        node = test_utils.extract_node(builtin_name + '  #@')
        with self.assertAddsMessages(testutils.Message(message, node=node)):
            self.checker.visit_name(node)

    @python2_only
    def test_map_builtin(self):
        self.check_not_builtin('map', 'map-builtin')

    @python2_only
    def test_range_builtin(self):
        self.check_not_builtin('range', 'range-builtin')

    @python2_only
    def test_zip_builtin(self):
        self.check_not_builtin('zip', 'zip-builtin')

    @python2_only
    def test_round_builtin(self):
        self.check_not_builtin('round', 'round-builtin')

    @python2_only
    def test_open_builtin(self):
        self.check_not_builtin('open', 'open-builtin')

class UnicodeCheckerTest(CheckerTestCase):

    CHECKER_CLASS = UnicodeChecker

    def tokenize(self, source):
        return tokenize.generate_tokens(io.StringIO(source).readline)

    def test_bytes_okay(self):
        tokens = self.tokenize("b'abc'")
        with self.assertNoMessages():
            self.checker.process_tokens(tokens)

    def test_unicode_okay(self):
        tokens = self.tokenize("u'abc'")
        with self.assertNoMessages():
            self.checker.process_tokens(tokens)

    def test_native_string(self):
        arg = "val = 'abc'"
        tokens = self.tokenize(arg)
        with self.assertAddsMessages(testutils.Message('native-string', line=1)):
            self.checker.process_tokens(tokens)

    def test_future_unicode(self):
        arg = "from __future__ import unicode_literals; val = 'abc'"
        tokens = self.tokenize(arg)
        with self.assertNoMessages():
            self.checker.process_tokens(tokens)

    def test_future_unicode_after_module_docstring(self):
        module = '"""Module docstring"""\nfrom __future__ import unicode_literals'
        tokens = self.tokenize(module)
        with self.assertNoMessages():
            self.checker.process_tokens(tokens)

    def test_future_unicode_after_shebang_and_module_docstring(self):
        module = '#! /usr/bin/python2.7\n"""Module docstring"""\nfrom __future__ import unicode_literals'
        tokens = self.tokenize(module)
        with self.assertNoMessages():
            self.checker.process_tokens(tokens)

if __name__ == '__main__':
    import unittest
    unittest.main()
<|endoftext|>


<commit_before>import tornado.web
import brukva.adisp
import logging
import settings
import simplejson as json
import urlparse
import tasks
import cPickle as pickle
import zlib

class Handler(tornado.web.RequestHandler):

    def cors(self):
        self.set_header('Access-Control-Allow-Origin', '*')
        self.set_header('Access-Control-Allow-Methods', 'GET, POST, OPTIONS')
        self.set_header('Access-Control-Allow-Headers', 'Origin, Content-Type, Accept, X-Spider')

class Redirect(Handler):

    def get(self, *args, **kwargs):
        self.redirect('http://ziyan.github.io/spider/')

class Capture(Handler):

    @tornado.web.asynchronous
    @brukva.adisp.process
    def post(self, *args, **kwargs):
        data = json.loads(self.request.body)

        # parse url and get site
        parts = urlparse.urlparse(data['url'])
        assert parts.scheme in ['http', 'https']
        site = '%s://%s' % (parts.scheme.lower(), parts.netloc.lower())

        @brukva.adisp.async
        def process(site, data, callback):
            pipeline = settings.REDIS_ASYNC.pipeline()
            pipeline.exists('spider:site:%s' % site)
            pipeline.hset('spider:site:%s' % site, data['url'], zlib.compress(pickle.dumps(data, pickle.HIGHEST_PROTOCOL)))
            pipeline.hlen('spider:site:%s' % site)
            pipeline.get('spider:site:%s:selectors' % site)
            pipeline.execute(callback)

        is_old_site, is_new_page, count, selectors = yield process(site, data)

        # stats
        settings.REDIS_ASYNC.incr('spider:stats:usage')
        if not is_old_site:
            settings.REDIS_ASYNC.incr('spider:stats:sites')
        if is_new_page:
            settings.REDIS_ASYNC.incr('spider:stats:pages')

        # schedule learner
        if is_new_page and count > 1:
            tasks.learn.delay(site)

        self.cors()
        self.content_type = 'application/json'
        self.write(json.dumps({
            'is_new_site': not is_old_site,
            'is_new_page': is_new_page > 0,
            'count': count,
            'selectors': selectors,
        }))
        self.finish()

    def options(self, *args, **kwargs):
        self.cors()
        self.finish()

class Site(Handler):

    @tornado.web.asynchronous
    @brukva.adisp.process
    def get(self, *args, **kwargs):
        url = self.get_argument('url')

        # parse url and get site
        parts = urlparse.urlparse(url)
        assert parts.scheme in ['http', 'https']
        site = '%s://%s' % (parts.scheme, parts.netloc)

        @brukva.adisp.async
        def process(site, callback):
            pipeline = settings.REDIS_ASYNC.pipeline()
            pipeline.hlen('spider:site:%s' % site)
            pipeline.get('spider:site:%s:selectors' % site)
            pipeline.execute(callback)

        count, selectors = yield process(site)

        self.cors()
        self.content_type = 'application/json'
        self.write(json.dumps({
            'count': count,
            'selectors': selectors,
        }))
        self.finish()

    def options(self, *args, **kwargs):
        self.cors()
        self.finish()

class Stats(Handler):

    @tornado.web.asynchronous
    @brukva.adisp.process
    def get(self, *args, **kwargs):

        @brukva.adisp.async
        def process(callback):
            pipeline = settings.REDIS_ASYNC.pipeline()
            pipeline.get('spider:stats:usage')
            pipeline.get('spider:stats:sites')
            pipeline.get('spider:stats:pages')
            pipeline.execute(callback)

        usage, sites, pages = yield process()

        self.cors()
        self.content_type = 'application/json'
        self.write(json.dumps({
            'usage': int(usage),
            'sites': int(sites),
            'pages': int(pages),
        }))
        self.finish()

    def options(self, *args, **kwargs):
        self.cors()
        self.finish()
<commit_msg>Tornado writing the wrong content type.<commit_after>import tornado.web
import brukva.adisp
import logging
import settings
import simplejson as json
import urlparse
import tasks
import cPickle as pickle
import zlib

class Handler(tornado.web.RequestHandler):

    def cors(self):
        self.set_header('Access-Control-Allow-Origin', '*')
        self.set_header('Access-Control-Allow-Methods', 'GET, POST, OPTIONS')
        self.set_header('Access-Control-Allow-Headers', 'Origin, Content-Type, Accept, X-Spider')

class Redirect(Handler):

    def get(self, *args, **kwargs):
        self.redirect('http://ziyan.github.io/spider/')

class Capture(Handler):

    @tornado.web.asynchronous
    @brukva.adisp.process
    def post(self, *args, **kwargs):
        data = json.loads(self.request.body)

        # parse url and get site
        parts = urlparse.urlparse(data['url'])
        assert parts.scheme in ['http', 'https']
        site = '%s://%s' % (parts.scheme.lower(), parts.netloc.lower())

        @brukva.adisp.async
        def process(site, data, callback):
            pipeline = settings.REDIS_ASYNC.pipeline()
            pipeline.exists('spider:site:%s' % site)
            pipeline.hset('spider:site:%s' % site, data['url'], zlib.compress(pickle.dumps(data, pickle.HIGHEST_PROTOCOL)))
            pipeline.hlen('spider:site:%s' % site)
            pipeline.get('spider:site:%s:selectors' % site)
            pipeline.execute(callback)

        is_old_site, is_new_page, count, selectors = yield process(site, data)

        # stats
        settings.REDIS_ASYNC.incr('spider:stats:usage')
        if not is_old_site:
            settings.REDIS_ASYNC.incr('spider:stats:sites')
        if is_new_page:
            settings.REDIS_ASYNC.incr('spider:stats:pages')

        # schedule learner
        if is_new_page and count > 1:
            tasks.learn.delay(site)

        self.content_type = 'application/json'
        self.cors()
        self.write(json.dumps({
            'is_new_site': not is_old_site,
            'is_new_page': is_new_page > 0,
            'count': count,
            'selectors': selectors,
        }))
        self.finish()

    def options(self, *args, **kwargs):
        self.cors()
        self.finish()

class Site(Handler):

    @tornado.web.asynchronous
    @brukva.adisp.process
    def get(self, *args, **kwargs):
        url = self.get_argument('url')

        # parse url and get site
        parts = urlparse.urlparse(url)
        assert parts.scheme in ['http', 'https']
        site = '%s://%s' % (parts.scheme, parts.netloc)

        @brukva.adisp.async
        def process(site, callback):
            pipeline = settings.REDIS_ASYNC.pipeline()
            pipeline.hlen('spider:site:%s' % site)
            pipeline.get('spider:site:%s:selectors' % site)
            pipeline.execute(callback)

        count, selectors = yield process(site)

        self.content_type = 'application/json'
        self.cors()
        self.write(json.dumps({
            'count': count,
            'selectors': selectors,
        }))
        self.finish()

    def options(self, *args, **kwargs):
        self.cors()
        self.finish()

class Stats(Handler):

    @tornado.web.asynchronous
    @brukva.adisp.process
    def get(self, *args, **kwargs):

        @brukva.adisp.async
        def process(callback):
            pipeline = settings.REDIS_ASYNC.pipeline()
            pipeline.get('spider:stats:usage')
            pipeline.get('spider:stats:sites')
            pipeline.get('spider:stats:pages')
            pipeline.execute(callback)

        usage, sites, pages = yield process()

        self.content_type = 'application/json'
        self.cors()
        self.write(json.dumps({
            'usage': int(usage),
            'sites': int(sites),
            'pages': int(pages),
        }))
        self.finish()

    def options(self, *args, **kwargs):
        self.cors()
        self.finish()
<|endoftext|>