<commit_before># -*- coding: utf-8 -*-
"""
    pygments.styles.tango
    ~~~~~~~~~~~~~~~~~~~~~

    :copyright: 2008 by André Roberge.
    :license: BSD

    The Crunchy default Style inspired from the color palette from
    the Tango Icon Theme Guidelines.

    http://tango.freedesktop.org/Tango_Icon_Theme_Guidelines

    Butter:     #fce94f     #edd400     #c4a000
    Orange:     #fcaf3e     #f57900     #ce5c00
    Chocolate:  #e9b96e     #c17d11     #8f5902
    Chameleon:  #8ae234     #73d216     #4e9a06
    Sky Blue:   #729fcf     #3465a4     #204a87
    Plum:       #ad7fa8     #75507b     #5c35cc
    Scarlet Red:#ef2929     #cc0000     #a40000
    Aluminium:  #eeeeec     #d3d7cf     #babdb6
                #888a85     #555753     #2e3436

    Not all of the above colors are used; other colors added:
        very light grey: #f8f8f8  (for background)

    This style can be used as a template as it includes all the known
    Token types, unlike most (if not all) of the styles included in the
    Pygments distribution.

    However, since Crunchy is intended to be used by beginners, we have strived
    to create a style that gloss over subtle distinctions between different
    categories.

    Taking Python for example, comments (Comment.*) and docstrings (String.Doc)
    have been chosen to have the same style.  Similarly, keywords (Keyword.*),
    and Operator.Word (and, or, in) have been assigned the same style.
"""

from pygments.style import Style
from pygments.token import Keyword, Name, Comment, String, Error, \
     Number, Operator, Generic, Whitespace, Punctuation, Other, Literal

class TangoStyle(Style):
    """
    The Crunchy default Style inspired from the color palette from
    the Tango Icon Theme Guidelines.
    """

    # work in progress...

    background_color = "#f8f8f8"
    default_style = ""

    styles = {
        # No corresponding class for the following:
        #Text:              "", # class:  '',
        Whitespace:                "underline #f8f8f8", # class: 'w',
        Error:                     "#a40000 border:#ef2929", # class: 'err',
        Other:                     "#000000",        # class 'x',

        Comment:                   "italic #8f5902", # class: 'c',
        Comment.Multiline:         "italic #8f5902", # class: 'cm',
        Comment.Preproc:           "italic #8f5902", # class: 'cp',
        Comment.Single:            "italic #8f5902", # class: 'c1',
        Comment.Special:           "italic #8f5902", # class: 'cs',

        Keyword:                   "bold #204a87", # class: 'k',
        Keyword.Constant:          "bold #204a87", # class: 'kc',
        Keyword.Declaration:       "bold #204a87", # class: 'kd',
        Keyword.Namespace:         "bold #204a87", # class: 'kn',
        Keyword.Pseudo:            "bold #204a87", # class: 'kp',
        Keyword.Reserved:          "bold #204a87", # class: 'kr',
        Keyword.Type:              "bold #204a87", # class: 'kt',

        Operator:                  "bold #ce5c00", # class: 'o'
        Operator.Word:             "bold #204a87", # class: 'ow' - like keywords

        Punctuation:               "bold #000000", # class: 'p'

        # because special names such as Name.Class, Name.Function, etc.
        # are not recognized as such later in the parsing, we choose them
        # to look the same as ordinary variables.
        Name:                          "#000000",      # class: 'n'
        Name.Attribute:                "#c4a000",      # class: 'na', - to be revised
        Name.Builtin:                  "#204a87",      # class: 'nb'
        Name.Builtin.Pseudo:           "#3465a4", # class: 'bp'
        Name.Class:                    "#000000", # class: 'nc' - to be revised
        Name.Constant:                 "#000000",      # class: 'no', - to be revised
        Name.Decorator:                "bold #5c35cc",      # class: 'nd', - to be revised
        Name.Entity:                   "#ce5c00",      # class: 'ni',
        Name.Exception:                "bold #cc0000", # class: 'ne',
        Name.Function:                 "#000000",      # class: 'nf'
        Name.Property:                 "#000000",      # class: 'py',
        Name.Label:                    "#f57900",      # class: 'nl',
        Name.Namespace:                "#000000",      # class: 'nn' - to be revised
        Name.Other:                    "#000000",      # class: 'nx',
        Name.Tag:                      "bold #204a87", # class'nt' -- like a keyword
        Name.Variable:                 "#000000",      # class: 'nv', - to be revised
        Name.Variable.Class:           "#000000",      # class: 'vc', - to be revised
        Name.Variable.Global:          "#000000",      # class: 'vg', - to be revised
        Name.Variable.Instance:        "#000000",      # class: 'vi', - to be revised

        # since the tango light blue does not show up well in text, we choose
        # a pure blue instead.
        Number:                        "bold #0000cf", # class: 'm'
        Number.Float:                  "bold #0000cf", # class: ''mf',
        Number.Hex:                    "bold #0000cf", # class: ''mh',
        Number.Integer:                "bold #0000cf", # class: ''mi',
        Number.Integer.Long:           "bold #0000cf", # class: ''il',
        Number.Oct:                    "bold #0000cf", # class: ''mo',

        Literal:                   "#000000",      # class: 'l',
        Literal.Date:              "#000000",      # class: 'ld',

        String:                    "#4e9a06",      # class: 's',
        String.Backtick:           "#4e9a06",      # class: 'sb',
        String.Char:               "#4e9a06",      # class: 'sc',
        String.Doc:                "italic #8f5902",  # class: 'sd' - like a comment
        String.Double:             "#4e9a06",      # class: 's2',
        String.Escape:             "#4e9a06",      # class: 'se',
        String.Heredoc:            "#4e9a06",      # class: 'sh',
        String.Interpol:           "#4e9a06",      # class: 'si',
        String.Other:              "#4e9a06",      # class: 'sx',
        String.Regex:              "#4e9a06",      # class: 'sr',
        String.Single:             "#4e9a06",      # class: 's1',
        String.Symbol:             "#4e9a06",      # class: 'ss',

        Generic:                   "#000000",      # class: 'g',
        Generic.Deleted:           "#a40000",      # class: 'gd',
        Generic.Emph:              "italic #000000",      # class: 'ge',
        Generic.Error:             "#ef2929",      # class: 'gr',
        Generic.Heading:           "bold #000080",      # class: 'gh',
        Generic.Inserted:          "#00A000",      # class: 'gi',
        Generic.Output:            "italic #000000",      # class: 'go',
        Generic.Prompt:            "#8f5902",      # class: 'gp',
        Generic.Strong:            "bold #000000",      # class: 'gs',
        Generic.Subheading:        "bold #800080",      # class: 'gu',
        Generic.Traceback:         "bold #a40000",      # class: 'gt',
    }
<commit_msg>Move tango style's copyright info to the bottom of the docstring, so `check_sources.py` is happy.<commit_after># -*- coding: utf-8 -*-
"""
    pygments.styles.tango
    ~~~~~~~~~~~~~~~~~~~~~

    The Crunchy default Style inspired from the color palette from
    the Tango Icon Theme Guidelines.

    http://tango.freedesktop.org/Tango_Icon_Theme_Guidelines

    Butter:     #fce94f     #edd400     #c4a000
    Orange:     #fcaf3e     #f57900     #ce5c00
    Chocolate:  #e9b96e     #c17d11     #8f5902
    Chameleon:  #8ae234     #73d216     #4e9a06
    Sky Blue:   #729fcf     #3465a4     #204a87
    Plum:       #ad7fa8     #75507b     #5c35cc
    Scarlet Red:#ef2929     #cc0000     #a40000
    Aluminium:  #eeeeec     #d3d7cf     #babdb6
                #888a85     #555753     #2e3436

    Not all of the above colors are used; other colors added:
        very light grey: #f8f8f8  (for background)

    This style can be used as a template as it includes all the known
    Token types, unlike most (if not all) of the styles included in the
    Pygments distribution.

    However, since Crunchy is intended to be used by beginners, we have strived
    to create a style that gloss over subtle distinctions between different
    categories.

    Taking Python for example, comments (Comment.*) and docstrings (String.Doc)
    have been chosen to have the same style.  Similarly, keywords (Keyword.*),
    and Operator.Word (and, or, in) have been assigned the same style.

    :copyright: 2008 by André Roberge.
    :license: BSD, see LICENSE for more details.
"""

from pygments.style import Style
from pygments.token import Keyword, Name, Comment, String, Error, \
     Number, Operator, Generic, Whitespace, Punctuation, Other, Literal

class TangoStyle(Style):
    """
    The Crunchy default Style inspired from the color palette from
    the Tango Icon Theme Guidelines.
    """

    # work in progress...

    background_color = "#f8f8f8"
    default_style = ""

    styles = {
        # No corresponding class for the following:
        #Text:              "", # class:  '',
        Whitespace:                "underline #f8f8f8", # class: 'w',
        Error:                     "#a40000 border:#ef2929", # class: 'err',
        Other:                     "#000000",        # class 'x',

        Comment:                   "italic #8f5902", # class: 'c',
        Comment.Multiline:         "italic #8f5902", # class: 'cm',
        Comment.Preproc:           "italic #8f5902", # class: 'cp',
        Comment.Single:            "italic #8f5902", # class: 'c1',
        Comment.Special:           "italic #8f5902", # class: 'cs',

        Keyword:                   "bold #204a87", # class: 'k',
        Keyword.Constant:          "bold #204a87", # class: 'kc',
        Keyword.Declaration:       "bold #204a87", # class: 'kd',
        Keyword.Namespace:         "bold #204a87", # class: 'kn',
        Keyword.Pseudo:            "bold #204a87", # class: 'kp',
        Keyword.Reserved:          "bold #204a87", # class: 'kr',
        Keyword.Type:              "bold #204a87", # class: 'kt',

        Operator:                  "bold #ce5c00", # class: 'o'
        Operator.Word:             "bold #204a87", # class: 'ow' - like keywords

        Punctuation:               "bold #000000", # class: 'p'

        # because special names such as Name.Class, Name.Function, etc.
        # are not recognized as such later in the parsing, we choose them
        # to look the same as ordinary variables.
        Name:                          "#000000",      # class: 'n'
        Name.Attribute:                "#c4a000",      # class: 'na', - to be revised
        Name.Builtin:                  "#204a87",      # class: 'nb'
        Name.Builtin.Pseudo:           "#3465a4", # class: 'bp'
        Name.Class:                    "#000000", # class: 'nc' - to be revised
        Name.Constant:                 "#000000",      # class: 'no', - to be revised
        Name.Decorator:                "bold #5c35cc",      # class: 'nd', - to be revised
        Name.Entity:                   "#ce5c00",      # class: 'ni',
        Name.Exception:                "bold #cc0000", # class: 'ne',
        Name.Function:                 "#000000",      # class: 'nf'
        Name.Property:                 "#000000",      # class: 'py',
        Name.Label:                    "#f57900",      # class: 'nl',
        Name.Namespace:                "#000000",      # class: 'nn' - to be revised
        Name.Other:                    "#000000",      # class: 'nx',
        Name.Tag:                      "bold #204a87", # class'nt' -- like a keyword
        Name.Variable:                 "#000000",      # class: 'nv', - to be revised
        Name.Variable.Class:           "#000000",      # class: 'vc', - to be revised
        Name.Variable.Global:          "#000000",      # class: 'vg', - to be revised
        Name.Variable.Instance:        "#000000",      # class: 'vi', - to be revised

        # since the tango light blue does not show up well in text, we choose
        # a pure blue instead.
        Number:                        "bold #0000cf", # class: 'm'
        Number.Float:                  "bold #0000cf", # class: ''mf',
        Number.Hex:                    "bold #0000cf", # class: ''mh',
        Number.Integer:                "bold #0000cf", # class: ''mi',
        Number.Integer.Long:           "bold #0000cf", # class: ''il',
        Number.Oct:                    "bold #0000cf", # class: ''mo',

        Literal:                   "#000000",      # class: 'l',
        Literal.Date:              "#000000",      # class: 'ld',

        String:                    "#4e9a06",      # class: 's',
        String.Backtick:           "#4e9a06",      # class: 'sb',
        String.Char:               "#4e9a06",      # class: 'sc',
        String.Doc:                "italic #8f5902",  # class: 'sd' - like a comment
        String.Double:             "#4e9a06",      # class: 's2',
        String.Escape:             "#4e9a06",      # class: 'se',
        String.Heredoc:            "#4e9a06",      # class: 'sh',
        String.Interpol:           "#4e9a06",      # class: 'si',
        String.Other:              "#4e9a06",      # class: 'sx',
        String.Regex:              "#4e9a06",      # class: 'sr',
        String.Single:             "#4e9a06",      # class: 's1',
        String.Symbol:             "#4e9a06",      # class: 'ss',

        Generic:                   "#000000",      # class: 'g',
        Generic.Deleted:           "#a40000",      # class: 'gd',
        Generic.Emph:              "italic #000000",      # class: 'ge',
        Generic.Error:             "#ef2929",      # class: 'gr',
        Generic.Heading:           "bold #000080",      # class: 'gh',
        Generic.Inserted:          "#00A000",      # class: 'gi',
        Generic.Output:            "italic #000000",      # class: 'go',
        Generic.Prompt:            "#8f5902",      # class: 'gp',
        Generic.Strong:            "bold #000000",      # class: 'gs',
        Generic.Subheading:        "bold #800080",      # class: 'gu',
        Generic.Traceback:         "bold #a40000",      # class: 'gt',
    }
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Support for the softwareupdate command on MacOS.
'''

# Import python libs
import re

__virtualname__ = softwareupdate

def __virtual__():
    '''
    Only for MacOS
    '''
    return __virtualname__ if __grains__['os'] == 'MacOS' else False

def _get_upgradable():
    '''
    Utility function to get upgradable packages.

    Sample return date:
    { 'updatename': '1.2.3-45', ... }
    '''
    cmd = 'softwareupdate --list'
    out = __salt__['cmd.run_stdout'](cmd, output_loglevel='debug')

    # rexp parses lines that look like the following:
    #    * Safari6.1.2MountainLion-6.1.2
    #    Safari (6.1.2), 51679K [recommended]
    rexp = re.compile('(?m)^   [*|-] '
                      r'([^ ].*)[\r\n].*\(([^\)]+)')

    keys = ['name', 'version']
    _get = lambda l, k: l[keys.index(k)]

    upgrades = rexp.findall(out)

    ret = {}
    for line in upgrades:
        name = _get(line, 'name')
        version_num = _get(line, 'version')
        ret[name] = version_num

    return ret

def list_upgrades():
    '''
    List all available program upgrades.

    CLI Example:

    .. code-block:: bash

       salt '*' softwareupdate.list_upgrades
    '''

    return _get_upgradable()

def ignore(*updates):
    '''
    Ignore a specific program update.

    CLI Example:

    .. code-block:: bash

       salt '*' softwareupdate.ignore <update-name>
       salt '*' softwareupdate.ignore "<update with whitespace>"
       salt '*' softwareupdate.ignore <update1> <update2> <update3>
    '''

    ret = []

    if len(updates) == 0:
        return ''

    for name in updates:
        cmd = ['softwareupdate', '--ignore', name]
        __salt__['cmd.run_stdout'](cmd, python_shell=False,
                                      output_loglevel='debug')

        all_ignored = list_ignored()

        if name in all_ignored:
            ret.append(name)

    return ret

def list_ignored():
    '''
    List all upgrades that has been ignored.

    CLI Example:

    .. code-block:: bash

       salt '*' softwareupdate.list_ignored
    '''
    cmd = 'softwareupdate --list --ignore'
    out = __salt__['cmd.run_stdout'](cmd, output_loglevel='debug')

    # rep parses lines that look like the following:
    #     "Safari6.1.2MountainLion-6.1.2",
    # or:
    #     Safari6.1.2MountainLion-6.1.2
    rexp = re.compile('(?m)^    ["]?'
                      r'([^,|\s|"].*[^"])[,|"]')

    ignored_updates = rexp.findall(out)

    if ignored_updates:
        ret = ignored_updates
    else:
        ret = None
    return ret

def reset_ignored():
    '''
    Make sure the ignored updates are not ignored anymore,
    returns a list of the updates that are no longer ignored.

    CLI Example:

    .. code-block:: bash

       salt '*' softwareupdate.reset_ignored
    '''
    cmd = 'softwareupdate', '--reset-ignored'
    ignored_updates = list_ignored()

    if ignored_updates:
        __salt__['cmd.run_stdout'](cmd, output_loglevel='debug')
        ret = ignored_updates
    else:
        ret = None

    return ret

def schedule(*status):
    '''
    Decide if automatic checking for upgrades should be on or off.
    If no argumentsare given it will return the current status.
    Appaend on or off to change the status.

    Return values:
    - ``True``: Automatic checking is now on,
    - ``False``: Automatic checking is now off,
    - ``None``: Invalid argument.

    CLI Example:

    .. code-block:: bash

       salt '*' softwareupdate.schedule
       salt '*' softwareupdate.schedule on|off
    '''
    if len(status) == 0:
        cmd = 'softwareupdate --schedule'
    elif str(status[0]) == 'True':
        cmd = 'softwareupdate --schedule on'
    elif str(status[0]) == 'False':
        cmd = 'softwareupdate --schedule off'
    else:
        return None

    out = __salt__['cmd.run_stdout'](cmd, output_loglevel='debug')

    current_status = out.split()[-1]
    if current_status == 'off':
        return False
    elif current_status == 'on':
        return True

def upgrade():
    '''
    Installs all available upgrades.

    Return values:
    - ``True``: The update was installed.
    - ``False``: The update was not installed.

    CLI Example:

    .. code-block:: bash

       salt '*' softwareupdate.upgrade
    '''
    ret = {}

    available_upgrades = list_upgrades()
    cmd = 'softwareupdate --install --all'

    __salt__['cmd.run_stdout'](cmd, output_loglevel='debug')

    upgrades_left = list_upgrades()

    for name in available_upgrades:
        if name not in upgrades_left:
            ret[name] = True
        else:
            ret[name] = False

    return ret

def install(*updates):
    '''
    Install a named upgrade.

    Return values:
    - ``True``: The update was installed.
    - ``False``: The update was not installed.
    - ``None``: There is no update avaliable with that name.

    CLI Example:

    .. code-block:: bash

       salt '*' softwareupdate.install <update name>
       salt '*' softwareupdate.install <update1> <update2> <update3>
    '''
    ret = {}

    if len(updates) == 0:
        return ''

    avaliable_upgrades = list_upgrades()

    for name in updates:
        cmd = ['softwareupdate', '--install', name]
        __salt__['cmd.run_stdout'](cmd, python_shell=False,
                                      output_loglevel='debug')

    upgrades_left = list_upgrades()

    for name in updates:
        if name not in avaliable_upgrades:
            ret[name] = None
        elif name not in upgrades_left:
            ret[name] = True
        else:
            ret[name] = False

    return ret

def upgrade_available(update):
    '''
    Check whether or not an upgrade is available with a given name.

    CLI Example:

    .. code-block:: bash

       salt '*' softwareupdate.upgrade_available <update name>
    '''

    return update in list_upgrades()
<commit_msg>Adding boilerplate too quickly....<commit_after># -*- coding: utf-8 -*-
'''
Support for the softwareupdate command on MacOS.
'''

# Import python libs
import re

__virtualname__ = 'softwareupdate'

def __virtual__():
    '''
    Only for MacOS
    '''
    return __virtualname__ if __grains__['os'] == 'MacOS' else False

def _get_upgradable():
    '''
    Utility function to get upgradable packages.

    Sample return date:
    { 'updatename': '1.2.3-45', ... }
    '''
    cmd = 'softwareupdate --list'
    out = __salt__['cmd.run_stdout'](cmd, output_loglevel='debug')

    # rexp parses lines that look like the following:
    #    * Safari6.1.2MountainLion-6.1.2
    #    Safari (6.1.2), 51679K [recommended]
    rexp = re.compile('(?m)^   [*|-] '
                      r'([^ ].*)[\r\n].*\(([^\)]+)')

    keys = ['name', 'version']
    _get = lambda l, k: l[keys.index(k)]

    upgrades = rexp.findall(out)

    ret = {}
    for line in upgrades:
        name = _get(line, 'name')
        version_num = _get(line, 'version')
        ret[name] = version_num

    return ret

def list_upgrades():
    '''
    List all available program upgrades.

    CLI Example:

    .. code-block:: bash

       salt '*' softwareupdate.list_upgrades
    '''

    return _get_upgradable()

def ignore(*updates):
    '''
    Ignore a specific program update.

    CLI Example:

    .. code-block:: bash

       salt '*' softwareupdate.ignore <update-name>
       salt '*' softwareupdate.ignore "<update with whitespace>"
       salt '*' softwareupdate.ignore <update1> <update2> <update3>
    '''

    ret = []

    if len(updates) == 0:
        return ''

    for name in updates:
        cmd = ['softwareupdate', '--ignore', name]
        __salt__['cmd.run_stdout'](cmd, python_shell=False,
                                      output_loglevel='debug')

        all_ignored = list_ignored()

        if name in all_ignored:
            ret.append(name)

    return ret

def list_ignored():
    '''
    List all upgrades that has been ignored.

    CLI Example:

    .. code-block:: bash

       salt '*' softwareupdate.list_ignored
    '''
    cmd = 'softwareupdate --list --ignore'
    out = __salt__['cmd.run_stdout'](cmd, output_loglevel='debug')

    # rep parses lines that look like the following:
    #     "Safari6.1.2MountainLion-6.1.2",
    # or:
    #     Safari6.1.2MountainLion-6.1.2
    rexp = re.compile('(?m)^    ["]?'
                      r'([^,|\s|"].*[^"])[,|"]')

    ignored_updates = rexp.findall(out)

    if ignored_updates:
        ret = ignored_updates
    else:
        ret = None
    return ret

def reset_ignored():
    '''
    Make sure the ignored updates are not ignored anymore,
    returns a list of the updates that are no longer ignored.

    CLI Example:

    .. code-block:: bash

       salt '*' softwareupdate.reset_ignored
    '''
    cmd = 'softwareupdate', '--reset-ignored'
    ignored_updates = list_ignored()

    if ignored_updates:
        __salt__['cmd.run_stdout'](cmd, output_loglevel='debug')
        ret = ignored_updates
    else:
        ret = None

    return ret

def schedule(*status):
    '''
    Decide if automatic checking for upgrades should be on or off.
    If no argumentsare given it will return the current status.
    Appaend on or off to change the status.

    Return values:
    - ``True``: Automatic checking is now on,
    - ``False``: Automatic checking is now off,
    - ``None``: Invalid argument.

    CLI Example:

    .. code-block:: bash

       salt '*' softwareupdate.schedule
       salt '*' softwareupdate.schedule on|off
    '''
    if len(status) == 0:
        cmd = 'softwareupdate --schedule'
    elif str(status[0]) == 'True':
        cmd = 'softwareupdate --schedule on'
    elif str(status[0]) == 'False':
        cmd = 'softwareupdate --schedule off'
    else:
        return None

    out = __salt__['cmd.run_stdout'](cmd, output_loglevel='debug')

    current_status = out.split()[-1]
    if current_status == 'off':
        return False
    elif current_status == 'on':
        return True

def upgrade():
    '''
    Installs all available upgrades.

    Return values:
    - ``True``: The update was installed.
    - ``False``: The update was not installed.

    CLI Example:

    .. code-block:: bash

       salt '*' softwareupdate.upgrade
    '''
    ret = {}

    available_upgrades = list_upgrades()
    cmd = 'softwareupdate --install --all'

    __salt__['cmd.run_stdout'](cmd, output_loglevel='debug')

    upgrades_left = list_upgrades()

    for name in available_upgrades:
        if name not in upgrades_left:
            ret[name] = True
        else:
            ret[name] = False

    return ret

def install(*updates):
    '''
    Install a named upgrade.

    Return values:
    - ``True``: The update was installed.
    - ``False``: The update was not installed.
    - ``None``: There is no update avaliable with that name.

    CLI Example:

    .. code-block:: bash

       salt '*' softwareupdate.install <update name>
       salt '*' softwareupdate.install <update1> <update2> <update3>
    '''
    ret = {}

    if len(updates) == 0:
        return ''

    avaliable_upgrades = list_upgrades()

    for name in updates:
        cmd = ['softwareupdate', '--install', name]
        __salt__['cmd.run_stdout'](cmd, python_shell=False,
                                      output_loglevel='debug')

    upgrades_left = list_upgrades()

    for name in updates:
        if name not in avaliable_upgrades:
            ret[name] = None
        elif name not in upgrades_left:
            ret[name] = True
        else:
            ret[name] = False

    return ret

def upgrade_available(update):
    '''
    Check whether or not an upgrade is available with a given name.

    CLI Example:

    .. code-block:: bash

       salt '*' softwareupdate.upgrade_available <update name>
    '''

    return update in list_upgrades()
<|endoftext|>


<commit_before><commit_msg>Added file size into result<commit_after><|endoftext|>


<commit_before>from unittest import  TextTestRunner
try:
    from .loader import TestLoader
except:
    from loader import TestLoader

import sublime, sublime_plugin
import threading
import imp
import os
import re

# script directory
__dir__ = os.path.dirname(os.path.abspath(__file__))

class OutputPanelInsertCommand(sublime_plugin.TextCommand):
    def run(self, edit, characters):
        self.view.set_read_only(False)
        self.view.insert(edit, self.view.size(), characters)
        self.view.set_read_only(True)
        self.view.show(self.view.size())

class OutputPanel:
    def __init__(self, name, file_regex='', line_regex = '', base_dir = None, word_wrap = False, line_numbers = False,
        gutter = False, scroll_past_end = False, syntax = 'Packages/Text/Plain text.tmLanguage'):

        self.name = name
        self.window = sublime.active_window()
        self.output_view = self.window.get_output_panel(name)

        # default to the current file directory
        if (not base_dir and self.window.active_view() and self.window.active_view().file_name()):
            base_dir = os.path.dirname(self.window.active_view().file_name())

        self.output_view.settings().set("result_file_regex", file_regex)
        self.output_view.settings().set("result_line_regex", line_regex)
        self.output_view.settings().set("result_base_dir", base_dir)
        self.output_view.settings().set("word_wrap", word_wrap)
        self.output_view.settings().set("line_numbers", line_numbers)
        self.output_view.settings().set("gutter", gutter)
        self.output_view.settings().set("scroll_past_end", scroll_past_end)
        self.output_view.settings().set("syntax", syntax)

    def write(self, s):
        self.output_view.run_command('output_panel_insert', {'characters': s})

    def flush(self):
        pass

    def show(self):
        self.window.run_command("show_panel", {"panel": "output."+self.name})

    def close(self):
        pass

class UnitTestingCommand(sublime_plugin.ApplicationCommand):
    def run(self, package=None, output=None):
        settingsFileName = "UnitTesting.sublime-settings"
        settingsName = "recent_package"
        settings = sublime.load_settings(settingsFileName)
        if package:
            tests_dir = 'tests'

            if output=="panel":
                output_panel = OutputPanel('unittests', file_regex = r'File "([^"]*)", line (\d+)')
                output_panel.show()
                stream = output_panel
            else:
                if output:
                    outfile = output
                else:
                    outputdir = os.path.join(sublime.packages_path(), 'User', 'UnitTesting', "tests_output")
                    if not os.path.isdir(outputdir):
                        os.makedirs(outputdir)
                    outfile = os.path.join(outputdir, package)

                if os.path.exists(outfile): os.remove(outfile)
                stream = open(outfile, "w")

            try:
                # use custom loader which support ST2 and reloading modules
                loader = TestLoader()
                test = loader.discover(os.path.join(sublime.packages_path(),package, tests_dir))
                testRunner = TextTestRunner(stream, verbosity=2)
                testRunner.run(test)
            except Exception as e:
                stream.write("ERROR: %s\n" % e)

            stream.close()
            # save the package name
            settings.set(settingsName, package)
            sublime.save_settings(settingsFileName)
        else:
            recent_package = settings.get(settingsName, "Package Name")
            view = sublime.active_window().show_input_panel('Package:', recent_unittesting_package,
                lambda x: sublime.run_command("unit_testing", {"package":x, "output":output}), None, None )
            view.run_command("select_all")
<commit_msg>fix up<commit_after>from unittest import  TextTestRunner
try:
    from .loader import TestLoader
except:
    from loader import TestLoader

import sublime, sublime_plugin
import threading
import imp
import os
import re

# script directory
__dir__ = os.path.dirname(os.path.abspath(__file__))

class OutputPanelInsertCommand(sublime_plugin.TextCommand):
    def run(self, edit, characters):
        self.view.set_read_only(False)
        self.view.insert(edit, self.view.size(), characters)
        self.view.set_read_only(True)
        self.view.show(self.view.size())

class OutputPanel:
    def __init__(self, name, file_regex='', line_regex = '', base_dir = None, word_wrap = False, line_numbers = False,
        gutter = False, scroll_past_end = False, syntax = 'Packages/Text/Plain text.tmLanguage'):

        self.name = name
        self.window = sublime.active_window()
        self.output_view = self.window.get_output_panel(name)

        # default to the current file directory
        if (not base_dir and self.window.active_view() and self.window.active_view().file_name()):
            base_dir = os.path.dirname(self.window.active_view().file_name())

        self.output_view.settings().set("result_file_regex", file_regex)
        self.output_view.settings().set("result_line_regex", line_regex)
        self.output_view.settings().set("result_base_dir", base_dir)
        self.output_view.settings().set("word_wrap", word_wrap)
        self.output_view.settings().set("line_numbers", line_numbers)
        self.output_view.settings().set("gutter", gutter)
        self.output_view.settings().set("scroll_past_end", scroll_past_end)
        self.output_view.settings().set("syntax", syntax)

    def write(self, s):
        self.output_view.run_command('output_panel_insert', {'characters': s})

    def flush(self):
        pass

    def show(self):
        self.window.run_command("show_panel", {"panel": "output."+self.name})

    def close(self):
        pass

class UnitTestingCommand(sublime_plugin.ApplicationCommand):
    def run(self, package=None, output=None):
        settingsFileName = "UnitTesting.sublime-settings"
        settingsName = "recent_package"
        settings = sublime.load_settings(settingsFileName)
        if package:
            tests_dir = 'tests'

            if output=="panel":
                output_panel = OutputPanel('unittests', file_regex = r'File "([^"]*)", line (\d+)')
                output_panel.show()
                stream = output_panel
            else:
                if output:
                    outfile = output
                else:
                    outputdir = os.path.join(sublime.packages_path(), 'User', 'UnitTesting', "tests_output")
                    if not os.path.isdir(outputdir):
                        os.makedirs(outputdir)
                    outfile = os.path.join(outputdir, package)

                if os.path.exists(outfile): os.remove(outfile)
                stream = open(outfile, "w")

            try:
                # use custom loader which support ST2 and reloading modules
                loader = TestLoader()
                test = loader.discover(os.path.join(sublime.packages_path(),package, tests_dir))
                testRunner = TextTestRunner(stream, verbosity=2)
                testRunner.run(test)
            except Exception as e:
                stream.write("ERROR: %s\n" % e)

            stream.close()
            # save the package name
            settings.set(settingsName, package)
            sublime.save_settings(settingsFileName)
        else:
            recent_package = settings.get(settingsName, "Package Name")
            view = sublime.active_window().show_input_panel('Package:', recent_package,
                lambda x: sublime.run_command("unit_testing", {"package":x, "output":output}), None, None )
            view.run_command("select_all")
<|endoftext|>


<commit_before>#Python reminders

# Collapse the list
''.join(list)

#Scan list for longest string and print max item
print(max(len(i) for i in string.split()))

#Count occurrences of item in list
[a,b,c,d,e].count(item)

#put anything in square brackets to search list index
a[0]

#Strange list rotation
def rotate_left3(nums):
  a = nums[0]
  nums[0] = nums[1]
  nums[1] = nums[2]
  nums[2] = a
  return nums -> [1,2,3] = [2,3,1]

#Evaluator positioning
if someting:
	return (item 1 >= 10)
else:
	return (item > 1 and item <= 5)

#Use F strings
x = "hello"
w = "world"
print(f"{x} {w}")

#List comprehensions
	#example CW solution used to reverse all words >= 5 letters and print out the modified sentence
	[word[::-1] if len(word) >= 5 else word for word in words]
	#Example CW solution to create a dictionary of letters and their frequency of occurrence in a string
	{i : message.count(i) for i in set(message)}

#Collapse multiple inputs into one
	#from this
	def predict_age(age_1, age_2, age_3, age_4, age_5, age_6, age_7, age_8):
	#to this
	def predict_age(*ages)

#Remove char
def remove_exclamation_marks(s):
    return s.replace('!', '')

#Quick Lambda implementation on multiple args - with f"string"
def newvibe(a,b,c):
	var1 = lambda x: (x * 5)
	print(f"This is a: {var1(a)}, This is b: {var1(b)}, This is c: {var1(c)}")
<commit_msg>Update REMINDERS.py<commit_after>#Python reminders

# Collapse the list
''.join(list)

#Scan list for longest string and print max item
print(max(len(i) for i in string.split()))

#Dont forget min and max
max(x) for largest
min(x) for smallest

#Count occurrences of item in list
[a,b,c,d,e].count(item)

#put anything in square brackets to search list index
a[0]

#Strange list rotation
def rotate_left3(nums):
  a = nums[0]
  nums[0] = nums[1]
  nums[1] = nums[2]
  nums[2] = a
  return nums -> [1,2,3] = [2,3,1]

#Evaluator positioning
if someting:
	return (item 1 >= 10)
else:
	return (item > 1 and item <= 5)

#Use F strings
x = "hello"
w = "world"
print(f"{x} {w}")

#List comprehensions
	#example CW solution used to reverse all words >= 5 letters and print out the modified sentence
	[word[::-1] if len(word) >= 5 else word for word in words]
	#Example CW solution to create a dictionary of letters and their frequency of occurrence in a string
	{i : message.count(i) for i in set(message)}

#Collapse multiple inputs into one
	#from this
	def predict_age(age_1, age_2, age_3, age_4, age_5, age_6, age_7, age_8):
	#to this
	def predict_age(*ages)

#Remove char
def remove_exclamation_marks(s):
    return s.replace('!', '')

#Quick Lambda implementation on multiple args - with f"string"
def newvibe(a,b,c):
	var1 = lambda x: (x * 5)
	print(f"This is a: {var1(a)}, This is b: {var1(b)}, This is c: {var1(c)}")
<|endoftext|>


<commit_before>import os
import matplotlib.pyplot as plt
from matplotlib import rc
from matplotlib.backends.backend_pdf import PdfPages

font = {'family' : 'normal',
        'weight' : 'bold',
        'size'   : 4}

rc('font', **font)

class FigureError(Exception):
    """
    Error for figure opening/closing in plotting lib
    """
    pass

class LegendError(Exception):
    """
    Error for figure opening/closing in plotting lib
    """
    pass

def plot_color_picker(i):
    from matplotlib.colors import cnames
    from random import shuffle
    colorcodes = ['b', 'g', 'r', 'c', 'm', 'y', 'k']
    colorcodes.extend(cnames.keys())
    i = i % len(colorcodes)

    return colorcodes[i]

class Plot(object):
    def __init__(self, outputpath, name):
        #TODO: Validate Directory
        self.pdf = PdfPages(os.path.join(outputpath, name + ".pdf"))
        self.figure = None
        self.lines = []

    def close_plot(self):
        try:
            self.close_figure()
        except FigureError:
            pass

        self.pdf.close()

    def create_figure(self):
        if not self.figure:
            self.figure = plt.figure()
        else:
            raise FigureError("Error: figure already exists. Close the figure to create a new figure.")

    def close_figure(self, save=True):
        if self.figure:
            if save:
                self.pdf.savefig()
            plt.close(self.figure)
            self.figure = None
        else:
            raise FigureError("Error: no figure is open. Cannot close nothing.")

    def add_legend(self):
        if self.lines:
            plt.legend()#labels=self.labels)
        else:
            raise LegendError("Error: No labels to add to the legend.")

class PixelPlot(Plot):
    def add_pixel(self, pixelObject, color=None, closefigure=False):

        if color:
            color = '{0}'.format(color)
        elif color is None and pixelObject.color:
            color = '{0}'.format(pixelObject.color)
        else:
            color = "black"

        try:
            self.create_figure()
        except FigureError:
            pass

        axes = self.figure.add_subplot(1, 1, 1)
        print("Plotting {0}, {1}".format(pixelObject.values, pixelObject.bandDOYs))
        axes.plot(pixelObject.bandDOYs, pixelObject.values, color=color, linestyle='-')
        axes.set_xlabel("Pixel row {0} col {1}: {2} as {3} in classified".format(pixelObject.row,
                                                                                pixelObject.col,
                                                                                pixelObject.actualclass,
                                                                                pixelObject.classificationclass))
        if closefigure is True:
            self.close_figure()

class SignaturePlot(Plot):
    def plot_signature(self, temporalSignature, color=None, closefigure=False):

        if not color:
            color = "k"

        try:
            self.create_figure()
        except FigureError:
            pass

        print color
        axes = self.figure.add_subplot(1, 1, 1)
        self.lines.append(axes.plot(temporalSignature.daysofyear, temporalSignature.vivalues,
                                    color=color, linestyle='-', label=temporalSignature.name))
        #axes.set_xlabel("Temporal signature {0}".format(temporalSignature.name))

        if closefigure is True:
            self.close_figure()

    def plot_collection(self, signatureCollection, insamefigure=True, legend=True, closeplot=True):

        if insamefigure:
            closefig = False

        for i, signature in enumerate(signatureCollection.signatures):
            color = plot_color_picker(i)
            self.plot_signature(signature, color=color, closefigure=closefig)

        if legend:
            self.add_legend()

        if closeplot:
            self.close_plot()

def plot_correct_incorrect_px(accuracyimage, multidateimageclipped, cropimage, classifiedimage, outputdir, plotcorrectpx=True, plotincorrectpx=True):
    # TODO: Finish below
    """Pesudocode:

    open accuracyimage as array
    open multidateimageclipped as array
    open cropimage as array
    open classifiedimage as array

    for row in accuracyimage
        for col in accuracyimage
            if accuracyimage[row, col] == 1 and plotcorrectpx:

    okay, need to create pixel objects for each non-nodata pixel in the accuracy image, get truth value and classified id,
    save into two lists, one of correct and one of incorrect
    plot each list"""

    if plotcorrectpx:
        correctpxplot = PixelPlot(outputdir, "correctpxplot")

    if plotincorrectpx:
        incorrectpxplot = PixelPlot(outputdir, "incorrectpxplot")<commit_msg>removed debug print statement<commit_after>import os
import matplotlib.pyplot as plt
from matplotlib import rc
from matplotlib.backends.backend_pdf import PdfPages

font = {'family' : 'normal',
        'weight' : 'bold',
        'size'   : 4}

rc('font', **font)

class FigureError(Exception):
    """
    Error for figure opening/closing in plotting lib
    """
    pass

class LegendError(Exception):
    """
    Error for figure opening/closing in plotting lib
    """
    pass

def plot_color_picker(i):
    from matplotlib.colors import cnames
    from random import shuffle
    colorcodes = ['b', 'g', 'r', 'c', 'm', 'y', 'k']
    colorcodes.extend(cnames.keys())
    i = i % len(colorcodes)

    return colorcodes[i]

class Plot(object):
    def __init__(self, outputpath, name):
        #TODO: Validate Directory
        self.pdf = PdfPages(os.path.join(outputpath, name + ".pdf"))
        self.figure = None
        self.lines = []

    def close_plot(self):
        try:
            self.close_figure()
        except FigureError:
            pass

        self.pdf.close()

    def create_figure(self):
        if not self.figure:
            self.figure = plt.figure()
        else:
            raise FigureError("Error: figure already exists. Close the figure to create a new figure.")

    def close_figure(self, save=True):
        if self.figure:
            if save:
                self.pdf.savefig()
            plt.close(self.figure)
            self.figure = None
        else:
            raise FigureError("Error: no figure is open. Cannot close nothing.")

    def add_legend(self):
        if self.lines:
            plt.legend()#labels=self.labels)
        else:
            raise LegendError("Error: No labels to add to the legend.")

class PixelPlot(Plot):
    def add_pixel(self, pixelObject, color=None, closefigure=False):

        if color:
            color = '{0}'.format(color)
        elif color is None and pixelObject.color:
            color = '{0}'.format(pixelObject.color)
        else:
            color = "black"

        try:
            self.create_figure()
        except FigureError:
            pass

        axes = self.figure.add_subplot(1, 1, 1)
        print("Plotting {0}, {1}".format(pixelObject.values, pixelObject.bandDOYs))
        axes.plot(pixelObject.bandDOYs, pixelObject.values, color=color, linestyle='-')
        axes.set_xlabel("Pixel row {0} col {1}: {2} as {3} in classified".format(pixelObject.row,
                                                                                pixelObject.col,
                                                                                pixelObject.actualclass,
                                                                                pixelObject.classificationclass))
        if closefigure is True:
            self.close_figure()

class SignaturePlot(Plot):
    def plot_signature(self, temporalSignature, color=None, closefigure=False):

        if not color:
            color = "k"

        try:
            self.create_figure()
        except FigureError:
            pass

        axes = self.figure.add_subplot(1, 1, 1)
        self.lines.append(axes.plot(temporalSignature.daysofyear, temporalSignature.vivalues,
                                    color=color, linestyle='-', label=temporalSignature.name))
        #axes.set_xlabel("Temporal signature {0}".format(temporalSignature.name))

        if closefigure is True:
            self.close_figure()

    def plot_collection(self, signatureCollection, insamefigure=True, legend=True, closeplot=True):

        if insamefigure:
            closefig = False

        for i, signature in enumerate(signatureCollection.signatures):
            color = plot_color_picker(i)
            self.plot_signature(signature, color=color, closefigure=closefig)

        if legend:
            self.add_legend()

        if closeplot:
            self.close_plot()

def plot_correct_incorrect_px(accuracyimage, multidateimageclipped, cropimage, classifiedimage, outputdir, plotcorrectpx=True, plotincorrectpx=True):
    # TODO: Finish below
    """Pesudocode:

    open accuracyimage as array
    open multidateimageclipped as array
    open cropimage as array
    open classifiedimage as array

    for row in accuracyimage
        for col in accuracyimage
            if accuracyimage[row, col] == 1 and plotcorrectpx:

    okay, need to create pixel objects for each non-nodata pixel in the accuracy image, get truth value and classified id,
    save into two lists, one of correct and one of incorrect
    plot each list"""

    if plotcorrectpx:
        correctpxplot = PixelPlot(outputdir, "correctpxplot")

    if plotincorrectpx:
        incorrectpxplot = PixelPlot(outputdir, "incorrectpxplot")<|endoftext|>


<commit_before><commit_msg>Fix config overloading<commit_after><|endoftext|>


<commit_before><commit_msg>Stair location tweaks<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import unicode_literals

import sys

from distutils.version import StrictVersion

from django.template.base import (
    Lexer, Template as BaseTemplate, InvalidTemplateLibrary)

from .settings import (DJANGO_VERSION, BUILT_IN_TAGS, BUILT_IN_TAG_VALUES,
                       BUILT_IN_FILTERS)
from .utils import get_filters, update_dictionary

if StrictVersion(DJANGO_VERSION) > StrictVersion('1.8'):
    from django.template.base import get_library

class Template(BaseTemplate):

    def __init__(self, template_string, origin=None, name=None, engine=None):
        super(Template, self).__init__(template_string, origin, name, engine)

        # Used for backwards compatibility (implemented in Django 1.9)
        if not hasattr(self, 'source'):
            self.source = template_string

        self.tokens = self._get_tokens()
        # The manually specified (loaded) modules and members (tags/filters)
        self.loaded_modules, self.loaded_members = self._parse_load_block()
        self.used_tags = self._get_used_tags()
        self.used_filters = self._get_used_filters()
        # Get the tags and filters available to this template
        self.tags, self.filters = self._get_templatetags_members()
        # Find utilized modules, tags and filters
        self.utilized_modules = self._get_utilized_modules()
        self.utilized_members = self._get_utilized_members()

    def list_duplicates(self):
        """
        List duplicate results, i.e. duplicate library, tag or filter loads.
        If possible, tries to combine the results in the same row.

        :returns: table (list of lists), header (list of header names)
        """

        temp_table = {}
        # Find duplicate library loads
        for module in self.loaded_modules:
            lines = self.loaded_modules[module]
            lines_str = ', '.join(map(str, lines))
            if len(lines) > 1 and lines_str not in temp_table.keys():
                temp_table[lines_str] = [module, None]

        # Find duplicate member loads
        for member in self.loaded_members:
            lines = self.loaded_members[member]
            lines_str = ', '.join(map(str, lines))
            if len(lines) > 1:
                if lines_str not in temp_table.keys():
                    temp_table[lines_str] = [None, member]
                else:
                    temp_table[lines_str][1] = member

        # Prepare output format
        headers = ['Duplicate module', 'Duplicate tag/filter', 'Line number']
        table = []
        if temp_table:
            for key in temp_table:
                row = temp_table[key]
                row.append(key)
                table.append(row)

        return table, headers

    def list_unutilized_items(self):
        """
        List unutilized modules, tags and filters in a single table
        """
        modules = []
        members = []
        # List unutilized modules
        if self.utilized_modules:
            for module in self.utilized_modules:
                if not self.utilized_modules[module]:
                    modules.append(module)

        # List unutilized tags/filters
        if self.utilized_members:
            for member in self.utilized_members:
                if not self.utilized_members[member]:
                    members.append(member)

        if len(modules) > len(members):
            diff = len(modules) - len(members)
            members += [None] * diff
        else:
            diff = len(members) - len(modules)
            modules += [None] * diff

        headers = ['Unutilized module', 'Unutilized tag/filter']
        table = zip(modules, members)

        return list(table), headers

    def _get_utilized_members(self):
        """
        Separates the loaded tags based on their utilization.

        :returns: {'tag_name': Boolean}
        """
        utilized_members = {}

        for member in self.loaded_members:
            utilized = False
            if member in self.used_tags or member in self.used_filters:
                utilized = True
            utilized_members[member] = utilized

        return utilized_members

    def _get_utilized_modules(self):
        """
        Separates the loaded modules based on their utilization.

        :returns: {'module_name': Boolean}
        """
        utilized_modules = {}

        for module in self.loaded_modules:
            utilized = False

            for tag in self.used_tags:
                if tag in self.tags[module]:
                    utilized = True
                    break

            if not utilized and self.used_filters:
                for custom_filter in self.used_filters:
                    if custom_filter in self.filters[module]:
                        utilized = True
                        break

            utilized_modules[module] = utilized

        return utilized_modules

    def _get_templatetags_members(self):
        """
        Get the names of tags and filters from available templatetags modules.

        :returns: {'somelib': [tags]}, {'somelib': [filters]}
        """
        tags = {}
        filters = {}
        for module in self.loaded_modules:
            try:
                lib = get_library(module)
            except InvalidTemplateLibrary:
                msg = ('Unable to locate the loaded library! Library: {}; '
                       'Template: {}\n').format(module, self.name)
                sys.stdout.write(msg)
                tags[module] = []
                continue
            tags[module] = lib.tags.keys()
            filters[module] = lib.filters.keys()

        return tags, filters

    def _get_tokens(self):
        """
        Get the list of tokens from the template source.

        A modified version of Django's compile_nodelist method (Template class)
        https://github.com/django/django/blob/master/django/template/base.py#L221

        :returns: a list of Tokens
        """
        # From Django's source code
        if self.engine.debug:
            from django.template.debug import DebugLexer
            lexer = DebugLexer(self.source, self.origin)
        else:
            lexer = Lexer(self.source, self.origin)

        return lexer.tokenize()

    def _parse_load_block(self):
        """
        Get the names of loaded templatetags modules as well as individually
        loaded tags and the line numbers they are located at.

        :returns: {'module': [line_numbers]}, {'member': [line_numbers]}
        """

        modules = {}
        members = {}

        for token in self.tokens:
            token_content = token.split_contents()
            # Extract load blocks
            if token.token_type == 2 and token_content[0] == 'load':
                # FROM syntax is used; individual members are loaded
                if len(token_content) >= 4 and token_content[-2] == 'from':
                    # Add loaded module
                    module = token_content[-1]
                    modules = update_dictionary(modules, module, token.lineno)
                    # Add loaded members
                    loaded_members = token_content[1:-2]
                    for member in loaded_members:
                        members = update_dictionary(members, member, token.lineno)
                # Regular syntax
                else:
                    # Multiple modules can be imported in the same load block
                    templatetags_modules = token_content[1:]
                    for module in templatetags_modules:
                        modules = update_dictionary(modules, module, token.lineno)

        return modules, members

    def _get_used_tags(self):
        """
        Get the list of custom template tags used in the template.

        :returns: a list of custom tag names
        """
        used_tags = []
        for token in self.tokens:
            token_content = token.split_contents()
            # Extract blocks that do not contain one of the built-in tags
            if (token.token_type == 2 and
                    token_content[0] not in BUILT_IN_TAGS.keys() and
                    # Skip built-in 'end' tags
                    token_content[0] not in BUILT_IN_TAG_VALUES):
                # Extract only the name of the template tag (ignore arguments)
                used_tags.append(token_content[0])

        return used_tags

    def _get_used_filters(self):
        """
        Get the list of custom filters used in the template.

        :returns: a list of custom filter names
        """
        used_filters = []

        for token in self.tokens:
            filters = []
            token_content = token.split_contents()

            # Variable token
            if token.token_type == 1 and '|' in token_content[0]:
                filters += get_filters(token_content[0])

            # Tag token
            elif token.token_type == 2:
                if '|' in ' '.join(token_content):
                    for item in token_content:
                        if '|' in item:
                            filters += get_filters(item)

            # Exclude built-in filters
            for filter_name in filters:
                if (filter_name not in BUILT_IN_FILTERS and
                        filter_name not in used_filters):
                    used_filters.append(filter_name)

        return used_filters
<commit_msg>Update documentation<commit_after># -*- coding: utf-8 -*-

from __future__ import unicode_literals

import sys

from distutils.version import StrictVersion

from django.template.base import (
    Lexer, Template as BaseTemplate, InvalidTemplateLibrary)

from .settings import (DJANGO_VERSION, BUILT_IN_TAGS, BUILT_IN_TAG_VALUES,
                       BUILT_IN_FILTERS)
from .utils import get_filters, update_dictionary

if StrictVersion(DJANGO_VERSION) > StrictVersion('1.8'):
    from django.template.base import get_library

class Template(BaseTemplate):
    """
    An override of Django's Template class.

    After calling the parent class, the template is analyzed for duplicates
    and unnecessary loads.

    Additional attributes:
    :tokens: a list of tokens found in the template
    :loaded_modules: a dictionary of loaded modules
    :loaded_members: a dictionary of loaded tags/filters
    :used_tags: a list of custom tags used in the template
    :used_filters: a list of custom filters used in the template
    :tags: a dictionary of custom tags loaded into the template
    :filters: a dictionary of custom filters loaded into the template
    :utilized_modules: a dictionary of utilization statuses
    :utilized_members: a dictionary of utilization statuses
    """

    def __init__(self, template_string, origin=None, name=None, engine=None):
        super(Template, self).__init__(template_string, origin, name, engine)

        # Used for backwards compatibility (implemented in Django 1.9)
        if not hasattr(self, 'source'):
            self.source = template_string

        self.tokens = self._get_tokens()
        # The manually specified (loaded) modules and members (tags/filters)
        self.loaded_modules, self.loaded_members = self._parse_load_block()
        self.used_tags = self._get_used_tags()
        self.used_filters = self._get_used_filters()
        # Get the tags and filters available to this template
        self.tags, self.filters = self._get_templatetags_members()
        # Find utilized modules, tags and filters
        self.utilized_modules = self._get_utilized_modules()
        self.utilized_members = self._get_utilized_members()

    def list_duplicates(self):
        """
        List duplicate results, i.e. duplicate library, tag or filter loads.
        If possible, tries to combine the results in the same row.

        :returns: table (list of lists), header (list of header names)
        """

        temp_table = {}
        # Find duplicate library loads
        for module in self.loaded_modules:
            lines = self.loaded_modules[module]
            lines_str = ', '.join(map(str, lines))
            if len(lines) > 1 and lines_str not in temp_table.keys():
                temp_table[lines_str] = [module, None]

        # Find duplicate member loads
        for member in self.loaded_members:
            lines = self.loaded_members[member]
            lines_str = ', '.join(map(str, lines))
            if len(lines) > 1:
                if lines_str not in temp_table.keys():
                    temp_table[lines_str] = [None, member]
                else:
                    temp_table[lines_str][1] = member

        # Prepare output format
        headers = ['Duplicate module', 'Duplicate tag/filter', 'Line number']
        table = []
        if temp_table:
            for key in temp_table:
                row = temp_table[key]
                row.append(key)
                table.append(row)

        return table, headers

    def list_unutilized_items(self):
        """
        List unutilized modules, tags and filters in a single table
        """
        modules = []
        members = []
        # List unutilized modules
        if self.utilized_modules:
            for module in self.utilized_modules:
                if not self.utilized_modules[module]:
                    modules.append(module)

        # List unutilized tags/filters
        if self.utilized_members:
            for member in self.utilized_members:
                if not self.utilized_members[member]:
                    members.append(member)

        if len(modules) > len(members):
            diff = len(modules) - len(members)
            members += [None] * diff
        else:
            diff = len(members) - len(modules)
            modules += [None] * diff

        headers = ['Unutilized module', 'Unutilized tag/filter']
        table = zip(modules, members)

        return list(table), headers

    def _get_utilized_members(self):
        """
        Separates the loaded tags based on their utilization.

        :returns: {'tag_name': Boolean}
        """
        utilized_members = {}

        for member in self.loaded_members:
            utilized = False
            if member in self.used_tags or member in self.used_filters:
                utilized = True
            utilized_members[member] = utilized

        return utilized_members

    def _get_utilized_modules(self):
        """
        Separates the loaded modules based on their utilization.

        :returns: {'module_name': Boolean}
        """
        utilized_modules = {}

        for module in self.loaded_modules:
            utilized = False

            for tag in self.used_tags:
                if tag in self.tags[module]:
                    utilized = True
                    break

            if not utilized and self.used_filters:
                for custom_filter in self.used_filters:
                    if custom_filter in self.filters[module]:
                        utilized = True
                        break

            utilized_modules[module] = utilized

        return utilized_modules

    def _get_templatetags_members(self):
        """
        Get the names of tags and filters from available templatetags modules.

        :returns: {'somelib': [tags]}, {'somelib': [filters]}
        """
        tags = {}
        filters = {}
        for module in self.loaded_modules:
            try:
                lib = get_library(module)
            except InvalidTemplateLibrary:
                msg = ('Unable to locate the loaded library! Library: {}; '
                       'Template: {}\n').format(module, self.name)
                sys.stdout.write(msg)
                tags[module] = []
                continue
            tags[module] = lib.tags.keys()
            filters[module] = lib.filters.keys()

        return tags, filters

    def _get_tokens(self):
        """
        Get the list of tokens from the template source.

        A modified version of Django's compile_nodelist method (Template class)
        https://github.com/django/django/blob/master/django/template/base.py#L221

        :returns: a list of Tokens
        """
        # From Django's source code
        if self.engine.debug:
            from django.template.debug import DebugLexer
            lexer = DebugLexer(self.source, self.origin)
        else:
            lexer = Lexer(self.source, self.origin)

        return lexer.tokenize()

    def _parse_load_block(self):
        """
        Get the names of loaded templatetags modules as well as individually
        loaded tags and the line numbers they are located at.

        :returns: {'module': [line_numbers]}, {'member': [line_numbers]}
        """

        modules = {}
        members = {}

        for token in self.tokens:
            token_content = token.split_contents()
            # Extract load blocks
            if token.token_type == 2 and token_content[0] == 'load':
                # FROM syntax is used; individual members are loaded
                if len(token_content) >= 4 and token_content[-2] == 'from':
                    # Add loaded module
                    module = token_content[-1]
                    modules = update_dictionary(modules, module, token.lineno)
                    # Add loaded members
                    loaded_members = token_content[1:-2]
                    for member in loaded_members:
                        members = update_dictionary(members, member, token.lineno)
                # Regular syntax
                else:
                    # Multiple modules can be imported in the same load block
                    templatetags_modules = token_content[1:]
                    for module in templatetags_modules:
                        modules = update_dictionary(modules, module, token.lineno)

        return modules, members

    def _get_used_tags(self):
        """
        Get the list of custom template tags used in the template.

        :returns: a list of custom tag names
        """
        used_tags = []
        for token in self.tokens:
            token_content = token.split_contents()
            # Extract blocks that do not contain one of the built-in tags
            if (token.token_type == 2 and
                    token_content[0] not in BUILT_IN_TAGS.keys() and
                    # Skip built-in 'end' tags
                    token_content[0] not in BUILT_IN_TAG_VALUES):
                # Extract only the name of the template tag (ignore arguments)
                used_tags.append(token_content[0])

        return used_tags

    def _get_used_filters(self):
        """
        Get the list of custom filters used in the template.

        :returns: a list of custom filter names
        """
        used_filters = []

        for token in self.tokens:
            filters = []
            token_content = token.split_contents()

            # Variable token
            if token.token_type == 1 and '|' in token_content[0]:
                filters += get_filters(token_content[0])

            # Tag token
            elif token.token_type == 2:
                if '|' in ' '.join(token_content):
                    for item in token_content:
                        if '|' in item:
                            filters += get_filters(item)

            # Exclude built-in filters
            for filter_name in filters:
                if (filter_name not in BUILT_IN_FILTERS and
                        filter_name not in used_filters):
                    used_filters.append(filter_name)

        return used_filters
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Concurrency controls in zookeeper
=========================================================================

This module allows you to acquire and release a slot. This is primarily useful
for ensureing that no more than N hosts take a specific action at once. This can
also be used to coordinate between masters.
'''
from __future__ import absolute_import

import logging

try:
    from kazoo.client import KazooClient

    from kazoo.retry import (
        ForceRetryError
    )
    import kazoo.recipe.lock
    import kazoo.recipe.barrier
    import kazoo.recipe.party
    from kazoo.exceptions import CancelledError
    from kazoo.exceptions import NoNodeError

    # TODO: use the kazoo one, waiting for pull req:
    # https://github.com/python-zk/kazoo/pull/206
    class _Semaphore(kazoo.recipe.lock.Semaphore):
        def __init__(self,
                    client,
                    path,
                    identifier=None,
                    max_leases=1,
                    ephemeral_lease=True,
                    ):
            kazoo.recipe.lock.Semaphore.__init__(self,
                                                client,
                                                path,
                                                identifier=identifier,
                                                max_leases=max_leases)
            self.ephemeral_lease = ephemeral_lease

            # if its not ephemeral, make sure we didn't already grab it
            if not self.ephemeral_lease:
                try:
                    for child in self.client.get_children(self.path):
                        try:
                            data, stat = self.client.get(self.path + "/" + child)
                            if identifier == data.decode('utf-8'):
                                self.create_path = self.path + "/" + child
                                self.is_acquired = True
                                break
                        except NoNodeError:  # pragma: nocover
                            pass
                except NoNodeError:  # pragma: nocover
                    pass

        def _get_lease(self, data=None):
            # Make sure the session is still valid
            if self._session_expired:
                raise ForceRetryError("Retry on session loss at top")

            # Make sure that the request hasn't been canceled
            if self.cancelled:
                raise CancelledError("Semaphore cancelled")

            # Get a list of the current potential lock holders. If they change,
            # notify our wake_event object. This is used to unblock a blocking
            # self._inner_acquire call.
            children = self.client.get_children(self.path,
                                                self._watch_lease_change)

            # If there are leases available, acquire one
            if len(children) < self.max_leases:
                self.client.create(self.create_path, self.data, ephemeral=self.ephemeral_lease)

            # Check if our acquisition was successful or not. Update our state.
            if self.client.exists(self.create_path):
                self.is_acquired = True
            else:
                self.is_acquired = False

            # Return current state
            return self.is_acquired

    HAS_DEPS = True
except ImportError:
    HAS_DEPS = False

ZK_CONNECTION = None
SEMAPHORE_MAP = {}

__virtualname__ = 'zk_concurrency'

def __virtual__():
    if not HAS_DEPS:
        return (False, "Module zk_concurrency: dependencies failed")

    return __virtualname__

def _get_zk_conn(hosts):
    global ZK_CONNECTION
    if ZK_CONNECTION is None:
        ZK_CONNECTION = KazooClient(hosts=hosts)
        ZK_CONNECTION.start()

    return ZK_CONNECTION

def _close_zk_conn():
    global ZK_CONNECTION
    if ZK_CONNECTION is None:
        return

    ZK_CONNECTION.stop()
    ZK_CONNECTION = None

def lock_holders(path,
                 zk_hosts,
                 identifier=None,
                 max_concurrency=1,
                 timeout=None,
                 ephemeral_lease=False):
    '''
    Return an un-ordered list of lock holders

    path
        The path in zookeeper where the lock is

    zk_hosts
        zookeeper connect string

    identifier
        Name to identify this minion

    max_concurrency
        Maximum number of lock holders

    timeout
        timeout to wait for the lock. A None timeout will block forever

    ephemeral_lease
        Whether the locks in zookeper should be ephemeral

    Example:

    ... code-block: bash

        salt minion zk_concurrency.lock_holders /lock/path host1:1234,host2:1234
    '''

    zk = _get_zk_conn(zk_hosts)
    if path not in SEMAPHORE_MAP:
        SEMAPHORE_MAP[path] = _Semaphore(zk,
                                        path,
                                        identifier,
                                        max_leases=max_concurrency,
                                        ephemeral_lease=ephemeral_lease)
    return SEMAPHORE_MAP[path].lease_holders()

def lock(path,
         zk_hosts,
         identifier=None,
         max_concurrency=1,
         timeout=None,
         ephemeral_lease=False,
         force=False,  # foricble get the lock regardless of open slots
         ):
    '''
    Get lock (with optional timeout)

    path
        The path in zookeeper where the lock is

    zk_hosts
        zookeeper connect string

    identifier
        Name to identify this minion

    max_concurrency
        Maximum number of lock holders

    timeout
        timeout to wait for the lock. A None timeout will block forever

    ephemeral_lease
        Whether the locks in zookeper should be ephemeral

    force
        Forcibly acquire the lock regardless of available slots

    Example:

    ... code-block: bash

        salt minion zk_concurrency.lock /lock/path host1:1234,host2:1234
    '''
    zk = _get_zk_conn(zk_hosts)
    if path not in SEMAPHORE_MAP:
        SEMAPHORE_MAP[path] = _Semaphore(zk,
                                        path,
                                        identifier,
                                        max_leases=max_concurrency,
                                        ephemeral_lease=ephemeral_lease)

    # forcibly get the lock regardless of max_concurrency
    if force:
        SEMAPHORE_MAP[path].assured_path = True

    # block waiting for lock acquisition
    if timeout:
        logging.info('Acquiring lock {0} with timeout={1}'.format(path, timeout))
        SEMAPHORE_MAP[path].acquire(timeout=timeout)
    else:
        logging.info('Acquiring lock {0} with no timeout'.format(path))
        SEMAPHORE_MAP[path].acquire()

    return SEMAPHORE_MAP[path].is_acquired

def unlock(path,
           zk_hosts=None,  # in case you need to unlock without having run lock (failed execution for example)
           identifier=None,
           max_concurrency=1,
           ephemeral_lease=False
           ):
    '''
    Remove lease from semaphore

    path
        The path in zookeeper where the lock is

    zk_hosts
        zookeeper connect string

    identifier
        Name to identify this minion

    max_concurrency
        Maximum number of lock holders

    timeout
        timeout to wait for the lock. A None timeout will block forever

    ephemeral_lease
        Whether the locks in zookeper should be ephemeral

    Example:

    ... code-block: bash

        salt minion zk_concurrency.unlock /lock/path host1:1234,host2:1234
    '''
    # if someone passed in zk_hosts, and the path isn't in SEMAPHORE_MAP, lets
    # see if we can find it
    if zk_hosts is not None and path not in SEMAPHORE_MAP:
        zk = _get_zk_conn(zk_hosts)
        SEMAPHORE_MAP[path] = _Semaphore(zk,
                                        path,
                                        identifier,
                                        max_leases=max_concurrency,
                                        ephemeral_lease=ephemeral_lease)

    if path in SEMAPHORE_MAP:
        SEMAPHORE_MAP[path].release()
        del SEMAPHORE_MAP[path]
        return True
    else:
        logging.error('Unable to find lease for path {0}'.format(path))
        return False

def party_members(path,
                  zk_hosts,
                  min_nodes=1,
                  blocking=False
                  ):
    '''
    Get the List of identifiers in a particular party, optionally waiting for the specified
    minimum number of nodes (min_nodes) to appear

    path
        The path in zookeeper where the lock is

    zk_hosts
        zookeeper connect string

    min_nodes
        The minimum number of nodes needed to be present in the party

    blocking
        The flag indicating if we need to block until min_nodes are available

    Example:

    ... code-block: bash

        salt minion zk_concurrency.party_members /lock/path host1:1234,host2:1234
    '''
    zk = _get_zk_conn(zk_hosts)
    party = kazoo.recipe.party.ShallowParty(zk, path)
    if blocking:
        barrier = kazoo.recipe.barrier.DoubleBarrier(zk, path, min_nodes)
        barrier.enter()
        party = kazoo.recipe.party.ShallowParty(zk, path)
        barrier.leave()
    return list(party)
<commit_msg>Adding more examples for blocking/min_nodes<commit_after># -*- coding: utf-8 -*-
'''
Concurrency controls in zookeeper
=========================================================================

This module allows you to acquire and release a slot. This is primarily useful
for ensureing that no more than N hosts take a specific action at once. This can
also be used to coordinate between masters.
'''
from __future__ import absolute_import

import logging

try:
    from kazoo.client import KazooClient

    from kazoo.retry import (
        ForceRetryError
    )
    import kazoo.recipe.lock
    import kazoo.recipe.barrier
    import kazoo.recipe.party
    from kazoo.exceptions import CancelledError
    from kazoo.exceptions import NoNodeError

    # TODO: use the kazoo one, waiting for pull req:
    # https://github.com/python-zk/kazoo/pull/206
    class _Semaphore(kazoo.recipe.lock.Semaphore):
        def __init__(self,
                    client,
                    path,
                    identifier=None,
                    max_leases=1,
                    ephemeral_lease=True,
                    ):
            kazoo.recipe.lock.Semaphore.__init__(self,
                                                client,
                                                path,
                                                identifier=identifier,
                                                max_leases=max_leases)
            self.ephemeral_lease = ephemeral_lease

            # if its not ephemeral, make sure we didn't already grab it
            if not self.ephemeral_lease:
                try:
                    for child in self.client.get_children(self.path):
                        try:
                            data, stat = self.client.get(self.path + "/" + child)
                            if identifier == data.decode('utf-8'):
                                self.create_path = self.path + "/" + child
                                self.is_acquired = True
                                break
                        except NoNodeError:  # pragma: nocover
                            pass
                except NoNodeError:  # pragma: nocover
                    pass

        def _get_lease(self, data=None):
            # Make sure the session is still valid
            if self._session_expired:
                raise ForceRetryError("Retry on session loss at top")

            # Make sure that the request hasn't been canceled
            if self.cancelled:
                raise CancelledError("Semaphore cancelled")

            # Get a list of the current potential lock holders. If they change,
            # notify our wake_event object. This is used to unblock a blocking
            # self._inner_acquire call.
            children = self.client.get_children(self.path,
                                                self._watch_lease_change)

            # If there are leases available, acquire one
            if len(children) < self.max_leases:
                self.client.create(self.create_path, self.data, ephemeral=self.ephemeral_lease)

            # Check if our acquisition was successful or not. Update our state.
            if self.client.exists(self.create_path):
                self.is_acquired = True
            else:
                self.is_acquired = False

            # Return current state
            return self.is_acquired

    HAS_DEPS = True
except ImportError:
    HAS_DEPS = False

ZK_CONNECTION = None
SEMAPHORE_MAP = {}

__virtualname__ = 'zk_concurrency'

def __virtual__():
    if not HAS_DEPS:
        return (False, "Module zk_concurrency: dependencies failed")

    return __virtualname__

def _get_zk_conn(hosts):
    global ZK_CONNECTION
    if ZK_CONNECTION is None:
        ZK_CONNECTION = KazooClient(hosts=hosts)
        ZK_CONNECTION.start()

    return ZK_CONNECTION

def _close_zk_conn():
    global ZK_CONNECTION
    if ZK_CONNECTION is None:
        return

    ZK_CONNECTION.stop()
    ZK_CONNECTION = None

def lock_holders(path,
                 zk_hosts,
                 identifier=None,
                 max_concurrency=1,
                 timeout=None,
                 ephemeral_lease=False):
    '''
    Return an un-ordered list of lock holders

    path
        The path in zookeeper where the lock is

    zk_hosts
        zookeeper connect string

    identifier
        Name to identify this minion

    max_concurrency
        Maximum number of lock holders

    timeout
        timeout to wait for the lock. A None timeout will block forever

    ephemeral_lease
        Whether the locks in zookeper should be ephemeral

    Example:

    ... code-block: bash

        salt minion zk_concurrency.lock_holders /lock/path host1:1234,host2:1234
    '''

    zk = _get_zk_conn(zk_hosts)
    if path not in SEMAPHORE_MAP:
        SEMAPHORE_MAP[path] = _Semaphore(zk,
                                        path,
                                        identifier,
                                        max_leases=max_concurrency,
                                        ephemeral_lease=ephemeral_lease)
    return SEMAPHORE_MAP[path].lease_holders()

def lock(path,
         zk_hosts,
         identifier=None,
         max_concurrency=1,
         timeout=None,
         ephemeral_lease=False,
         force=False,  # foricble get the lock regardless of open slots
         ):
    '''
    Get lock (with optional timeout)

    path
        The path in zookeeper where the lock is

    zk_hosts
        zookeeper connect string

    identifier
        Name to identify this minion

    max_concurrency
        Maximum number of lock holders

    timeout
        timeout to wait for the lock. A None timeout will block forever

    ephemeral_lease
        Whether the locks in zookeper should be ephemeral

    force
        Forcibly acquire the lock regardless of available slots

    Example:

    ... code-block: bash

        salt minion zk_concurrency.lock /lock/path host1:1234,host2:1234
    '''
    zk = _get_zk_conn(zk_hosts)
    if path not in SEMAPHORE_MAP:
        SEMAPHORE_MAP[path] = _Semaphore(zk,
                                        path,
                                        identifier,
                                        max_leases=max_concurrency,
                                        ephemeral_lease=ephemeral_lease)

    # forcibly get the lock regardless of max_concurrency
    if force:
        SEMAPHORE_MAP[path].assured_path = True

    # block waiting for lock acquisition
    if timeout:
        logging.info('Acquiring lock {0} with timeout={1}'.format(path, timeout))
        SEMAPHORE_MAP[path].acquire(timeout=timeout)
    else:
        logging.info('Acquiring lock {0} with no timeout'.format(path))
        SEMAPHORE_MAP[path].acquire()

    return SEMAPHORE_MAP[path].is_acquired

def unlock(path,
           zk_hosts=None,  # in case you need to unlock without having run lock (failed execution for example)
           identifier=None,
           max_concurrency=1,
           ephemeral_lease=False
           ):
    '''
    Remove lease from semaphore

    path
        The path in zookeeper where the lock is

    zk_hosts
        zookeeper connect string

    identifier
        Name to identify this minion

    max_concurrency
        Maximum number of lock holders

    timeout
        timeout to wait for the lock. A None timeout will block forever

    ephemeral_lease
        Whether the locks in zookeper should be ephemeral

    Example:

    ... code-block: bash

        salt minion zk_concurrency.unlock /lock/path host1:1234,host2:1234
    '''
    # if someone passed in zk_hosts, and the path isn't in SEMAPHORE_MAP, lets
    # see if we can find it
    if zk_hosts is not None and path not in SEMAPHORE_MAP:
        zk = _get_zk_conn(zk_hosts)
        SEMAPHORE_MAP[path] = _Semaphore(zk,
                                        path,
                                        identifier,
                                        max_leases=max_concurrency,
                                        ephemeral_lease=ephemeral_lease)

    if path in SEMAPHORE_MAP:
        SEMAPHORE_MAP[path].release()
        del SEMAPHORE_MAP[path]
        return True
    else:
        logging.error('Unable to find lease for path {0}'.format(path))
        return False

def party_members(path,
                  zk_hosts,
                  min_nodes=1,
                  blocking=False
                  ):
    '''
    Get the List of identifiers in a particular party, optionally waiting for the specified
    minimum number of nodes (min_nodes) to appear

    path
        The path in zookeeper where the lock is

    zk_hosts
        zookeeper connect string

    min_nodes
        The minimum number of nodes needed to be present in the party

    blocking
        The flag indicating if we need to block until min_nodes are available

    Example:

    ... code-block: bash

        salt minion zk_concurrency.party_members /lock/path host1:1234,host2:1234
        salt minion zk_concurrency.party_members /lock/path host1:1234,host2:1234 min_nodes=3 blocking=True
    '''
    zk = _get_zk_conn(zk_hosts)
    party = kazoo.recipe.party.ShallowParty(zk, path)
    if blocking:
        barrier = kazoo.recipe.barrier.DoubleBarrier(zk, path, min_nodes)
        barrier.enter()
        party = kazoo.recipe.party.ShallowParty(zk, path)
        barrier.leave()
    return list(party)
<|endoftext|>