<commit_before>from setuptools import setup
from setuptools import find_packages

# Used in CI and by deveopers
build_requires = [
    'wheel',
    'twine',
]

# Used in CI and by deveopers
test_requires = [
    'pytest',
    'pytest-runner',
    'flake8',
]

# Used in developers' PCs only
dev_requires = [
    'pytest-sugar',
    'sphinx',
    'sphinx-rtd-theme',
]

# Used in ReadTheDocs build servers
# (actually they already have Sphinx, but let's specify it explicitly.)
docs_requires = [
    'sphinx',
]

setup(
    name='wtte',
    version='1.0.2',
    description='Weibull Time To Event model. A Deep Learning model for churn- and failure prediction and everything else.',
    author='Egil Martinsson',
    author_email='egil.martinsson@gmail.com',
    url='https://github.com/ragulpr/wtte-rnn/',
    license='MIT',
    install_requires=[
        'keras>=2.0',
        'numpy',
        'pandas',
        'scipy',
        'six==1.10.0',
    ],
    extras_require={
        'plot': ['matplotlib'],
        'tf': ["tensorflow>=1.1.0"],
        'tf_gpu': ["tensorflow-gpu>=1.1.0"],
        'build': build_requires,
        'test': test_requires,
        'dev': dev_requires,
        'docs': docs_requires,
    },
    setup_requires=['pytest-runner'],
    tests_require=['pytest'],
    packages=find_packages('.', exclude=['examples', 'tests']),
)
<commit_msg>version -> 1.0.3<commit_after>from setuptools import setup
from setuptools import find_packages

# Used in CI and by deveopers
build_requires = [
    'wheel',
    'twine',
]

# Used in CI and by deveopers
test_requires = [
    'pytest',
    'pytest-runner',
    'flake8',
]

# Used in developers' PCs only
dev_requires = [
    'pytest-sugar',
    'sphinx',
    'sphinx-rtd-theme',
]

# Used in ReadTheDocs build servers
# (actually they already have Sphinx, but let's specify it explicitly.)
docs_requires = [
    'sphinx',
]

setup(
    name='wtte',
    version='1.0.3',
    description='Weibull Time To Event model. A Deep Learning model for churn- and failure prediction and everything else.',
    author='Egil Martinsson',
    author_email='egil.martinsson@gmail.com',
    url='https://github.com/ragulpr/wtte-rnn/',
    license='MIT',
    install_requires=[
        'keras>=2.0',
        'numpy',
        'pandas',
        'scipy',
        'six==1.10.0',
    ],
    extras_require={
        'plot': ['matplotlib'],
        'tf': ["tensorflow>=1.1.0"],
        'tf_gpu': ["tensorflow-gpu>=1.1.0"],
        'build': build_requires,
        'test': test_requires,
        'dev': dev_requires,
        'docs': docs_requires,
    },
    setup_requires=['pytest-runner'],
    tests_require=['pytest'],
    packages=find_packages('.', exclude=['examples', 'tests']),
)
<|endoftext|>


<commit_before>from setuptools import setup

setup(
    name='sparktestingbase',
    version='0.0.7-snapshot',
    author='Holden Karau',
    author_email='holden@pigscanfly.ca',
    packages=['sparktestingbase', 'sparktestingbase.test'],
    url='https://github.com/holdenk/spark-testing-base',
    license='LICENSE.txt',
    description='Spark testing for python',
    long_description='',
    install_requires=[
        'unittest2',
        'findspark'
    ],
    test_requires=[
        'nose',
        'coverage',
        'unittest2'
    ],
)
<commit_msg>Add py.test and hypothesis<commit_after>from setuptools import setup

setup(
    name='sparktestingbase',
    version='0.0.7-snapshot',
    author='Holden Karau',
    author_email='holden@pigscanfly.ca',
    packages=['sparktestingbase', 'sparktestingbase.test'],
    url='https://github.com/holdenk/spark-testing-base',
    license='LICENSE.txt',
    description='Spark testing for python',
    long_description='',
    install_requires=[
        'unittest2',
        'findspark',
        'py.test',
        'hypothesis'
    ],
    test_requires=[
        'nose',
        'coverage',
        'unittest2'
    ],
)
<|endoftext|>


<commit_before>import sublime
import sublime_plugin

try:
    from SublimeJEDI.sublime_jedi import get_script, JediEnvMixin
except ImportError:
    from sublime_jedi import get_script, JediEnvMixin

from jedi.api import NotFoundError

def check_if_string(view):
    """ Checks if the current selection is a string

        :param view: `sublime.View` object

        :return: bool
    """
    sels = view.sel()
    region = view.word(sels[0])

    line = view.line(region)
    currently_string = False
    current_string_quotes = []

    for x in range(line.a, line.b):
        char = view.substr(x)
        if char in ('\'', '"'):

            if len(current_string_quotes) == 1 and current_string_quotes[-1] == char:
                currently_string = False
                current_string_quotes.pop()

            else:
                currently_string = True
                current_string_quotes.append(char)

        if x >= region.a:
            return currently_string

    return currently_string

def get_current_location(view):
    return view.sel()[0].begin()

def related_names(view):
    script = get_script(view, get_current_location(view))
    related_names = script.related_names()
    return filter(lambda x: not x.in_builtin_module(), related_names)

class BaseLookUpJediCommand(JediEnvMixin):
    def is_enabled(self):
        """ check if we can run the command """

        # nothing to do with non-python code
        if 'python' not in self.view.settings().get('syntax').lower():
            return False

        # check if this is python string, then avoid goto event
        location = get_current_location(self.view)
        if 'python string' in self.view.scope_name(location):
            return False
        return True

    def _jump_to_in_window(self, filename, line_number=None, column_number=None):
        """ Opens a new window and jumps to declaration if possible

            :param filename: string or int
            :param line_number: int
            :param column_number: int
        """
        active_window = self.view.window()

        # If the file was selected from a drop down list
        if isinstance(filename, int):
            if filename == -1:  # cancelled
                return
            filename, line_number, column_number = self.options_map[filename]
        active_window.open_file('%s:%s:%s' % (filename, line_number or 0,
                                column_number or 0), sublime.ENCODED_POSITION)

    def _window_quick_panel_open_window(self, options):
        """ Shows the active `sublime.Window` quickpanel (dropdown) for
            user selection.

            :param option: list of `jedi.api_classes.BasDefinition`
        """
        options = list(options)
        active_window = self.view.window()

        # Map the filenames to line and column numbers
        self.options_map = dict((i, (o.module_path, o.line, o.column))
                                for i, o in enumerate(options))

        # Show the user a selection of filenames
        active_window.show_quick_panel(
            [self.prepare_option(o) for o in options],
            self._jump_to_in_window
        )

    def prepare_option(self, option):
        """ prepare option to display out in quick panel """
        raise NotImplementedError(
            "{} require `prepare_option` definition".format(self.__class__)
        )

class SublimeJediGoto(BaseLookUpJediCommand, sublime_plugin.TextCommand):
    """ go to object definition """
    def run(self, edit):
        with self.env:
            script = get_script(self.view, get_current_location(self.view))

            # If we have a possible python declaration
            # use jedi to find possible declarations.
            # found = self.attempt_get_definition(script)
            # if not found:
            #     found = self.attempt_go_to(script)
            for method in ['get_definition', 'goto']:
                try:
                    defns = getattr(script, method)()
                except NotFoundError:
                    return
                else:
                    self.handle_definitions(defns)
                    break

    def handle_definitions(self, defns):
        # filter out builtin
        defns = [i for i in defns if not i.in_builtin_module()]
        if not defns:
            return False
        if len(defns) == 1:
            defn = defns[0]
            self._jump_to_in_window(defn.module_path, defn.line, defn.column)
        else:
            self._window_quick_panel_open_window(defns)

    def prepare_option(self, option):
        return self.prepare_definition(option)

    def prepare_definition(self, option):
        return option.module_path

class SublimeJediFindUsages(BaseLookUpJediCommand, sublime_plugin.TextCommand):
    """ find object usages """
    def run(self, edit):
        usages = self.find_usages()
        self._window_quick_panel_open_window(usages)

    def find_usages(self):
        with self.env:
            return related_names(self.view)

    def prepare_option(self, option):
        return self.prepare_related_name(option)

    def prepare_related_name(self, option):
        return [option.module_path,
                "line: %d column: %d" % (option.line, option.column)]
<commit_msg>issue #43<commit_after>import sublime
import sublime_plugin

try:
    from SublimeJEDI.sublime_jedi import get_script, JediEnvMixin
except ImportError:
    from sublime_jedi import get_script, JediEnvMixin

from jedi.api import NotFoundError

def check_if_string(view):
    """ Checks if the current selection is a string

        :param view: `sublime.View` object

        :return: bool
    """
    sels = view.sel()
    region = view.word(sels[0])

    line = view.line(region)
    currently_string = False
    current_string_quotes = []

    for x in range(line.a, line.b):
        char = view.substr(x)
        if char in ('\'', '"'):

            if len(current_string_quotes) == 1 and current_string_quotes[-1] == char:
                currently_string = False
                current_string_quotes.pop()

            else:
                currently_string = True
                current_string_quotes.append(char)

        if x >= region.a:
            return currently_string

    return currently_string

def get_current_location(view):
    return view.sel()[0].begin()

def related_names(view):
    script = get_script(view, get_current_location(view))
    related_names = script.related_names()
    return filter(lambda x: not x.in_builtin_module(), related_names)

class BaseLookUpJediCommand(JediEnvMixin):
    def is_enabled(self):
        """ check if we can run the command """

        # nothing to do with non-python code
        if 'python' not in self.view.settings().get('syntax').lower():
            return False

        # check if this is python string, then avoid goto event
        location = get_current_location(self.view)
        if 'python string' in self.view.scope_name(location):
            return False
        return True

    def _jump_to_in_window(self, filename, line_number=None, column_number=None):
        """ Opens a new window and jumps to declaration if possible

            :param filename: string or int
            :param line_number: int
            :param column_number: int
        """
        active_window = self.view.window()

        # If the file was selected from a drop down list
        if isinstance(filename, int):
            if filename == -1:  # cancelled
                return
            filename, line_number, column_number = self.options_map[filename]
        active_window.open_file('%s:%s:%s' % (filename, line_number or 0,
                                column_number or 0), sublime.ENCODED_POSITION)

    def _window_quick_panel_open_window(self, options):
        """ Shows the active `sublime.Window` quickpanel (dropdown) for
            user selection.

            :param option: list of `jedi.api_classes.BasDefinition`
        """
        options = list(options)
        active_window = self.view.window()

        # Map the filenames to line and column numbers
        self.options_map = dict((i, (o.module_path, o.line, o.column))
                                for i, o in enumerate(options))

        # Show the user a selection of filenames
        active_window.show_quick_panel(
            [self.prepare_option(o) for o in options],
            self._jump_to_in_window
        )

    def prepare_option(self, option):
        """ prepare option to display out in quick panel """
        raise NotImplementedError(
            "{} require `prepare_option` definition".format(self.__class__)
        )

class SublimeJediGoto(BaseLookUpJediCommand, sublime_plugin.TextCommand):
    """ go to object definition """
    def run(self, edit):
        with self.env:
            script = get_script(self.view, get_current_location(self.view))

            try:
                defns = script.goto()
            except NotFoundError:
                return
            else:
                self.handle_definitions(defns)

    def handle_definitions(self, defns):
        # filter out builtin
        defns = [i for i in defns if not i.in_builtin_module()]
        if not defns:
            return False
        if len(defns) == 1:
            defn = defns[0]
            self._jump_to_in_window(defn.module_path, defn.line, defn.column)
        else:
            self._window_quick_panel_open_window(defns)

    def prepare_option(self, option):
        return self.prepare_definition(option)

    def prepare_definition(self, option):
        return option.module_path

class SublimeJediFindUsages(BaseLookUpJediCommand, sublime_plugin.TextCommand):
    """ find object usages """
    def run(self, edit):
        usages = self.find_usages()
        self._window_quick_panel_open_window(usages)

    def find_usages(self):
        with self.env:
            return related_names(self.view)

    def prepare_option(self, option):
        return self.prepare_related_name(option)

    def prepare_related_name(self, option):
        return [option.module_path,
                "line: %d column: %d" % (option.line, option.column)]
<|endoftext|>


<commit_before>#!/usr/bin/env python

import PyV8

import os
import sys
import compiler

from os.path import join, dirname, basename, abspath
from optparse import OptionParser

usage = """
  usage: %prog [options] <application module name or path>
"""

currentdir = abspath(dirname(__file__))
pyjspth = abspath(join(dirname(__file__), ".."))
sys.path = [(join(pyjspth, "pyjs", "src"))] + sys.path

import pyjs

pyjs.pyjspth = pyjspth
pyjs.path += [os.path.join(pyjspth, 'library'),
            os.path.join(pyjspth, 'addons'),
]

from pyjs import translator
from linker import PLATFORM, PyV8Linker, add_linker_options
from jsglobal import Global

class JSRuntimeError(Exception):
    def __init__(self, ctxt, exc):
        self.module = ctxt.locals['$pyjs']['track']['module']
        self.lineno = ctxt.locals['$pyjs']['track']['lineno']
        self.traceback = self.get_traceback(ctxt.locals['$pyjs']['trackstack'])
        errortype, errortext = exc.message.split(':', 1)
        if not ' ' in errortype:
            self.jserrortype = errortype
            self.jserrortext = errortext
        else:
            self.jserrortype = exc.__class__.__name__
            self.jserrortext = exc.message

    def __str__(self):
        return "{0.jserrortype}: {0.module}.py:{0.lineno} {0.jserrortext}".\
               format(self)

    def full(self):
        return ("{0.jserrortype}: {0.module}.py:{0.lineno}\n"
                "Traceback:\n{trace}\n"
                "{0.jserrortext}".format(self, trace=self.get_traceback_text()))

    def get_traceback(self, jstb):
        tb = []
        for x in jstb:
            tb.append(dict(module=x['module'], lineno=x['lineno']))
        return tb

    def get_traceback_text(self):
        trace = []
        for x in self.traceback:
            trace.append("  {0}.py:{1}".format(x['module'], x['lineno']))
        return "\n".join(trace)

def main():
    usage = """
    usage: %prog [ options ] [ -c command | module_name | script | - ] [ -- arguments ]
    """
    parser = OptionParser(usage = usage)
    translator.add_compile_options(parser)

    parser.add_option(
        "--dynamic",
        dest="unlinked_modules",
        action="append",
        help="regular expression for modules that will not be linked"
        "and thus loaded dynamically"
        )
    parser.add_option(
        "-c",
        dest="command",
        help="Python command to run")

    # override the default because we want print
    parser.set_defaults(print_statements=True)
    add_linker_options(parser)
    options, args = parser.parse_args()
    IS_REPL = False
    if len(args) == 0 or args[0] == '-':
        IS_REPL = True
        modules = ['main']
    else:
        modules = args

    _modules = []
    for mod in modules:
        if mod.startswith('.') or mod.startswith('/') or mod.endswith('.py'):
            pyjs.path[0:0] = [os.path.dirname(os.path.abspath(mod))]
            _modules.append(os.path.basename(mod).split('.')[0])
        else:
            _modules.append(mod)
    modules = _modules

    for d in options.library_dirs:
        pyjs.path.append(os.path.abspath(d))

    #print "paths:", pyjs.path
    translator_arguments = translator.get_compile_options(options)
    linker = PyV8Linker(modules, output=options.output,
                        platforms=[PLATFORM],
                        path=pyjs.path,
                        compiler=compiler,
                        translator_arguments=translator_arguments)
    linker()

    fp = open(linker.out_file_mod, 'r')
    txt = fp.read()
    fp.close()

    #PyV8.debugger.enabled = True

    # create a context with an explicit global
    g = Global()
    ctxt = PyV8.JSContext(g)
    g.__context__ = ctxt
    # enter the context
    ctxt.enter()
    try:
        x = ctxt.eval(txt)
    except Exception, e:
        print JSRuntimeError(ctxt, e).full()

    if IS_REPL:
        from repl import REPL
        REPL(compiler, linker, translator_arguments, g, ctxt)()

if __name__ == '__main__':
    main()

<commit_msg>Pyv8 should use stdlib/ first<commit_after>#!/usr/bin/env python

import PyV8

import os
import sys
import compiler

from os.path import join, dirname, basename, abspath
from optparse import OptionParser

usage = """
  usage: %prog [options] <application module name or path>
"""

currentdir = abspath(dirname(__file__))
pyjspth = abspath(join(dirname(__file__), ".."))
sys.path = [(join(pyjspth, "pyjs", "src"))] + sys.path

import pyjs

pyjs.pyjspth = pyjspth
pyjs.path += [os.path.join(pyjspth, 'library'),
            os.path.join(pyjspth, 'addons'),
]

from pyjs import translator
from linker import PLATFORM, PyV8Linker, add_linker_options
from jsglobal import Global

class JSRuntimeError(Exception):
    def __init__(self, ctxt, exc):
        self.module = ctxt.locals['$pyjs']['track']['module']
        self.lineno = ctxt.locals['$pyjs']['track']['lineno']
        self.traceback = self.get_traceback(ctxt.locals['$pyjs']['trackstack'])
        errortype, errortext = exc.message.split(':', 1)
        if not ' ' in errortype:
            self.jserrortype = errortype
            self.jserrortext = errortext
        else:
            self.jserrortype = exc.__class__.__name__
            self.jserrortext = exc.message

    def __str__(self):
        return "{0.jserrortype}: {0.module}.py:{0.lineno} {0.jserrortext}".\
               format(self)

    def full(self):
        return ("{0.jserrortype}: {0.module}.py:{0.lineno}\n"
                "Traceback:\n{trace}\n"
                "{0.jserrortext}".format(self, trace=self.get_traceback_text()))

    def get_traceback(self, jstb):
        tb = []
        for x in jstb:
            tb.append(dict(module=x['module'], lineno=x['lineno']))
        return tb

    def get_traceback_text(self):
        trace = []
        for x in self.traceback:
            trace.append("  {0}.py:{1}".format(x['module'], x['lineno']))
        return "\n".join(trace)

def main():
    usage = """
    usage: %prog [ options ] [ -c command | module_name | script | - ] [ -- arguments ]
    """
    parser = OptionParser(usage = usage)
    translator.add_compile_options(parser)

    parser.add_option(
        "--dynamic",
        dest="unlinked_modules",
        action="append",
        help="regular expression for modules that will not be linked"
        "and thus loaded dynamically"
        )
    parser.add_option(
        "-c",
        dest="command",
        help="Python command to run")

    # override the default because we want print
    parser.set_defaults(print_statements=True)
    add_linker_options(parser)
    options, args = parser.parse_args()
    IS_REPL = False
    if len(args) == 0 or args[0] == '-':
        IS_REPL = True
        modules = ['main']
    else:
        modules = args

    _modules = []
    for mod in modules:
        if mod.startswith('.') or mod.startswith('/') or mod.endswith('.py'):
            pyjs.path[0:0] = [os.path.dirname(os.path.abspath(mod))]
            _modules.append(os.path.basename(mod).split('.')[0])
        else:
            _modules.append(mod)
    modules = _modules

    pyjs.path[0:0] = [join(pyjs.pyjspth, 'stdlib')]
    pyjs.path.append(join(pyjs.pyjspth, 'pyjs', 'src'))

    for d in options.library_dirs:
        pyjs.path.append(os.path.abspath(d))

    #print "paths:", pyjs.path
    translator_arguments = translator.get_compile_options(options)
    linker = PyV8Linker(modules, output=options.output,
                        platforms=[PLATFORM],
                        path=pyjs.path,
                        compiler=compiler,
                        translator_arguments=translator_arguments)
    linker()

    fp = open(linker.out_file_mod, 'r')
    txt = fp.read()
    fp.close()

    #PyV8.debugger.enabled = True

    # create a context with an explicit global
    g = Global()
    ctxt = PyV8.JSContext(g)
    g.__context__ = ctxt
    # enter the context
    ctxt.enter()
    try:
        x = ctxt.eval(txt)
    except Exception, e:
        print JSRuntimeError(ctxt, e).full()

    if IS_REPL:
        from repl import REPL
        REPL(compiler, linker, translator_arguments, g, ctxt)()

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before>#
# (C) Copyright 2011 Jacek Konieczny <jajcus@jajcus.net>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License Version
# 2.1 as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#

"""ElementTree API selection.

The rest of PyXMPP2 package imports the ElementTree API from this module.

The actual API can be selected in one of two ways:

By importing this module (before anything else) its `ElementTree` variable:

    >> import pyxmpp2.etree
    >> import xml.etree.cElementTree
    >> pyxmpp2.etree.ElementTree = xml.etree.cElementTree

Or by setting the 'PYXMPP2_ETREE' variable, e.g.:

    $ PYXMPP2_ETREE="xml.etree"

By default the standard Python ElementTree implementation is used
(`xml.etree.ElementTree`)
"""

from __future__ import absolute_import

__docformat__ = "restructuredtext en"

import os
from abc import ABCMeta

if "PYXMPP2_ETREE" in os.environ:
    ElementTree = __import__(os.environ["PYXMPP2_ETREE"], fromlist=[""])
else:
    from xml.etree import ElementTree

class ElementClass:
    __metaclass__ = ABCMeta
    @classmethod
    def __subclasshook__(cls, other):
        if cls is ElementClass:
            return hasattr(other, "tag")
        return NotImplemented

<commit_msg>cElementTree compatibility fix<commit_after>#
# (C) Copyright 2011 Jacek Konieczny <jajcus@jajcus.net>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License Version
# 2.1 as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#

"""ElementTree API selection.

The rest of PyXMPP2 package imports the ElementTree API from this module.

The actual API can be selected in one of two ways:

By importing this module (before anything else) its `ElementTree` variable:

    >> import pyxmpp2.etree
    >> import xml.etree.cElementTree
    >> pyxmpp2.etree.ElementTree = xml.etree.cElementTree

Or by setting the 'PYXMPP2_ETREE' variable, e.g.:

    $ PYXMPP2_ETREE="xml.etree"

By default the standard Python ElementTree implementation is used
(`xml.etree.ElementTree`)
"""

from __future__ import absolute_import

__docformat__ = "restructuredtext en"

import os
from abc import ABCMeta

if "PYXMPP2_ETREE" in os.environ:
    ElementTree = __import__(os.environ["PYXMPP2_ETREE"], fromlist=[""])
else:
    from xml.etree import ElementTree

class ElementClass:
    __metaclass__ = ABCMeta
    element_type = type(ElementTree.Element("x"))
    @classmethod
    def __subclasshook__(cls, other):
        if cls is ElementClass:
            return other is cls.element_type or hasattr(other, "tag")
        return NotImplemented

<|endoftext|>


<commit_before>#   Foremast - Pipeline Tooling
#
#   Copyright 2016 Gogo, LLC
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

"""Construct a block section of Stages in a Spinnaker Pipeline."""
import copy
import json
import logging
from pprint import pformat

from ..consts import ASG_WHITELIST, DEFAULT_EC2_SECURITYGROUPS
from ..utils import generate_encoded_user_data, get_template

LOG = logging.getLogger(__name__)

def construct_pipeline_block_lambda(env='',
                             generated=None,
                             previous_env=None,
                             region='us-east-1',
                             region_subnets=None,
                             settings=None,
                             pipeline_data=None):
    """Create the Pipeline JSON from template.

    This handles the common repeatable patterns in a pipeline, such as
    judgement, infrastructure, tagger and qe.

    Args:
        env (str): Deploy environment name, e.g. dev, stage, prod.
        generated (gogoutils.Generator): Gogo Application name generator.
        previous_env (str): The previous deploy environment to use as
            Trigger.
        region (str): AWS Region to deploy to.
        settings (dict): Environment settings from configurations.
        region_subnets (dict): Subnets for a Region, e.g.
            {'us-west-2': ['us-west-2a', 'us-west-2b', 'us-west-2c']}.

    Returns:
        dict: Pipeline JSON template rendered with configurations.
    """
    LOG.info('%s block for [%s].', env, region)

    if env.startswith('prod'):
        template_name = 'pipeline/pipeline_{}.json.j2'.format(env)
    else:
        template_name = 'pipeline/pipeline_stages_lambda.json.j2'

    LOG.debug('%s info:\n%s', env, pformat(settings))

    gen_app_name = generated.app_name()
    user_data = generate_encoded_user_data(env=env,
                                           region=region,
                                           app_name=gen_app_name,
                                           group_name=generated.project)

    # Use different variable to keep template simple
    instance_security_groups = list(DEFAULT_EC2_SECURITYGROUPS)
    instance_security_groups.append(gen_app_name)
    instance_security_groups.extend(settings['security_group']['instance_extras'])

    LOG.info('Instance security groups to attach: {0}'.format(instance_security_groups))

    # check if scaling policy exists
    if settings['asg']['scaling_policy']:
        scalingpolicy = True
        LOG.info('Found scaling policy')
    else:
        scalingpolicy = False
        LOG.info('No scaling policy found')

    if settings['app']['eureka_enabled']:
        elb = []
    else:
        elb = ['{0}'.format(gen_app_name)]
    LOG.info('Attaching the following ELB: {0}'.format(elb))

    provider_healthcheck = []
    for provider, active in settings['asg']['provider_healthcheck'].items():
        if active:
            provider_healthcheck.append(provider.capitalize())
    LOG.info('Provider healthchecks: {0}'.format(provider_healthcheck))

    has_provider_healthcheck = False
    if len(provider_healthcheck) > 0:
        has_provider_healthcheck = True

    data = copy.deepcopy(settings)

    # Default HC type in DEV to EC2, default to EC2 if eureka enabled
    # FIXME: Need to also set `provider_healthcheck` when `eureka_enabled`
    if env == 'dev' or settings['app']['eureka_enabled']:
        data['asg'].update({
            'hc_type': 'EC2'
        })
        LOG.info('Switching health check type to: EC2')

    LOG.info('White listed dev asg apps: {0}'.format(ASG_WHITELIST))
    if env == 'dev' and gen_app_name not in ASG_WHITELIST:
        data['asg'].update({
            'max_inst': '1',
            'min_inst': '1'
        })
        LOG.info('App {0} is not white listed, using default dev ASG settings'.format(gen_app_name))

    data['app'].update({
        'appname': gen_app_name,
        'repo_name': generated.repo,
        'group_name': generated.project,
        'environment': env,
        'region': region,
        'az_dict': json.dumps(region_subnets),
        'previous_env': previous_env,
        'encoded_user_data': user_data,
        'instance_security_groups': json.dumps(instance_security_groups),
        'elb': json.dumps(elb),
        'promote_restrict': pipeline_data['promote_restrict'],
        'owner_email': pipeline_data['owner_email'],
        'scalingpolicy': scalingpolicy
    })

    data['asg'].update({
        'hc_type': data['asg'].get('hc_type').upper(),
        'provider_healthcheck': json.dumps(provider_healthcheck),
        "has_provider_healthcheck": has_provider_healthcheck,
    })

    LOG.debug('Block data:\n%s', pformat(data))

    pipeline_json = get_template(template_file=template_name, data=data)
    return pipeline_json
<commit_msg>Removed ec2 specific code<commit_after>#   Foremast - Pipeline Tooling
#
#   Copyright 2016 Gogo, LLC
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

"""Construct a block section of Stages in a Spinnaker Pipeline."""
import copy
import json
import logging
from pprint import pformat

from ..consts import DEFAULT_EC2_SECURITYGROUPS
from ..utils import generate_encoded_user_data, get_template

LOG = logging.getLogger(__name__)

def construct_pipeline_block_lambda(env='',
                             generated=None,
                             previous_env=None,
                             region='us-east-1',
                             region_subnets=None,
                             settings=None,
                             pipeline_data=None):
    """Create the Pipeline JSON from template.

    This handles the common repeatable patterns in a pipeline, such as
    judgement, infrastructure, tagger and qe.

    Args:
        env (str): Deploy environment name, e.g. dev, stage, prod.
        generated (gogoutils.Generator): Gogo Application name generator.
        previous_env (str): The previous deploy environment to use as
            Trigger.
        region (str): AWS Region to deploy to.
        settings (dict): Environment settings from configurations.
        region_subnets (dict): Subnets for a Region, e.g.
            {'us-west-2': ['us-west-2a', 'us-west-2b', 'us-west-2c']}.

    Returns:
        dict: Pipeline JSON template rendered with configurations.
    """
    LOG.info('%s block for [%s].', env, region)

    if env.startswith('prod'):
        template_name = 'pipeline/pipeline_{}.json.j2'.format(env)
    else:
        template_name = 'pipeline/pipeline_stages_lambda.json.j2'

    LOG.debug('%s info:\n%s', env, pformat(settings))

    gen_app_name = generated.app_name()
    user_data = generate_encoded_user_data(env=env,
                                           region=region,
                                           app_name=gen_app_name,
                                           group_name=generated.project)

    # Use different variable to keep template simple
    instance_security_groups = list(DEFAULT_EC2_SECURITYGROUPS)
    instance_security_groups.append(gen_app_name)
    instance_security_groups.extend(settings['security_group']['instance_extras'])

    LOG.info('Instance security groups to attach: {0}'.format(instance_security_groups))

    data = copy.deepcopy(settings)

    data['app'].update({
        'appname': gen_app_name,
        'repo_name': generated.repo,
        'group_name': generated.project,
        'environment': env,
        'region': region,
        'az_dict': json.dumps(region_subnets),
        'previous_env': previous_env,
        'encoded_user_data': user_data,
        'instance_security_groups': json.dumps(instance_security_groups),
        'promote_restrict': pipeline_data['promote_restrict'],
        'owner_email': pipeline_data['owner_email']
    })

    LOG.debug('Block data:\n%s', pformat(data))

    pipeline_json = get_template(template_file=template_name, data=data)
    return pipeline_json
<|endoftext|>


<commit_before># coding=UTF8

import os
import shutil
import pyexiv2
import time
import utils

# noinspection PyBroadException
class PictureArchiver:
    _move_files = False
    _diagnostics = False

    _verbose = True
    _debug = False

    _imgCount = 0
    _currImgIndex = 0
    _success_count = 0
    _currImgFileName = None
    _correct_dates_only = False
    _start_size = 0

    onAdvance = None

    def __init__(self, src_path, dest_path):
        self._srcPath = src_path
        self._destPath = dest_path

    def _do_advance(self):
        try:
            if self.onAdvance is not None:
                self.onAdvance(self)
        except Exception:
            pass

    def _log(self, text):
        if self._verbose:
            print text

    def _debug(self, text):
        if self._debug:
            print text

    def _error(self, msg):
        print "ERROR:", msg

    def _correct_exif_date(self, filename, date):
        try:
            exif_data = pyexiv2.ImageMetadata(filename)
            need_write = False
            exif_data.read()

            if utils.get_exif_value(exif_data, 'Exif.Image.DateTime') is None:
                exif_data['Exif.Image.DateTime'] = date
                need_write = True

            if utils.get_exif_value(exif_data, 'Exif.Photo.DateTimeOriginal') is None:
                exif_data['Exif.Photo.DateTimeOriginal'] = date
                need_write = True

            if utils.get_exif_value(exif_data, 'Exif.Photo.DateTimeDigitized') is None:
                exif_data['Exif.Photo.DateTimeDigitized'] = date
                need_write = True

            if need_write:
                exif_data.write(True)
        except Exception as e:
            self._error(e)

    def _is_valid_backup_file(self, file_name):
        fname, fext = os.path.splitext(file_name)
        return fext.lower().lstrip(".") in utils.MIME_TYPES.keys()

    def _get_dest_folder_name(self, obj_date):
        if obj_date is not None:
            year = obj_date.strftime("%Y")
            month = obj_date.strftime("%m")
            day = obj_date.strftime("%Y-%m-%d")
            destpath = os.path.join(year, month, day)
            return destpath
        else:
            return ""

    def _correct_picture_date(self, picture_path, datetime):
        self._correct_exif_date(picture_path, datetime)

        filetime = time.mktime(datetime.timetuple())
        os.utime(picture_path, (filetime, filetime))

        self._debug("Corrected: " + picture_path)

    def _walk_dir_correct_date(self, root_dir):
        dir_list = os.listdir(root_dir)
        for filename in dir_list:
            src_file = os.path.join(root_dir, filename)

            if os.path.isdir(src_file):
                self._walk_dir_correct_date(src_file)
                continue

            if os.path.isfile(src_file):
                picture_date = utils.get_picture_date(src_file)
                self._correct_picture_date(src_file, picture_date)

    def _walk_dir(self, root_dir):
        dir_list = os.listdir(root_dir)
        self._imgCount += len(dir_list)

        files_left = len(dir_list)
        for filename in dir_list:
            self._currImgIndex += 1
            self._currImgFileName = filename
            self._do_advance()

            src_file = os.path.join(root_dir, filename)

            if os.path.isdir(src_file):
                self._walk_dir(src_file)
                continue

            if not os.path.isfile(src_file):
                continue

            if not self._is_valid_backup_file(src_file):
                self._log("SKIPING: '" + src_file + "' is not a picture or video")
                continue

            if self._start_size > 0:
                fs = os.path.getsize(src_file)
                if self._start_size > fs:
                    self._log("SKIPING: " + src_file + " is not larger than " + utils.sizeof_fmt(self._start_size) + " bytes (" +
                              utils.sizeof_fmt(fs) + ")")
                    continue

            picture_date = utils.get_picture_date(src_file)
            if picture_date is None:
                self._log("SKIPING: '" + src_file + "' Couldn't determine file date")
                continue

            dest_folder_name = self._get_dest_folder_name(picture_date)

            dest_folder = os.path.join(self._destPath, dest_folder_name)
            dest_file = os.path.join(dest_folder, filename)
            move = self._move_files or src_file.startswith(u"/home/hm/Imágenes/Camara")

            try:
                if os.path.isfile(dest_file) and os.path.samefile(src_file, dest_file):
                    self._log("SKIPING: '" + dest_file + "' Source and destination are the same.")
                    continue

                if os.path.isfile(dest_file):
                    self._log("SKIPING: '" + dest_file + "' already exists.")
                    if not self._diagnostics:
                        self._correct_picture_date(dest_file, picture_date)
                    continue

                if not os.path.isdir(dest_folder):
                    self._log("CREATING: Folder '" + dest_folder + "'")
                    if not self._diagnostics:
                        os.makedirs(dest_folder)

                if move:
                    self._log("MOVING: '" + src_file + "' to '" + dest_file + "'")
                    if not self._diagnostics:
                        shutil.move(src_file, dest_folder)

                    files_left -= 1
                    if files_left == 0:
                        try:
                            if not self._diagnostics:
                                os.rmdir(root_dir)
                        except:
                            self._error("Error removing dir")

                else:
                    self._log("COPING: '" + src_file + "' to '" + dest_file + "'")
                    if not self._diagnostics:
                        shutil.copy(src_file, dest_folder)

                if not self._diagnostics:
                    self._correct_picture_date(dest_file, picture_date)

                self._success_count += 1
            except Exception as exp:
                self._error(exp)
                continue

    def archive_pictures(self):
        self._imgCount = 0
        self._currImgIndex = 0
        self._success_count = 0
        self._walk_dir(self._srcPath)

        self._log(str(self._success_count) + " of " + str(self._currImgIndex) + " files copied.")

    @classmethod
    def do(cls, src_path, dest_path, diagnostics, move, start_size):
        obj = cls(src_path, dest_path)
        obj._diagnostics = diagnostics
        obj._move_files = move
        obj._start_size = int(start_size) * 1024 * 1024
        print obj._start_size

        if obj._diagnostics:
            obj._log("WARING: Diagnostics mode activated.")
        obj.archive_pictures()

    @classmethod
    def correct_dates(cls, src_path):
        obj = cls(src_path, src_path)
        obj._walk_dir_correct_date(src_path)<commit_msg>Moved "static" members<commit_after># coding=UTF8

import os
import shutil
import pyexiv2
import time
import utils

# noinspection PyBroadException
class PictureArchiver:
    _verbose = True
    _debug = False

    def __init__(self, src_path, dest_path):
        self._srcPath = src_path
        self._destPath = dest_path
        self._move_files = False
        self._diagnostics = False
        self._imgCount = 0
        self._currImgIndex = 0
        self._success_count = 0
        self._currImgFileName = None
        self._correct_dates_only = False
        self._start_size = 0

        self.onAdvance = None

    def _do_advance(self):
        try:
            if self.onAdvance is not None:
                self.onAdvance(self)
        except Exception:
            pass

    def _log(self, text):
        if self._verbose:
            print text

    def _debug(self, text):
        if self._debug:
            print text

    def _error(self, msg):
        print "ERROR:", msg

    def _correct_exif_date(self, filename, date):
        try:
            exif_data = pyexiv2.ImageMetadata(filename)
            need_write = False
            exif_data.read()

            if utils.get_exif_value(exif_data, 'Exif.Image.DateTime') is None:
                exif_data['Exif.Image.DateTime'] = date
                need_write = True

            if utils.get_exif_value(exif_data, 'Exif.Photo.DateTimeOriginal') is None:
                exif_data['Exif.Photo.DateTimeOriginal'] = date
                need_write = True

            if utils.get_exif_value(exif_data, 'Exif.Photo.DateTimeDigitized') is None:
                exif_data['Exif.Photo.DateTimeDigitized'] = date
                need_write = True

            if need_write:
                exif_data.write(True)
        except Exception as e:
            self._error(e)

    def _is_valid_backup_file(self, file_name):
        fname, fext = os.path.splitext(file_name)
        return fext.lower().lstrip(".") in utils.MIME_TYPES.keys()

    def _get_dest_folder_name(self, obj_date):
        if obj_date is not None:
            year = obj_date.strftime("%Y")
            month = obj_date.strftime("%m")
            day = obj_date.strftime("%Y-%m-%d")
            destpath = os.path.join(year, month, day)
            return destpath
        else:
            return ""

    def _correct_picture_date(self, picture_path, datetime):
        self._correct_exif_date(picture_path, datetime)

        filetime = time.mktime(datetime.timetuple())
        os.utime(picture_path, (filetime, filetime))

        self._debug("Corrected: " + picture_path)

    def _walk_dir_correct_date(self, root_dir):
        dir_list = os.listdir(root_dir)
        for filename in dir_list:
            src_file = os.path.join(root_dir, filename)

            if os.path.isdir(src_file):
                self._walk_dir_correct_date(src_file)
                continue

            if os.path.isfile(src_file):
                picture_date = utils.get_picture_date(src_file)
                self._correct_picture_date(src_file, picture_date)

    def _walk_dir(self, root_dir):
        dir_list = os.listdir(root_dir)
        self._imgCount += len(dir_list)

        files_left = len(dir_list)
        for filename in dir_list:
            self._currImgIndex += 1
            self._currImgFileName = filename
            self._do_advance()

            src_file = os.path.join(root_dir, filename)

            if os.path.isdir(src_file):
                self._walk_dir(src_file)
                continue

            if not os.path.isfile(src_file):
                continue

            if not self._is_valid_backup_file(src_file):
                self._log("SKIPING: '" + src_file + "' is not a picture or video")
                continue

            if self._start_size > 0:
                fs = os.path.getsize(src_file)
                if self._start_size > fs:
                    self._log("SKIPING: " + src_file + " is not larger than " + utils.sizeof_fmt(self._start_size) + " bytes (" +
                              utils.sizeof_fmt(fs) + ")")
                    continue

            picture_date = utils.get_picture_date(src_file)
            if picture_date is None:
                self._log("SKIPING: '" + src_file + "' Couldn't determine file date")
                continue

            dest_folder_name = self._get_dest_folder_name(picture_date)

            dest_folder = os.path.join(self._destPath, dest_folder_name)
            dest_file = os.path.join(dest_folder, filename)
            move = self._move_files or src_file.startswith(u"/home/hm/Imágenes/Camara")

            try:
                if os.path.isfile(dest_file) and os.path.samefile(src_file, dest_file):
                    self._log("SKIPING: '" + dest_file + "' Source and destination are the same.")
                    continue

                if os.path.isfile(dest_file):
                    self._log("SKIPING: '" + dest_file + "' already exists.")
                    if not self._diagnostics:
                        self._correct_picture_date(dest_file, picture_date)
                    continue

                if not os.path.isdir(dest_folder):
                    self._log("CREATING: Folder '" + dest_folder + "'")
                    if not self._diagnostics:
                        os.makedirs(dest_folder)

                if move:
                    self._log("MOVING: '" + src_file + "' to '" + dest_file + "'")
                    if not self._diagnostics:
                        shutil.move(src_file, dest_folder)

                    files_left -= 1
                    if files_left == 0:
                        try:
                            if not self._diagnostics:
                                os.rmdir(root_dir)
                        except:
                            self._error("Error removing dir")

                else:
                    self._log("COPING: '" + src_file + "' to '" + dest_file + "'")
                    if not self._diagnostics:
                        shutil.copy(src_file, dest_folder)

                if not self._diagnostics:
                    self._correct_picture_date(dest_file, picture_date)

                self._success_count += 1
            except Exception as exp:
                self._error(exp)
                continue

    def archive_pictures(self):
        self._imgCount = 0
        self._currImgIndex = 0
        self._success_count = 0
        self._walk_dir(self._srcPath)

        self._log(str(self._success_count) + " of " + str(self._currImgIndex) + " files copied.")

    @classmethod
    def do(cls, src_path, dest_path, diagnostics, move, start_size):
        obj = cls(src_path, dest_path)
        obj._diagnostics = diagnostics
        obj._move_files = move
        obj._start_size = int(start_size) * 1024 * 1024
        print obj._start_size

        if obj._diagnostics:
            obj._log("WARING: Diagnostics mode activated.")
        obj.archive_pictures()

    @classmethod
    def correct_dates(cls, src_path):
        obj = cls(src_path, src_path)
        obj._walk_dir_correct_date(src_path)<|endoftext|>


<commit_before>"""This module implements a client to interface the pilight-daemon.

More information about pilight is here: https://www.pilight.org/.
"""

import threading
import socket
import json
import logging

class Client(threading.Thread):

    """This client interfaces with the pilight-daemon (https://www.pilight.org/).

    Sending and receiving codes is implemented in an asychronous way.
    A callback function can be defined that reacts on received data.

    All pilight-send commands can be used by this client. Documentation
    can be found here https://wiki.pilight.org/doku.php/psend.
    Also check https://manual.pilight.org/en/api.

    :param host: Address where the pilight-daemon intance runs
    :param port: Port of the pilight-daemon on the host
    :param timeout: Time until a time out exception is raised when connecting
    :param recv_ident: The identification of the receiver to sucribe
    to the pilight-daemon topics (https://manual.pilight.org/en/api)
    :param recv_codes_only: If True: only call the callback function when the
    pilight-daemon received a code, not for status messages etc.
    :param veto_repeats: If True: only call the callback function when the
    pilight-daemon received a new code, not the same code repeated.
    Repeated codes happen quickly when a button is pressed.
    """

    # pylint: disable=too-many-arguments, too-many-instance-attributes

    def __init__(self, host='127.0.0.1', port=5000, timeout=1,
                 recv_ident=None, recv_codes_only=True, veto_repeats=True):
        """Initialize the pilight client.

        The readout thread is not started automatically.
        """
        threading.Thread.__init__(self)
        self.daemon = True
        self._stop_thread = threading.Event()
        self._lock = threading.Lock()
        self.recv_ident = recv_ident
        self.recv_codes_only = recv_codes_only
        self.veto_repeats = veto_repeats

        self.host = host
        self.port = port
        self.timeout = timeout

        # Open 2 socket connections, one for sending one for receiving data
        # That is the simplest approach to allow asynchronus communication with
        # the pilight daemon
        self.connect_sender()
        self.connect_receiver()

        # Timeout to allow receiver thread termination and to restrict blocking
        # connection time
        self.callback = None

    def connect_receiver(self):
        if self.recv_ident:
            client_identification_receiver = self.recv_ident
        else:
            client_identification_receiver = {
                "action": "identify",
                "options": {
                    "core": 0,  # To get CPU load and RAM of pilight daemon
                    # To receive the RF data received by pilight
                    "receiver": 1,
                    "config": 0,
                    "forward": 0
                }
            }

        self.receive_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.receive_socket.settimeout(self.timeout)
        self.receive_socket.connect((self.host, self.port))
        # Identify this clients sockets at the pilight-deamon
        self.receive_socket.send(
            json.dumps(client_identification_receiver).encode())
        answer = json.loads(self.receive_socket.recv(1024).decode())
        # Check connections are acknowledged
        if ('success' not in answer['status']):
            raise IOError(
                'Connection to the pilight daemon failed. Reply %s',
                answer)

    def connect_sender(self):
        # Identify client (https://manual.pilight.org/en/api)
        client_identification_sender = {
            "action": "identify",
            "options": {
                # To get CPU load and RAM of pilight daemon, is neverless
                # ignored by daemon ...
                "core": 0,
                "receiver": 0,  # To receive the RF data received by pilight
                "config": 0
            }
        }

        self.send_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.send_socket.settimeout(self.timeout)
        self.send_socket.connect((self.host, self.port))
        self.send_socket.send(
            json.dumps(client_identification_sender).encode())
        answer = json.loads(self.send_socket.recv(1024).decode())
        if ('success' not in answer['status']):
            raise IOError(
                'Connection to the pilight daemon failed. Reply %s',
                answer)

    def set_callback(self, function):
        """Function to be called when data is received."""
        self.callback = function

    def stop(self):
        """Called to stop the reveiver thread."""
        self._stop_thread.set()
        # f you want to close the connection in a timely fashion,
        # call shutdown() before close().
        with self._lock:  # Receive thread might use the socket
            self.receive_socket.shutdown(socket.SHUT_RDWR)
            self.receive_socket.close()

        self.send_socket.shutdown(socket.SHUT_RDWR)
        self.send_socket.close()

    def run(self):  # Thread for receiving data from pilight
        """Receiver thread function called on Client.start()."""
        logging.debug('Pilight receiver thread started')
        if not self.callback:
            raise RuntimeError('No callback function set, cancel readout thread')

        def handle_messages(messages):
            """Call callback on each receive message."""
            for message in messages:  # Loop over received  messages
                if message:  # Can be empty due to splitlines
                    message_dict = json.loads(message.decode())
                    if self.recv_codes_only:
                        # Filter: Only use receiver messages
                        if 'receiver' in message_dict['origin']:
                            if self.veto_repeats:
                                if message_dict['repeats'] == 1:
                                    self.callback(message_dict)
                            else:
                                self.callback(message_dict)
                    else:
                        self.callback(message_dict)

        while not self._stop_thread.isSet():
            try:  # Read socket in a non blocking call and interpret data
                # Sometimes more than one JSON object is in the stream thus
                # split at \n
                with self._lock:
                    messages = self.receive_socket.recv(1024).splitlines()
                handle_messages(messages)
            # FIXME handle lost connection -> reconnect
            except (socket.timeout, ValueError):  # No data
                pass
        logging.debug('Pilight receiver thread stopped')

    def send_code(self, data, acknowledge=True):
        """Send a RF code known to the pilight-daemon.

        For protocols look at https://manual.pilight.org/en/api.
        When acknowledge is set, it is checked if the code was issued.
        :param data: Dictionary with the data
        :param acknowledge: Raise IO exception if the code is not
        send by the pilight-deamon
        """
        if "protocol" not in data:
            raise ValueError(
                'Pilight data to send does not contain a protocol info. '
                'Check the pilight-send doku!', str(data))

        # Create message to send
        message = {
            "action": "send",  # Tell pilight daemon to send the data
            "code": data,
        }

        # If connection is closed IOError is raised
        try:
            self.send_socket.sendall(json.dumps(message).encode())
        except socket.error:
            # reconnect...
            self.connect_sender()
            self.send_socket.sendall(json.dumps(message).encode())

        if acknowledge:  # Check if command is acknowledged by pilight daemon
            messages = self.send_socket.recv(1024).splitlines()
            received = False
            for message in messages:  # Loop over received messages
                if message:  # Can be empty due to splitlines
                    acknowledge_message = json.loads(message.decode())
                    # Filter correct message
                    if ('status' in acknowledge_message and
                            acknowledge_message['status'] == 'success'):
                        received = True
            if not received:
                raise IOError('Send code failed. Code: %s', str(data))
<commit_msg>Trying another approach...<commit_after>"""This module implements a client to interface the pilight-daemon.

More information about pilight is here: https://www.pilight.org/.
"""

import threading
import socket
import json
import logging

class Client(threading.Thread):

    """This client interfaces with the pilight-daemon (https://www.pilight.org/).

    Sending and receiving codes is implemented in an asychronous way.
    A callback function can be defined that reacts on received data.

    All pilight-send commands can be used by this client. Documentation
    can be found here https://wiki.pilight.org/doku.php/psend.
    Also check https://manual.pilight.org/en/api.

    :param host: Address where the pilight-daemon intance runs
    :param port: Port of the pilight-daemon on the host
    :param timeout: Time until a time out exception is raised when connecting
    :param recv_ident: The identification of the receiver to sucribe
    to the pilight-daemon topics (https://manual.pilight.org/en/api)
    :param recv_codes_only: If True: only call the callback function when the
    pilight-daemon received a code, not for status messages etc.
    :param veto_repeats: If True: only call the callback function when the
    pilight-daemon received a new code, not the same code repeated.
    Repeated codes happen quickly when a button is pressed.
    """

    # pylint: disable=too-many-arguments, too-many-instance-attributes

    def __init__(self, host='127.0.0.1', port=5000, timeout=1,
                 recv_ident=None, recv_codes_only=True, veto_repeats=True):
        """Initialize the pilight client.

        The readout thread is not started automatically.
        """
        threading.Thread.__init__(self)
        self.daemon = True
        self._stop_thread = threading.Event()
        self._lock = threading.Lock()
        self.recv_ident = recv_ident
        self.recv_codes_only = recv_codes_only
        self.veto_repeats = veto_repeats

        self.host = host
        self.port = port
        self.timeout = timeout

        # Open 2 socket connections, one for sending one for receiving data
        # That is the simplest approach to allow asynchronus communication with
        # the pilight daemon
        self.connect_sender()
        self.connect_receiver()

        # Timeout to allow receiver thread termination and to restrict blocking
        # connection time
        self.callback = None

    def connect_receiver(self):
        if self.recv_ident:
            client_identification_receiver = self.recv_ident
        else:
            client_identification_receiver = {
                "action": "identify",
                "options": {
                    "core": 0,  # To get CPU load and RAM of pilight daemon
                    # To receive the RF data received by pilight
                    "receiver": 1,
                    "config": 0,
                    "forward": 0
                }
            }

        self.receive_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.receive_socket.settimeout(self.timeout)
        self.receive_socket.connect((self.host, self.port))
        # Identify this clients sockets at the pilight-deamon
        self.receive_socket.send(
            json.dumps(client_identification_receiver).encode())
        answer = json.loads(self.receive_socket.recv(1024).decode())
        # Check connections are acknowledged
        if ('success' not in answer['status']):
            raise IOError(
                'Connection to the pilight daemon failed. Reply %s',
                answer)

    def connect_sender(self):
        # Identify client (https://manual.pilight.org/en/api)
        client_identification_sender = {
            "action": "identify",
            "options": {
                # To get CPU load and RAM of pilight daemon, is neverless
                # ignored by daemon ...
                "core": 0,
                "receiver": 0,  # To receive the RF data received by pilight
                "config": 0
            }
        }

        self.send_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.send_socket.settimeout(self.timeout)
        self.send_socket.connect((self.host, self.port))
        self.send_socket.send(
            json.dumps(client_identification_sender).encode())
        answer = json.loads(self.send_socket.recv(1024).decode())
        if ('success' not in answer['status']):
            raise IOError(
                'Connection to the pilight daemon failed. Reply %s',
                answer)

    def set_callback(self, function):
        """Function to be called when data is received."""
        self.callback = function

    def stop(self):
        """Called to stop the reveiver thread."""
        self._stop_thread.set()
        # f you want to close the connection in a timely fashion,
        # call shutdown() before close().
        with self._lock:  # Receive thread might use the socket
            self.receive_socket.shutdown(socket.SHUT_RDWR)
            self.receive_socket.close()

        self.send_socket.shutdown(socket.SHUT_RDWR)
        self.send_socket.close()

    def run(self):  # Thread for receiving data from pilight
        """Receiver thread function called on Client.start()."""
        logging.debug('Pilight receiver thread started')
        if not self.callback:
            raise RuntimeError('No callback function set, cancel readout thread')

        def handle_messages(messages):
            """Call callback on each receive message."""
            for message in messages:  # Loop over received  messages
                if message:  # Can be empty due to splitlines
                    message_dict = json.loads(message.decode())
                    if self.recv_codes_only:
                        # Filter: Only use receiver messages
                        if 'receiver' in message_dict['origin']:
                            if self.veto_repeats:
                                if message_dict['repeats'] == 1:
                                    self.callback(message_dict)
                            else:
                                self.callback(message_dict)
                    else:
                        self.callback(message_dict)

        while not self._stop_thread.isSet():
            try:  # Read socket in a non blocking call and interpret data
                # Sometimes more than one JSON object is in the stream thus
                # split at \n
                with self._lock:
                    messages = self.receive_socket.recv(1024).splitlines()
                handle_messages(messages)
            # FIXME handle lost connection -> reconnect
            except (socket.timeout, ValueError):  # No data
                pass
			except socket.error:
				self.connect_sender
				self.connect_receiver
        logging.debug('Pilight receiver thread stopped')

    def send_code(self, data, acknowledge=True):
        """Send a RF code known to the pilight-daemon.

        For protocols look at https://manual.pilight.org/en/api.
        When acknowledge is set, it is checked if the code was issued.
        :param data: Dictionary with the data
        :param acknowledge: Raise IO exception if the code is not
        send by the pilight-deamon
        """
        if "protocol" not in data:
            raise ValueError(
                'Pilight data to send does not contain a protocol info. '
                'Check the pilight-send doku!', str(data))

        # Create message to send
        message = {
            "action": "send",  # Tell pilight daemon to send the data
            "code": data,
        }

		# If connection is closed IOError is raised
        self.send_socket.sendall(json.dumps(message).encode())

        if acknowledge:  # Check if command is acknowledged by pilight daemon
            messages = self.send_socket.recv(1024).splitlines()
            received = False
            for message in messages:  # Loop over received messages
                if message:  # Can be empty due to splitlines
                    acknowledge_message = json.loads(message.decode())
                    # Filter correct message
                    if ('status' in acknowledge_message and
                            acknowledge_message['status'] == 'success'):
                        received = True
            if not received:
                raise IOError('Send code failed. Code: %s', str(data))
<|endoftext|>


<commit_before>"""Base Command class, and related routines"""

import os
import sys
import tempfile
import traceback
import time
import optparse

from pip import cmdoptions
from pip.locations import running_under_virtualenv
from pip.log import logger
from pip.download import PipSession
from pip.exceptions import (BadCommand, InstallationError, UninstallationError,
                            CommandError, PreviousBuildDirError)
from pip.compat import StringIO
from pip.baseparser import ConfigOptionParser, UpdatingDefaultsHelpFormatter
from pip.status_codes import (
    SUCCESS, ERROR, UNKNOWN_ERROR, VIRTUALENV_NOT_FOUND,
    PREVIOUS_BUILD_DIR_ERROR,
)
from pip.util import get_prog, normalize_path

__all__ = ['Command']

class Command(object):
    name = None
    usage = None
    hidden = False

    def __init__(self):
        parser_kw = {
            'usage': self.usage,
            'prog': '%s %s' % (get_prog(), self.name),
            'formatter': UpdatingDefaultsHelpFormatter(),
            'add_help_option': False,
            'name': self.name,
            'description': self.__doc__,
        }

        self.parser = ConfigOptionParser(**parser_kw)

        # Commands should add options to this option group
        optgroup_name = '%s Options' % self.name.capitalize()
        self.cmd_opts = optparse.OptionGroup(self.parser, optgroup_name)

        # Add the general options
        gen_opts = cmdoptions.make_option_group(
            cmdoptions.general_group,
            self.parser,
        )
        self.parser.add_option_group(gen_opts)

    def _build_session(self, options):
        session = PipSession(
            cache=normalize_path(os.path.join(options.cache_dir, "http")),
            retries=options.retries,
        )

        # Handle custom ca-bundles from the user
        if options.cert:
            session.verify = options.cert
        elif options.no_check_certificate:
            session.verify = False

        # Handle SSL client certificate
        if options.client_cert:
            session.cert = options.client_cert

        # Handle timeouts
        if options.timeout:
            session.timeout = options.timeout

        # Handle configured proxies
        if options.proxy:
            session.proxies = {
                "http": options.proxy,
                "https": options.proxy,
            }

        # Determine if we can prompt the user for authentication or not
        session.auth.prompting = not options.no_input

        return session

    def setup_logging(self):
        pass

    def parse_args(self, args):
        # factored out for testability
        return self.parser.parse_args(args)

    def main(self, args):
        options, args = self.parse_args(args)

        level = 1  # Notify
        level += options.verbose
        level -= options.quiet
        level = logger.level_for_integer(4 - level)
        complete_log = []
        logger.add_consumers(
            (level, sys.stdout),
            (logger.DEBUG, complete_log.append),
        )
        if options.log_explicit_levels:
            logger.explicit_levels = True

        self.setup_logging()

        # TODO: try to get these passing down from the command?
        #      without resorting to os.environ to hold these.

        if options.no_input:
            os.environ['PIP_NO_INPUT'] = '1'

        if options.exists_action:
            os.environ['PIP_EXISTS_ACTION'] = ' '.join(options.exists_action)

        if options.require_venv:
            # If a venv is required check if it can really be found
            if not running_under_virtualenv():
                logger.fatal(
                    'Could not find an activated virtualenv (required).'
                )
                sys.exit(VIRTUALENV_NOT_FOUND)

        if options.log:
            log_fp = open_logfile(options.log, 'a')
            logger.add_consumers((logger.DEBUG, log_fp))
        else:
            log_fp = None

        exit = SUCCESS
        store_log = False
        try:
            status = self.run(options, args)
            # FIXME: all commands should return an exit status
            # and when it is done, isinstance is not needed anymore
            if isinstance(status, int):
                exit = status
        except PreviousBuildDirError as exc:
            logger.fatal(str(exc))
            logger.info('Exception information:\n%s' % format_exc())
            store_log = True
            exit = PREVIOUS_BUILD_DIR_ERROR
        except (InstallationError, UninstallationError) as exc:
            logger.fatal(str(exc))
            logger.info('Exception information:\n%s' % format_exc())
            store_log = True
            exit = ERROR
        except BadCommand as exc:
            logger.fatal(str(exc))
            logger.info('Exception information:\n%s' % format_exc())
            store_log = True
            exit = ERROR
        except CommandError as exc:
            logger.fatal('ERROR: %s' % exc)
            logger.info('Exception information:\n%s' % format_exc())
            exit = ERROR
        except KeyboardInterrupt:
            logger.fatal('Operation cancelled by user')
            logger.info('Exception information:\n%s' % format_exc())
            store_log = True
            exit = ERROR
        except:
            logger.fatal('Exception:\n%s' % format_exc())
            store_log = True
            exit = UNKNOWN_ERROR
        if store_log:
            log_file_fn = options.log_file
            text = '\n'.join(complete_log)
            try:
                log_file_fp = open_logfile(log_file_fn, 'w')
            except IOError:
                temp = tempfile.NamedTemporaryFile(delete=False)
                log_file_fn = temp.name
                log_file_fp = open_logfile(log_file_fn, 'w')
            logger.fatal('Storing debug log for failure in %s' % log_file_fn)
            log_file_fp.write(text)
            log_file_fp.close()
        if log_fp is not None:
            log_fp.close()
        return exit

def format_exc(exc_info=None):
    if exc_info is None:
        exc_info = sys.exc_info()
    out = StringIO()
    traceback.print_exception(*exc_info, **dict(file=out))
    return out.getvalue()

def open_logfile(filename, mode='a'):
    """Open the named log file in append mode.

    If the file already exists, a separator will also be printed to
    the file to separate past activity from current activity.
    """
    filename = os.path.expanduser(filename)
    filename = os.path.abspath(filename)
    dirname = os.path.dirname(filename)
    if not os.path.exists(dirname):
        os.makedirs(dirname)
    exists = os.path.exists(filename)

    kw = {}
    if sys.version_info[0] != 2:
        kw['errors'] = 'replace'

    log_fp = open(filename, mode, **kw)
    if exists:
        log_fp.write('%s\n' % ('-' * 60))
        log_fp.write('%s run on %s\n' % (sys.argv[0], time.strftime('%c')))
    return log_fp
<commit_msg>Fix for unicode in the log<commit_after>"""Base Command class, and related routines"""

import os
import sys
import tempfile
import traceback
import time
import optparse

from pip import cmdoptions
from pip.locations import running_under_virtualenv
from pip.log import logger
from pip.download import PipSession
from pip.exceptions import (BadCommand, InstallationError, UninstallationError,
                            CommandError, PreviousBuildDirError)
from pip.compat import StringIO, native_str
from pip.baseparser import ConfigOptionParser, UpdatingDefaultsHelpFormatter
from pip.status_codes import (
    SUCCESS, ERROR, UNKNOWN_ERROR, VIRTUALENV_NOT_FOUND,
    PREVIOUS_BUILD_DIR_ERROR,
)
from pip.util import get_prog, normalize_path

__all__ = ['Command']

class Command(object):
    name = None
    usage = None
    hidden = False

    def __init__(self):
        parser_kw = {
            'usage': self.usage,
            'prog': '%s %s' % (get_prog(), self.name),
            'formatter': UpdatingDefaultsHelpFormatter(),
            'add_help_option': False,
            'name': self.name,
            'description': self.__doc__,
        }

        self.parser = ConfigOptionParser(**parser_kw)

        # Commands should add options to this option group
        optgroup_name = '%s Options' % self.name.capitalize()
        self.cmd_opts = optparse.OptionGroup(self.parser, optgroup_name)

        # Add the general options
        gen_opts = cmdoptions.make_option_group(
            cmdoptions.general_group,
            self.parser,
        )
        self.parser.add_option_group(gen_opts)

    def _build_session(self, options):
        session = PipSession(
            cache=normalize_path(os.path.join(options.cache_dir, "http")),
            retries=options.retries,
        )

        # Handle custom ca-bundles from the user
        if options.cert:
            session.verify = options.cert
        elif options.no_check_certificate:
            session.verify = False

        # Handle SSL client certificate
        if options.client_cert:
            session.cert = options.client_cert

        # Handle timeouts
        if options.timeout:
            session.timeout = options.timeout

        # Handle configured proxies
        if options.proxy:
            session.proxies = {
                "http": options.proxy,
                "https": options.proxy,
            }

        # Determine if we can prompt the user for authentication or not
        session.auth.prompting = not options.no_input

        return session

    def setup_logging(self):
        pass

    def parse_args(self, args):
        # factored out for testability
        return self.parser.parse_args(args)

    def main(self, args):
        options, args = self.parse_args(args)

        level = 1  # Notify
        level += options.verbose
        level -= options.quiet
        level = logger.level_for_integer(4 - level)
        complete_log = []
        logger.add_consumers(
            (level, sys.stdout),
            (logger.DEBUG, complete_log.append),
        )
        if options.log_explicit_levels:
            logger.explicit_levels = True

        self.setup_logging()

        # TODO: try to get these passing down from the command?
        #      without resorting to os.environ to hold these.

        if options.no_input:
            os.environ['PIP_NO_INPUT'] = '1'

        if options.exists_action:
            os.environ['PIP_EXISTS_ACTION'] = ' '.join(options.exists_action)

        if options.require_venv:
            # If a venv is required check if it can really be found
            if not running_under_virtualenv():
                logger.fatal(
                    'Could not find an activated virtualenv (required).'
                )
                sys.exit(VIRTUALENV_NOT_FOUND)

        if options.log:
            log_fp = open_logfile(options.log, 'a')
            logger.add_consumers((logger.DEBUG, log_fp))
        else:
            log_fp = None

        exit = SUCCESS
        store_log = False
        try:
            status = self.run(options, args)
            # FIXME: all commands should return an exit status
            # and when it is done, isinstance is not needed anymore
            if isinstance(status, int):
                exit = status
        except PreviousBuildDirError as exc:
            logger.fatal(str(exc))
            logger.info('Exception information:\n%s' % format_exc())
            store_log = True
            exit = PREVIOUS_BUILD_DIR_ERROR
        except (InstallationError, UninstallationError) as exc:
            logger.fatal(str(exc))
            logger.info('Exception information:\n%s' % format_exc())
            store_log = True
            exit = ERROR
        except BadCommand as exc:
            logger.fatal(str(exc))
            logger.info('Exception information:\n%s' % format_exc())
            store_log = True
            exit = ERROR
        except CommandError as exc:
            logger.fatal('ERROR: %s' % exc)
            logger.info('Exception information:\n%s' % format_exc())
            exit = ERROR
        except KeyboardInterrupt:
            logger.fatal('Operation cancelled by user')
            logger.info('Exception information:\n%s' % format_exc())
            store_log = True
            exit = ERROR
        except:
            logger.fatal('Exception:\n%s' % format_exc())
            store_log = True
            exit = UNKNOWN_ERROR
        if store_log:
            log_file_fn = options.log_file
            text = '\n'.join(native_str(l, True) for l in complete_log)
            try:
                log_file_fp = open_logfile(log_file_fn, 'w')
            except IOError:
                temp = tempfile.NamedTemporaryFile(delete=False)
                log_file_fn = temp.name
                log_file_fp = open_logfile(log_file_fn, 'w')
            logger.fatal('Storing debug log for failure in %s' % log_file_fn)
            log_file_fp.write(text)
            log_file_fp.close()
        if log_fp is not None:
            log_fp.close()
        return exit

def format_exc(exc_info=None):
    if exc_info is None:
        exc_info = sys.exc_info()
    out = StringIO()
    traceback.print_exception(*exc_info, **dict(file=out))
    return out.getvalue()

def open_logfile(filename, mode='a'):
    """Open the named log file in append mode.

    If the file already exists, a separator will also be printed to
    the file to separate past activity from current activity.
    """
    filename = os.path.expanduser(filename)
    filename = os.path.abspath(filename)
    dirname = os.path.dirname(filename)
    if not os.path.exists(dirname):
        os.makedirs(dirname)
    exists = os.path.exists(filename)

    kw = {}
    if sys.version_info[0] != 2:
        kw['errors'] = 'replace'

    log_fp = open(filename, mode, **kw)
    if exists:
        log_fp.write('%s\n' % ('-' * 60))
        log_fp.write('%s run on %s\n' % (sys.argv[0], time.strftime('%c')))
    return log_fp
<|endoftext|>