<commit_before>from contextlib import suppress
from uuid import UUID
import datetime
import ipaddress
import re

from jsonschema.exceptions import FormatError

class FormatChecker(object):
    """
    A ``format`` property checker.

    JSON Schema does not mandate that the ``format`` property actually do any
    validation. If validation is desired however, instances of this class can
    be hooked into validators to enable format validation.

    `FormatChecker` objects always return ``True`` when asked about
    formats that they do not know how to validate.

    To check a custom format using a function that takes an instance and
    returns a ``bool``, use the `FormatChecker.checks` or
    `FormatChecker.cls_checks` decorators.

    Arguments:

        formats (~collections.abc.Iterable):

            The known formats to validate. This argument can be used to
            limit which formats will be used during validation.
    """

    checkers = {}

    def __init__(self, formats=None):
        if formats is None:
            self.checkers = self.checkers.copy()
        else:
            self.checkers = dict((k, self.checkers[k]) for k in formats)

    def __repr__(self):
        return "<FormatChecker checkers={}>".format(sorted(self.checkers))

    def checks(self, format, raises=()):
        """
        Register a decorated function as validating a new format.

        Arguments:

            format (str):

                The format that the decorated function will check.

            raises (Exception):

                The exception(s) raised by the decorated function when an
                invalid instance is found.

                The exception object will be accessible as the
                `jsonschema.exceptions.ValidationError.cause` attribute of the
                resulting validation error.
        """

        def _checks(func):
            self.checkers[format] = (func, raises)
            return func
        return _checks

    cls_checks = classmethod(checks)

    def check(self, instance, format):
        """
        Check whether the instance conforms to the given format.

        Arguments:

            instance (*any primitive type*, i.e. str, number, bool):

                The instance to check

            format (str):

                The format that instance should conform to

        Raises:

            FormatError: if the instance does not conform to ``format``
        """

        if format not in self.checkers:
            return

        func, raises = self.checkers[format]
        result, cause = None, None
        try:
            result = func(instance)
        except raises as e:
            cause = e
        if not result:
            raise FormatError(f"{instance!r} is not a {format!r}", cause=cause)

    def conforms(self, instance, format):
        """
        Check whether the instance conforms to the given format.

        Arguments:

            instance (*any primitive type*, i.e. str, number, bool):

                The instance to check

            format (str):

                The format that instance should conform to

        Returns:

            bool: whether it conformed
        """

        try:
            self.check(instance, format)
        except FormatError:
            return False
        else:
            return True

draft3_format_checker = FormatChecker()
draft4_format_checker = FormatChecker()
draft6_format_checker = FormatChecker()
draft7_format_checker = FormatChecker()
draft201909_format_checker = FormatChecker()
draft202012_format_checker = FormatChecker()

_draft_checkers = dict(
    draft3=draft3_format_checker,
    draft4=draft4_format_checker,
    draft6=draft6_format_checker,
    draft7=draft7_format_checker,
    draft201909=draft201909_format_checker,
    draft202012=draft202012_format_checker,
)

def _checks_drafts(
    name=None,
    draft3=None,
    draft4=None,
    draft6=None,
    draft7=None,
    draft201909=None,
    draft202012=None,
    raises=(),
):
    draft3 = draft3 or name
    draft4 = draft4 or name
    draft6 = draft6 or name
    draft7 = draft7 or name
    draft201909 = draft201909 or name
    draft202012 = draft202012 or name

    def wrap(func):
        if draft3:
            func = _draft_checkers["draft3"].checks(draft3, raises)(func)
        if draft4:
            func = _draft_checkers["draft4"].checks(draft4, raises)(func)
        if draft6:
            func = _draft_checkers["draft6"].checks(draft6, raises)(func)
        if draft7:
            func = _draft_checkers["draft7"].checks(draft7, raises)(func)
        if draft201909:
            func = _draft_checkers["draft201909"].checks(draft201909, raises)(
                func,
            )
        if draft202012:
            func = _draft_checkers["draft202012"].checks(draft202012, raises)(
                func,
            )

        # Oy. This is bad global state, but relied upon for now, until
        # deprecation. See https://github.com/Julian/jsonschema/issues/519
        # and test_format_checkers_come_with_defaults
        FormatChecker.cls_checks(
            draft202012 or draft201909 or draft7 or draft6 or draft4 or draft3,
            raises,
        )(func)
        return func
    return wrap

@_checks_drafts(name="idn-email")
@_checks_drafts(name="email")
def is_email(instance):
    if not isinstance(instance, str):
        return True
    return "@" in instance

@_checks_drafts(
    draft3="ip-address",
    draft4="ipv4",
    draft6="ipv4",
    draft7="ipv4",
    draft201909="ipv4",
    draft202012="ipv4",
    raises=ipaddress.AddressValueError,
)
def is_ipv4(instance):
    if not isinstance(instance, str):
        return True
    return ipaddress.IPv4Address(instance)

@_checks_drafts(name="ipv6", raises=ipaddress.AddressValueError)
def is_ipv6(instance):
    if not isinstance(instance, str):
        return True
    address = ipaddress.IPv6Address(instance)
    return not getattr(address, "scope_id", "")

with suppress(ImportError):
    from fqdn import FQDN

    @_checks_drafts(
        draft3="host-name",
        draft4="hostname",
        draft6="hostname",
        draft7="hostname",
        draft201909="hostname",
        draft202012="hostname",
    )
    def is_host_name(instance):
        if not isinstance(instance, str):
            return True
        return FQDN(instance).is_valid

with suppress(ImportError):
    # The built-in `idna` codec only implements RFC 3890, so we go elsewhere.
    import idna

    @_checks_drafts(
        draft7="idn-hostname",
        draft201909="idn-hostname",
        draft202012="idn-hostname",
        raises=(idna.IDNAError, UnicodeError),
    )
    def is_idn_host_name(instance):
        if not isinstance(instance, str):
            return True
        idna.encode(instance)
        return True

try:
    import rfc3987
except ImportError:
    with suppress(ImportError):
        from rfc3986_validator import validate_rfc3986

        @_checks_drafts(name="uri")
        def is_uri(instance):
            if not isinstance(instance, str):
                return True
            return validate_rfc3986(instance, rule="URI")

        @_checks_drafts(
            draft6="uri-reference",
            draft7="uri-reference",
            draft201909="uri-reference",
            draft202012="uri-reference",
            raises=ValueError,
        )
        def is_uri_reference(instance):
            if not isinstance(instance, str):
                return True
            return validate_rfc3986(instance, rule="URI_reference")

else:
    @_checks_drafts(
        draft7="iri",
        draft201909="iri",
        draft202012="iri",
        raises=ValueError,
    )
    def is_iri(instance):
        if not isinstance(instance, str):
            return True
        return rfc3987.parse(instance, rule="IRI")

    @_checks_drafts(
        draft7="iri-reference",
        draft201909="iri-reference",
        draft202012="iri-reference",
        raises=ValueError,
    )
    def is_iri_reference(instance):
        if not isinstance(instance, str):
            return True
        return rfc3987.parse(instance, rule="IRI_reference")

    @_checks_drafts(name="uri", raises=ValueError)
    def is_uri(instance):
        if not isinstance(instance, str):
            return True
        return rfc3987.parse(instance, rule="URI")

    @_checks_drafts(
        draft6="uri-reference",
        draft7="uri-reference",
        draft201909="uri-reference",
        draft202012="uri-reference",
        raises=ValueError,
    )
    def is_uri_reference(instance):
        if not isinstance(instance, str):
            return True
        return rfc3987.parse(instance, rule="URI_reference")

with suppress(ImportError):
    from rfc3339_validator import validate_rfc3339

    @_checks_drafts(name="date-time")
    def is_datetime(instance):
        if not isinstance(instance, str):
            return True
        return validate_rfc3339(instance.upper())

    @_checks_drafts(
        draft7="time",
        draft201909="time",
        draft202012="time",
    )
    def is_time(instance):
        if not isinstance(instance, str):
            return True
        return is_datetime("1970-01-01T" + instance)

@_checks_drafts(name="regex", raises=re.error)
def is_regex(instance):
    if not isinstance(instance, str):
        return True
    return re.compile(instance)

@_checks_drafts(
    draft3="date",
    draft7="date",
    draft201909="date",
    draft202012="date",
    raises=ValueError,
)
def is_date(instance):
    if not isinstance(instance, str):
        return True
    return instance.isascii() and datetime.date.fromisoformat(instance)

@_checks_drafts(draft3="time", raises=ValueError)
def is_draft3_time(instance):
    if not isinstance(instance, str):
        return True
    return datetime.datetime.strptime(instance, "%H:%M:%S")

with suppress(ImportError):
    from webcolors import CSS21_NAMES_TO_HEX
    import webcolors

    def is_css_color_code(instance):
        return webcolors.normalize_hex(instance)

    @_checks_drafts(draft3="color", raises=(ValueError, TypeError))
    def is_css21_color(instance):
        if (
            not isinstance(instance, str)
            or instance.lower() in CSS21_NAMES_TO_HEX
        ):
            return True
        return is_css_color_code(instance)

    def is_css3_color(instance):
        if instance.lower() in webcolors.css3_names_to_hex:
            return True
        return is_css_color_code(instance)

with suppress(ImportError):
    import jsonpointer

    @_checks_drafts(
        draft6="json-pointer",
        draft7="json-pointer",
        draft201909="json-pointer",
        draft202012="json-pointer",
        raises=jsonpointer.JsonPointerException,
    )
    def is_json_pointer(instance):
        if not isinstance(instance, str):
            return True
        return jsonpointer.JsonPointer(instance)

    # TODO: I don't want to maintain this, so it
    #       needs to go either into jsonpointer (pending
    #       https://github.com/stefankoegl/python-json-pointer/issues/34) or
    #       into a new external library.
    @_checks_drafts(
        draft7="relative-json-pointer",
        draft201909="relative-json-pointer",
        draft202012="relative-json-pointer",
        raises=jsonpointer.JsonPointerException,
    )
    def is_relative_json_pointer(instance):
        # Definition taken from:
        # https://tools.ietf.org/html/draft-handrews-relative-json-pointer-01#section-3
        if not isinstance(instance, str):
            return True
        non_negative_integer, rest = [], ""
        for i, character in enumerate(instance):
            if character.isdigit():
                # digits with a leading "0" are not allowed
                if i > 0 and int(instance[i - 1]) == 0:
                    return False

                non_negative_integer.append(character)
                continue

            if not non_negative_integer:
                return False

            rest = instance[i:]
            break
        return (rest == "#") or jsonpointer.JsonPointer(rest)

with suppress(ImportError):
    import uri_template

    @_checks_drafts(
        draft6="uri-template",
        draft7="uri-template",
        draft201909="uri-template",
        draft202012="uri-template",
    )
    def is_uri_template(instance):
        if not isinstance(instance, str):
            return True
        return uri_template.validate(instance)

with suppress(ImportError):
    import isoduration

    @_checks_drafts(
        draft201909="duration",
        draft202012="duration",
        raises=isoduration.DurationParsingException,
    )
    def is_duration(instance):
        if not isinstance(instance, str):
            return True
        return isoduration.parse_duration(instance)

@_checks_drafts(
    draft201909="uuid",
    draft202012="uuid",
    raises=ValueError,
)
def is_uuid(instance):
    if not isinstance(instance, str):
        return True
    UUID(instance)
    return all(instance[position] == "-" for position in (8, 13, 18, 23))
<commit_msg>Remove dead code.<commit_after>from contextlib import suppress
from uuid import UUID
import datetime
import ipaddress
import re

from jsonschema.exceptions import FormatError

class FormatChecker(object):
    """
    A ``format`` property checker.

    JSON Schema does not mandate that the ``format`` property actually do any
    validation. If validation is desired however, instances of this class can
    be hooked into validators to enable format validation.

    `FormatChecker` objects always return ``True`` when asked about
    formats that they do not know how to validate.

    To check a custom format using a function that takes an instance and
    returns a ``bool``, use the `FormatChecker.checks` or
    `FormatChecker.cls_checks` decorators.

    Arguments:

        formats (~collections.abc.Iterable):

            The known formats to validate. This argument can be used to
            limit which formats will be used during validation.
    """

    checkers = {}

    def __init__(self, formats=None):
        if formats is None:
            self.checkers = self.checkers.copy()
        else:
            self.checkers = dict((k, self.checkers[k]) for k in formats)

    def __repr__(self):
        return "<FormatChecker checkers={}>".format(sorted(self.checkers))

    def checks(self, format, raises=()):
        """
        Register a decorated function as validating a new format.

        Arguments:

            format (str):

                The format that the decorated function will check.

            raises (Exception):

                The exception(s) raised by the decorated function when an
                invalid instance is found.

                The exception object will be accessible as the
                `jsonschema.exceptions.ValidationError.cause` attribute of the
                resulting validation error.
        """

        def _checks(func):
            self.checkers[format] = (func, raises)
            return func
        return _checks

    cls_checks = classmethod(checks)

    def check(self, instance, format):
        """
        Check whether the instance conforms to the given format.

        Arguments:

            instance (*any primitive type*, i.e. str, number, bool):

                The instance to check

            format (str):

                The format that instance should conform to

        Raises:

            FormatError: if the instance does not conform to ``format``
        """

        if format not in self.checkers:
            return

        func, raises = self.checkers[format]
        result, cause = None, None
        try:
            result = func(instance)
        except raises as e:
            cause = e
        if not result:
            raise FormatError(f"{instance!r} is not a {format!r}", cause=cause)

    def conforms(self, instance, format):
        """
        Check whether the instance conforms to the given format.

        Arguments:

            instance (*any primitive type*, i.e. str, number, bool):

                The instance to check

            format (str):

                The format that instance should conform to

        Returns:

            bool: whether it conformed
        """

        try:
            self.check(instance, format)
        except FormatError:
            return False
        else:
            return True

draft3_format_checker = FormatChecker()
draft4_format_checker = FormatChecker()
draft6_format_checker = FormatChecker()
draft7_format_checker = FormatChecker()
draft201909_format_checker = FormatChecker()
draft202012_format_checker = FormatChecker()

_draft_checkers = dict(
    draft3=draft3_format_checker,
    draft4=draft4_format_checker,
    draft6=draft6_format_checker,
    draft7=draft7_format_checker,
    draft201909=draft201909_format_checker,
    draft202012=draft202012_format_checker,
)

def _checks_drafts(
    name=None,
    draft3=None,
    draft4=None,
    draft6=None,
    draft7=None,
    draft201909=None,
    draft202012=None,
    raises=(),
):
    draft3 = draft3 or name
    draft4 = draft4 or name
    draft6 = draft6 or name
    draft7 = draft7 or name
    draft201909 = draft201909 or name
    draft202012 = draft202012 or name

    def wrap(func):
        if draft3:
            func = _draft_checkers["draft3"].checks(draft3, raises)(func)
        if draft4:
            func = _draft_checkers["draft4"].checks(draft4, raises)(func)
        if draft6:
            func = _draft_checkers["draft6"].checks(draft6, raises)(func)
        if draft7:
            func = _draft_checkers["draft7"].checks(draft7, raises)(func)
        if draft201909:
            func = _draft_checkers["draft201909"].checks(draft201909, raises)(
                func,
            )
        if draft202012:
            func = _draft_checkers["draft202012"].checks(draft202012, raises)(
                func,
            )

        # Oy. This is bad global state, but relied upon for now, until
        # deprecation. See https://github.com/Julian/jsonschema/issues/519
        # and test_format_checkers_come_with_defaults
        FormatChecker.cls_checks(
            draft202012 or draft201909 or draft7 or draft6 or draft4 or draft3,
            raises,
        )(func)
        return func
    return wrap

@_checks_drafts(name="idn-email")
@_checks_drafts(name="email")
def is_email(instance):
    if not isinstance(instance, str):
        return True
    return "@" in instance

@_checks_drafts(
    draft3="ip-address",
    draft4="ipv4",
    draft6="ipv4",
    draft7="ipv4",
    draft201909="ipv4",
    draft202012="ipv4",
    raises=ipaddress.AddressValueError,
)
def is_ipv4(instance):
    if not isinstance(instance, str):
        return True
    return ipaddress.IPv4Address(instance)

@_checks_drafts(name="ipv6", raises=ipaddress.AddressValueError)
def is_ipv6(instance):
    if not isinstance(instance, str):
        return True
    address = ipaddress.IPv6Address(instance)
    return not getattr(address, "scope_id", "")

with suppress(ImportError):
    from fqdn import FQDN

    @_checks_drafts(
        draft3="host-name",
        draft4="hostname",
        draft6="hostname",
        draft7="hostname",
        draft201909="hostname",
        draft202012="hostname",
    )
    def is_host_name(instance):
        if not isinstance(instance, str):
            return True
        return FQDN(instance).is_valid

with suppress(ImportError):
    # The built-in `idna` codec only implements RFC 3890, so we go elsewhere.
    import idna

    @_checks_drafts(
        draft7="idn-hostname",
        draft201909="idn-hostname",
        draft202012="idn-hostname",
        raises=(idna.IDNAError, UnicodeError),
    )
    def is_idn_host_name(instance):
        if not isinstance(instance, str):
            return True
        idna.encode(instance)
        return True

try:
    import rfc3987
except ImportError:
    with suppress(ImportError):
        from rfc3986_validator import validate_rfc3986

        @_checks_drafts(name="uri")
        def is_uri(instance):
            if not isinstance(instance, str):
                return True
            return validate_rfc3986(instance, rule="URI")

        @_checks_drafts(
            draft6="uri-reference",
            draft7="uri-reference",
            draft201909="uri-reference",
            draft202012="uri-reference",
            raises=ValueError,
        )
        def is_uri_reference(instance):
            if not isinstance(instance, str):
                return True
            return validate_rfc3986(instance, rule="URI_reference")

else:
    @_checks_drafts(
        draft7="iri",
        draft201909="iri",
        draft202012="iri",
        raises=ValueError,
    )
    def is_iri(instance):
        if not isinstance(instance, str):
            return True
        return rfc3987.parse(instance, rule="IRI")

    @_checks_drafts(
        draft7="iri-reference",
        draft201909="iri-reference",
        draft202012="iri-reference",
        raises=ValueError,
    )
    def is_iri_reference(instance):
        if not isinstance(instance, str):
            return True
        return rfc3987.parse(instance, rule="IRI_reference")

    @_checks_drafts(name="uri", raises=ValueError)
    def is_uri(instance):
        if not isinstance(instance, str):
            return True
        return rfc3987.parse(instance, rule="URI")

    @_checks_drafts(
        draft6="uri-reference",
        draft7="uri-reference",
        draft201909="uri-reference",
        draft202012="uri-reference",
        raises=ValueError,
    )
    def is_uri_reference(instance):
        if not isinstance(instance, str):
            return True
        return rfc3987.parse(instance, rule="URI_reference")

with suppress(ImportError):
    from rfc3339_validator import validate_rfc3339

    @_checks_drafts(name="date-time")
    def is_datetime(instance):
        if not isinstance(instance, str):
            return True
        return validate_rfc3339(instance.upper())

    @_checks_drafts(
        draft7="time",
        draft201909="time",
        draft202012="time",
    )
    def is_time(instance):
        if not isinstance(instance, str):
            return True
        return is_datetime("1970-01-01T" + instance)

@_checks_drafts(name="regex", raises=re.error)
def is_regex(instance):
    if not isinstance(instance, str):
        return True
    return re.compile(instance)

@_checks_drafts(
    draft3="date",
    draft7="date",
    draft201909="date",
    draft202012="date",
    raises=ValueError,
)
def is_date(instance):
    if not isinstance(instance, str):
        return True
    return instance.isascii() and datetime.date.fromisoformat(instance)

@_checks_drafts(draft3="time", raises=ValueError)
def is_draft3_time(instance):
    if not isinstance(instance, str):
        return True
    return datetime.datetime.strptime(instance, "%H:%M:%S")

with suppress(ImportError):
    from webcolors import CSS21_NAMES_TO_HEX
    import webcolors

    def is_css_color_code(instance):
        return webcolors.normalize_hex(instance)

    @_checks_drafts(draft3="color", raises=(ValueError, TypeError))
    def is_css21_color(instance):
        if (
            not isinstance(instance, str)
            or instance.lower() in CSS21_NAMES_TO_HEX
        ):
            return True
        return is_css_color_code(instance)

with suppress(ImportError):
    import jsonpointer

    @_checks_drafts(
        draft6="json-pointer",
        draft7="json-pointer",
        draft201909="json-pointer",
        draft202012="json-pointer",
        raises=jsonpointer.JsonPointerException,
    )
    def is_json_pointer(instance):
        if not isinstance(instance, str):
            return True
        return jsonpointer.JsonPointer(instance)

    # TODO: I don't want to maintain this, so it
    #       needs to go either into jsonpointer (pending
    #       https://github.com/stefankoegl/python-json-pointer/issues/34) or
    #       into a new external library.
    @_checks_drafts(
        draft7="relative-json-pointer",
        draft201909="relative-json-pointer",
        draft202012="relative-json-pointer",
        raises=jsonpointer.JsonPointerException,
    )
    def is_relative_json_pointer(instance):
        # Definition taken from:
        # https://tools.ietf.org/html/draft-handrews-relative-json-pointer-01#section-3
        if not isinstance(instance, str):
            return True
        non_negative_integer, rest = [], ""
        for i, character in enumerate(instance):
            if character.isdigit():
                # digits with a leading "0" are not allowed
                if i > 0 and int(instance[i - 1]) == 0:
                    return False

                non_negative_integer.append(character)
                continue

            if not non_negative_integer:
                return False

            rest = instance[i:]
            break
        return (rest == "#") or jsonpointer.JsonPointer(rest)

with suppress(ImportError):
    import uri_template

    @_checks_drafts(
        draft6="uri-template",
        draft7="uri-template",
        draft201909="uri-template",
        draft202012="uri-template",
    )
    def is_uri_template(instance):
        if not isinstance(instance, str):
            return True
        return uri_template.validate(instance)

with suppress(ImportError):
    import isoduration

    @_checks_drafts(
        draft201909="duration",
        draft202012="duration",
        raises=isoduration.DurationParsingException,
    )
    def is_duration(instance):
        if not isinstance(instance, str):
            return True
        return isoduration.parse_duration(instance)

@_checks_drafts(
    draft201909="uuid",
    draft202012="uuid",
    raises=ValueError,
)
def is_uuid(instance):
    if not isinstance(instance, str):
        return True
    UUID(instance)
    return all(instance[position] == "-" for position in (8, 13, 18, 23))
<|endoftext|>


<commit_before>########
# Copyright (c) 2016 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
############

import os
import sys
import functools

import sh
import yaml
from path import path

from cloudify import ctx
from cloudify import exceptions
from cloudify.decorators import operation

from common import bake

def repo_operation(fn):
    @operation
    @functools.wraps(fn)
    def wrapper(*args, **kwargs):
        repo = GitRepo()
        return fn(repo, *args, **kwargs)
    return wrapper

@repo_operation
def clone(repo, **_):
    repo.clone()

@repo_operation
def configure(repo, **_):
    repo.configure()

@repo_operation
def pull(repo, **_):
    repo.pull()

@repo_operation
def status(repo, **_):
    repo.status()

@repo_operation
def checkout(repo, branch, **_):
    repo.checkout(branch)

@repo_operation
def squash(repo, base, **_):
    repo.squash(base)

@repo_operation
def reset(repo, hard, origin, **_):
    repo.reset(hard, origin)

@repo_operation
def rebase(repo, base, **_):
    repo.rebase(base)

@repo_operation
def diff(repo, revision_range, **_):
    repo.diff(revision_range)

class GitRepo(object):

    def clone(self):
        git = bake(sh.git)
        ctx.instance.runtime_properties['repo_location'] = self.repo_location
        ctx.instance.runtime_properties['git_version'] = sh.git(
                version=True).stdout.strip()
        if os.path.isdir(self.repo_location):
            return
        if self.clone_method == 'https':
            clone_url = 'https://github.com/{}/{}.git'.format(
                self.organization, self.name)
        elif self.clone_method == 'ssh':
            clone_url = 'git@github.com:{}/{}.git'.format(
                self.organization, self.name)
        else:
            raise exceptions.NonRecoverableError(
                'Illegal clone method: {0}'.format(self.clone_method))
        git.clone(clone_url, self.repo_location, '-b', self.branch).wait()

    def configure(self):
        # configure commit-msg hook
        hook_path = path(self.repo_location) / '.git' / 'hooks' / 'commit-msg'
        ctx.download_resource_and_render(
                'resources/commit-msg',
                template_variables={'sys_executable': sys.executable},
                target_path=hook_path)
        os.chmod(hook_path, 0755)
        # git config
        for key, value in self.git_config.items():
            self.git.config(key, value).wait()

    def pull(self):
        kwargs = {
            'prune': True
        }
        git_version = ctx.instance.runtime_properties['git_version']
        if git_version >= 'git version 2.0.0':
            kwargs['tags'] = True
        self.git.pull(**kwargs).wait()

    def status(self):
        for git_prompt_path in self.git_prompt_paths:
            if os.path.exists(os.path.expanduser(git_prompt_path)):
                break
        else:
            git_prompt_path = None
        if git_prompt_path:
            script_path = ctx.download_resource_and_render(
                    'resources/git-branch-state.sh', template_variables={
                        'git_prompt_path': git_prompt_path,
                        'repo_location': self.repo_location})
            try:
                os.chmod(script_path, 0o0755)
                script = sh.Command(script_path)
                branch_state = script().stdout.strip()
            finally:
                os.remove(script_path)
        else:
            branch_state = self.current_branch
        ctx.logger.info(branch_state)
        self.git.status(s=True).wait()

    def checkout(self, branch):
        default_branch = self.branch
        current_branch_set = None
        branches = {}
        if self.branches_file.exists():
            branches = yaml.safe_load(self.branches_file.text()) or {}
        if branch in branches:
            branches_set = branches[branch]
            if all(k in branches_set for k in ['branch', 'repos']):
                if self.name in branches_set['repos']:
                    current_branch_set = branch
                    branch = branches_set['branch']
                else:
                    branch = default_branch
            else:
                if self.name in branches_set:
                    current_branch_set = branch
                    branch = branches_set[self.name]
                else:
                    branch = default_branch
        elif branch == 'default':
            branch = default_branch
        elif self.type == 'misc':
            return
        elif self.type not in ['core', 'plugin']:
            raise exceptions.NonRecoverableError('Unhandled repo type: {}'
                                                 .format(self.type))
        elif not branch:
            raise exceptions.NonRecoverableError('Branch is not defined')
        elif branch.startswith('.'):
            branch = self._fix_branch_name(branch)
        try:
            self.git.checkout(branch).wait()
            if current_branch_set:
                ctx.instance.runtime_properties[
                    'current_branch_set'] = current_branch_set
                ctx.instance.runtime_properties[
                    'current_branch_set_branch'] = branch
            else:
                ctx.instance.runtime_properties.pop('current_branch_set', None)
                ctx.instance.runtime_properties.pop(
                    'current_branch_set_branch', None)
        except sh.ErrorReturnCode:
            ctx.logger.error('Could not checkout branch {0}'.format(branch))

    def reset(self, hard, origin):
        if not self._validate_branch_set():
            return
        self.git.reset.bake(hard=hard)(
            '{0}/{1}'.format(origin, self.current_branch)).wait()

    def rebase(self, base):
        if not self._validate_branch_set():
            return
        try:
            base = self._fix_branch_name(base)
            self.git.rebase(base).wait()
        except Exception as e:
            ctx.logger.error('Failed rebase, aborting: {0}'.format(e))
            try:
                self.git.rebase(abort=True).wait()
            except:
                pass

    def squash(self, base):
        if not self._validate_branch_set():
            return
        git_version = ctx.instance.runtime_properties['git_version']
        if git_version < 'git version 1.9':
            ctx.logger.warn('git version >= 1.9 is required for squash.')
            return
        git = self.git_output
        base = self._fix_branch_name(base)
        merge_base = git.bake('merge-base', fork_point=True)(
                base).stdout.strip()
        commits = git.bake(
                'rev-list',
                ancestry_path=True,
        )('{0}..HEAD'.format(merge_base)).stdout.strip().split('\n')
        commits = [c.strip() for c in commits]
        if len(commits) == 1:
            ctx.logger.info('Single commit found. Skipping squash.')
            return
        commit_message_sha = commits[-1]
        commit_message = git.log(commit_message_sha,
                                 format='%B', n=1).stdout.strip()
        git = self.git
        ctx.logger.info(
            'Squashing with merge_base: {0} and commit message: {1}'
            .format(merge_base, commit_message))
        git.reset(merge_base, soft=True).wait()
        git.commit(message=commit_message).wait()

    def diff(self, revision_range):
        if self.type not in ['core', 'plugin']:
            return
        split_range = revision_range.split('..')
        if len(split_range) != 2:
            ctx.logger.error(
                'Invalid range supplied: {0}'.format(revision_range))
            return
        left = self._fix_branch_name(split_range[0])
        right = self._fix_branch_name(split_range[1])
        diff_range = '{0}..{1}'.format(left, right)
        try:
            self.git.diff(diff_range).wait()
        except sh.ErrorReturnCode:
            ctx.logger.error('{0} diff failed'.format(diff_range))

    @property
    def location(self):
        return os.path.expanduser(ctx.properties['location'])

    @property
    def name(self):
        return ctx.properties['name']

    @property
    def repo_location(self):
        return os.path.join(self.location, self.name)

    @property
    def organization(self):
        return ctx.properties['organization']

    @property
    def branch(self):
        return ctx.properties['branch']

    @property
    def clone_method(self):
        return ctx.properties['clone_method']

    @property
    def type(self):
        return ctx.properties['repo_type']

    @property
    def git_config(self):
        return ctx.properties['git_config']

    @property
    def git_prompt_paths(self):
        return ctx.properties['git_prompt_paths']

    @property
    def branches_file(self):
        return path(ctx.properties['branches_file'])

    @property
    def current_branch(self):
        return self.git_output('rev-parse', '--abbrev-ref',
                               'HEAD').stdout.strip()

    @property
    def git(self):
        return self._git(log_out=True)

    @property
    def git_output(self):
        return self._git(log_out=False)

    @staticmethod
    def _git(log_out):
        repo_location = ctx.instance.runtime_properties['repo_location']
        git = sh.git
        if log_out:
            git = bake(git)
        return git.bake(
                '--no-pager',
                '--git-dir', path(repo_location) / '.git',
                '--work-tree', repo_location)

    def _validate_branch_set(self):
        current_branch_set = ctx.instance.runtime_properties.get(
                'current_branch_set')
        current_branch_set_branch = ctx.instance.runtime_properties.get(
                'current_branch_set_branch')
        if not (current_branch_set and current_branch_set_branch):
            return False
        current_branch = self.current_branch
        if current_branch_set_branch != current_branch:
            ctx.logger.warn(
                    'Current branch: "{0}" does not match current branch set: '
                    '"{1}" branch "{2}"'.format(current_branch,
                                                current_branch_set,
                                                current_branch_set_branch))
            return False
        return True

    def _fix_branch_name(self, branch):
        if not branch.startswith('.'):
            return branch
        if self.type not in ['core', 'plugin']:
            return branch
        template = '3{}' if self.type == 'core' else '1{}'
        return template.format(branch)
<commit_msg>fix + refactor<commit_after>########
# Copyright (c) 2016 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
############

import os
import sys
import functools

import sh
import yaml
from path import path

from cloudify import ctx
from cloudify import exceptions
from cloudify.decorators import operation

from common import bake

def repo_operation(fn):
    @operation
    @functools.wraps(fn)
    def wrapper(*args, **kwargs):
        repo = GitRepo()
        return fn(repo, *args, **kwargs)
    return wrapper

@repo_operation
def clone(repo, **_):
    repo.clone()

@repo_operation
def configure(repo, **_):
    repo.configure()

@repo_operation
def pull(repo, **_):
    repo.pull()

@repo_operation
def status(repo, **_):
    repo.status()

@repo_operation
def checkout(repo, branch, **_):
    repo.checkout(branch)

@repo_operation
def squash(repo, base, **_):
    repo.squash(base)

@repo_operation
def reset(repo, hard, origin, **_):
    repo.reset(hard, origin)

@repo_operation
def rebase(repo, base, **_):
    repo.rebase(base)

@repo_operation
def diff(repo, revision_range, **_):
    repo.diff(revision_range)

class GitRepo(object):

    def clone(self):
        git = bake(sh.git)
        ctx.instance.runtime_properties['repo_location'] = self.repo_location
        self.git_version = sh.git(version=True).stdout.strip()
        if os.path.isdir(self.repo_location):
            return
        if self.clone_method == 'https':
            clone_url = 'https://github.com/{}/{}.git'.format(
                self.organization, self.name)
        elif self.clone_method == 'ssh':
            clone_url = 'git@github.com:{}/{}.git'.format(
                self.organization, self.name)
        else:
            raise exceptions.NonRecoverableError(
                'Illegal clone method: {0}'.format(self.clone_method))
        git.clone(clone_url, self.repo_location, '-b', self.branch).wait()

    def configure(self):
        # configure commit-msg hook
        hook_path = path(self.repo_location) / '.git' / 'hooks' / 'commit-msg'
        ctx.download_resource_and_render(
                'resources/commit-msg',
                template_variables={'sys_executable': sys.executable},
                target_path=hook_path)
        os.chmod(hook_path, 0755)
        # git config
        for key, value in self.git_config.items():
            self.git.config(key, value).wait()

    def pull(self):
        kwargs = {
            'prune': True
        }
        if self.git_version >= 'git version 2.0.0':
            kwargs['tags'] = True
        self.git.pull(**kwargs).wait()

    def status(self):
        for git_prompt_path in self.git_prompt_paths:
            if os.path.exists(os.path.expanduser(git_prompt_path)):
                break
        else:
            git_prompt_path = None
        if git_prompt_path:
            script_path = ctx.download_resource_and_render(
                    'resources/git-branch-state.sh', template_variables={
                        'git_prompt_path': git_prompt_path,
                        'repo_location': self.repo_location})
            try:
                os.chmod(script_path, 0o0755)
                script = sh.Command(script_path)
                branch_state = script().stdout.strip()
            finally:
                os.remove(script_path)
        else:
            branch_state = self.current_branch
        ctx.logger.info(branch_state)
        self.git.status(s=True).wait()

    def checkout(self, branch):
        default_branch = self.branch
        active_branch_set = None
        branches = {}
        if self.branches_file.exists():
            branches = yaml.safe_load(self.branches_file.text()) or {}
        if branch in branches:
            branches_set = branches[branch]
            if all(k in branches_set for k in ['branch', 'repos']):
                if self.name in branches_set['repos']:
                    active_branch_set = branch
                    branch = branches_set['branch']
                else:
                    branch = default_branch
            else:
                if self.name in branches_set:
                    active_branch_set = branch
                    branch = branches_set[self.name]
                else:
                    branch = default_branch
        elif branch == 'default':
            branch = default_branch
        elif self.type == 'misc':
            return
        elif self.type not in ['core', 'plugin']:
            raise exceptions.NonRecoverableError('Unhandled repo type: {}'
                                                 .format(self.type))
        elif not branch:
            raise exceptions.NonRecoverableError('Branch is not defined')
        elif branch.startswith('.'):
            branch = self._fix_branch_name(branch)
        try:
            self.git.checkout(branch).wait()
            if active_branch_set:
                self.active_branch_set = active_branch_set
                self.active_branch_set_branch = branch
            else:
                del self.active_branch_set
                del self.active_branch_set_branch
        except sh.ErrorReturnCode:
            ctx.logger.error('Could not checkout branch {0}'.format(branch))

    def reset(self, hard, origin):
        if not self._validate_branch_set():
            return
        self.git.reset.bake(hard=hard)(
            '{0}/{1}'.format(origin, self.current_branch)).wait()

    def rebase(self, base):
        if not self._validate_branch_set():
            return
        try:
            base = self._fix_branch_name(base)
            self.git.rebase(base).wait()
        except Exception as e:
            ctx.logger.error('Failed rebase, aborting: {0}'.format(e))
            try:
                self.git.rebase(abort=True).wait()
            except:
                pass

    def squash(self, base):
        if not self._validate_branch_set():
            return
        if self.git_version < 'git version 1.9':
            ctx.logger.warn('git version >= 1.9 is required for squash.')
            return
        git = self.git_output
        base = self._fix_branch_name(base)
        merge_base = git.bake('merge-base', fork_point=True)(
                base).stdout.strip()
        commits = git.bake(
                'rev-list',
                ancestry_path=True,
        )('{0}..HEAD'.format(merge_base)).stdout.strip().split('\n')
        commits = [c.strip() for c in commits]
        if len(commits) == 1:
            ctx.logger.info('Single commit found. Skipping squash.')
            return
        commit_message_sha = commits[-1]
        commit_message = git.log(commit_message_sha,
                                 format='%B', n=1).stdout.strip()
        git = self.git
        ctx.logger.info(
            'Squashing with merge_base: {0} and commit message: {1}'
            .format(merge_base, commit_message))
        git.reset(merge_base, soft=True).wait()
        git.commit(message=commit_message).wait()

    def diff(self, revision_range):
        if self.type not in ['core', 'plugin']:
            return
        split_range = revision_range.split('..')
        if len(split_range) != 2:
            ctx.logger.error(
                'Invalid range supplied: {0}'.format(revision_range))
            return
        left = self._fix_branch_name(split_range[0])
        right = self._fix_branch_name(split_range[1])
        diff_range = '{0}..{1}'.format(left, right)
        try:
            self.git.diff(diff_range).wait()
        except sh.ErrorReturnCode:
            ctx.logger.error('{0} diff failed'.format(diff_range))

    @property
    def location(self):
        return os.path.expanduser(ctx.node.properties['location'])

    @property
    def name(self):
        return ctx.node.properties['name']

    @property
    def repo_location(self):
        return os.path.join(self.location, self.name)

    @property
    def organization(self):
        return ctx.node.properties['organization']

    @property
    def branch(self):
        return ctx.node.properties['branch']

    @property
    def clone_method(self):
        return ctx.node.properties['clone_method']

    @property
    def type(self):
        return ctx.node.properties['repo_type']

    @property
    def git_config(self):
        return ctx.node.properties['git_config']

    @property
    def git_prompt_paths(self):
        return ctx.node.properties['git_prompt_paths']

    @property
    def branches_file(self):
        return path(ctx.node.properties['branches_file'])

    @property
    def active_branch_set(self):
        return ctx.instance.runtime_properties.get('current_branch_set')

    @active_branch_set.setter
    def active_branch_set(self, value):
        ctx.instance.runtime_properties['current_branch_set'] = value

    @active_branch_set.deleter
    def active_branch_set(self):
        ctx.instance.runtime_properties.pop('current_branch_set', None)

    @property
    def active_branch_set_branch(self):
        return ctx.instance.runtime_properties.get('current_branch_set_branch')

    @active_branch_set_branch.setter
    def active_branch_set_branch(self, value):
        ctx.instance.runtime_properties['current_branch_set_branch'] = value

    @active_branch_set_branch.deleter
    def active_branch_set_branch(self):
        ctx.instance.runtime_properties.pop('current_branch_set_branch', None)

    @property
    def git_version(self):
        return ctx.instance.runtime_properties['git_version']

    @git_version.setter
    def git_version(self, value):
        ctx.instance.runtime_properties['git_version'] = value

    @property
    def current_branch(self):
        return self.git_output('rev-parse', '--abbrev-ref',
                               'HEAD').stdout.strip()

    @property
    def git(self):
        return self._git(log_out=True)

    @property
    def git_output(self):
        return self._git(log_out=False)

    def _git(self, log_out):
        git = sh.git
        if log_out:
            git = bake(git)
        return git.bake(
                '--no-pager',
                '--git-dir', path(self.repo_location) / '.git',
                '--work-tree', self.repo_location)

    def _validate_branch_set(self):
        if not (self.active_branch_set and self.active_branch_set_branch):
            return False
        current_branch = self.current_branch
        if self.active_branch_set_branch != current_branch:
            ctx.logger.warn(
                    'Current branch: "{0}" does not match current branch set: '
                    '"{1}" branch "{2}"'.format(current_branch,
                                                self.active_branch_set,
                                                self.active_branch_set_branch))
            return False
        return True

    def _fix_branch_name(self, branch):
        if not branch.startswith('.'):
            return branch
        if self.type not in ['core', 'plugin']:
            return branch
        template = '3{}' if self.type == 'core' else '1{}'
        return template.format(branch)
<|endoftext|>


<commit_before># Load in core dependencies
import code
import os
import sublime

# Set up constants
__dir__ = os.path.dirname(os.path.abspath(__file__))

def run():
    # On every run, re-import the test class
    # TODO: Determine if this is necessary
    filepath = __dir__ + '/plugin_runner.py'
    plugin_dict = {
        '__dir__': __dir__,
        '__file__': filepath,
        '__name__': '%s.plugin_runner' % __package__,
        '__package__': __package__,
        '__builtins__': __builtins__,
    }

    # DEV: In Python 2.x, use execfile. In 3.x, use compile + exec.
    # if getattr(__builtins__, 'execfile', None):
    if sublime.version() < '3000':
        execfile(filepath, plugin_dict, plugin_dict)
    else:
        f = open(filepath)
        script = f.read()
        interpretter = code.InteractiveInterpreter(plugin_dict)
        interpretter.runcode(compile(script, filepath, 'exec'))
    test = plugin_dict['Test']()
    test.run(__dir__)

# TODO: Set timeout loop that checks if `run` has set a global variable
sublime.set_timeout(run, 1000)
<commit_msg>Clarifying init TODO<commit_after># Load in core dependencies
import code
import os
import sublime

# Set up constants
__dir__ = os.path.dirname(os.path.abspath(__file__))

def run():
    # On every run, re-import the test class
    # TODO: Determine if this is necessary
    filepath = __dir__ + '/plugin_runner.py'
    plugin_dict = {
        '__dir__': __dir__,
        '__file__': filepath,
        '__name__': '%s.plugin_runner' % __package__,
        '__package__': __package__,
        '__builtins__': __builtins__,
    }

    # DEV: In Python 2.x, use execfile. In 3.x, use compile + exec.
    # if getattr(__builtins__, 'execfile', None):
    if sublime.version() < '3000':
        execfile(filepath, plugin_dict, plugin_dict)
    else:
        f = open(filepath)
        script = f.read()
        interpretter = code.InteractiveInterpreter(plugin_dict)
        interpretter.runcode(compile(script, filepath, 'exec'))
    test = plugin_dict['Test']()
    test.run(__dir__)

# TODO: Set timeout loop that checks if `run` has set a global variable
# TODO: This thought was along side a plugin hook so we can guarantee most plugins are loaded
sublime.set_timeout(run, 1000)
<|endoftext|>


<commit_before>#! /usr/bin/env python
# $Id$

"""test.py -- Exercise the Gnuplot.py module.

This module is not meant to be a flashy demonstration; rather it is a
thorough test of many combinations of Gnuplot.py features.

Copyright (C) 1999 Michael Haggerty

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or (at
your option) any later version.  This program is distributed in the
hope that it will be useful, but WITHOUT ANY WARRANTY; without even
the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
PURPOSE.  See the GNU General Public License for more details; it is
available at <http://www.fsf.org/copyleft/gpl.html>, or by writing to
the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
Boston, MA 02111-1307, USA.

"""

__version__ = '1.4'
__cvs_version__ = '$Revision$'

import math, time
import Numeric
from Numeric import NewAxis

try:
    import Gnuplot, Gnuplot.funcutils
except ImportError:
    # kludge in case Gnuplot hasn't been installed as a module yet:
    import __init__
    Gnuplot = __init__
    import funcutils
    Gnuplot.funcutils = funcutils

gp = Gnuplot # abbreviation

def wait(str=None, prompt='Press return to show results...\n'):
    if str is not None:
        print str
    raw_input(prompt)

def main():
    """Exercise the Gnuplot module."""

    wait('Popping up a blank gnuplot window on your screen.')
    g = gp.Gnuplot()
    g.clear()

    # Make a temporary file:
    file1 = gp.TempFile() # will be deleted upon exit
    f = open(file1.filename, 'w')
    for x in Numeric.arange(100)/5. - 10.:
        f.write('%s %s %s\n' % (x, math.cos(x), math.sin(x)))
    f.close()

    print '############### test Func ########################################'
    wait('Plot a gnuplot-generated function')
    g.plot(gp.Func('sin(x)'))

    wait('Set title and axis labels and try replot()')
    g.title('Title')
    g.xlabel('x')
    g.ylabel('y')
    g.replot()

    wait('Style linespoints')
    g.plot(gp.Func('sin(x)', with='linespoints'))
    wait('title=None')
    g.plot(gp.Func('sin(x)', title=None))
    wait('title="Sine of x"')
    g.plot(gp.Func('sin(x)', title='Sine of x'))

    print 'Change Func attributes after construction:'
    f = gp.Func('sin(x)')
    wait('Original')
    g.plot(f)
    wait('Style linespoints')
    f.set_option(with='linespoints')
    g.plot(f)
    wait('title=None')
    f.set_option(title=None)
    g.plot(f)
    wait('title="Sine of x"')
    f.set_option(title='Sine of x')
    g.plot(f)

    print '############### test File ########################################'
    wait('Generate a File from a filename')
    g.plot(gp.File(file1.filename))
    wait('Generate a File given a TempFile object')
    g.plot(gp.File(file1))

    wait('Style lines')
    g.plot(gp.File(file1.filename, with='lines'))
    wait('using=1, using=(1,)')
    g.plot(gp.File(file1.filename, using=1, with='lines'),
           gp.File(file1.filename, using=(1,), with='points'))
    wait('using=(1,2), using="1:3"')
    g.plot(gp.File(file1.filename, using=(1,2)),
           gp.File(file1.filename, using='1:3'))
    wait('title=None')
    g.plot(gp.File(file1.filename, title=None))
    wait('title="title"')
    g.plot(gp.File(file1.filename, title='title'))

    print 'Change File attributes after construction:'
    f = gp.File(file1.filename)
    wait('Original')
    g.plot(f)
    wait('Style linespoints')
    f.set_option(with='linespoints')
    g.plot(f)
    wait('using=(1,3)')
    f.set_option(using=(1,3))
    g.plot(f)
    wait('title=None')
    f.set_option(title=None)
    g.plot(f)

    print '############### test Data ########################################'
    x = Numeric.arange(100)/5. - 10.
    y1 = Numeric.cos(x)
    y2 = Numeric.sin(x)
    d = Numeric.transpose((x,y1,y2))

    wait('Plot Data, specified column-by-column')
    g.plot(gp.Data(x,y2, inline=0))
    wait('Same thing, inline data')
    g.plot(gp.Data(x,y2, inline=1))

    wait('Plot Data, specified by an array')
    g.plot(gp.Data(d, inline=0))
    wait('Same thing, inline data')
    g.plot(gp.Data(d, inline=1))
    wait('with="lp 4 4"')
    g.plot(gp.Data(d, with='lp 4 4'))
    wait('cols=0')
    g.plot(gp.Data(d, cols=0))
    wait('cols=(0,1), cols=(0,2)')
    g.plot(gp.Data(d, cols=(0,1), inline=0),
           gp.Data(d, cols=(0,2), inline=0))
    wait('Same thing, inline data')
    g.plot(gp.Data(d, cols=(0,1), inline=1),
           gp.Data(d, cols=(0,2), inline=1))
    wait('Change title and replot()')
    g.title('New title')
    g.replot()
    wait('title=None')
    g.plot(gp.Data(d, title=None))
    wait('title="Cosine of x"')
    g.plot(gp.Data(d, title='Cosine of x'))

    print '############### test compute_Data ################################'
    x = Numeric.arange(100)/5. - 10.

    wait('Plot Data, computed by Gnuplot.py')
    g.plot(gp.funcutils.compute_Data(x, lambda x: math.cos(x), inline=0))
    wait('Same thing, inline data')
    g.plot(gp.funcutils.compute_Data(x, lambda x: math.cos(x), inline=1))
    wait('with="lp 4 4"')
    g.plot(gp.funcutils.compute_Data(x, lambda x: math.cos(x), with='lp 4 4'))

    print '############### test hardcopy ####################################'
    print '******** Generating postscript file "gp_test.ps" ********'
    wait()
    g.plot(gp.Func('cos(0.5*x*x)', with='linespoints 2 2',
                   title='cos(0.5*x^2)'))
    g.hardcopy('gp_test.ps')

    wait('Testing hardcopy options: mode="landscape"')
    g.hardcopy('gp_test.ps', mode='landscape')
    wait('Testing hardcopy options: mode="portrait"')
    g.hardcopy('gp_test.ps', mode='portrait')
    wait('Testing hardcopy options: mode="eps"')
    g.hardcopy('gp_test.ps', mode='eps')
    wait('Testing hardcopy options: eps=1')
    g.hardcopy('gp_test.ps', eps=1)
    wait('Testing hardcopy options: enhanced=1')
    g.hardcopy('gp_test.ps', eps=0, enhanced=1)
    wait('Testing hardcopy options: enhanced=0')
    g.hardcopy('gp_test.ps', enhanced=0)
    wait('Testing hardcopy options: color=1')
    g.hardcopy('gp_test.ps', color=1)
    wait('Testing hardcopy options: solid=1')
    g.hardcopy('gp_test.ps', color=0, solid=1)
    wait('Testing hardcopy options: duplexing="duplex"')
    g.hardcopy('gp_test.ps', solid=0, duplexing='duplex')
    wait('Testing hardcopy options: duplexing="defaultplex"')
    g.hardcopy('gp_test.ps', duplexing='defaultplex')
    wait('Testing hardcopy options: fontname="Times-Italic"')
    g.hardcopy('gp_test.ps', fontname='Times-Italic')
    wait('Testing hardcopy options: fontsize=20')
    g.hardcopy('gp_test.ps', fontsize=20)
    wait('Testing hardcopy options: mode="default"')
    g.hardcopy('gp_test.ps', mode='default')

    print '############### test shortcuts ###################################'
    wait('plot Func and Data using shortcuts')
    g.plot('sin(x)', d)

    print '############### test splot #######################################'
    g.splot(gp.Data(d, with='linesp', inline=0))
    wait('Same thing, inline data')
    g.splot(gp.Data(d, with='linesp', inline=1))

    print '############### test GridData and compute_GridData ###############'
    # set up x and y values at which the function will be tabulated:
    x = Numeric.arange(35)/2.0
    y = Numeric.arange(30)/10.0 - 1.5
    # Make a 2-d array containing a function of x and y.  First create
    # xm and ym which contain the x and y values in a matrix form that
    # can be `broadcast' into a matrix of the appropriate shape:
    xm = x[:,NewAxis]
    ym = y[NewAxis,:]
    m = (Numeric.sin(xm) + 0.1*xm) - ym**2
    wait('a function of two variables from a GridData file')
    g('set parametric')
    g('set data style lines')
    g('set hidden')
    g('set contour base')
    g.xlabel('x')
    g.ylabel('y')
    g.splot(gp.GridData(m,x,y, binary=0, inline=0))
    wait('Same thing, inline data')
    g.splot(gp.GridData(m,x,y, binary=0, inline=1))

    wait('The same thing using binary mode')
    g.splot(gp.GridData(m,x,y, binary=1))

    wait('The same thing using compute_GridData to tabulate function')
    g.splot(gp.funcutils.compute_GridData(
        x,y, lambda x,y: math.sin(x) + 0.1*x - y**2,
        ))

    wait('Use compute_GridData in ufunc and binary mode')
    g.splot(gp.funcutils.compute_GridData(
        x,y, lambda x,y: Numeric.sin(x) + 0.1*x - y**2,
        ufunc=1, binary=1,
        ))

    wait('And now rotate it a bit')
    for view in range(35,70,5):
        g('set view 60, %d' % view)
        g.replot()
        time.sleep(1.0)

    wait(prompt='Press return to end the test.\n')

# when executed, just run main():
if __name__ == '__main__':
    main()

<commit_msg>Forgot a `wait'.<commit_after>#! /usr/bin/env python
# $Id$

"""test.py -- Exercise the Gnuplot.py module.

This module is not meant to be a flashy demonstration; rather it is a
thorough test of many combinations of Gnuplot.py features.

Copyright (C) 1999 Michael Haggerty

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or (at
your option) any later version.  This program is distributed in the
hope that it will be useful, but WITHOUT ANY WARRANTY; without even
the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
PURPOSE.  See the GNU General Public License for more details; it is
available at <http://www.fsf.org/copyleft/gpl.html>, or by writing to
the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
Boston, MA 02111-1307, USA.

"""

__version__ = '1.4'
__cvs_version__ = '$Revision$'

import math, time
import Numeric
from Numeric import NewAxis

try:
    import Gnuplot, Gnuplot.funcutils
except ImportError:
    # kludge in case Gnuplot hasn't been installed as a module yet:
    import __init__
    Gnuplot = __init__
    import funcutils
    Gnuplot.funcutils = funcutils

gp = Gnuplot # abbreviation

def wait(str=None, prompt='Press return to show results...\n'):
    if str is not None:
        print str
    raw_input(prompt)

def main():
    """Exercise the Gnuplot module."""

    wait('Popping up a blank gnuplot window on your screen.')
    g = gp.Gnuplot()
    g.clear()

    # Make a temporary file:
    file1 = gp.TempFile() # will be deleted upon exit
    f = open(file1.filename, 'w')
    for x in Numeric.arange(100)/5. - 10.:
        f.write('%s %s %s\n' % (x, math.cos(x), math.sin(x)))
    f.close()

    print '############### test Func ########################################'
    wait('Plot a gnuplot-generated function')
    g.plot(gp.Func('sin(x)'))

    wait('Set title and axis labels and try replot()')
    g.title('Title')
    g.xlabel('x')
    g.ylabel('y')
    g.replot()

    wait('Style linespoints')
    g.plot(gp.Func('sin(x)', with='linespoints'))
    wait('title=None')
    g.plot(gp.Func('sin(x)', title=None))
    wait('title="Sine of x"')
    g.plot(gp.Func('sin(x)', title='Sine of x'))

    print 'Change Func attributes after construction:'
    f = gp.Func('sin(x)')
    wait('Original')
    g.plot(f)
    wait('Style linespoints')
    f.set_option(with='linespoints')
    g.plot(f)
    wait('title=None')
    f.set_option(title=None)
    g.plot(f)
    wait('title="Sine of x"')
    f.set_option(title='Sine of x')
    g.plot(f)

    print '############### test File ########################################'
    wait('Generate a File from a filename')
    g.plot(gp.File(file1.filename))
    wait('Generate a File given a TempFile object')
    g.plot(gp.File(file1))

    wait('Style lines')
    g.plot(gp.File(file1.filename, with='lines'))
    wait('using=1, using=(1,)')
    g.plot(gp.File(file1.filename, using=1, with='lines'),
           gp.File(file1.filename, using=(1,), with='points'))
    wait('using=(1,2), using="1:3"')
    g.plot(gp.File(file1.filename, using=(1,2)),
           gp.File(file1.filename, using='1:3'))
    wait('title=None')
    g.plot(gp.File(file1.filename, title=None))
    wait('title="title"')
    g.plot(gp.File(file1.filename, title='title'))

    print 'Change File attributes after construction:'
    f = gp.File(file1.filename)
    wait('Original')
    g.plot(f)
    wait('Style linespoints')
    f.set_option(with='linespoints')
    g.plot(f)
    wait('using=(1,3)')
    f.set_option(using=(1,3))
    g.plot(f)
    wait('title=None')
    f.set_option(title=None)
    g.plot(f)

    print '############### test Data ########################################'
    x = Numeric.arange(100)/5. - 10.
    y1 = Numeric.cos(x)
    y2 = Numeric.sin(x)
    d = Numeric.transpose((x,y1,y2))

    wait('Plot Data, specified column-by-column')
    g.plot(gp.Data(x,y2, inline=0))
    wait('Same thing, inline data')
    g.plot(gp.Data(x,y2, inline=1))

    wait('Plot Data, specified by an array')
    g.plot(gp.Data(d, inline=0))
    wait('Same thing, inline data')
    g.plot(gp.Data(d, inline=1))
    wait('with="lp 4 4"')
    g.plot(gp.Data(d, with='lp 4 4'))
    wait('cols=0')
    g.plot(gp.Data(d, cols=0))
    wait('cols=(0,1), cols=(0,2)')
    g.plot(gp.Data(d, cols=(0,1), inline=0),
           gp.Data(d, cols=(0,2), inline=0))
    wait('Same thing, inline data')
    g.plot(gp.Data(d, cols=(0,1), inline=1),
           gp.Data(d, cols=(0,2), inline=1))
    wait('Change title and replot()')
    g.title('New title')
    g.replot()
    wait('title=None')
    g.plot(gp.Data(d, title=None))
    wait('title="Cosine of x"')
    g.plot(gp.Data(d, title='Cosine of x'))

    print '############### test compute_Data ################################'
    x = Numeric.arange(100)/5. - 10.

    wait('Plot Data, computed by Gnuplot.py')
    g.plot(gp.funcutils.compute_Data(x, lambda x: math.cos(x), inline=0))
    wait('Same thing, inline data')
    g.plot(gp.funcutils.compute_Data(x, lambda x: math.cos(x), inline=1))
    wait('with="lp 4 4"')
    g.plot(gp.funcutils.compute_Data(x, lambda x: math.cos(x), with='lp 4 4'))

    print '############### test hardcopy ####################################'
    print '******** Generating postscript file "gp_test.ps" ********'
    wait()
    g.plot(gp.Func('cos(0.5*x*x)', with='linespoints 2 2',
                   title='cos(0.5*x^2)'))
    g.hardcopy('gp_test.ps')

    wait('Testing hardcopy options: mode="landscape"')
    g.hardcopy('gp_test.ps', mode='landscape')
    wait('Testing hardcopy options: mode="portrait"')
    g.hardcopy('gp_test.ps', mode='portrait')
    wait('Testing hardcopy options: mode="eps"')
    g.hardcopy('gp_test.ps', mode='eps')
    wait('Testing hardcopy options: eps=1')
    g.hardcopy('gp_test.ps', eps=1)
    wait('Testing hardcopy options: enhanced=1')
    g.hardcopy('gp_test.ps', eps=0, enhanced=1)
    wait('Testing hardcopy options: enhanced=0')
    g.hardcopy('gp_test.ps', enhanced=0)
    wait('Testing hardcopy options: color=1')
    g.hardcopy('gp_test.ps', color=1)
    wait('Testing hardcopy options: solid=1')
    g.hardcopy('gp_test.ps', color=0, solid=1)
    wait('Testing hardcopy options: duplexing="duplex"')
    g.hardcopy('gp_test.ps', solid=0, duplexing='duplex')
    wait('Testing hardcopy options: duplexing="defaultplex"')
    g.hardcopy('gp_test.ps', duplexing='defaultplex')
    wait('Testing hardcopy options: fontname="Times-Italic"')
    g.hardcopy('gp_test.ps', fontname='Times-Italic')
    wait('Testing hardcopy options: fontsize=20')
    g.hardcopy('gp_test.ps', fontsize=20)
    wait('Testing hardcopy options: mode="default"')
    g.hardcopy('gp_test.ps', mode='default')

    print '############### test shortcuts ###################################'
    wait('plot Func and Data using shortcuts')
    g.plot('sin(x)', d)

    print '############### test splot #######################################'
    wait('a 3-d curve')
    g.splot(gp.Data(d, with='linesp', inline=0))
    wait('Same thing, inline data')
    g.splot(gp.Data(d, with='linesp', inline=1))

    print '############### test GridData and compute_GridData ###############'
    # set up x and y values at which the function will be tabulated:
    x = Numeric.arange(35)/2.0
    y = Numeric.arange(30)/10.0 - 1.5
    # Make a 2-d array containing a function of x and y.  First create
    # xm and ym which contain the x and y values in a matrix form that
    # can be `broadcast' into a matrix of the appropriate shape:
    xm = x[:,NewAxis]
    ym = y[NewAxis,:]
    m = (Numeric.sin(xm) + 0.1*xm) - ym**2
    wait('a function of two variables from a GridData file')
    g('set parametric')
    g('set data style lines')
    g('set hidden')
    g('set contour base')
    g.xlabel('x')
    g.ylabel('y')
    g.splot(gp.GridData(m,x,y, binary=0, inline=0))
    wait('Same thing, inline data')
    g.splot(gp.GridData(m,x,y, binary=0, inline=1))

    wait('The same thing using binary mode')
    g.splot(gp.GridData(m,x,y, binary=1))

    wait('The same thing using compute_GridData to tabulate function')
    g.splot(gp.funcutils.compute_GridData(
        x,y, lambda x,y: math.sin(x) + 0.1*x - y**2,
        ))

    wait('Use compute_GridData in ufunc and binary mode')
    g.splot(gp.funcutils.compute_GridData(
        x,y, lambda x,y: Numeric.sin(x) + 0.1*x - y**2,
        ufunc=1, binary=1,
        ))

    wait('And now rotate it a bit')
    for view in range(35,70,5):
        g('set view 60, %d' % view)
        g.replot()
        time.sleep(1.0)

    wait(prompt='Press return to end the test.\n')

# when executed, just run main():
if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before>import asyncio
import threading
from typing import (
    Optional,
    Iterable,
    AsyncGenerator,
    TypeVar,
    Type,
    NamedTuple, Callable)

import pykube
import structlog
from aiochannel import Channel

from k8s_snapshots.context import Context

_logger = structlog.get_logger(__name__)

Resource = TypeVar(
    'Resource',
    bound=pykube.objects.APIObject,
)

ClientFactory = Callable[[], pykube.HTTPClient]

# Copy of a locally-defined namedtuple in
# pykube.query.WatchQuery.object_stream()
_WatchEvent = NamedTuple(
    '_WatchEvent',
    [
        ('type', str),
        ('object', Resource),
    ]
)

class SnapshotRule(pykube.objects.APIObject):
    version = "k8s-snapshots.elsdoerfer.com/v1"
    endpoint = "snapshotrules"
    kind = "SnapshotRule"

class Kubernetes:
    """
    Allows for easier mocking of Kubernetes resources.
    """
    def __init__(
            self,
            client_factory: Optional[ClientFactory]=None
    ):
        """

        Parameters
        ----------
        client_factory
            Used in threaded operations to create a local
            :any:`pykube.HTTPClient` instance.
        """
        # Used for threaded operations
        self.client_factory = client_factory

    def get_or_none(
            self,
            resource_type: Type[Resource],
            name: str,
            namespace: Optional[str]=None,
    ) -> Optional[Resource]:
        """
        Sync wrapper for :any:`pykube.query.Query().get_or_none`
         """
        resource_query = resource_type.objects(self.client_factory())
        if namespace is not None:
            resource_query = resource_query.filter(
                namespace=namespace
            )

        return resource_query.get_or_none(name=name)

    def watch(
            self,
            resource_type: Type[Resource],
    ) -> Iterable[_WatchEvent]:
        """
        Sync wrapper for :any:`pykube.query.Query().watch().object_stream()`
        """
        return resource_type.objects(self.client_factory())\
            .filter(namespace=pykube.all).watch().object_stream()

def get_resource_or_none_sync(
        client_factory: ClientFactory,
        resource_type: Type[Resource],
        name: str,
        namespace: Optional[str]=None,
) -> Optional[Resource]:
    return Kubernetes(client_factory).get_or_none(
        resource_type,
        name,
        namespace,
    )

async def get_resource_or_none(
        client_factory: ClientFactory,
        resource_type: Type[Resource],
        name: str,
        namespace: Optional[str]=None,
        *,
        loop=None
) -> Optional[Resource]:
    loop = loop or asyncio.get_event_loop()

    def _get():
        return get_resource_or_none_sync(
            client_factory=client_factory,
            resource_type=resource_type,
            name=name,
            namespace=namespace,
        )

    return await loop.run_in_executor(
        None,
        _get,
    )

def watch_resources_sync(
        client_factory: ClientFactory,
        resource_type: pykube.objects.APIObject,
) -> Iterable:
    return Kubernetes(client_factory).watch(
        resource_type=resource_type
    )

async def watch_resources(
        ctx: Context,
        resource_type: Resource,
        *,
        delay: int,
        allow_missing: bool = False,
        loop=None
) -> AsyncGenerator[_WatchEvent, None]:
    """ Asynchronously watch Kubernetes resources """
    async_gen = _watch_resources_thread_wrapper(
        ctx.kube_client,
        resource_type,
        allow_missing=allow_missing,
        loop=loop
    )

    # Workaround a race condition in pykube:
    # https: // github.com / kelproject / pykube / issues / 138
    await asyncio.sleep(delay)

    async for item in async_gen:
        yield item

async def _watch_resources_thread_wrapper(
        client_factory: Callable[[], pykube.HTTPClient],
        resource_type: Type[Resource],
        allow_missing: bool = False,
        *,
        loop=None
) -> AsyncGenerator[_WatchEvent, None]:
    """ Async wrapper for pykube.watch().object_stream() """
    loop = loop or asyncio.get_event_loop()
    _log = _logger.bind(
        resource_type_name=resource_type.__name__,
    )
    channel = Channel()

    def worker():
        try:
            _log.debug('watch-resources.worker.start')
            while True:
                sync_iterator = watch_resources_sync(
                    client_factory=client_factory,
                    resource_type=resource_type
                )
                _log.debug('watch-resources.worker.watch-opened')
                for event in sync_iterator:
                    # only put_nowait seems to cause SIGSEGV
                    loop.call_soon_threadsafe(channel.put_nowait, event)
                _log.debug('watch-resources.worker.watch-closed')
        except pykube.exceptions.HTTPError as e:
            # TODO: It's possible that the user creates the resource
            # while we are already running. We should pick this up
            # automatically, i.e. watch ThirdPartyResource, or just
            # check every couple of seconds.
            if e.code == 404 and allow_missing:
                _log.info('watch-resources.worker.skipped')
            else:
                _log.exception('watch-resources.worker.error')
        except:
            _log.exception('watch-resources.worker.error')
        finally:
            _log.debug('watch-resources.worker.finalized')
            channel.close()

    thread = threading.Thread(
        target=worker,
        daemon=True,
    )
    thread.start()

    async for channel_event in channel:
        yield channel_event

    _log.debug('watch-resources.done')

<commit_msg>Run yapf over a file as an experiment.<commit_after>import asyncio
import threading
from typing import (Optional, Iterable, AsyncGenerator, TypeVar, Type,
                    NamedTuple, Callable)

import pykube
import structlog
from aiochannel import Channel

from k8s_snapshots.context import Context

_logger = structlog.get_logger(__name__)

Resource = TypeVar(
    'Resource',
    bound=pykube.objects.APIObject,
)

ClientFactory = Callable[[], pykube.HTTPClient]

# Copy of a locally-defined namedtuple in
# pykube.query.WatchQuery.object_stream()
_WatchEvent = NamedTuple('_WatchEvent', [
    ('type', str),
    ('object', Resource),
])

class SnapshotRule(pykube.objects.APIObject):
    version = "k8s-snapshots.elsdoerfer.com/v1"
    endpoint = "snapshotrules"
    kind = "SnapshotRule"

class Kubernetes:
    """
    Allows for easier mocking of Kubernetes resources.
    """

    def __init__(self, client_factory: Optional[ClientFactory] = None):
        """

        Parameters
        ----------
        client_factory
            Used in threaded operations to create a local
            :any:`pykube.HTTPClient` instance.
        """
        # Used for threaded operations
        self.client_factory = client_factory

    def get_or_none(self,
                    resource_type: Type[Resource],
                    name: str,
                    namespace: Optional[str] = None) -> Optional[Resource]:
        """
        Sync wrapper for :any:`pykube.query.Query().get_or_none`
         """
        resource_query = resource_type.objects(self.client_factory())
        if namespace is not None:
            resource_query = resource_query.filter(namespace=namespace)

        return resource_query.get_or_none(name=name)

    def watch(
            self,
            resource_type: Type[Resource],
    ) -> Iterable[_WatchEvent]:
        """
        Sync wrapper for :any:`pykube.query.Query().watch().object_stream()`
        """
        return resource_type.objects(self.client_factory())\
            .filter(namespace=pykube.all).watch().object_stream()

def get_resource_or_none_sync(
        client_factory: ClientFactory,
        resource_type: Type[Resource],
        name: str,
        namespace: Optional[str] = None) -> Optional[Resource]:
    return Kubernetes(client_factory).get_or_none(
        resource_type,
        name,
        namespace,
    )

async def get_resource_or_none(client_factory: ClientFactory,
                               resource_type: Type[Resource],
                               name: str,
                               namespace: Optional[str] = None,
                               *,
                               loop=None) -> Optional[Resource]:
    loop = loop or asyncio.get_event_loop()

    def _get():
        return get_resource_or_none_sync(
            client_factory=client_factory,
            resource_type=resource_type,
            name=name,
            namespace=namespace,
        )

    return await loop.run_in_executor(
        None,
        _get,
    )

def watch_resources_sync(
        client_factory: ClientFactory,
        resource_type: pykube.objects.APIObject,
) -> Iterable:
    return Kubernetes(client_factory).watch(resource_type=resource_type)

async def watch_resources(ctx: Context,
                          resource_type: Resource,
                          *,
                          delay: int,
                          allow_missing: bool = False,
                          loop=None) -> AsyncGenerator[_WatchEvent, None]:
    """ Asynchronously watch Kubernetes resources """
    async_gen = _watch_resources_thread_wrapper(
        ctx.kube_client, resource_type, allow_missing=allow_missing, loop=loop)

    # Workaround a race condition in pykube:
    # https: // github.com / kelproject / pykube / issues / 138
    await asyncio.sleep(delay)

    async for item in async_gen:
        yield item

async def _watch_resources_thread_wrapper(
        client_factory: Callable[[], pykube.HTTPClient],
        resource_type: Type[Resource],
        allow_missing: bool = False,
        *,
        loop=None) -> AsyncGenerator[_WatchEvent, None]:
    """ Async wrapper for pykube.watch().object_stream() """
    loop = loop or asyncio.get_event_loop()
    _log = _logger.bind(resource_type_name=resource_type.__name__, )
    channel = Channel()

    def worker():
        try:
            _log.debug('watch-resources.worker.start')
            while True:
                sync_iterator = watch_resources_sync(
                    client_factory=client_factory, resource_type=resource_type)
                _log.debug('watch-resources.worker.watch-opened')
                for event in sync_iterator:
                    # only put_nowait seems to cause SIGSEGV
                    loop.call_soon_threadsafe(channel.put_nowait, event)
                _log.debug('watch-resources.worker.watch-closed')
        except pykube.exceptions.HTTPError as e:
            # TODO: It's possible that the user creates the resource
            # while we are already running. We should pick this up
            # automatically, i.e. watch ThirdPartyResource, or just
            # check every couple of seconds.
            if e.code == 404 and allow_missing:
                _log.info('watch-resources.worker.skipped')
            else:
                _log.exception('watch-resources.worker.error')
        except:
            _log.exception('watch-resources.worker.error')
        finally:
            _log.debug('watch-resources.worker.finalized')
            channel.close()

    thread = threading.Thread(
        target=worker,
        daemon=True,
    )
    thread.start()

    async for channel_event in channel:
        yield channel_event

    _log.debug('watch-resources.done')
<|endoftext|>


<commit_before>import os
import json
from featurize_clusters import TextFeaturizer
import numpy as np
from itertools import chain
from pyproj import Proj,transform
import time
from datetime import datetime
from pandas import Series, DataFrame, Panel
import pandas as pd

INPROJ = Proj(init='epsg:4326')
OUTPROJ = Proj(init='epsg:3857')
MV = 'None' # handle missing values this way

def get_static_features(lattice_cluster):
    ret_time_loc = list((time,coord,prob) for time,coord,prob in get_locations_and_time(lattice_cluster))
    ret_time_loc.sort(key=lambda x:x[0])#sort by date
    posttimes,coords,probs = zip(*ret_time_loc)

    posttimes_clean = [x for x in posttimes if str(x) != 'nan']
    coords_clean = [x for x in coords if not np.isnan(x).any()]
    probs_clean = [x for x in probs if str(x) != 'nan']
    ages = tuple(res for res in get_age(lattice_cluster))
    len_cluster = len(lattice_cluster)
    return np.hstack((cluster_age_summary(ages,len_cluster),cluster_location_summary(coords_clean,probs_clean,len_cluster),temporal_summary(posttimes_clean)))

def temporal_summary(posttimes):
    #use strings of dates as they can be sorted without conversion
    if len(posttimes)>1:
        dind = pd.to_datetime(sorted(posttimes))
        daterange = (dind[-1]-dind[0]).days#difference between first and last post in days
        df = Series(np.ones(len(posttimes)), index=dind)

        #1day -aggregate by day
        df1 = df.resample('d').sum().fillna(0.0)
        #2days -aggregate by 2-day window
        df2 = df1.resample("2d",closed='left').sum()
        #monthly -aggregate by month
        dfm = df1.resample("M",closed='left').sum()

        return np.array((0.0,daterange,df1.mean(),df1.min(),df1.max(),df1.median(),df1.std(),df2.mean(),df2.min(),df2.max(),df2.median(),df2.std(),dfm.mean(),dfm.min(),dfm.max(),dfm.median(),dfm.std()))
    else:
        return np.hstack((np.array([1.0]),np.zeros(16)))

def cluster_text_summary(textfeaturizer,text):
    '''
    create vector embedding for all text in a cluster of ads
    '''
    #Text features
    X = textfeaturizer.get_text_features(text)

    X = X.tocsc() #transform into column-sparse object
    mean_text = X.mean(axis=0) #mean for cluster
    #sqaure elements to then compute variance
    X.data**=2
    variance_text = X.mean(axis=0)-np.multiply(mean_text,mean_text)  #variance of each word
    agg_variance = np.sum(variance_text) #aggregate variance

    #create vector of text features for the cluster
    return np.hstack((np.squeeze(np.array(mean_text)),agg_variance))

def cluster_age_summary(ages,len_cluster):
    missing_age = len_cluster-len(ages) #number of ads that do not contain age
    ages = np.array(tuple(chain(*ages)))

    #vector that summarizes the age features of cluster
    return np.array((ages.mean(),ages.std(),np.median(ages),missing_age))

def cluster_location_summary(locations_webMer,loc_prob,len_cluster):
    if locations_webMer:
        num_found_loc = len(locations_webMer)
        missing_lat = len_cluster-num_found_loc
        missing_lat_proportion = float(missing_lat)/len_cluster

        locations_webMer=np.array(locations_webMer)

        #Extract mean of xaxis
        #centroid_cluster = np.mean(locations_webMer,0)
        centroid_cluster = np.average(locations_webMer,0,loc_prob)

        ###Measure the area the cluster covers
        min_coord = np.amin(locations_webMer,0)
        max_coord = np.amax(locations_webMer,0)

        dif_extreme_coord = np.subtract(max_coord,min_coord)
        area_cluster = dif_extreme_coord[0]*dif_extreme_coord[1]

        ###Measure how concentrated ads are
        #distance from each point to the centroid - add 1 to avoid errors when all ads are in same location

        sqdif_coord = np.zeros(shape=(num_found_loc,2))
        sqdif_coord[:,0] = np.square(np.abs(locations_webMer[:,0]-centroid_cluster[0]))
        sqdif_coord[:,1] = np.square(np.abs(locations_webMer[:,1]-centroid_cluster[1]))
        dist_centroid = np.sqrt(np.sum(sqdif_coord,1))

        disp_metric = np.sum(np.multiply(dist_centroid,loc_prob))/num_found_loc  ###alternatively we can sum over the log

        #return np.hstack((missing_lat_proportion,np.squeeze(centroid_cluster),area_cluster,disp_metric))
        return np.hstack(np.array([0.0]),missing_lat_proportion,area_cluster,disp_metric))
    else:
        return np.hstack((np.array([1.0]),np.zeros(3)))

def spatiotemporal_summary(posttimes,coords,probs):
    #time_coords_clean = [x for x in time_coords if not np.isnan(x).any()]
    if len(posttimes)>1:
        dind = pd.to_datetime(posttimes)
        df_coords = Series(coords, index=dind)
        df_probs = Series(probs, index=dind)
        df = pd.concat([df_coords,df_probs],axis=1)
        df = df[pd.notnull(df.index)]
        df = df.dropna(axis=0,how='any')

        #1day -aggregate by day
        df1 = df.resample('d').sum().fillna(0.0)
        #2days -aggregate by 2-day window
        df2 = df1.resample("2d",closed='left').sum()
        #monthly -aggregate by month
        dfm = df1.resample("M",closed='left').sum()

        return np.array((0.0,daterange,df1.mean(),df1.min(),df1.max(),df1.median(),df1.std(),df2.mean(),df2.min(),df2.max(),df2.median(),df2.std(),dfm.mean(),dfm.min(),dfm.max(),dfm.median(),dfm.std()))
    else:
        return np.hstack((np.array([1.0]),np.zeros(16)))

def get_locations_and_time(cluster):
    for item in cluster:
        if u'lattice-postdatetime' in item[u'extractions']:
            posttime = item[u'extractions'][u'lattice-postdatetime'][u'results'][0][u'value']
        else:
            posttime = np.nan
        if u'lattice-location' in item[u'extractions']:
            ret = tuple((coord,prob) for coord,prob in load_cities(item[u'extractions'][u'lattice-location'][u'results']))
            if ret:
                coords,probs = zip(*ret)
                coords = np.array(coords)
                probs = np.array(probs)
                weigavg_coords = np.average(coords,0,probs)
                mean_probs = probs.mean()
            else:
                weigavg_coords = np.nan
                mean_probs = np.nan
        else:
            weigavg_coords = np.nan
            mean_probs = np.nan
        yield posttime,weigavg_coords,mean_probs

def get_age(cluster):
    for item in cluster:
        if 'lattice-age' in item[u'extractions']:
            yield map(float,(d[u'value'] for d in item[u'extractions'][u'lattice-age'][u'results']))

def load_cities(iterable):
    for d in iterable:
        if u'city' in d[u'context']:
            yield transform(INPROJ,OUTPROJ,d[u'context'][u'city'][u'centroid_lon'],
                            d[u'context'][u'city'][u'centroid_lat']),d[u'probability']

#####What is below is old

#def get_coord(cluster):
#        for item in cluster:
#            if u'lattice-location' in item[u'extractions']:
#
#                tup = tuple(transform(INPROJ,OUTPROJ,d[u'context'][u'city'][u'centroid_lon'],
#                                                     d[u'context'][u'city'][u'centroid_lat'])
#                            for d in item[u'extractions'][u'lattice-location'][u'results']
#                                if d[u'context'][u'city'][u'centroid_lat'] is not None)
#                if tup:#add code to aggregate multiple locations into one
#                    yield tup[0]

<commit_msg>features fixed without spatiotemporal<commit_after>import os
import json
from featurize_clusters import TextFeaturizer
import numpy as np
from itertools import chain, groupby
from pyproj import Proj,transform
import time
from datetime import datetime
from pandas import Series, DataFrame, Panel
import pandas as pd

INPROJ = Proj(init='epsg:4326')
OUTPROJ = Proj(init='epsg:3857')
MV = 'None' # handle missing values this way

def get_static_features(lattice_cluster):
    ret_time_loc = list((time,coord,prob) for time,coord,prob in get_locations_and_time(lattice_cluster))
    ret_time_loc.sort(key=lambda x:x[0])#sort by date
    posttimes,coords,probs = zip(*ret_time_loc)
    ret_time_loc = [items for items in ret_time_loc if not None in items]
    posttimes = [x for x in posttimes if not x is None]
    coords = [x for x in coords if not x is None]
    probs = [x for x in probs if not x is None]
    ages = tuple(res for res in get_age(lattice_cluster))
    len_cluster = len(lattice_cluster)
    return np.hstack((cluster_age_summary(ages,len_cluster),cluster_location_summary(coords,probs,len_cluster),temporal_summary(posttimes)))

def spatiotemporal_summary(ret_time_loc):
    #time_coords_clean = [x for x in time_coords if not np.isnan(x).any()]

    if len(ret_time_loc)>1:
        spatiotemp = list((area,disp) for area,disp in locationFeat_daily(ret_time_loc))
        #calculate min, mean,max, std area and disp
        spatiotemp_mean = np.average(spatiotemp,0)

        return np.array((0.0,daterange,df1.mean(),df1.min(),df1.max(),df1.median(),df1.std(),df2.mean(),df2.min(),df2.max(),df2.median(),df2.std(),dfm.mean(),dfm.min(),dfm.max(),dfm.median(),dfm.std()))
    else:
        return np.hstack((np.array([1.0]),np.zeros(16)))

def locationFeat_daily(ret_time_loc):
    for k,g in groupby(ret_time_loc,lambda x:x[0]):
        ret_time_loc_daily = list(g)
        posttimes_daily,coords_daily,probs_daily = zip(*ret_time_loc_daily)
        yield cluster_location_summary_daily(coords_daily,probs_daily)

def temporal_summary(posttimes):
    #use strings of dates as they can be sorted without conversion
    if len(posttimes)>1:
        dind = pd.to_datetime(sorted(posttimes))
        daterange = (dind[-1]-dind[0]).days#difference between first and last post in days
        df = Series(np.ones(len(posttimes)), index=dind)

        #1day -aggregate by day
        df1 = df.resample('d').sum().fillna(0.0)
        #2days -aggregate by 2-day window
        df2 = df1.resample("2d",closed='left').sum()
        #monthly -aggregate by month
        dfm = df1.resample("M",closed='left').sum()

        return np.array((0.0,daterange,df1.mean(),df1.min(),df1.max(),df1.median(),df1.std(),df2.mean(),df2.min(),df2.max(),df2.median(),df2.std(),dfm.mean(),dfm.min(),dfm.max(),dfm.median(),dfm.std()))
    else:
        return np.hstack((np.array([1.0]),np.zeros(16)))

def cluster_text_summary(textfeaturizer,text):
    '''
    create vector embedding for all text in a cluster of ads
    '''
    #Text features
    X = textfeaturizer.get_text_features(text)

    X = X.tocsc() #transform into column-sparse object
    mean_text = X.mean(axis=0) #mean for cluster
    #sqaure elements to then compute variance
    X.data**=2
    variance_text = X.mean(axis=0)-np.multiply(mean_text,mean_text)  #variance of each word
    agg_variance = np.sum(variance_text) #aggregate variance

    #create vector of text features for the cluster
    return np.hstack((np.squeeze(np.array(mean_text)),agg_variance))

def cluster_age_summary(ages,len_cluster):
    missing_age = len_cluster-len(ages) #number of ads that do not contain age
    ages = np.array(tuple(chain(*ages)))

    #vector that summarizes the age features of cluster
    return np.array((ages.mean(),ages.std(),np.median(ages),missing_age))

def cluster_location_summary(locations_webMer,loc_prob,len_cluster):
    if locations_webMer:
        num_found_loc = len(locations_webMer)
        missing_lat = len_cluster-num_found_loc
        missing_lat_proportion = float(missing_lat)/len_cluster

        locations_webMer=np.array(locations_webMer)

        #Extract mean of xaxis
        #centroid_cluster = np.mean(locations_webMer,0)
        centroid_cluster = np.average(locations_webMer,0,loc_prob)

        ###Measure the area the cluster covers
        min_coord = np.amin(locations_webMer,0)
        max_coord = np.amax(locations_webMer,0)

        dif_extreme_coord = np.subtract(max_coord,min_coord)
        area_cluster = dif_extreme_coord[0]*dif_extreme_coord[1]

        ###Measure how concentrated ads are
        #distance from each point to the centroid - add 1 to avoid errors when all ads are in same location

        sqdif_coord = np.zeros(shape=(num_found_loc,2))
        sqdif_coord[:,0] = np.square(np.abs(locations_webMer[:,0]-centroid_cluster[0]))
        sqdif_coord[:,1] = np.square(np.abs(locations_webMer[:,1]-centroid_cluster[1]))
        dist_centroid = np.sqrt(np.sum(sqdif_coord,1))

        disp_metric = np.sum(np.multiply(dist_centroid,loc_prob))/num_found_loc  ###alternatively we can sum over the log

        #return np.hstack((missing_lat_proportion,np.squeeze(centroid_cluster),area_cluster,disp_metric))
        return np.hstack((np.array([0.0]),missing_lat_proportion,area_cluster,disp_metric))
    else:
        return np.hstack((np.array([1.0]),np.zeros(3)))

def cluster_location_summary_daily(locations_webMer,loc_prob):
    num_loc = len(locations_webMer)
    locations_webMer=np.array(locations_webMer)
    #Extract mean of xaxis
    #centroid_cluster = np.mean(locations_webMer,0)
    centroid_cluster = np.average(locations_webMer,0,loc_prob)
    ###Measure the area the cluster covers
    min_coord = np.amin(locations_webMer,0)
    max_coord = np.amax(locations_webMer,0)

    dif_extreme_coord = np.subtract(max_coord,min_coord)
    area_cluster = dif_extreme_coord[0]*dif_extreme_coord[1]

    ###Measure how concentrated ads are
    #distance from each point to the centroid - add 1 to avoid errors when all ads are in same location

    sqdif_coord = np.zeros(shape=(num_loc,2))
    sqdif_coord[:,0] = np.square(np.abs(locations_webMer[:,0]-centroid_cluster[0]))
    sqdif_coord[:,1] = np.square(np.abs(locations_webMer[:,1]-centroid_cluster[1]))
    dist_centroid = np.sqrt(np.sum(sqdif_coord,1))

    disp_metric = np.sum(np.multiply(dist_centroid,loc_prob))/num_loc  ###alternatively we can sum over the log

    #return np.hstack((missing_lat_proportion,np.squeeze(centroid_cluster),area_cluster,disp_metric))
    return np.hstack((area_cluster,disp_metric))

def get_locations_and_time(cluster):
    for item in cluster:
        if u'lattice-postdatetime' in item[u'extractions']:
            posttime = item[u'extractions'][u'lattice-postdatetime'][u'results'][0][u'value']
        else:
            posttime = None
        if u'lattice-location' in item[u'extractions']:
            ret = tuple((coord,prob) for coord,prob in load_cities(item[u'extractions'][u'lattice-location'][u'results']))
            if ret:
                coords,probs = zip(*ret)
                coords = np.array(coords)
                probs = np.array(probs)
                weigavg_coords = np.average(coords,0,probs)
                mean_probs = probs.mean()
            else:
                weigavg_coords = None
                mean_probs = None
        else:
            weigavg_coords = None
            mean_probs = None
        yield posttime,weigavg_coords,mean_probs

def get_age(cluster):
    for item in cluster:
        if 'lattice-age' in item[u'extractions']:
            yield map(float,(d[u'value'] for d in item[u'extractions'][u'lattice-age'][u'results']))

def load_cities(iterable):
    for d in iterable:
        if u'city' in d[u'context']:
            yield transform(INPROJ,OUTPROJ,d[u'context'][u'city'][u'centroid_lon'],
                            d[u'context'][u'city'][u'centroid_lat']),d[u'probability']

#####What is below is old

def get_locations_and_time_old(cluster):
    for item in cluster:
        if u'lattice-postdatetime' in item[u'extractions']:
            posttime = item[u'extractions'][u'lattice-postdatetime'][u'results'][0][u'value']
        else:
            posttime = np.nan
        if u'lattice-location' in item[u'extractions']:
            ret = tuple((coord,prob) for coord,prob in load_cities(item[u'extractions'][u'lattice-location'][u'results']))
            if ret:
                coords,probs = zip(*ret)
                coords = np.array(coords)
                probs = np.array(probs)
                weigavg_coords = np.average(coords,0,probs)
                mean_probs = probs.mean()
            else:
                weigavg_coords = np.nan
                mean_probs = np.nan
        else:
            weigavg_coords = np.nan
            mean_probs = np.nan
        yield posttime,weigavg_coords,mean_probs

def get_static_features_old(lattice_cluster):
    ret_time_loc = list((time,coord,prob) for time,coord,prob in get_locations_and_time(lattice_cluster))
    ret_time_loc.sort(key=lambda x:x[0])#sort by date
    posttimes,coords,probs = zip(*ret_time_loc)

    posttimes_clean = [x for x in posttimes if str(x) != 'nan']
    coords_clean = [x for x in coords if not np.isnan(x).any()]
    probs_clean = [x for x in probs if str(x) != 'nan']
    ages = tuple(res for res in get_age(lattice_cluster))
    len_cluster = len(lattice_cluster)
    return np.hstack((cluster_age_summary(ages,len_cluster),cluster_location_summary(coords_clean,probs_clean,len_cluster),temporal_summary(posttimes_clean)))
#def get_coord(cluster):
#        for item in cluster:
#            if u'lattice-location' in item[u'extractions']:
#
#                tup = tuple(transform(INPROJ,OUTPROJ,d[u'context'][u'city'][u'centroid_lon'],
#                                                     d[u'context'][u'city'][u'centroid_lat'])
#                            for d in item[u'extractions'][u'lattice-location'][u'results']
#                                if d[u'context'][u'city'][u'centroid_lat'] is not None)
#                if tup:#add code to aggregate multiple locations into one
#                    yield tup[0]

<|endoftext|>