<commit_before># -*- coding: utf-8 -*-
"""
    lantz.foreign
    ~~~~~~~~~~~~~

    Implements classes and methods to interface to foreign functions.

    :copyright: 2012 by Lantz Authors, see AUTHORS for more details.
    :license: BSD, see LICENSE for more details.
"""

import ctypes
from ctypes.util import find_library
from itertools import chain

from lantz import Driver

class Library(object):
    """Library wrapper

    :param library: ctypes library
    :param wrapper: callable that takes two arguments the name of the function
                    and the function itself. It should return a callable.
    """

    def __init__(self, library, wrapper):
        if isinstance(library, str):
            self.library_name = library

            if library.lower().endswith('.dll'):
                library = ctypes.WinDLL(library)
            else:
                library = ctypes.CDLL(library)

        self.internal = library
        self.wrapper = wrapper

    def __getattr__(self, name):
        if name.startswith('__') and name.endswith('__'):
            raise AttributeError(name)
        func = self.wrapper(name, getattr(self.internal, name))
        setattr(self, name, func)
        return func

TYPES = {'c': ctypes.c_char,
         'b': ctypes.c_byte,
         'B': ctypes.c_ubyte,
         '?': ctypes.c_bool,
         'h': ctypes.c_short,
         'H': ctypes.c_ushort,
         'i': ctypes.c_int,
         'I': ctypes.c_uint,
         'l': ctypes.c_long,
         'L': ctypes.c_ulong,
         'q': ctypes.c_longlong,
         'Q': ctypes.c_ulonglong,
         'f': ctypes.c_float,
         'd': ctypes.c_double}

class RetStr(object):

    def __init__(self, length, encoding='ascii'):
        self.length = length
        self.buffer = ctypes.create_string_buffer(b'', length)
        self.encoding = encoding

    def __iter__(self):
        yield self
        yield self.length

    @property
    def value(self):
        if self.encoding:
            return self.buffer.value.decode(self.encoding)
        else:
            return self.buffer.value

class RetValue(object):

    def __init__(self, type):
        try:
            self.buffer = (TYPES[type] * 1)()
        except KeyError:
            raise KeyError('The type {} is not defined ()'.format(type, TYPES.keys()))

    def __iter__(self):
        yield self

    @property
    def value(self):
        return self.buffer[0]

class RetTuple(object):

    def __init__(self, type, length=1):
        try:
            self.buffer = (TYPES[type] * length)()
        except KeyError:
            raise KeyError('The type {} is not defined ()'.format(type, TYPES.keys()))
        self.length = length

    def __iter__(self):
        yield self
        yield self.length

    @property
    def value(self):
        return tuple(self.buffer[:])

class LibraryDriver(Driver):
    """Base class for drivers that communicate with instruments
    calling a library (dll or others)

    To use this class you must override LIBRARY_NAME
    """

    #:Name of the library
    LIBRARY_NAME = ''

    def __init__(self, *args, **kwargs):
        library_name = kwargs.pop('library_name', None)
        super().__init__(*args, **kwargs)

        for name in chain(iter_lib(library_name), iter_lib(self.LIBRARY_NAME)):
            if name is None:
                continue
            try:
                self.lib = Library(name, self._wrapper)
                self.log_debug('Loaded library {}')
                break
            except OSError:
                pass
        else:
            raise OSError('library not found')

        self.log_info('LibraryDriver created with {}', name)
        self._add_types()

    def _add_types(self):
        pass

    def _return_handler(self, func_name, ret_value):
        return ret_value

    def _wrapper(self, name, func):
        def _inner(*args):
            new_args = []
            collect = []
            for arg in args:
                if isinstance(arg, (RetStr, RetTuple, RetValue)):
                    collect.append(arg)
                    #new_args.append(ctypes.byref(arg.buffer))
                    new_args.append(arg.buffer)
                elif isinstance(arg, str):
                    new_args.append(bytes(arg, 'ascii'))
                else:
                    new_args.append(arg)

            ret = self._return_handler(name, func(*new_args))

            if collect:
                values = [item.value for item in collect]
                values.insert(0, ret)
                self.log_debug('Function call {} returned {}. Collected: {}', name, ret, collect)
                return tuple(values)

            self.log_debug('Function call {} returned {}.', name, ret)
            return ret

        return _inner

def iter_lib(library_name):
    if not library_name:
        raise StopIteration
    if isinstance(library_name, str):
        yield library_name
        yield find_library(library_name.split('.')[0])
    else:
        for name in library_name:
            yield name
            yield find_library(name.split('.')[0])
<commit_msg>We changed some options concerning the operating system.<commit_after># -*- coding: utf-8 -*-
"""
    lantz.foreign
    ~~~~~~~~~~~~~

    Implements classes and methods to interface to foreign functions.

    :copyright: 2012 by Lantz Authors, see AUTHORS for more details.
    :license: BSD, see LICENSE for more details.
"""

import os
import ctypes
from ctypes.util import find_library
from itertools import chain

from lantz import Driver

class Library(object):
    """Library wrapper

    :param library: ctypes library
    :param wrapper: callable that takes two arguments the name of the function
                    and the function itself. It should return a callable.
    """

    def __init__(self, library, wrapper):
        if isinstance(library, str):
            self.library_name = library

            if os.name == 'nt':
                library = ctypes.WinDLL(library)
            else:
                library = ctypes.CDLL(library)

        self.internal = library
        self.wrapper = wrapper

    def __getattr__(self, name):
        if name.startswith('__') and name.endswith('__'):
            raise AttributeError(name)
        func = self.wrapper(name, getattr(self.internal, name))
        setattr(self, name, func)
        return func

TYPES = {'c': ctypes.c_char,
         'b': ctypes.c_byte,
         'B': ctypes.c_ubyte,
         '?': ctypes.c_bool,
         'h': ctypes.c_short,
         'H': ctypes.c_ushort,
         'i': ctypes.c_int,
         'I': ctypes.c_uint,
         'l': ctypes.c_long,
         'L': ctypes.c_ulong,
         'q': ctypes.c_longlong,
         'Q': ctypes.c_ulonglong,
         'f': ctypes.c_float,
         'd': ctypes.c_double}

class RetStr(object):

    def __init__(self, length, encoding='ascii'):
        self.length = length
        self.buffer = ctypes.create_string_buffer(b'', length)
        self.encoding = encoding

    def __iter__(self):
        yield self
        yield self.length

    @property
    def value(self):
        if self.encoding:
            return self.buffer.value.decode(self.encoding)
        else:
            return self.buffer.value

class RetValue(object):

    def __init__(self, type):
        try:
            self.buffer = (TYPES[type] * 1)()
        except KeyError:
            raise KeyError('The type {} is not defined ()'.format(type, TYPES.keys()))

    def __iter__(self):
        yield self

    @property
    def value(self):
        return self.buffer[0]

class RetTuple(object):

    def __init__(self, type, length=1):
        try:
            self.buffer = (TYPES[type] * length)()
        except KeyError:
            raise KeyError('The type {} is not defined ()'.format(type, TYPES.keys()))
        self.length = length

    def __iter__(self):
        yield self
        yield self.length

    @property
    def value(self):
        return tuple(self.buffer[:])

class LibraryDriver(Driver):
    """Base class for drivers that communicate with instruments
    calling a library (dll or others)

    To use this class you must override LIBRARY_NAME
    """

    #:Name of the library
    LIBRARY_NAME = ''

    def __init__(self, *args, **kwargs):
        library_name = kwargs.pop('library_name', None)
        super().__init__(*args, **kwargs)

        for name in chain(iter_lib(library_name), iter_lib(self.LIBRARY_NAME)):
            if name is None:
                continue
            try:
                self.lib = Library(name, self._wrapper)
                self.log_debug('Loaded library {}')
                break
            except OSError:
                pass
        else:
            raise OSError('library not found')

        self.log_info('LibraryDriver created with {}', name)
        self._add_types()

    def _add_types(self):
        pass

    def _return_handler(self, func_name, ret_value):
        return ret_value

    def _wrapper(self, name, func):
        def _inner(*args):
            new_args = []
            collect = []
            for arg in args:
                if isinstance(arg, (RetStr, RetTuple, RetValue)):
                    collect.append(arg)
                    #new_args.append(ctypes.byref(arg.buffer))
                    new_args.append(arg.buffer)
                elif isinstance(arg, str):
                    new_args.append(bytes(arg, 'ascii'))
                else:
                    new_args.append(arg)

            ret = self._return_handler(name, func(*new_args))

            if collect:
                values = [item.value for item in collect]
                values.insert(0, ret)
                self.log_debug('Function call {} returned {}. Collected: {}', name, ret, collect)
                return tuple(values)

            self.log_debug('Function call {} returned {}.', name, ret)
            return ret

        return _inner

def iter_lib(library_name):
    if not library_name:
        raise StopIteration
    if isinstance(library_name, str):
        yield library_name
        yield find_library(library_name.split('.')[0])
    else:
        for name in library_name:
            yield name
            yield find_library(name.split('.')[0])
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright 2015 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Simian network backoff detection module."""

import logging
import platform
import re
import urlparse
import requests

from simian.mac.client import flight_common

LINUX = 'Linux'
DARWIN = 'Darwin'
PLATFORM = platform.system()

ROUTE = {LINUX: ['/sbin/ip', 'route'], DARWIN: ['/usr/sbin/netstat', '-nr']}
ARP = {LINUX: '/usr/sbin/arp', DARWIN: '/usr/sbin/arp'}
HOST = '/usr/bin/host'
IFCONFIG = '/sbin/ifconfig'
INTERFACE_ANDROID_WAP = 'android_wap'
INTERFACE_WWAN = 'wwan'
INTERFACE_VPN = 'vpn'

def _GetPlatform():
  """Returns a str like constants LINUX or DARWIN."""
  platform_str = platform.system()
  assert platform_str in [LINUX, DARWIN]
  return platform_str

def GetAllInterfaceNames():
  """Get network interfaces info for this host.

  Note that this list may include all types of interfaces
  that are not normally interesting to this script, e.g. fw0.

  Returns:
    list, e.g. ['en0', 'en1', 'fw0', 'eth0']
  """
  this_platform = _GetPlatform()

  # Note slight difference in regex.
  # BSD ifconfig writes "interface_name:\s+"
  # while Linux writes "interface_name\s+"
  if this_platform == LINUX:
    intf_header = re.compile(r'^([a-z]+(?:[0-9]+)?)\s+')
  elif this_platform == DARWIN:
    intf_header = re.compile(r'^([a-z]+(?:[0-9]+)?):\s+')

  return_code, stdout, stderr = flight_common.Exec('/sbin/ifconfig')
  if return_code != 0 or stderr:
    return []

  interfaces = []

  if stdout:
    for l in stdout.splitlines():  # pylint: disable=maybe-no-member
      m = intf_header.search(str(l))
      if m:
        interfaces.append(m.group(1))

  return interfaces

def GetInterfaceNames(interface_type):
  """Get the network interface names for an interface type.

  Args:
    interface_type: str, like INTERFACE_* constant
  Returns:
    list of str, like ['ppp0'] or ['en0', 'en1']
  Raises:
    ValueError: if interface_type is unknown
    PlatformError: if platform is not implemented
  """
  this_platform = _GetPlatform()
  all_interfaces = GetAllInterfaceNames()

  if interface_type == INTERFACE_WWAN:
    return [x for x in all_interfaces if x.startswith('ppp')
            or x.startswith('bnep')]
  elif interface_type == INTERFACE_ANDROID_WAP:
    if this_platform == DARWIN:
      return [x for x in all_interfaces if x.startswith('en')]
    elif this_platform == LINUX:
      return [x for x in all_interfaces if x.startswith('wlan')]
  elif interface_type == INTERFACE_VPN:
    if this_platform in [DARWIN, LINUX]:
      return [x for x in all_interfaces if x.endswith('tun0')]
    else:
      raise ValueError('Unknown Platform: %s' % this_platform)
  else:
    raise ValueError(interface_type)

def GetNetworkGateway(network):
  """Get the gateway for a network.

  Uses "netstat -nr" on Darwin and "ip route" on Linux to read the routing
  table.

  It searches for a route with destination exactly matching the network
  parameter!

  Args:
    network: str, likely in CIDR format or default gateway,
        e.g. "1.2.3/24" or "0.0.0.0"
  Returns:
    a string like "1.2.3.4" or "link#1" or "01:02:03:04:05:06" or
    "dev wlan0", depending on the type of route and platform.
  """
  route = ROUTE.get(_GetPlatform(), None)
  logging.debug('Route: %s', str(route))
  if not route:
    return

  try:
    return_code, stdout, stderr = flight_common.Exec(route)
  except OSError:
    return_code = None

  if return_code != 0 or stderr or not stdout:
    return

  gateway_pattern = (
      r'^%s\s+(via[\s\t])?'
      r'([\d\.]+|[0-9a-f:]+|link#\d+|dev [a-z\d]+)[\s\t]+' % network)
  gateway = re.search(gateway_pattern, str(stdout), re.MULTILINE)

  if gateway:
    return gateway.group(2)

  return

def GetDefaultGateway():
  """Gets the default gateway.

  Returns:
    a string like "192.168.0.1" or None if default gateway is unknown.
  """
  if _GetPlatform() in [DARWIN, LINUX]:
    default = 'default'
  else:
    logging.error('Unknown platform %s', _GetPlatform())

  return GetNetworkGateway(default)

def GetMacAddress(ip):
  """Returns the mac address in the arp table for an ip.

  Args:
    ip: str, like "1.2.3.4"
  Returns:
    a string like "00:01:02:0a:0b:cd" or None if the mac is unknown.
    Note that the padding of mac address digits might vary on each platform.
  """
  arp = ARP.get(_GetPlatform(), None)
  if not arp:
    return

  try:
    return_code, stdout, stderr = flight_common.Exec([arp, '-an'])
  except OSError:
    return_code = None

  if return_code != 0 or stderr or not stdout:
    return

  # Darwin:
  # ? (172.26.112.25) at 70:cd:60:aa:f9:80 on en0 ifscope [ethernet]
  # Linux:
  # ? (172.26.113.162) at 10:9a:dd:60:fb:46 [ether] on eth0
  #
  # one partial match will suffice for both.

  ip = ip.replace('.', '\\.')

  arp_entry = re.search(
      r'^\?\s+\(%s\)\s+at\s+([0-9a-f:]+)\s+' % ip, stdout, re.MULTILINE)

  if arp_entry:
    return arp_entry.group(1)

  return

def GetHttpResource(host, path='/', port=80, redir=False):
  """Gets HTTP resource.

  Args:
    host: str, like "example.com", but not "http://example.com".
    path: optional, str, like "/path", default "/".
    port: optional, int, default 80.
    redir: optional, bool, whether to follow redirects.
  Returns:
    (int response code, str response body)
    (int -1, str error from http exception)
  """
  if port != 80:
    port_str = ':%d' % port
  else:
    port_str = ''
  url = 'http://%s%s' % (host, port_str)
  url = urlparse.urljoin(url, path)
  try:
    response = requests.get(url, allow_redirects=redir)
    code = response.status_code
    body = response.text
    return code, body
  except requests.RequestException as e:
    return -1, str(e)

def IsOnWwan():
  """"Checks WWAN device connection status.

  Note: this may produce false-positives, and may not catch all WWAN
    devices.  Several Sprint and Verizon devices were tested, all of which
    create ppp0 upon connection.  However, L2TP VPN also creates ppp0
    (Google no longer uses this as of Q2-2010 in favor of SSLVPN).  A
    stronger check is probably needed at some point.

    As of 2011-12-6 OpenVPN interface is tun0 on Linux and Darwin.

  Returns:
    Boolean. True if WWAN device is active, False otherwise.
  """
  wwan_ifaces = GetInterfaceNames(INTERFACE_WWAN)
  for wwan_iface in wwan_ifaces:
    try:
      return_code, unused_out, unused_err = flight_common.Exec(
          [IFCONFIG, wwan_iface])
    except OSError:
      return_code = None

    # ifconfig exits with 1 if interface doesn't exist.
    if return_code == 0:
      return True

  return False

def GetNetworkName(dev):
  """Return network name (SSID for WLANs) seen by nmcli.

  If dev is set: Find network name (SSID) associated with device dev.
  Otherwise return first connected network name.

  Args:
    dev: network device name as seen by NetworkManager.
  Returns:
    name of the matching network name if possible, None otherwise.
  """
  cmdline = '/usr/bin/nmcli -t -f NAME,DEVICES conn status'
  # Ignore "Auto " prefix on automatically connecting networks.
  ssid_re = re.compile(r'^(Auto )?([^:]*):(.*)$')
  try:
    return_code, out, _ = flight_common.Exec(cmdline)
  except OSError:
    return_code = -1
  if out and not return_code:
    for l in out.splitlines():
      res = ssid_re.match(l)
      if res:
        if not dev or res.groups()[2] == dev:
          return res.groups()[1]
  return None

def IsOnAndroidWap():
  """Checks if Android WiFi or Bluetooth tethering is connected.

  Returns:
    Boolean. True if Android tethering is connected, False otherwise.
  """
  # ifconfig output looks a little bit different on Darwin vs Linux.
  #
  # Darwin:
  # inet 169.254.135.20 netmask 0xffff0000 broadcast 169.254.255.255
  # Linux:
  # inet addr:172.26.113.45  Bcast:172.26.115.255  Mask:255.255.252.0
  android_wap_match_regex = re.compile(
      r'inet[\w\s]*[\s:]+192\.168\.(42|43|44)\.\d{1,3}\s+'
      r'.*(?:netmask\s+0xffffff00\s+|Mask:255\.255\.255\.0)')

  ifaces = GetInterfaceNames(INTERFACE_ANDROID_WAP)

  for wifi_iface in ifaces:
    # Android tethering uses very specific subnets*, as well as dnsmasq which
    # reveals itself via the TXT VERSION.BIND record.
    # * 192.168.42.0/24 for wired, 192.168.43.0/24 for WiFi, and
    #   192.168.44.0/24 for Bluetooth.
    try:
      return_code, stdout, stderr = flight_common.Exec([IFCONFIG, wifi_iface])
    except OSError:
      return_code = None

    if return_code != 0 or stderr:  # interface was likely not found.
      continue

    android_wap_match = android_wap_match_regex.search(stdout)

    # Look for an interface on 192.168.4[2-4].0/24.
    if android_wap_match is not None:
      # If the default gateway is not through a likely Android WAN interface,
      # tethering may be active but is not likely to be used.
      default_gateway = GetDefaultGateway()
      logging.debug('Default gateway: %s', str(default_gateway))
      default_gateway_prefix = '192.168.%s.' % android_wap_match.group(1)
      if not default_gateway.startswith(default_gateway_prefix):
        return False

      # IP, netmask, gateway look like Android WAP, so check dnsmasq.

      # Request needs to be explicitly top level, as Linux uses
      # ndots:2 which would turn VERSION.BIND (without trailing dot) into
      # VERSION.BIND.foo.example.com in some cases.
      cmd = [HOST, '-W', '5', '-c', 'CHAOS', '-t', 'txt', 'VERSION.BIND.',
             default_gateway]
      try:
        return_code, stdout, unused_err = flight_common.Exec(cmd)
      except OSError:
        return_code = None
      if return_code != 0:
        continue
      dnsmasq_match = re.search(
          r'VERSION\.BIND descriptive text "dnsmasq-.*"', stdout)
      if dnsmasq_match is not None:
        # IP, netmask and dnsmasq all match Android WAP tethering.
        return True

  return False

def IsOnIosWap():
  """Checks if the wireless connection is to an iOS WAP tether.

  Returns:
    Boolean. True if iOS WAP is connected, False otherwise.
  """
  # iOS WAP looks like a 172.20.10/28 network. Gateway is
  # 172.20.10.1 with mac address ac:de:48:*.
  gateway = GetNetworkGateway('172.20.10/28')
  if not gateway:
    return False

  ip = GetDefaultGateway()
  if not ip:
    return False

  mac = GetMacAddress(ip)
  if not mac:
    return False

  if mac.startswith('ac:de:48:'):
    return True

  return False

def IsOnMifi():
  """Checks if the wireless connection is to a MiFi-like device.

  These devices are available from Verizon, Sprint, others, and usually
  offer some kind of web access portal that says MiFi or Jetpack as a text
  string.

  Returns:
    Bool, True if the connection is a likely MiFi-like device, False if not.
  """
  ip = GetDefaultGateway()
  if not ip:
    return False

  if ip.startswith('192.168.1.'):  # Verizon and Sprint devices
    http_status, body = GetHttpResource(ip, redir=True)
    # MiFi-like devices usually run a http interface. It returns a long http
    # response with various easily found "MiFi" or "Jetpack" strings in it
    # when loaded. No http auth challenge is issued.
    if http_status == 200 and body and ('MiFi' in body or 'Jetpack' in body):
      return True
  elif ip == '192.168.8.1':  # common Huawei gateway
    http_status, _ = GetHttpResource(ip, redir=False)
    return http_status == 307

  return False
<commit_msg>removed MAC address checking, added check for unique open port (62078) on iPhone gateway<commit_after>#!/usr/bin/env python
#
# Copyright 2015 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Simian network backoff detection module."""

import logging
import platform
import re
import urlparse
import requests
import socket

from simian.mac.client import flight_common

LINUX = 'Linux'
DARWIN = 'Darwin'
PLATFORM = platform.system()

ROUTE = {LINUX: ['/sbin/ip', 'route'], DARWIN: ['/usr/sbin/netstat', '-nr']}
ARP = {LINUX: '/usr/sbin/arp', DARWIN: '/usr/sbin/arp'}
HOST = '/usr/bin/host'
IFCONFIG = '/sbin/ifconfig'
INTERFACE_ANDROID_WAP = 'android_wap'
INTERFACE_WWAN = 'wwan'
INTERFACE_VPN = 'vpn'

def _GetPlatform():
  """Returns a str like constants LINUX or DARWIN."""
  platform_str = platform.system()
  assert platform_str in [LINUX, DARWIN]
  return platform_str

def GetAllInterfaceNames():
  """Get network interfaces info for this host.

  Note that this list may include all types of interfaces
  that are not normally interesting to this script, e.g. fw0.

  Returns:
    list, e.g. ['en0', 'en1', 'fw0', 'eth0']
  """
  this_platform = _GetPlatform()

  # Note slight difference in regex.
  # BSD ifconfig writes "interface_name:\s+"
  # while Linux writes "interface_name\s+"
  if this_platform == LINUX:
    intf_header = re.compile(r'^([a-z]+(?:[0-9]+)?)\s+')
  elif this_platform == DARWIN:
    intf_header = re.compile(r'^([a-z]+(?:[0-9]+)?):\s+')

  return_code, stdout, stderr = flight_common.Exec('/sbin/ifconfig')
  if return_code != 0 or stderr:
    return []

  interfaces = []

  if stdout:
    for l in stdout.splitlines():  # pylint: disable=maybe-no-member
      m = intf_header.search(str(l))
      if m:
        interfaces.append(m.group(1))

  return interfaces

def GetInterfaceNames(interface_type):
  """Get the network interface names for an interface type.

  Args:
    interface_type: str, like INTERFACE_* constant
  Returns:
    list of str, like ['ppp0'] or ['en0', 'en1']
  Raises:
    ValueError: if interface_type is unknown
    PlatformError: if platform is not implemented
  """
  this_platform = _GetPlatform()
  all_interfaces = GetAllInterfaceNames()

  if interface_type == INTERFACE_WWAN:
    return [x for x in all_interfaces if x.startswith('ppp')
            or x.startswith('bnep')]
  elif interface_type == INTERFACE_ANDROID_WAP:
    if this_platform == DARWIN:
      return [x for x in all_interfaces if x.startswith('en')]
    elif this_platform == LINUX:
      return [x for x in all_interfaces if x.startswith('wlan')]
  elif interface_type == INTERFACE_VPN:
    if this_platform in [DARWIN, LINUX]:
      return [x for x in all_interfaces if x.endswith('tun0')]
    else:
      raise ValueError('Unknown Platform: %s' % this_platform)
  else:
    raise ValueError(interface_type)

def GetNetworkGateway(network):
  """Get the gateway for a network.

  Uses "netstat -nr" on Darwin and "ip route" on Linux to read the routing
  table.

  It searches for a route with destination exactly matching the network
  parameter!

  Args:
    network: str, likely in CIDR format or default gateway,
        e.g. "1.2.3/24" or "0.0.0.0"
  Returns:
    a string like "1.2.3.4" or "link#1" or "01:02:03:04:05:06" or
    "dev wlan0", depending on the type of route and platform.
  """
  route = ROUTE.get(_GetPlatform(), None)
  logging.debug('Route: %s', str(route))
  if not route:
    return

  try:
    return_code, stdout, stderr = flight_common.Exec(route)
  except OSError:
    return_code = None

  if return_code != 0 or stderr or not stdout:
    return

  gateway_pattern = (
      r'^%s\s+(via[\s\t])?'
      r'([\d\.]+|[0-9a-f:]+|link#\d+|dev [a-z\d]+)[\s\t]+' % network)
  gateway = re.search(gateway_pattern, str(stdout), re.MULTILINE)

  if gateway:
    return gateway.group(2)

  return

def GetDefaultGateway():
  """Gets the default gateway.

  Returns:
    a string like "192.168.0.1" or None if default gateway is unknown.
  """
  if _GetPlatform() in [DARWIN, LINUX]:
    default = 'default'
  else:
    logging.error('Unknown platform %s', _GetPlatform())

  return GetNetworkGateway(default)

def GetHttpResource(host, path='/', port=80, redir=False):
  """Gets HTTP resource.

  Args:
    host: str, like "example.com", but not "http://example.com".
    path: optional, str, like "/path", default "/".
    port: optional, int, default 80.
    redir: optional, bool, whether to follow redirects.
  Returns:
    (int response code, str response body)
    (int -1, str error from http exception)
  """
  if port != 80:
    port_str = ':%d' % port
  else:
    port_str = ''
  url = 'http://%s%s' % (host, port_str)
  url = urlparse.urljoin(url, path)
  try:
    response = requests.get(url, allow_redirects=redir)
    code = response.status_code
    body = response.text
    return code, body
  except requests.RequestException as e:
    return -1, str(e)

def IsOnWwan():
  """"Checks WWAN device connection status.

  Note: this may produce false-positives, and may not catch all WWAN
    devices.  Several Sprint and Verizon devices were tested, all of which
    create ppp0 upon connection.  However, L2TP VPN also creates ppp0
    (Google no longer uses this as of Q2-2010 in favor of SSLVPN).  A
    stronger check is probably needed at some point.

    As of 2011-12-6 OpenVPN interface is tun0 on Linux and Darwin.

  Returns:
    Boolean. True if WWAN device is active, False otherwise.
  """
  wwan_ifaces = GetInterfaceNames(INTERFACE_WWAN)
  for wwan_iface in wwan_ifaces:
    try:
      return_code, unused_out, unused_err = flight_common.Exec(
          [IFCONFIG, wwan_iface])
    except OSError:
      return_code = None

    # ifconfig exits with 1 if interface doesn't exist.
    if return_code == 0:
      return True

  return False

def GetNetworkName(dev):
  """Return network name (SSID for WLANs) seen by nmcli.

  If dev is set: Find network name (SSID) associated with device dev.
  Otherwise return first connected network name.

  Args:
    dev: network device name as seen by NetworkManager.
  Returns:
    name of the matching network name if possible, None otherwise.
  """
  cmdline = '/usr/bin/nmcli -t -f NAME,DEVICES conn status'
  # Ignore "Auto " prefix on automatically connecting networks.
  ssid_re = re.compile(r'^(Auto )?([^:]*):(.*)$')
  try:
    return_code, out, _ = flight_common.Exec(cmdline)
  except OSError:
    return_code = -1
  if out and not return_code:
    for l in out.splitlines():
      res = ssid_re.match(l)
      if res:
        if not dev or res.groups()[2] == dev:
          return res.groups()[1]
  return None

def IsOnAndroidWap():
  """Checks if Android WiFi or Bluetooth tethering is connected.

  Returns:
    Boolean. True if Android tethering is connected, False otherwise.
  """
  # ifconfig output looks a little bit different on Darwin vs Linux.
  #
  # Darwin:
  # inet 169.254.135.20 netmask 0xffff0000 broadcast 169.254.255.255
  # Linux:
  # inet addr:172.26.113.45  Bcast:172.26.115.255  Mask:255.255.252.0
  android_wap_match_regex = re.compile(
      r'inet[\w\s]*[\s:]+192\.168\.(42|43|44)\.\d{1,3}\s+'
      r'.*(?:netmask\s+0xffffff00\s+|Mask:255\.255\.255\.0)')

  ifaces = GetInterfaceNames(INTERFACE_ANDROID_WAP)

  for wifi_iface in ifaces:
    # Android tethering uses very specific subnets*, as well as dnsmasq which
    # reveals itself via the TXT VERSION.BIND record.
    # * 192.168.42.0/24 for wired, 192.168.43.0/24 for WiFi, and
    #   192.168.44.0/24 for Bluetooth.
    try:
      return_code, stdout, stderr = flight_common.Exec([IFCONFIG, wifi_iface])
    except OSError:
      return_code = None

    if return_code != 0 or stderr:  # interface was likely not found.
      continue

    android_wap_match = android_wap_match_regex.search(stdout)

    # Look for an interface on 192.168.4[2-4].0/24.
    if android_wap_match is not None:
      # If the default gateway is not through a likely Android WAN interface,
      # tethering may be active but is not likely to be used.
      default_gateway = GetDefaultGateway()
      logging.debug('Default gateway: %s', str(default_gateway))
      default_gateway_prefix = '192.168.%s.' % android_wap_match.group(1)
      if not default_gateway.startswith(default_gateway_prefix):
        return False

      # IP, netmask, gateway look like Android WAP, so check dnsmasq.

      # Request needs to be explicitly top level, as Linux uses
      # ndots:2 which would turn VERSION.BIND (without trailing dot) into
      # VERSION.BIND.foo.example.com in some cases.
      cmd = [HOST, '-W', '5', '-c', 'CHAOS', '-t', 'txt', 'VERSION.BIND.',
             default_gateway]
      try:
        return_code, stdout, unused_err = flight_common.Exec(cmd)
      except OSError:
        return_code = None
      if return_code != 0:
        continue
      dnsmasq_match = re.search(
          r'VERSION\.BIND descriptive text "dnsmasq-.*"', stdout)
      if dnsmasq_match is not None:
        # IP, netmask and dnsmasq all match Android WAP tethering.
        return True

  return False

def IsOnIosWap():
  """Checks if the wireless connection is to an iOS WAP tether.

  Returns:
    Boolean. True if iOS WAP is connected, False otherwise.
  """
  # iOS WAP looks like a 172.20.10/28 network. Gateway is
  # 172.20.10.1 with tcp port 62078 open.
  gateway = GetNetworkGateway('172.20.10/28')
  if not gateway:
    return False

  ip = GetDefaultGateway()
  if ip != '172.20.10.1':
    return False

  sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  result = sock.connect_ex((ip, 62078))
  if result == 0:
    return True

  return False

def IsOnMifi():
  """Checks if the wireless connection is to a MiFi-like device.

  These devices are available from Verizon, Sprint, others, and usually
  offer some kind of web access portal that says MiFi or Jetpack as a text
  string.

  Returns:
    Bool, True if the connection is a likely MiFi-like device, False if not.
  """
  ip = GetDefaultGateway()
  if not ip:
    return False

  if ip.startswith('192.168.1.'):  # Verizon and Sprint devices
    http_status, body = GetHttpResource(ip, redir=True)
    # MiFi-like devices usually run a http interface. It returns a long http
    # response with various easily found "MiFi" or "Jetpack" strings in it
    # when loaded. No http auth challenge is issued.
    if http_status == 200 and body and ('MiFi' in body or 'Jetpack' in body):
      return True
  elif ip == '192.168.8.1':  # common Huawei gateway
    http_status, _ = GetHttpResource(ip, redir=False)
    return http_status == 307

  return False
<|endoftext|>


<commit_before>#!/usr/bin/python
# coding=utf-8

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import os
import csv
import codecs
import sys

def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs):
        csv_reader = csv.reader(utf8_data, dialect=dialect, **kwargs)
        for row in csv_reader:
            yield [unicode(cell, 'utf-8') for cell in row]

def remove_first_line_from_file(authors_file_name):
    with codecs.open(authors_file_name, 'r', encoding='utf-8') as fin:
        data = fin.read().splitlines(True)
    with codecs.open(authors_file_name, 'w', encoding='utf-8') as fout:
        fout.writelines(data[1:])

def assure_path_exists(path):
        dir = os.path.dirname(path)
        if not os.path.exists(dir):
                os.makedirs(dir)

def convertCaptchatoBase64(drive):
    imgBase64 = drive.execute_script("""
    var img = window.document.getElementById('image_captcha');
    // Create an empty canvas element
    var canvas = window.document.createElement("canvas");
    canvas.width = img.width;
    canvas.height = img.height;

    // Copy the image contents to the canvas
    var ctx = canvas.getContext("2d");
    ctx.drawImage(img, 0, 0);

    var dataURL = canvas.toDataURL("image/png");

    return dataURL.replace(/^data:image\/(png|jpg);base64,/, "");
    """)
    return imgBase64

if __name__ == '__main__':

    if (len(sys.argv)>1):
        prefix_file_name = sys.argv[1]
        authors_file_name = prefix_file_name+'_authors.csv'
        downloaded_file_name = prefix_file_name+'_downloaded.csv'
        error_file_name = prefix_file_name+'_error.csv'
        not_found_file_name = prefix_file_name+'_not_found.csv'
    else:
        authors_file_name = 'authors.csv'
        downloaded_file_name = 'downloaded.csv'
        error_file_name = 'error.csv'
        not_found_file_name = 'not_found.csv'

    try:
        assure_path_exists(os.getcwd()+'/data')
    except:
        sys.exit('Erro ao criar o diretório '+os.getcwd()+'/data')

    ids_downloaded = os.listdir(os.getcwd()+'/data')

    profile = webdriver.FirefoxProfile()
    profile.set_preference('permissions.default.stylesheet', 2)
    profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'false')
    profile.set_preference('browser.download.dir', os.getcwd()+'/data')
    profile.set_preference('browser.download.folderList', 2)
    profile.set_preference('browser.helperApps.neverAsk.saveToDisk', 'application/zip')
    driver = webdriver.Firefox(firefox_profile=profile)
    driver.implicitly_wait(30)
    base_url = 'http://buscatextual.cnpq.br/buscatextual'

    while os.stat(authors_file_name).st_size != 0:
        try:
            csv_reader = unicode_csv_reader(open(authors_file_name))
        except:
            sys.exit('Erro ao abrir o arquivo '+authors_file_name)

        row = next(csv_reader)
        author_name = row[0]
        paper_title = row[1].encode('utf-8')

        driver.get(base_url)
        driver.find_element_by_id('textoBusca').clear()
        driver.find_element_by_id('textoBusca').send_keys(author_name)
        driver.find_element_by_id('buscarDemais').click()
        driver.find_element_by_id('botaoBuscaFiltros').click()

        if not 'Nenhum resultado foi encontrado para:' in driver.page_source.encode('utf-8'):
            resultado_geral = driver.find_element_by_class_name('resultado')
            resultados = resultado_geral.find_elements_by_tag_name('a')
            ids = []

            for resultado in resultados:
                ids.append(resultado.get_attribute('href')[24:34])

            paper_found = False
            for id in ids:
                if not paper_found:
                    print(author_name.encode('utf-8')+' - '+paper_title)
                    print(base_url+'/visualizacv.do?id='+id)
                    driver.get(base_url+'/visualizacv.do?id='+id)

                    # QUEBRAR CAPTCHA #
                    imgBase64 = convertCaptchatoBase64(drive)
                    params = urllib.urlencode({'imgBase64': imgBase64})
                    headers = {"Content-type": "application/x-www-form-urlencoded","Accept": "text/plain"}
                    conn = httplib.HTTPConnection("")
                    conn.request("POST", "", params, headers)
                    response = conn.getresponse()
                    print response.status, response.reason

                    driver.implicitly_wait(31622400)
                    informacoes_autor = driver.find_element_by_class_name('informacoes-autor')
                    page_source = driver.page_source.encode('utf-8')
                    if paper_title.lower().replace(' ', '') in page_source.lower().replace(' ', ''):
                        paper_found = True
                        id_autor_xml = informacoes_autor.find_element_by_tag_name('li')
                        id_autor_xml = id_autor_xml.text[-16:]

                        if not (id_autor_xml+'.zip') in ids_downloaded:
                            print(base_url+'/download.do?idcnpq='+id_autor_xml)
                            driver.get(base_url+'/download.do?idcnpq='+id_autor_xml)

                            # QUEBRAR CAPTCHA #
                            WebDriverWait(driver, 31622400).until(EC.invisibility_of_element_located((By.ID, 'btn_validar_captcha')))
                        ids_downloaded.append(id_autor_xml+'.zip')

                        with open(downloaded_file_name, 'a') as downloaded_file:
                            try:
                                downloaded_file.write('"'+author_name.encode('utf-8')+'","'+paper_title+'"\n')
                            except:
                                driver.quit()
                                sys.exit('Erro ao escrever no arquivo '+downloaded_file_name)
                        break

            if not paper_found:
                with open(error_file_name, 'a') as error_file:
                    try:
                        error_file.write('"'+author_name.encode('utf-8')+'","'+paper_title+'"\n')
                    except:
                        driver.quit()
                        sys.exit('Erro ao escrever no arquivo '+error_file_name)

        else:
            with open(not_found_file_name, 'a') as not_found_file:
                try:
                    not_found_file.write('"'+author_name.encode('utf-8')+'","'+paper_title+'"\n')
                except:
                    driver.quit()
                    sys.exit('Erro ao escrever no arquivo '+not_found_file_name)

        try:
            remove_first_line_from_file(authors_file_name)
        except:
            driver.quit()
            sys.exit('Erro ao tentar remover a primeira linha do arquivo '+authors_file_name)

    driver.quit()
    print ('\n\nOK!\n')
<commit_msg>Consuming service to broke captcha<commit_after>#!/usr/bin/python
# coding=utf-8

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import os
import csv
import codecs
import sys
import urllib
import httplib
import requests
import json

def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs):
        csv_reader = csv.reader(utf8_data, dialect=dialect, **kwargs)
        for row in csv_reader:
            yield [unicode(cell, 'utf-8') for cell in row]

def remove_first_line_from_file(authors_file_name):
    with codecs.open(authors_file_name, 'r', encoding='utf-8') as fin:
        data = fin.read().splitlines(True)
    with codecs.open(authors_file_name, 'w', encoding='utf-8') as fout:
        fout.writelines(data[1:])

def assure_path_exists(path):
        dir = os.path.dirname(path)
        if not os.path.exists(dir):
                os.makedirs(dir)

def convertCaptchatoBase64(drive):
    imgBase64 = drive.execute_script("""
    var img = window.document.getElementById('image_captcha');
    // Create an empty canvas element
    var canvas = window.document.createElement("canvas");
    canvas.width = img.width;
    canvas.height = img.height;

    // Copy the image contents to the canvas
    var ctx = canvas.getContext("2d");
    ctx.drawImage(img, 0, 0);

    var dataURL = canvas.toDataURL("image/png");

    return dataURL.replace(/^data:image\/(png|jpg);base64,/, "");
    """)
    return imgBase64

if __name__ == '__main__':

    if (len(sys.argv)>1):
        prefix_file_name = sys.argv[1]
        authors_file_name = prefix_file_name+'_authors.csv'
        downloaded_file_name = prefix_file_name+'_downloaded.csv'
        error_file_name = prefix_file_name+'_error.csv'
        not_found_file_name = prefix_file_name+'_not_found.csv'
    else:
        authors_file_name = 'authors.csv'
        downloaded_file_name = 'downloaded.csv'
        error_file_name = 'error.csv'
        not_found_file_name = 'not_found.csv'

    try:
        assure_path_exists(os.getcwd()+'/data')
    except:
        sys.exit('Erro ao criar o diretório '+os.getcwd()+'/data')

    ids_downloaded = os.listdir(os.getcwd()+'/data')

    profile = webdriver.FirefoxProfile()
    profile.set_preference('permissions.default.stylesheet', 2)
    profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'false')
    profile.set_preference('browser.download.dir', os.getcwd()+'/data')
    profile.set_preference('browser.download.folderList', 2)
    profile.set_preference('browser.helperApps.neverAsk.saveToDisk', 'application/zip')
    driver = webdriver.Firefox(firefox_profile=profile)
    driver.implicitly_wait(3)
    base_url = 'http://buscatextual.cnpq.br/buscatextual'

    while os.stat(authors_file_name).st_size != 0:
        try:
            csv_reader = unicode_csv_reader(open(authors_file_name))
        except:
            sys.exit('Erro ao abrir o arquivo '+authors_file_name)

        row = next(csv_reader)
        author_name = row[0]
        paper_title = row[1].encode('utf-8')

        driver.get(base_url)
        driver.find_element_by_id('textoBusca').clear()
        driver.find_element_by_id('textoBusca').send_keys(author_name)
        driver.find_element_by_id('buscarDemais').click()
        driver.find_element_by_id('botaoBuscaFiltros').click()

        if not 'Nenhum resultado foi encontrado para:' in driver.page_source.encode('utf-8'):
            resultado_geral = driver.find_element_by_class_name('resultado')
            resultados = resultado_geral.find_elements_by_tag_name('a')
            ids = []

            for resultado in resultados:
                ids.append(resultado.get_attribute('href')[24:34])

            paper_found = False
            for id in ids:
                if not paper_found:
                    print(author_name.encode('utf-8')+' - '+paper_title)
                    print(base_url+'/visualizacv.do?id='+id)
                    driver.get(base_url+'/visualizacv.do?id='+id)

                    # QUEBRAR CAPTCHA #

                    button = driver.find_element_by_id('btn_validar_captcha')
                    while True:

                        print button.is_displayed()
                        imgBase64 = convertCaptchatoBase64(driver)

                        r = requests.post('http://192.168.200.84:8080/CaptchaService/captcha', data = json.dumps({'imgBase64':imgBase64}))

                        r.encoding = "UTF-8"
                        jsonCaptcha = r.json()

                        driver.find_element_by_id('informado').clear()
                        driver.find_element_by_id('informado').send_keys(jsonCaptcha['captcha'])
                        driver.find_element_by_id('btn_validar_captcha').click()
                        try:
                            page_source = driver.page_source.encode('utf-8')
                            informacoes_autor = driver.find_element_by_class_name('informacoes-autor')
                            break
                        except:
                            print "Tentar Novamente"
                    print "Quebrou"

                    if paper_title.lower().replace(' ', '') in page_source.lower().replace(' ', ''):
                        paper_found = True
                        id_autor_xml = informacoes_autor.find_element_by_tag_name('li')
                        id_autor_xml = id_autor_xml.text[-16:]

                        if not (id_autor_xml+'.zip') in ids_downloaded:
                            print(base_url+'/download.do?idcnpq='+id_autor_xml)
                            driver.get(base_url+'/download.do?idcnpq='+id_autor_xml)
                            while driver.findElement(By.ID("btn_validar_captcha")):

                                imgBase64 = convertCaptchatoBase64(driver)

                                r = requests.post('http://192.168.200.84:8080/CaptchaService/captcha', data = json.dumps({'imgBase64':imgBase64}))

                                r.encoding = "UTF-8"
                                jsonCaptcha = r.json()

                                driver.find_element_by_id('informado').clear()
                                driver.find_element_by_id('informado').send_keys(jsonCaptcha['captcha'])
                                driver.find_element_by_id('btn_validar_captcha').click()

                                ids_downloaded.append(id_autor_xml+'.zip')
                                WebDriverWait(driver, 1)
                        with open(downloaded_file_name, 'a') as downloaded_file:
                            try:
                                downloaded_file.write('"'+author_name.encode('utf-8')+'","'+paper_title+'"\n'                                                               )
                            except:
                                driver.quit()
                                sys.exit('Erro ao escrever no arquivo '+downloaded_file_name)
                        break

            if not paper_found:
                with open(error_file_name, 'a') as error_file:
                    try:
                        error_file.write('"'+author_name.encode('utf-8')+'","'+paper_title+'"\n')
                    except:
                        driver.quit()
                        sys.exit('Erro ao escrever no arquivo '+error_file_name)

        else:
            with open(not_found_file_name, 'a') as not_found_file:
                try:
                    not_found_file.write('"'+author_name.encode('utf-8')+'","'+paper_title+'"\n')
                except:
                    driver.quit()
                    sys.exit('Erro ao escrever no arquivo '+not_found_file_name)

        try:
            remove_first_line_from_file(authors_file_name)
        except:
            driver.quit()
            sys.exit('Erro ao tentar remover a primeira linha do arquivo '+authors_file_name)

    driver.quit()
    print ('\n\nOK!\n')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
Helpful decorators to use with tasks.

Example usage:

.. code-block:: python

    class MyTask(law.Task):

        @log
        @safe_output(skip=KeyboardInterrupt)
        def run(self):
            ...

The usage of a decorator without invocation (e.g. ``@log``) is equivalent to the one *with*
invocation (``@log()``), for law to distuinguish between the two cases **always** use keyword
arguments when configuring decorators. Default arguments are applied in either case.
"""

__all__ = ["factory", "log", "safe_output", "delay", "notify"]

import sys
import time
import traceback
import functools
import random
import socket
import collections
import logging

import luigi

from law.task.base import ProxyTask
from law.parameter import get_param, NotifyParameter
from law.target.local import LocalFileTarget
from law.util import human_time_diff, TeeStream

logger = logging.getLogger(__name__)

def factory(**default_opts):
    """
    Factory function to create decorators for tasks's run methods. Default options for the decorator
    function can be given in *default_opts*. The returned decorator can be used with or without
    actual invocation. Example:

    .. code-block:: python

        @factory(digits=2)
        def runtime(fn, opts, task, *args, **kwargs):
            t0 = time.time()
            try:
                return fn(task, *args, **kwargs)
            finally:
                t1 = time.time()
                diff = round(t1 - t0, opts["digits"])
                print("runtime:")
                print(diff)

        ...

        class MyTask(law.Task):

            @runtime
            def run(self):
                ...

            # or
            @runtime(digits=3):
            def run(self):
                ...

    .. note::

        Decorators might not have the expected behavior when used to decorate generator functions
        such as ``Task.run()`` methods that yield dynamic dependency.
    """
    def wrapper(decorator):
        @functools.wraps(decorator)
        def wrapper(fn=None, **opts):
            _opts = default_opts.copy()
            _opts.update(opts)

            def wrapper(fn):
                @functools.wraps(fn)
                def wrapper(*args, **kwargs):
                    return decorator(fn, _opts, *args, **kwargs)
                return wrapper

            return wrapper if fn is None else wrapper(fn)
        return wrapper
    return wrapper

def get_task(task):
    return task if not isinstance(task, ProxyTask) else task.task

@factory()
def log(fn, opts, task, *args, **kwargs):
    """ log()
    Wraps a bound method of a task and redirects output of both stdin and stdout to the file
    defined by the tasks's *log* parameter. It its value is ``"-"`` or *None*, the output is not
    redirected.
    """
    _task = get_task(task)
    log = get_param(_task.log_file, _task.default_log_file)

    if log == "-" or not log:
        return fn(task, *args, **kwargs)
    else:
        # use the local target functionality to create the parent directory
        LocalFileTarget(log).parent.touch()
        with open(log, "a", 1) as f:
            tee = TeeStream(f, sys.__stdout__)
            sys.stdout = tee
            sys.stderr = tee
            try:
                ret = fn(task, *args, **kwargs)
            except Exception as e:
                traceback.print_exc(file=tee)
                raise e
            finally:
                sys.stdout = sys.__stdout__
                sys.stderr = sys.__stderr__
                tee.flush()
        return ret

@factory(skip=None)
def safe_output(fn, opts, task, *args, **kwargs):
    """ safe_output(skip=None)
    Wraps a bound method of a task and guards its execution. If an exception occurs, and it is not
    an instance of *skip*, the task's output is removed prior to the actual raising.
    """
    try:
        return fn(task, *args, **kwargs)
    except Exception as e:
        if opts["skip"] is None or not isinstance(e, opts["skip"]):
            for outp in luigi.task.flatten(task.output()):
                outp.remove()
        raise

@factory(t=5, stddev=0, pdf="gauss")
def delay(fn, opts, task, *args, **kwargs):
    """ delay(t=5, stddev=0., pdf="gauss")
    Wraps a bound method of a task and delays its execution by *t* seconds.
    """
    if opts["stddev"] <= 0:
        t = opts["t"]
    elif opts["pdf"] == "gauss":
        t = random.gauss(opts["t"], opts["stddev"])
    elif opts["pdf"] == "uniform":
        t = random.uniform(opts["t"], opts["stddev"])
    else:
        raise ValueError("unknown delay decorator pdf '{}'".format(opts["pdf"]))

    time.sleep(t)

    return fn(task, *args, **kwargs)

@factory(on_success=True, on_failure=True)
def notify(fn, opts, task, *args, **kwargs):
    """ notify(on_success=True, on_failure=True, **kwargs)
    Wraps a bound method of a task and guards its execution. Information about the execution (task
    name, duration, etc) is collected and dispatched to all notification transports registered on
    wrapped task via adding :py:class:`law.NotifyParameter` parameters. Example:

    .. code-block:: python

        class MyTask(law.Task):

            notify_mail = law.NotifyMailParameter()

            @notify
            # or
            @notify(sender="foo@bar.com", recipient="user@host.tld")
            def run(self):
                ...

    When the *notify_mail* parameter is *True*, a notification is sent to the configured email
    address. Also see :ref:`config-notifications`.
    """
    _task = get_task(task)

    # get notification transports
    transports = []
    for param_name, param in _task.get_params():
        if isinstance(param, NotifyParameter) and getattr(_task, param_name):
            try:
                transport = param.get_transport()
                if transport:
                    transports.append(transport)
            except Exception as e:
                logger.warning("get_transport() failed for '{}' parameter: {}".format(
                    param_name, e))

    # nothing to do when there is no transport
    if not transports:
        return fn(task, *args, **kwargs)

    # guard the fn call and gather infos
    error = None
    t0 = time.time()
    try:
        return fn(task, *args, **kwargs)
    except (Exception, KeyboardInterrupt) as e:
        error = e
        raise
    finally:
        success = error is None
        # do nothing on KeyboardInterrupt, or when on_success / on_failure do not match the status
        if isinstance(error, KeyboardInterrupt):
            return
        elif success and not opts["on_success"]:
            return
        elif not success and not opts["on_failure"]:
            return

        duration = human_time_diff(seconds=round(time.time() - t0, 1))
        status_string = "succeeded" if success else "failed"
        title = "Task {} {}!".format(_task.get_task_family(), status_string)
        parts = collections.OrderedDict([
            ("Host", socket.gethostname()),
            ("Duration", duration),
            ("Last message", "-" if not len(_task._message_cache) else _task._message_cache[-1]),
            ("Task", str(_task)),
        ])
        if not success:
            parts["Traceback"] = traceback.format_exc()
        message = "\n".join("{}: {}".format(*tpl) for tpl in parts.items())

        # dispatch via all transports
        for transport in transports:
            fn = transport["func"]
            raw = transport.get("raw", False)
            try:
                fn(success, title, parts if raw else message, **opts)
            except Exception as e:
                t = traceback.format_exc()
                logger.warning("notification failed via transport '{}': {}\n{}".format(fn, e, t))
<commit_msg>Fix re-raising in log decorator.<commit_after># -*- coding: utf-8 -*-

"""
Helpful decorators to use with tasks.

Example usage:

.. code-block:: python

    class MyTask(law.Task):

        @log
        @safe_output(skip=KeyboardInterrupt)
        def run(self):
            ...

The usage of a decorator without invocation (e.g. ``@log``) is equivalent to the one *with*
invocation (``@log()``), for law to distuinguish between the two cases **always** use keyword
arguments when configuring decorators. Default arguments are applied in either case.
"""

__all__ = ["factory", "log", "safe_output", "delay", "notify"]

import sys
import time
import traceback
import functools
import random
import socket
import collections
import logging

import luigi

from law.task.base import ProxyTask
from law.parameter import get_param, NotifyParameter
from law.target.local import LocalFileTarget
from law.util import human_time_diff, TeeStream

logger = logging.getLogger(__name__)

def factory(**default_opts):
    """
    Factory function to create decorators for tasks's run methods. Default options for the decorator
    function can be given in *default_opts*. The returned decorator can be used with or without
    actual invocation. Example:

    .. code-block:: python

        @factory(digits=2)
        def runtime(fn, opts, task, *args, **kwargs):
            t0 = time.time()
            try:
                return fn(task, *args, **kwargs)
            finally:
                t1 = time.time()
                diff = round(t1 - t0, opts["digits"])
                print("runtime:")
                print(diff)

        ...

        class MyTask(law.Task):

            @runtime
            def run(self):
                ...

            # or
            @runtime(digits=3):
            def run(self):
                ...

    .. note::

        Decorators might not have the expected behavior when used to decorate generator functions
        such as ``Task.run()`` methods that yield dynamic dependency.
    """
    def wrapper(decorator):
        @functools.wraps(decorator)
        def wrapper(fn=None, **opts):
            _opts = default_opts.copy()
            _opts.update(opts)

            def wrapper(fn):
                @functools.wraps(fn)
                def wrapper(*args, **kwargs):
                    return decorator(fn, _opts, *args, **kwargs)
                return wrapper

            return wrapper if fn is None else wrapper(fn)
        return wrapper
    return wrapper

def get_task(task):
    return task if not isinstance(task, ProxyTask) else task.task

@factory()
def log(fn, opts, task, *args, **kwargs):
    """ log()
    Wraps a bound method of a task and redirects output of both stdin and stdout to the file
    defined by the tasks's *log* parameter. It its value is ``"-"`` or *None*, the output is not
    redirected.
    """
    _task = get_task(task)
    log = get_param(_task.log_file, _task.default_log_file)

    if log == "-" or not log:
        return fn(task, *args, **kwargs)
    else:
        # use the local target functionality to create the parent directory
        LocalFileTarget(log).parent.touch()
        with open(log, "a", 1) as f:
            tee = TeeStream(f, sys.__stdout__)
            sys.stdout = tee
            sys.stderr = tee
            try:
                ret = fn(task, *args, **kwargs)
            except:
                traceback.print_exc(file=tee)
                raise
            finally:
                sys.stdout = sys.__stdout__
                sys.stderr = sys.__stderr__
                tee.flush()
        return ret

@factory(skip=None)
def safe_output(fn, opts, task, *args, **kwargs):
    """ safe_output(skip=None)
    Wraps a bound method of a task and guards its execution. If an exception occurs, and it is not
    an instance of *skip*, the task's output is removed prior to the actual raising.
    """
    try:
        return fn(task, *args, **kwargs)
    except Exception as e:
        if opts["skip"] is None or not isinstance(e, opts["skip"]):
            for outp in luigi.task.flatten(task.output()):
                outp.remove()
        raise

@factory(t=5, stddev=0, pdf="gauss")
def delay(fn, opts, task, *args, **kwargs):
    """ delay(t=5, stddev=0., pdf="gauss")
    Wraps a bound method of a task and delays its execution by *t* seconds.
    """
    if opts["stddev"] <= 0:
        t = opts["t"]
    elif opts["pdf"] == "gauss":
        t = random.gauss(opts["t"], opts["stddev"])
    elif opts["pdf"] == "uniform":
        t = random.uniform(opts["t"], opts["stddev"])
    else:
        raise ValueError("unknown delay decorator pdf '{}'".format(opts["pdf"]))

    time.sleep(t)

    return fn(task, *args, **kwargs)

@factory(on_success=True, on_failure=True)
def notify(fn, opts, task, *args, **kwargs):
    """ notify(on_success=True, on_failure=True, **kwargs)
    Wraps a bound method of a task and guards its execution. Information about the execution (task
    name, duration, etc) is collected and dispatched to all notification transports registered on
    wrapped task via adding :py:class:`law.NotifyParameter` parameters. Example:

    .. code-block:: python

        class MyTask(law.Task):

            notify_mail = law.NotifyMailParameter()

            @notify
            # or
            @notify(sender="foo@bar.com", recipient="user@host.tld")
            def run(self):
                ...

    When the *notify_mail* parameter is *True*, a notification is sent to the configured email
    address. Also see :ref:`config-notifications`.
    """
    _task = get_task(task)

    # get notification transports
    transports = []
    for param_name, param in _task.get_params():
        if isinstance(param, NotifyParameter) and getattr(_task, param_name):
            try:
                transport = param.get_transport()
                if transport:
                    transports.append(transport)
            except Exception as e:
                logger.warning("get_transport() failed for '{}' parameter: {}".format(
                    param_name, e))

    # nothing to do when there is no transport
    if not transports:
        return fn(task, *args, **kwargs)

    # guard the fn call and gather infos
    error = None
    t0 = time.time()
    try:
        return fn(task, *args, **kwargs)
    except (Exception, KeyboardInterrupt) as e:
        error = e
        raise
    finally:
        success = error is None
        # do nothing on KeyboardInterrupt, or when on_success / on_failure do not match the status
        if isinstance(error, KeyboardInterrupt):
            return
        elif success and not opts["on_success"]:
            return
        elif not success and not opts["on_failure"]:
            return

        duration = human_time_diff(seconds=round(time.time() - t0, 1))
        status_string = "succeeded" if success else "failed"
        title = "Task {} {}!".format(_task.get_task_family(), status_string)
        parts = collections.OrderedDict([
            ("Host", socket.gethostname()),
            ("Duration", duration),
            ("Last message", "-" if not len(_task._message_cache) else _task._message_cache[-1]),
            ("Task", str(_task)),
        ])
        if not success:
            parts["Traceback"] = traceback.format_exc()
        message = "\n".join("{}: {}".format(*tpl) for tpl in parts.items())

        # dispatch via all transports
        for transport in transports:
            fn = transport["func"]
            raw = transport.get("raw", False)
            try:
                fn(success, title, parts if raw else message, **opts)
            except Exception as e:
                t = traceback.format_exc()
                logger.warning("notification failed via transport '{}': {}\n{}".format(fn, e, t))
<|endoftext|>


<commit_before>import sklearn.mixture as sk_m
import numpy as np

class MeanCov_GMM(sk_m.GMM):
""" class to enable GMM fitting starting with mean and covariance
    matrices initialised to those of the data.

    Uses an affine transformation to transform data to 0 mean and
    identity covariance in fit(), then transforms means and
    covariances found back to original coordinate system.

    All functions and parameters are the same as sklearn.mixture.GMM
"""

    def fit(self,X):

        X_mean = np.mean(X, axis=0)

        X_modified = (X - X_mean[np.newaxis,:])
        X_covar=np.dot(X_modified.T,X_modified)/(X_modified.shape[0])
        X_covar_L=np.linalg.cholesky(X_covar)
        X_covar_invL=np.linalg.solve(X_covar_L,np.eye(X_covar.shape[0]))
        X_modified = np.dot(X_covar_invL, X_modified.T).T

        self = super(MeanCov_GMM,self).fit(X_modified)

        means=self.means_.copy()
        for i in range(self.covars_.shape[0]):
          self.means_[i] = X_mean + np.dot(X_covar_L, means[i])

        covars=self.covars_.copy()
        for i in range(self.covars_.shape[0]):
                self.covars_[i] = np.dot(X_covar_L, np.dot(covars[i], X_covar_L.T) )

        return self

<commit_msg>Fixing indentation<commit_after>import sklearn.mixture as sk_m
import numpy as np

class MeanCov_GMM(sk_m.GMM):
    """
    Class to enable GMM fitting starting with mean and covariance
    matrices initialised to those of the data.

    Uses an affine transformation to transform data to 0 mean and
    identity covariance in fit(), then transforms means and
    covariances found back to original coordinate system.

    All functions and parameters are the same as sklearn.mixture.GMM
    """

    def fit(self,X):

        X_mean = np.mean(X, axis=0)

        X_modified = (X - X_mean[np.newaxis,:])
        X_covar=np.dot(X_modified.T,X_modified)/(X_modified.shape[0])
        X_covar_L=np.linalg.cholesky(X_covar)
        X_covar_invL=np.linalg.solve(X_covar_L,np.eye(X_covar.shape[0]))
        X_modified = np.dot(X_covar_invL, X_modified.T).T

        self = super(MeanCov_GMM,self).fit(X_modified)

        means=self.means_.copy()
        for i in range(self.covars_.shape[0]):
          self.means_[i] = X_mean + np.dot(X_covar_L, means[i])

        covars=self.covars_.copy()
        for i in range(self.covars_.shape[0]):
                self.covars_[i] = np.dot(X_covar_L, np.dot(covars[i], X_covar_L.T) )

        return self

<|endoftext|>


<commit_before>import numpy as np
from multiphenotype_utils import (get_continuous_features_as_matrix, add_id, remove_id_and_get_mat,
    partition_dataframe_into_binary_and_continuous, divide_idxs_into_batches)
import pandas as pd
import tensorflow as tf
from dimreducer import DimReducer

from general_autoencoder import GeneralAutoencoder
from standard_autoencoder import StandardAutoencoder
from variational_autoencoder import VariationalAutoencoder
from variational_age_autoencoder import VariationalAgeAutoencoder

class SparseVariationalAgeAutoencoder(VariationalAgeAutoencoder):
    """
    Implements a variational autoencoder with an age prior and sparsity.
    The loss function for this only really makes sense when we have a single decoder layer, so we assert that.
    """
    def __init__(self,
                 k_age,
                 Z_age_coef,
                 sparsity_weighting = .1,
                 **kwargs):

        super(SparseVariationalAgeAutoencoder, self).__init__(k_age = k_age,
                                                        Z_age_coef = Z_age_coef,
                                                        **kwargs)
        # Does not include input_dim, but includes last hidden layer
        # self.encoder_layer_sizes = encoder_layer_sizes
        # self.k = self.encoder_layer_sizes[-1]

        # self.decoder_layer_sizes = decoder_layer_sizes
        self.k_age = k_age
        self.Z_age_coef = Z_age_coef
        assert self.k >= self.k_age
        self.need_ages = True

        self.initialization_function = self.glorot_init
        self.kl_weighting = 1
        #self.non_linearity = tf.nn.sigmoid
        self.sigma_scaling = .1
        self.sparsity_weighting = sparsity_weighting

        # assert we only have a single decoder layer (otherwise the sparsity loss doesn't make sense).
        assert(len([layer_name for layer_name in self.weights if 'decoder' in layer_name]) == 1)

    def get_loss(self):
        """
        Uses self.X, self.Xr, self.Z_sigma, self.Z_mu, self.kl_weighting
        """
        _, binary_loss, continuous_loss, _ = super(VariationalAutoencoder, self).get_loss()

        # Subtract off self.Z_age_coef * self.ages from the components of self.Z_mu
        # that are supposed to correlate with age
        # This assumes that the age-related components are drawn from N(Z_age_coef * age, 1)
        Z_mu_age = self.Z_mu[:, :self.k_age] - self.Z_age_coef * tf.reshape(self.ages, (-1, 1)) # Relies on broadcasting

        # Leave the other components untouched
        Z_mu_others = self.Z_mu[:, self.k_age:]

        # Z_mu_diffs is the difference between Z_mu and the priors
        Z_mu_diffs = tf.concat((Z_mu_age, Z_mu_others), axis=1)

        kl_div_loss = -.5 * (
            1 +
            2 * tf.log(self.Z_sigma) - tf.square(Z_mu_diffs) - tf.square(self.Z_sigma))
        kl_div_loss = tf.reduce_mean(
            tf.reduce_sum(
                kl_div_loss,
                axis=1),
            axis=0) * self.kl_weighting

        sparsity_loss = 0
        for layer_name in self.weights:
            if 'decoder' in layer_name:
                sparsity_loss += tf.reduce_sum(tf.abs(self.weights[layer_name]))
        sparsity_loss = sparsity_loss * self.sparsity_weighting

        combined_loss = combined_loss = self.combine_loss_components(binary_loss, continuous_loss, kl_div_loss + sparsity_loss)

        return combined_loss, binary_loss, continuous_loss, kl_div_loss
<commit_msg>minor tweak<commit_after>import numpy as np
from multiphenotype_utils import (get_continuous_features_as_matrix, add_id, remove_id_and_get_mat,
    partition_dataframe_into_binary_and_continuous, divide_idxs_into_batches)
import pandas as pd
import tensorflow as tf
from dimreducer import DimReducer

from general_autoencoder import GeneralAutoencoder
from standard_autoencoder import StandardAutoencoder
from variational_autoencoder import VariationalAutoencoder
from variational_age_autoencoder import VariationalAgeAutoencoder

class SparseVariationalAgeAutoencoder(VariationalAgeAutoencoder):
    """
    Implements a variational autoencoder with an age prior and sparsity.
    The loss function for this only really makes sense when we have a single decoder layer, so we assert that.
    """
    def __init__(self,
                 k_age,
                 Z_age_coef,
                 sparsity_weighting = .1,
                 **kwargs):

        super(SparseVariationalAgeAutoencoder, self).__init__(k_age = k_age,
                                                        Z_age_coef = Z_age_coef,
                                                        **kwargs)
        # Does not include input_dim, but includes last hidden layer
        # self.encoder_layer_sizes = encoder_layer_sizes
        # self.k = self.encoder_layer_sizes[-1]

        # self.decoder_layer_sizes = decoder_layer_sizes
        self.k_age = k_age
        self.Z_age_coef = Z_age_coef
        assert self.k >= self.k_age
        self.need_ages = True

        self.initialization_function = self.glorot_init
        self.kl_weighting = 1
        #self.non_linearity = tf.nn.sigmoid
        self.sigma_scaling = .1
        self.sparsity_weighting = sparsity_weighting

        # assert we only have a single decoder layer (otherwise the sparsity loss doesn't make sense).
        assert(len([layer_name for layer_name in self.weights if 'decoder' in layer_name]) == 1)

    def get_loss(self):
        """
        Uses self.X, self.Xr, self.Z_sigma, self.Z_mu, self.kl_weighting
        """
        _, binary_loss, continuous_loss, _ = super(VariationalAutoencoder, self).get_loss()

        # Subtract off self.Z_age_coef * self.ages from the components of self.Z_mu
        # that are supposed to correlate with age
        # This assumes that the age-related components are drawn from N(Z_age_coef * age, 1)
        Z_mu_age = self.Z_mu[:, :self.k_age] - self.Z_age_coef * tf.reshape(self.ages, (-1, 1)) # Relies on broadcasting

        # Leave the other components untouched
        Z_mu_others = self.Z_mu[:, self.k_age:]

        # Z_mu_diffs is the difference between Z_mu and the priors
        Z_mu_diffs = tf.concat((Z_mu_age, Z_mu_others), axis=1)

        kl_div_loss = -.5 * (
            1 +
            2 * tf.log(self.Z_sigma) - tf.square(Z_mu_diffs) - tf.square(self.Z_sigma))
        kl_div_loss = tf.reduce_mean(
            tf.reduce_sum(
                kl_div_loss,
                axis=1),
            axis=0) * self.kl_weighting

        sparsity_loss = 0
        for layer_name in self.weights:
            if 'decoder' in layer_name:
                sparsity_loss += tf.reduce_sum(tf.abs(self.weights[layer_name]))
        sparsity_loss = sparsity_loss * self.sparsity_weighting

        combined_loss = self.combine_loss_components(binary_loss, continuous_loss, kl_div_loss + sparsity_loss)

        return combined_loss, binary_loss, continuous_loss, kl_div_loss
<|endoftext|>


<commit_before>from . import gs
from . import kv
import sublime
import threading
import time
import traceback

class Event(object):
	def __init__(self):
		self.lst = []
		self.lck = threading.Lock()
		self.post_add = None

	def __call__(self, *args, **kwargs):
		with self.lck:
			l = self.lst[:]

		for f in l:
			try:
				f(*args, **kwargs)
			except Exception:
				print(traceback.format_exc())

		return self

	def __iadd__(self, f):
		with self.lck:
			self.lst.append(f)

		if self.post_add:
			try:
				self.post_add(self, f)
			except Exception:
				print(traceback.format_exc())

		return self

	def __isub__(self, f):
		with self.lck:
			self.lst.remove(f)

		return self

class Signal(Event):
	def __init__(self, delay_ms):
		Event.__init__(self)

		self.n = 0

		self.delay = max(10, delay_ms)
		self.deadline = 0
		self.running = False

	def __call__(self):
		with self.lck:
			self.deadline = (self.tms() + self.delay)

		self._sched()

	def tms(self):
		return int(time.time() * 1000)

	def _sched(self, ms=0):
		with self.lck:
			if not self.running:
				if ms <= 0:
					ms = self.delay

				self.running = True
				sublime.set_timeout(self._timedout, ms)

	def _timedout(self):
		with self.lck:
			self.running = False
			deadline = self.deadline

		ms = deadline - self.tms()
		if ms <= 0:
			Event.__call__(self)
		else:
			self._sched(ms)

kvs = kv.M()

def kvm(k):
	return kvs.get(k, lambda: (kv.M(), True))

def df_mod_sig(view):
	def f():
		sublime.set_timeout(lambda: view_updated(view), 0)

	sig = Signal(500)
	sig += f
	return (sig, True)

def sig_mod(view):
	sig = kvm(view.id()).get('mod-sig', lambda: df_mod_sig(view))
	sig()

def lc_reset(view):
	kvm(view.id()).remove('last-row')

def lc(view):
	m = kvm(view.id())
	sel = gs.sel(view).begin()
	row, _ = view.rowcol(sel)
	if m.put('last-row', row) != row:
		sublime.set_timeout(lambda: line_changed(view), 0)

def df_mov_sig(view):
	def f():
		sublime.set_timeout(lambda: cursor_moved(view), 0)
		sublime.set_timeout(lambda: lc(view), 0)

	sig = Signal(500)
	sig += f
	return (sig, True)

def sig_mov(view):
	sig = kvm(view.id()).get('mov-sig', lambda: df_mov_sig(view))
	sig()

debug = Event()
init = Event()
view_updated = Event()
line_changed = Event()
cursor_moved = Event()
<commit_msg>*  move last-row clearing into sig_mov to avoid an extra kvm() call<commit_after>from . import gs
from . import kv
import sublime
import threading
import time
import traceback

class Event(object):
	def __init__(self):
		self.lst = []
		self.lck = threading.Lock()
		self.post_add = None

	def __call__(self, *args, **kwargs):
		with self.lck:
			l = self.lst[:]

		for f in l:
			try:
				f(*args, **kwargs)
			except Exception:
				print(traceback.format_exc())

		return self

	def __iadd__(self, f):
		with self.lck:
			self.lst.append(f)

		if self.post_add:
			try:
				self.post_add(self, f)
			except Exception:
				print(traceback.format_exc())

		return self

	def __isub__(self, f):
		with self.lck:
			self.lst.remove(f)

		return self

class Signal(Event):
	def __init__(self, delay_ms):
		Event.__init__(self)

		self.n = 0

		self.delay = max(10, delay_ms)
		self.deadline = 0
		self.running = False

	def __call__(self):
		with self.lck:
			self.deadline = (self.tms() + self.delay)

		self._sched()

	def tms(self):
		return int(time.time() * 1000)

	def _sched(self, ms=0):
		with self.lck:
			if not self.running:
				if ms <= 0:
					ms = self.delay

				self.running = True
				sublime.set_timeout(self._timedout, ms)

	def _timedout(self):
		with self.lck:
			self.running = False
			deadline = self.deadline

		ms = deadline - self.tms()
		if ms <= 0:
			Event.__call__(self)
		else:
			self._sched(ms)

kvs = kv.M()

def kvm(k):
	return kvs.get(k, lambda: (kv.M(), True))

def df_mod_sig(view):
	def f():
		sublime.set_timeout(lambda: view_updated(view), 0)

	sig = Signal(500)
	sig += f
	return (sig, True)

def sig_mod(view):
	sig = kvm(view.id()).get('mod-sig', lambda: df_mod_sig(view))
	sig()

def lc(view):
	m = kvm(view.id())
	sel = gs.sel(view).begin()
	row, _ = view.rowcol(sel)
	if m.put('last-row', row) != row:
		sublime.set_timeout(lambda: line_changed(view), 0)

def df_mov_sig(view):
	def f():
		sublime.set_timeout(lambda: cursor_moved(view), 0)
		sublime.set_timeout(lambda: lc(view), 0)

	sig = Signal(500)
	sig += f
	return (sig, True)

def sig_mov(view, reset_last_row=False):
	m = kvm(view.id())

	if reset_last_row:
		m.remove('last-row')

	sig = m.get('mov-sig', lambda: df_mov_sig(view))
	sig()

debug = Event()
init = Event()
view_updated = Event()
line_changed = Event()
cursor_moved = Event()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright 2011 Tomo Krajina
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import math as mod_math

from . import utils as mod_utils

# Generic geo related function and class(es)

# One degree in meters:
ONE_DEGREE = 1000. * 10000.8 / 90.

def length(locations=None, _3d=None):
    locations = locations or []
    if not locations:
        return 0
    length = 0
    for i in range(len(locations)):
        if i > 0:
            previous_location = locations[i - 1]
            location = locations[i]

            if _3d:
                d = location.distance_3d(previous_location)
            else:
                d = location.distance_2d(previous_location)
            if d != 0 and not d:
                pass
            else:
                length += d
    return length

def length_2d(locations=None):
    """ 2-dimensional length of locations (only latitude and longitude, no elevation """
    locations = locations or []
    return length(locations, False)

def length_3d(locations):
    """ 3-dimensional length of locations (is uses latitude, longitude and elevation). """
    locations = locations or []
    return length(locations, True)

def distance(latitude_1, longitude_1, elevation_1, latitude_2, longitude_2, elevation_2):
    """ Distance between two points. If elevation is None compute a 2d distance """

    coef = mod_math.cos(latitude_1 / 180. * mod_math.pi)
    x = latitude_1 - latitude_2
    y = (longitude_1 - longitude_2) * coef

    distance_2d = mod_math.sqrt(x * x + y * y) * ONE_DEGREE

    if elevation_1 is None or elevation_2 is None or elevation_1 == elevation_2:
        return distance_2d

    return mod_math.sqrt(distance_2d ** 2 + (elevation_1 - elevation_2) ** 2)

def elevation_angle(location1, location2, radians=False):
    """ Uphill/downhill angle between two locations. """
    if location1.elevation is None or location2.elevation is None:
        return None

    b = float(location2.elevation - location1.elevation)
    a = location2.distance_2d(location1)

    if a == 0:
        return 0

    angle = mod_math.atan(b / a)

    if radians:
        return angle

    return 180 * angle / mod_math.pi

class Location:
    """ Generic geographical location """

    latitude = None
    longitude = None
    elevation = None

    def __init__(self, latitude, longitude, elevation=None):
        self.latitude = latitude
        self.longitude = longitude
        self.elevation = elevation

    def has_elevation(self):
        return self.elevation or self.elevation == 0

    def remove_elevation(self):
        self.elevation = None

    def distance_2d(self, location):
        if not location:
            return None

        return distance(self.latitude, self.longitude, None, location.latitude, location.longitude, None)

    def distance_3d(self, location):
        if not location:
            return None

        return distance(self.latitude, self.longitude, self.elevation, location.latitude, location.longitude, location.elevation)

    def elevation_angle(self, location, radians=False):
        return elevation_angle(self, location, radians)

    def move(self, latitude_diff, longitude_diff):
        self.latitude += latitude_diff
        self.longitude += longitude_diff

    def __str__(self):
        return '[loc:%s,%s@%s]' % (self.latitude, self.longitude, self.elevation)

    def __hash__(self):
                return mod_utils.hash_object(self, 'latitude', 'longitude', 'elevation')

<commit_msg>Added a missing default parameter.<commit_after># -*- coding: utf-8 -*-

# Copyright 2011 Tomo Krajina
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import math as mod_math

from . import utils as mod_utils

# Generic geo related function and class(es)

# One degree in meters:
ONE_DEGREE = 1000. * 10000.8 / 90.

def length(locations=None, _3d=None):
    locations = locations or []
    if not locations:
        return 0
    length = 0
    for i in range(len(locations)):
        if i > 0:
            previous_location = locations[i - 1]
            location = locations[i]

            if _3d:
                d = location.distance_3d(previous_location)
            else:
                d = location.distance_2d(previous_location)
            if d != 0 and not d:
                pass
            else:
                length += d
    return length

def length_2d(locations=None):
    """ 2-dimensional length of locations (only latitude and longitude, no elevation """
    locations = locations or []
    return length(locations, False)

def length_3d(locations=None):
    """ 3-dimensional length of locations (is uses latitude, longitude and elevation). """
    locations = locations or []
    return length(locations, True)

def distance(latitude_1, longitude_1, elevation_1, latitude_2, longitude_2, elevation_2):
    """ Distance between two points. If elevation is None compute a 2d distance """

    coef = mod_math.cos(latitude_1 / 180. * mod_math.pi)
    x = latitude_1 - latitude_2
    y = (longitude_1 - longitude_2) * coef

    distance_2d = mod_math.sqrt(x * x + y * y) * ONE_DEGREE

    if elevation_1 is None or elevation_2 is None or elevation_1 == elevation_2:
        return distance_2d

    return mod_math.sqrt(distance_2d ** 2 + (elevation_1 - elevation_2) ** 2)

def elevation_angle(location1, location2, radians=False):
    """ Uphill/downhill angle between two locations. """
    if location1.elevation is None or location2.elevation is None:
        return None

    b = float(location2.elevation - location1.elevation)
    a = location2.distance_2d(location1)

    if a == 0:
        return 0

    angle = mod_math.atan(b / a)

    if radians:
        return angle

    return 180 * angle / mod_math.pi

class Location:
    """ Generic geographical location """

    latitude = None
    longitude = None
    elevation = None

    def __init__(self, latitude, longitude, elevation=None):
        self.latitude = latitude
        self.longitude = longitude
        self.elevation = elevation

    def has_elevation(self):
        return self.elevation or self.elevation == 0

    def remove_elevation(self):
        self.elevation = None

    def distance_2d(self, location):
        if not location:
            return None

        return distance(self.latitude, self.longitude, None, location.latitude, location.longitude, None)

    def distance_3d(self, location):
        if not location:
            return None

        return distance(self.latitude, self.longitude, self.elevation, location.latitude, location.longitude, location.elevation)

    def elevation_angle(self, location, radians=False):
        return elevation_angle(self, location, radians)

    def move(self, latitude_diff, longitude_diff):
        self.latitude += latitude_diff
        self.longitude += longitude_diff

    def __str__(self):
        return '[loc:%s,%s@%s]' % (self.latitude, self.longitude, self.elevation)

    def __hash__(self):
                return mod_utils.hash_object(self, 'latitude', 'longitude', 'elevation')

<|endoftext|>