<commit_before>"""
sockless
========

A friendlier interface to `socket`.

``sockless`` emulates file-like objects, allowing you to call familiar methods
& iterate over the lines. It also includes

Usage::

    import sockless

    with sockless.open('irc.freenode.net:6665', mode='rw') as sock:
        # Writing.
        sock.write('NICK atestbot\r\n')
        sock.write('USER atestbot bot@aserver.com unused :atestbot\r\n')
        sock.write('JOIN #testbot\r\n')

        # Reading lines from a socket.
        for line in sock:
            if not line:
                break

            if 'End of /NAMES list' in line:
                print "Successfully connected & joined. Quitting."
                break

            print line.strip()

"""
import contextlib

try:
    from gevent import socket
except ImportError:
    import socket

__author__ = 'Daniel Lindsley'
__license__ = 'BSD'
__version__ = (0, 8, 0)

DEFAULT_TIMEOUT = 60
DEFAULT_MAX_CONNS = 5

class SocklessException(Exception): pass
class TimedOut(SocklessException): pass
class AddressNotFound(SocklessException): pass
class BrokenConnection(SocklessException): pass
class NotConnected(SocklessException): pass

class Socket(object):
    def __init__(
        self,
        address,
        timeout=DEFAULT_TIMEOUT,
    ):
        self.address = address
        self.timeout = timeout

        self.closed = True

        self._conn = None
        self._conn_file = None
        self._readable = True
        self._writable = False

    def split_address(self, address):
        host, port = address.split(':')
        return host, int(port)

    def open(self, mode='rw'):
        host, port = self.split_address(self.address)

        if mode == 'r':
            self._set_readable()
        elif mode == 'w':
            self._set_writable()
        elif mode == 'rw':
            self._set_read_write()

        try:
            self._conn = socket.create_connection(
                (host, port),
                timeout=self.timeout
            )
        except socket.gaierror:
            raise AddressNotFound("Could connect to {}:{}".format(
                host,
                port
            ))
        except socket.timeout:
            raise TimedOut("Connection to {}:{} timed out".format(
                host,
                port
            ))

        self._conn_file = self._conn.makefile(mode)

    def _set_readable(self):
        self._readable = True
        self._writable = False

    def _set_writable(self):
        self._readable = False
        self._writable = True

    def _set_read_write(self):
        self._readable = True
        self._writable = True

    def _send(self, data):
        self._conn_file.write(data)
        self._conn_file.flush()

    # File-like methods

    def _check_conn(self):
        if not self._conn or not self._conn_file:
            raise NotConnected("Not connected to {}".format(
                self.address
            ))

    def close(self):
        self._check_conn()
        # Need to close both to be doing the right thing.
        self._conn_file.close()
        self._conn.close()

    def readable(self):
        return self._readable

    def writable(self):
        return self._writable

    def read(self, size=-1):
        if size <= -1:
            return self.readall()

        self._check_conn()
        return self._conn_file.read(size)

    def readall(self):
        self._check_conn()
        return self._conn_file.read()

    def readline(self):
        self._check_conn()
        return self._conn_file.readline()

    def readlines(self):
        self._check_conn()
        return self._conn_file.readlines()

    def write(self, data):
        return self._send(data)

    def __iter__(self):
        return self

    def __next__(self):
        return self.readline()

    def next(self):
        return self.__next__()

    # Socket-specific methods

    @property
    def hostname(self):
        return socket.gethostname()

    @property
    def fully_qualified_domain_name(self):
        return socket.getfqdn()

    fqdn = fully_qualified_domain_name

    @property
    def remote_ip(self):
        self._check_conn()
        return self._conn.getpeername()[0]

    @property
    def remote_port(self):
        self._check_conn()
        return self._conn.getpeername()[1]

    @property
    def local_ip(self):
        self._check_conn()
        return self._conn.getsockname()[0]

    @property
    def local_port(self):
        self._check_conn()
        return self._conn.getsockname()[1]

    def resolve_dns(self, address=None):
        if address is None:
            address = self.address

        host, port = self.split_address(address)
        bits = socket.getaddrinfo(host, port)
        return [bit[4] for bit in bits]

@contextlib.contextmanager
def open(address, timeout=DEFAULT_TIMEOUT, mode='rw'):
    sock = Socket(address, timeout=timeout)
    sock.open(mode)

    try:
        yield sock
    finally:
        sock.close()
<commit_msg>Added a first blush at a non-blocking socket.<commit_after>"""
sockless
========

A friendlier interface to `socket`.

``sockless`` emulates file-like objects, allowing you to call familiar methods
& iterate over the lines. It also includes

Usage::

    import sockless

    with sockless.open('irc.freenode.net:6665', mode='rw') as sock:
        # Writing.
        sock.write('NICK atestbot\r\n')
        sock.write('USER atestbot bot@aserver.com unused :atestbot\r\n')
        sock.write('JOIN #testbot\r\n')

        # Reading lines from a socket.
        for line in sock:
            if not line:
                break

            if 'End of /NAMES list' in line:
                print "Successfully connected & joined. Quitting."
                break

            print line.strip()

"""
import contextlib
import select
import socket

__author__ = 'Daniel Lindsley'
__license__ = 'BSD'
__version__ = (0, 8, 0)

DEFAULT_TIMEOUT = 60
DEFAULT_MAX_CONNS = 5

class SocklessException(Exception): pass
class TimedOut(SocklessException): pass
class AddressNotFound(SocklessException): pass
class BrokenConnection(SocklessException): pass
class NotConnected(SocklessException): pass

def split_address(address):
    host, port = address.split(':')
    return host, int(port)

class Socket(object):
    def __init__(
        self,
        address,
        timeout=DEFAULT_TIMEOUT,
    ):
        self.address = address
        self.timeout = timeout

        self.closed = True

        self._conn = None
        self._conn_file = None
        self._readable = True
        self._writable = False

    def split_address(self, address):
        return split_address(address)

    def open(self, mode='rw'):
        host, port = self.split_address(self.address)

        if mode == 'r':
            self._set_readable()
        elif mode == 'w':
            self._set_writable()
        elif mode == 'rw':
            self._set_read_write()

        try:
            self._conn = socket.create_connection(
                (host, port),
                timeout=self.timeout
            )
        except socket.gaierror:
            raise AddressNotFound("Could connect to {}:{}".format(
                host,
                port
            ))
        except socket.timeout:
            raise TimedOut("Connection to {}:{} timed out".format(
                host,
                port
            ))

        self._conn_file = self._conn.makefile(mode)

    def _set_readable(self):
        self._readable = True
        self._writable = False

    def _set_writable(self):
        self._readable = False
        self._writable = True

    def _set_read_write(self):
        self._readable = True
        self._writable = True

    def _send(self, data):
        self._conn_file.write(data)
        self._conn_file.flush()

    # File-like methods

    def _check_conn(self):
        if not self._conn or not self._conn_file:
            raise NotConnected("Not connected to {}".format(
                self.address
            ))

    def close(self):
        self._check_conn()
        # Need to close both to be doing the right thing.
        self._conn_file.close()
        self._conn.close()

    def readable(self):
        return self._readable

    def writable(self):
        return self._writable

    def read(self, size=-1):
        if size <= -1:
            return self.readall()

        self._check_conn()
        return self._conn_file.read(size)

    def readall(self):
        self._check_conn()
        return self._conn_file.read()

    def readline(self):
        self._check_conn()
        return self._conn_file.readline()

    def readlines(self):
        self._check_conn()
        return self._conn_file.readlines()

    def write(self, data):
        return self._send(data)

    def __iter__(self):
        return self

    def __next__(self):
        return self.readline()

    def next(self):
        return self.__next__()

    # Socket-specific methods

    @property
    def hostname(self):
        return socket.gethostname()

    @property
    def fully_qualified_domain_name(self):
        return socket.getfqdn()

    fqdn = fully_qualified_domain_name

    @property
    def remote_ip(self):
        self._check_conn()
        return self._conn.getpeername()[0]

    @property
    def remote_port(self):
        self._check_conn()
        return self._conn.getpeername()[1]

    @property
    def local_ip(self):
        self._check_conn()
        return self._conn.getsockname()[0]

    @property
    def local_port(self):
        self._check_conn()
        return self._conn.getsockname()[1]

    def resolve_dns(self, address=None):
        if address is None:
            address = self.address

        host, port = self.split_address(address)
        bits = socket.getaddrinfo(host, port)
        return [bit[4] for bit in bits]

class NonBlockingSocket(object):
    def __init__(self, address):
        self.address = address
        self._conn = None
        self._buffer = ''
        self._readable = False
        self._writable = False

    def split_address(self, address):
        return split_address(address)

    def _check_conn(self):
        if not self._conn:
            raise NotConnected("Not connected to {}".format(
                self.address
            ))

    def open(self, mode='rw'):
        host, port = self.split_address(self.address)

        if mode == 'rw':
            self._readable = True
            self._writable = True
        elif mode == 'r':
            self._readable = True
        elif mode == 'w':
            self._writable = True

        try:
            self._conn = socket.create_connection((host, port))
            self._conn.setblocking(0)
        except socket.gaierror:
            raise AddressNotFound("Could connect to {}:{}".format(
                host,
                port
            ))
        except socket.timeout:
            raise TimedOut("Connection to {}:{} timed out".format(
                host,
                port
            ))

    def close(self):
        self._check_conn()
        self._conn.close()

    def select(self):
        self._check_conn()
        rlist = []
        wlist = []

        if self._readable:
            rlist.append(self._conn)

        if self._writable:
            wlist.append(self._conn)

        return select.select(rlist, wlist, [])

    def readable(self):
        rlist, wlist, xlist = self.select()
        return len(rlist) > 0

    def writable(self):
        rlist, wlist, xlist = self.select()
        return len(wlist) > 0

    def read(self, size=4096):
        rlist, wlist, xlist = self.select()
        amount_read = 0

        if not rlist:
            return amount_read

        rsock = rlist[0]

        while rsock:
            received = rsock.recv(size)
            # Unfortunately, we can't detect broken connections here, since
            # this method doesn't block until the send *actually* happens.
            # Lots of false positivites trying to detect zero conditions.
            amount_read += len(received)
            self._buffer += received
            rlist, wlist, xlist = self.select()

            if rlist:
                rsock = rlist[0]
            else:
                rsock = None

        return amount_read

    def readline(self):
        return self.readlines(limit=1)

    def readlines(self, limit=-1):
        if limit == 0:
            return []

        while True:
            rsize = self.read()

            if not rsize:
                break

        if not self._buffer:
            return []

        if limit >= 0:
            line, self._buffer = self._buffer.split('\n', 1)
            return [line]

        lines = self._buffer.split('\n')

        if not self._buffer.endswith('\n'):
            self._buffer = lines.pop()
        else:
            self._buffer = ''

        return lines

    def write(self, data):
        rlist, wlist, xlist = self.select()

        if not wlist:
            # FIXME: Not sure this should just fail. Should I buffer instead?
            return False

        wsock = wlist[0]
        amount_sent = wsock.sendall(data)
        return amount_sent

    def __iter__(self):
        return self

    def __next__(self):
        return self.readline()

@contextlib.contextmanager
def open(address, timeout=DEFAULT_TIMEOUT, mode='rw'):
    sock = Socket(address, timeout=timeout)
    sock.open(mode)

    try:
        yield sock
    finally:
        sock.close()
<|endoftext|>


<commit_before>class Dataset(object):
  """A dataset in the Cloud Datastore.

  This class acts as an abstraction of a single dataset
  in the Cloud Datastore.

  A dataset is analogous to a database
  in relational database world,
  and corresponds to a single project
  using the Cloud Datastore.

  Typically, you would only have one of these per connection
  however it didn't seem right to collapse the functionality
  of a connection and a dataset together into a single class.

  Datasets (like :class:`gclouddatastore.query.Query`s)
  are immutable.
  That is, you cannot change the ID and connection
  references.
  If you need to modify the connection or ID,
  it's recommended to construct a new :class:`Dataset`.

  :type id: string
  :param id: The ID of the dataset (your project ID)

  :type connection: :class:`gclouddatastore.connection.Connection`
  :param connection: The connection to use for executing API calls.
  """

  def __init__(self, id, connection=None):
    self._connection = connection
    self._id = id

  def connection(self):
    """Get the current connection.

      >>> dataset = Dataset('dataset-id', connection=conn)
      >>> dataset.connection()
      <Connection object>

    :rtype: :class:`gclouddatastore.connection.Connection`
    :returns: Returns the current connection.
    """

    return self._connection

  def id(self):
    """Get the current dataset ID.

      >>> dataset = Dataset('dataset-id', connection=conn)
      >>> dataset.id()
      'dataset-id'

    :rtype: string
    :returns: The current dataset ID.
    """

    return self._id

  def query(self, *args, **kwargs):
    from gclouddatastore.query import Query
    kwargs['dataset'] = self
    return Query(*args, **kwargs)

  def entity(self, kind):
    from gclouddatastore.entity import Entity
    return Entity(dataset=self, kind=kind)

  def get_entity(self, key):
    """
    Retrieves an entity from the dataset, along with all of its attributes.

    :type key: :class:`gclouddatastore.key.Key`
    :param item_name: The name of the item to retrieve.

    :rtype: :class:`gclouddatastore.entity.Entity` or ``None``
    :return: The requested entity, or ``None`` if there was no match found.
    """
    return self.get_entities([key])

  def get_entities(self, keys):
    # This import is here to avoid circular references.
    from gclouddatastore.entity import Entity

    entity_pbs = self.connection().lookup(dataset_id=self.id(),
        key_pbs=[k.to_protobuf() for k in keys])

    entities = []
    for entity_pb in entity_pbs:
      entities.append(Entity.from_protobuf(entity_pb, dataset=self))
    return entities
<commit_msg>Fixed #11 - Documentation syntax error.<commit_after>class Dataset(object):
  """A dataset in the Cloud Datastore.

  This class acts as an abstraction of a single dataset
  in the Cloud Datastore.

  A dataset is analogous to a database
  in relational database world,
  and corresponds to a single project
  using the Cloud Datastore.

  Typically, you would only have one of these per connection
  however it didn't seem right to collapse the functionality
  of a connection and a dataset together into a single class.

  Datasets (like :class:`gclouddatastore.query.Query`)
  are immutable.
  That is, you cannot change the ID and connection
  references.
  If you need to modify the connection or ID,
  it's recommended to construct a new :class:`Dataset`.

  :type id: string
  :param id: The ID of the dataset (your project ID)

  :type connection: :class:`gclouddatastore.connection.Connection`
  :param connection: The connection to use for executing API calls.
  """

  def __init__(self, id, connection=None):
    self._connection = connection
    self._id = id

  def connection(self):
    """Get the current connection.

      >>> dataset = Dataset('dataset-id', connection=conn)
      >>> dataset.connection()
      <Connection object>

    :rtype: :class:`gclouddatastore.connection.Connection`
    :returns: Returns the current connection.
    """

    return self._connection

  def id(self):
    """Get the current dataset ID.

      >>> dataset = Dataset('dataset-id', connection=conn)
      >>> dataset.id()
      'dataset-id'

    :rtype: string
    :returns: The current dataset ID.
    """

    return self._id

  def query(self, *args, **kwargs):
    from gclouddatastore.query import Query
    kwargs['dataset'] = self
    return Query(*args, **kwargs)

  def entity(self, kind):
    from gclouddatastore.entity import Entity
    return Entity(dataset=self, kind=kind)

  def get_entity(self, key):
    """
    Retrieves an entity from the dataset, along with all of its attributes.

    :type key: :class:`gclouddatastore.key.Key`
    :param item_name: The name of the item to retrieve.

    :rtype: :class:`gclouddatastore.entity.Entity` or ``None``
    :return: The requested entity, or ``None`` if there was no match found.
    """
    return self.get_entities([key])

  def get_entities(self, keys):
    # This import is here to avoid circular references.
    from gclouddatastore.entity import Entity

    entity_pbs = self.connection().lookup(dataset_id=self.id(),
        key_pbs=[k.to_protobuf() for k in keys])

    entities = []
    for entity_pb in entity_pbs:
      entities.append(Entity.from_protobuf(entity_pb, dataset=self))
    return entities
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
################################################################################
# Copyright 2014, Distributed Meta-Analysis System
################################################################################

"""
This file provides methods for extracting data from impact bundles (.nc4 files)
"""

__copyright__ = "Copyright 2014, Distributed Meta-Analysis System"

__author__ = "James Rising"
__credits__ = ["James Rising"]
__maintainer__ = "James Rising"
__email__ = "jar2234@columbia.edu"

__status__ = "Production"
__version__ = "$Revision$"
# $Source$

import os, csv
import numpy as np
from netCDF4 import Dataset
import configs

def read(filepath, column='rebased'):

    rootgrp = Dataset(filepath, 'r', format='NETCDF4')

    years = rootgrp.variables['year'][:]
    regions = rootgrp.variables['regions'][:]
    data = rootgrp.variables[column][:, :]

    rootgrp.close()

    # Correct bad regions in costs
    if filepath[-10:] == '-costs.nc4' and not isinstance(regions[0], str) and not isinstance(regions[0], unicode) and np.isnan(regions[0]):
        rootgrp = Dataset(filepath.replace('-costs.nc4', '.nc4'), 'r', format='NETCDF4')
        regions = rootgrp.variables['regions'][:]
        rootgrp.close()

    return years, regions, data

def iterate_regions(filepath, config={}):
    """
    Config options: column
    """

    if 'column' in config or 'costs' not in filepath:
        years, regions, data = read(filepath, config.get('column', 'rebased'))
    else:
        years, regions, data1 = read(filepath, 'costs_lb')
        years, regions, data2 = read(filepath, 'costs_ub')
        data = ((data1 + data2) / 2) / 1e5

    config['regionorder'] = list(regions)

    if configs.is_allregions(config):
        yield 'all', years, data
        return

    regions = list(regions)
    for region in configs.get_regions(config, regions):
        if region == 'global':
            region = ''
        ii = regions.index(region)
        yield regions[ii], years, data[:, ii]

def iterate_values(years, values, config={}):
    """
    Config options: yearsets, years
    """

    if 'yearsets' in config and config['yearsets']:
        yearsets = config['yearsets']
        if yearsets == True:
            yearsets = [(2000, 2019), (2020, 2039), (2040, 2059), (2080, 2099)]

        for yearset in yearsets:
            if isinstance(yearset, list):
                yearset = tuple(yearset)
            if values.ndim == 1:
                yield "%d-%d" % yearset, np.mean(values[np.logical_and(years >= yearset[0], years < yearset[1])])
            else: # multiple regions included
                yield "%d-%d" % yearset, np.mean(values[np.logical_and(years >= yearset[0], years < yearset[1]), :], axis=0)
        return

    years = list(years)
    for year in configs.get_years(config, years):
        if values.ndim == 1:
            yield year, values[years.index(year)]
        else:
            yield year, values[years.index(year), :]
<commit_msg>make bundles.py extract only upper bound costs<commit_after># -*- coding: utf-8 -*-
################################################################################
# Copyright 2014, Distributed Meta-Analysis System
################################################################################

"""
This file provides methods for extracting data from impact bundles (.nc4 files)
"""

__copyright__ = "Copyright 2014, Distributed Meta-Analysis System"

__author__ = "James Rising"
__credits__ = ["James Rising"]
__maintainer__ = "James Rising"
__email__ = "jar2234@columbia.edu"

__status__ = "Production"
__version__ = "$Revision$"
# $Source$

import os, csv
import numpy as np
from netCDF4 import Dataset
import configs

def read(filepath, column='rebased'):

    rootgrp = Dataset(filepath, 'r', format='NETCDF4')

    years = rootgrp.variables['year'][:]
    regions = rootgrp.variables['regions'][:]
    data = rootgrp.variables[column][:, :]

    rootgrp.close()

    # Correct bad regions in costs
    if filepath[-10:] == '-costs.nc4' and not isinstance(regions[0], str) and not isinstance(regions[0], unicode) and np.isnan(regions[0]):
        rootgrp = Dataset(filepath.replace('-costs.nc4', '.nc4'), 'r', format='NETCDF4')
        regions = rootgrp.variables['regions'][:]
        rootgrp.close()

    return years, regions, data

def iterate_regions(filepath, config={}):
    """
    Config options: column
    """

    if 'column' in config or 'costs' not in filepath:
        years, regions, data = read(filepath, config.get('column', 'rebased'))
    else:
        years, regions, data1 = read(filepath, 'costs_lb')
        years, regions, data2 = read(filepath, 'costs_ub')
        data = (data2 / 1e5)

    config['regionorder'] = list(regions)

    if configs.is_allregions(config):
        yield 'all', years, data
        return

    regions = list(regions)
    for region in configs.get_regions(config, regions):
        if region == 'global':
            region = ''
        ii = regions.index(region)
        yield regions[ii], years, data[:, ii]

def iterate_values(years, values, config={}):
    """
    Config options: yearsets, years
    """

    if 'yearsets' in config and config['yearsets']:
        yearsets = config['yearsets']
        if yearsets == True:
            yearsets = [(2000, 2019), (2020, 2039), (2040, 2059), (2080, 2099)]

        for yearset in yearsets:
            if isinstance(yearset, list):
                yearset = tuple(yearset)
            if values.ndim == 1:
                yield "%d-%d" % yearset, np.mean(values[np.logical_and(years >= yearset[0], years < yearset[1])])
            else: # multiple regions included
                yield "%d-%d" % yearset, np.mean(values[np.logical_and(years >= yearset[0], years < yearset[1]), :], axis=0)
        return

    years = list(years)
    for year in configs.get_years(config, years):
        if values.ndim == 1:
            yield year, values[years.index(year)]
        else:
            yield year, values[years.index(year), :]
<|endoftext|>


<commit_before>"""
Constants for Memcached binary protocol

Originally generated from protocol_binary.h:
  egrep "( = |^[ ]{3,}\* )" doc/protocol_binary.h \
   | sed -e "s:^[ ]*::" -e "s:^\*:#:" -e "s:^PROTOCOL_BINARY_::" -e "s:,$::" \
   > gmcbs/const.py

"""
import struct

# Definition of the legal "magic" values used in a packet.
# See section 3.1 Magic byte
MAGIC_REQ = 0x80
MAGIC_RES = 0x81

# Definition of the valid response status numbers.
# See section 3.2 Response Status
RESPONSE_SUCCESS = 0x00
RESPONSE_KEY_ENOENT = 0x01
RESPONSE_KEY_EEXISTS = 0x02
RESPONSE_E2BIG = 0x03
RESPONSE_EINVAL = 0x04
RESPONSE_NOT_STORED = 0x05
RESPONSE_DELTA_BADVAL = 0x06
RESPONSE_AUTH_ERROR = 0x20
RESPONSE_AUTH_CONTINUE = 0x21
RESPONSE_UNKNOWN_COMMAND = 0x81
RESPONSE_ENOMEM = 0x82

# Defintion of the different command opcodes.
# See section 3.3 Command Opcodes
CMD_GET = 0x00
CMD_SET = 0x01
CMD_ADD = 0x02
CMD_REPLACE = 0x03
CMD_DELETE = 0x04
CMD_INCREMENT = 0x05
CMD_DECREMENT = 0x06
CMD_QUIT = 0x07
CMD_FLUSH = 0x08
CMD_GETQ = 0x09
CMD_NOOP = 0x0a
CMD_VERSION = 0x0b
CMD_GETK = 0x0c
CMD_GETKQ = 0x0d
CMD_APPEND = 0x0e
CMD_PREPEND = 0x0f
CMD_STAT = 0x10
CMD_SETQ = 0x11
CMD_ADDQ = 0x12
CMD_REPLACEQ = 0x13
CMD_DELETEQ = 0x14
CMD_INCREMENTQ = 0x15
CMD_DECREMENTQ = 0x16
CMD_QUITQ = 0x17
CMD_FLUSHQ = 0x18
CMD_APPENDQ = 0x19
CMD_PREPENDQ = 0x1a
CMD_TOUCH = 0x1c
CMD_GAT = 0x1d
CMD_GATQ = 0x1e
CMD_GATK = 0x23
CMD_GATKQ = 0x24
CMD_SASL_LIST_MECHS = 0x20
CMD_SASL_AUTH = 0x21
CMD_SASL_STEP = 0x22

# this header for use in other projects.  Range operations are
# not expected to be implemented in the memcached server itself.
CMD_RGET = 0x30
CMD_RSET = 0x31
CMD_RSETQ = 0x32
CMD_RAPPEND = 0x33
CMD_RAPPENDQ = 0x34
CMD_RPREPEND = 0x35
CMD_RPREPENDQ = 0x36
CMD_RDELETE = 0x37
CMD_RDELETEQ = 0x38
CMD_RINCR = 0x39
CMD_RINCRQ = 0x3a
CMD_RDECR = 0x3b
CMD_RDECRQ = 0x3c

# Definition of the data types in the packet
# See section 3.4 Data Types
RAW_BYTES = 0x00

# Helper dictionaries etc for Python.
# Yes, a bit hacky... but I'm a lazy person.
COMMANDS = {}
RESPONSES = {}
MAGIC = {}
QUIET = []
for k, v in locals().items():
    if k.startswith('CMD_'):
        COMMANDS[v] = k[4:]
        if k.endswith('Q'):
            # 'Q' variants: commands which shouldn't flush output buffer
            QUIET.append(v)
    elif k.startswith('RESPONSE_'):
        RESPONSES[v] = k[9:]
    elif k.startswith('MAGIC_'):
        MAGIC[v] = k[6:]

assert COMMANDS[CMD_QUIT] == 'QUIT'
assert RESPONSES[RESPONSE_SUCCESS] == 'SUCCESS'
assert MAGIC[MAGIC_REQ] == 'REQ'
assert CMD_GETKQ in QUIET

# magic, opcode, keylen, extralen, datatype, [reserved], bodylen, opaque, cas
HEADER_REQUEST_FORMAT = '!BBHBBxxIIQ'
HEADER_REQUEST_LEN = struct.calcsize(HEADER_REQUEST_FORMAT)
assert HEADER_REQUEST_LEN == 24

# magic, opcode, keylen, extralen, datatype, status, bodylen, opaque, cas
HEADER_RESPONSE_FORMAT = '!BBHBBHIIQ'
HEADER_RESPONSE_LEN = struct.calcsize(HEADER_RESPONSE_FORMAT)
assert HEADER_RESPONSE_LEN == 24
<commit_msg>namedtuples for header formats<commit_after>"""
Constants for Memcached binary protocol

Originally generated from protocol_binary.h:
  egrep "( = |^[ ]{3,}\* )" doc/protocol_binary.h \
   | sed -e "s:^[ ]*::" -e "s:^\*:#:" -e "s:^PROTOCOL_BINARY_::" -e "s:,$::" \
   > gmcbs/const.py

"""
from collections import namedtuple
import struct

# Definition of the legal "magic" values used in a packet.
# See section 3.1 Magic byte
MAGIC_REQ = 0x80
MAGIC_RES = 0x81

# Definition of the valid response status numbers.
# See section 3.2 Response Status
RESPONSE_SUCCESS = 0x00
RESPONSE_KEY_ENOENT = 0x01
RESPONSE_KEY_EEXISTS = 0x02
RESPONSE_E2BIG = 0x03
RESPONSE_EINVAL = 0x04
RESPONSE_NOT_STORED = 0x05
RESPONSE_DELTA_BADVAL = 0x06
RESPONSE_AUTH_ERROR = 0x20
RESPONSE_AUTH_CONTINUE = 0x21
RESPONSE_UNKNOWN_COMMAND = 0x81
RESPONSE_ENOMEM = 0x82

# Defintion of the different command opcodes.
# See section 3.3 Command Opcodes
CMD_GET = 0x00
CMD_SET = 0x01
CMD_ADD = 0x02
CMD_REPLACE = 0x03
CMD_DELETE = 0x04
CMD_INCREMENT = 0x05
CMD_DECREMENT = 0x06
CMD_QUIT = 0x07
CMD_FLUSH = 0x08
CMD_GETQ = 0x09
CMD_NOOP = 0x0a
CMD_VERSION = 0x0b
CMD_GETK = 0x0c
CMD_GETKQ = 0x0d
CMD_APPEND = 0x0e
CMD_PREPEND = 0x0f
CMD_STAT = 0x10
CMD_SETQ = 0x11
CMD_ADDQ = 0x12
CMD_REPLACEQ = 0x13
CMD_DELETEQ = 0x14
CMD_INCREMENTQ = 0x15
CMD_DECREMENTQ = 0x16
CMD_QUITQ = 0x17
CMD_FLUSHQ = 0x18
CMD_APPENDQ = 0x19
CMD_PREPENDQ = 0x1a
CMD_TOUCH = 0x1c
CMD_GAT = 0x1d
CMD_GATQ = 0x1e
CMD_GATK = 0x23
CMD_GATKQ = 0x24
CMD_SASL_LIST_MECHS = 0x20
CMD_SASL_AUTH = 0x21
CMD_SASL_STEP = 0x22

# this header for use in other projects.  Range operations are
# not expected to be implemented in the memcached server itself.
CMD_RGET = 0x30
CMD_RSET = 0x31
CMD_RSETQ = 0x32
CMD_RAPPEND = 0x33
CMD_RAPPENDQ = 0x34
CMD_RPREPEND = 0x35
CMD_RPREPENDQ = 0x36
CMD_RDELETE = 0x37
CMD_RDELETEQ = 0x38
CMD_RINCR = 0x39
CMD_RINCRQ = 0x3a
CMD_RDECR = 0x3b
CMD_RDECRQ = 0x3c

# Definition of the data types in the packet
# See section 3.4 Data Types
TYPE_RAW_BYTES = 0x00

# Helper dictionaries etc for Python.
# Yes, a bit hacky... but I'm a lazy person.
COMMANDS = {}
RESPONSES = {}
MAGIC = {}
TYPES = {}
QUIET = []
for k, v in locals().items():
    if k.startswith('CMD_'):
        COMMANDS[v] = k[4:]
        if k.endswith('Q'):
            # 'Q' variants: commands which shouldn't flush output buffer
            QUIET.append(v)
    elif k.startswith('RESPONSE_'):
        RESPONSES[v] = k[9:]
    elif k.startswith('MAGIC_'):
        MAGIC[v] = k[6:]
    elif k.startswith('TYPE_'):
        TYPES[v] = k[5:]

assert COMMANDS[CMD_QUIT] == 'QUIT'
assert RESPONSES[RESPONSE_SUCCESS] == 'SUCCESS'
assert MAGIC[MAGIC_REQ] == 'REQ'
assert TYPES[TYPE_RAW_BYTES] == 'RAW_BYTES'
assert CMD_GETKQ in QUIET

RequestHeader = namedtuple('RequestHeader', ['magic', 'opcode', 'keylen', 'extlen',
                                             'datatype', 'bodylen', 'opaque', 'cas'])
ResponseHeader = namedtuple('ResponseHeader', ['magic', 'opcode', 'keylen', 'extlen',
                                               'datatype', 'status', 'bodylen', 'opaque', 'cas'])

# magic, opcode, keylen, extralen, datatype, [reserved], bodylen, opaque, cas
HEADER_REQUEST_FORMAT = '!BBHBBxxIIQ'
HEADER_REQUEST_LEN = struct.calcsize(HEADER_REQUEST_FORMAT)
assert HEADER_REQUEST_LEN == 24

# magic, opcode, keylen, extralen, datatype, status, bodylen, opaque, cas
HEADER_RESPONSE_FORMAT = '!BBHBBHIIQ'
HEADER_RESPONSE_LEN = struct.calcsize(HEADER_RESPONSE_FORMAT)
assert HEADER_RESPONSE_LEN == 24
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright 2016 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from __future__ import print_function

import subprocess
import sys
cc,cxx = sys.argv[1:3]

if ('clang' in subprocess.check_output('%s --version' % cc, shell=True) and
    'clang' in subprocess.check_output('%s --version' % cxx, shell=True)):
  print('true')
else:
  print('false')
<commit_msg>Fix is_clang python3 encoding errors<commit_after>#!/usr/bin/env python
#
# Copyright 2016 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from __future__ import print_function

import subprocess
import sys
cc,cxx = sys.argv[1:3]

if (b'clang' in subprocess.check_output('%s --version' % cc, shell=True) and
    b'clang' in subprocess.check_output('%s --version' % cxx, shell=True)):
  print('true')
else:
  print('false')
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# This file is part of the SPORCO package. Details of the copyright
# and user license can be found in the 'LICENSE.txt' file distributed
# with the package.

"""
Convolutional Dictionary Learning with Spatial Mask
===================================================

This example demonstrates the use of :class:`.cbpdndlmd.ConvBPDNMaskDictLearn` for convolutional dictionary learning with a spatial mask, from a set of greyscale training images. The dictionary learning algorithm is based on the hybrid mask decoupling / ADMM consensus dictionary update :cite:`garcia-2017-convolutional`.
"""

from __future__ import print_function
from builtins import input
from builtins import range

import pyfftw   # See https://github.com/pyFFTW/pyFFTW/issues/40
import numpy as np

from sporco.admm import tvl2
from sporco.dictlrn import cbpdndl
from sporco.dictlrn import cbpdndlmd
from sporco import util
from sporco import plot

"""
Load training images.
"""

exim = util.ExampleImages(scaled=True, zoom=0.25, gray=True)
S1 = exim.image('barbara.png', idxexp=np.s_[10:522, 100:612])
S2 = exim.image('kodim23.png', idxexp=np.s_[:, 60:572])
S = np.dstack((S1, S2))

"""
Construct initial dictionary.
"""

np.random.seed(12345)
D0 = np.random.randn(8, 8, 32)

"""
Create random mask and apply to training images.
"""

frc = 0.5
W = util.rndmask(S.shape[0:2] + (1,), frc, dtype=np.float32)
Sw = W * S

"""
$\ell_2$-TV denoising with a spatial mask as a non-linear lowpass filter.
"""

lmbda = 0.1
opt = tvl2.TVL2Denoise.Options({'Verbose': False, 'MaxMainIter': 200,
            'DFidWeight': W, 'gEvalY': False, 'AutoRho': {'Enabled': True}})
b = tvl2.TVL2Denoise(Sw, lmbda, opt)
sl = b.solve()
sh = Sw - sl

"""
CDL without a spatial mask using :class:`.dictlrn.cbpdndl.ConvBPDNDictLearn`.
"""

lmbda = 0.05
opt1 = cbpdndl.ConvBPDNDictLearn.Options({'Verbose': True,
            'MaxMainIter': 200, 'AccurateDFid': True,
            'CBPDN': {'rho': 50.0*lmbda + 0.5},
            'CCMOD': {'rho': 1e2}}, dmethod='cns')
d1 = cbpdndl.ConvBPDNDictLearn(D0, sh, lmbda, opt1, dmethod='cns')
D1 = d1.solve()

"""
Reconstruct from the CDL solution without a spatial mask.
"""

sr1 = d1.reconstruct().squeeze() + sl

"""
CDL with a spatial mask using :class:`.cbpdndlmd.ConvBPDNMaskDictLearn`. (Note that :class:`.prlcnscdl.ConvBPDNMaskDcplDictLearn_Consensus` solves the same problem, but is substantially faster on a multi-core architecture.)
"""

opt2 = cbpdndlmd.ConvBPDNMaskDictLearn.Options({'Verbose': True,
            'MaxMainIter': 200, 'AccurateDFid': True,
            'CBPDN': {'rho': 20.0*lmbda + 0.5},
            'CCMOD': {'rho': 2e-1}}, dmethod='cns')
d2 = cbpdndlmd.ConvBPDNMaskDictLearn(D0, sh, lmbda, W, opt2,
                                         dmethod='cns')
D2 = d2.solve()

"""
Reconstruct from the CDL solution with a spatial mask.
"""

sr2 = d2.reconstruct().squeeze() + sl

"""
Compare dictionaries.
"""

fig = plot.figure(figsize=(14, 7))
plot.subplot(1, 2, 1)
plot.imview(util.tiledict(D1.squeeze()), title='Without Mask Decoupling',
            fig=fig)
plot.subplot(1, 2, 2)
plot.imview(util.tiledict(D2.squeeze()), title='With Mask Decoupling',
            fig=fig)
fig.show()

"""
Display reference and training images.
"""

fig = plot.figure(figsize=(14, 14))
plot.subplot(2, 2, 1)
plot.imview(S[...,0], title='Reference', fig=fig)
plot.subplot(2, 2, 2)
plot.imview(Sw[...,0], title='Test', fig=fig)
plot.subplot(2, 2, 3)
plot.imview(S[...,1], title='Reference', fig=fig)
plot.subplot(2, 2, 4)
plot.imview(Sw[...,1], title='Test', fig=fig)
fig.show()

"""
Compare reconstructed images.
"""

fig = plot.figure(figsize=(14, 14))
plot.subplot(2, 2, 1)
plot.imview(sr1[...,0], title='Without Mask Decoupling', fig=fig)
plot.subplot(2, 2, 2)
plot.imview(sr2[...,0], title='With Mask Decoupling', fig=fig)
plot.subplot(2, 2, 3)
plot.imview(sr1[...,1], title='Without Mask Decoupling', fig=fig)
plot.subplot(2, 2, 4)
plot.imview(sr2[...,1], title='With Mask Decoupling', fig=fig)
fig.show()

# Wait for enter on keyboard
input()
<commit_msg>Trivial edit<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# This file is part of the SPORCO package. Details of the copyright
# and user license can be found in the 'LICENSE.txt' file distributed
# with the package.

"""
Convolutional Dictionary Learning with Spatial Mask
===================================================

This example demonstrates the use of :class:`.cbpdndlmd.ConvBPDNMaskDictLearn` for convolutional dictionary learning with a spatial mask, from a set of greyscale training images. The dictionary learning algorithm is based on the hybrid mask decoupling / ADMM consensus dictionary update :cite:`garcia-2017-convolutional`.
"""

from __future__ import print_function
from builtins import input
from builtins import range

import pyfftw   # See https://github.com/pyFFTW/pyFFTW/issues/40
import numpy as np

from sporco.admm import tvl2
from sporco.dictlrn import cbpdndl
from sporco.dictlrn import cbpdndlmd
from sporco import util
from sporco import plot

"""
Load training images.
"""

exim = util.ExampleImages(scaled=True, zoom=0.25, gray=True)
S1 = exim.image('barbara.png', idxexp=np.s_[10:522, 100:612])
S2 = exim.image('kodim23.png', idxexp=np.s_[:, 60:572])
S = np.dstack((S1, S2))

"""
Construct initial dictionary.
"""

np.random.seed(12345)
D0 = np.random.randn(8, 8, 32)

"""
Create random mask and apply to training images.
"""

frc = 0.5
W = util.rndmask(S.shape[0:2] + (1,), frc, dtype=np.float32)
Sw = W * S

"""
$\ell_2$-TV denoising with a spatial mask as a non-linear lowpass filter.
"""

lmbda = 0.1
opt = tvl2.TVL2Denoise.Options({'Verbose': False, 'MaxMainIter': 200,
            'DFidWeight': W, 'gEvalY': False, 'AutoRho': {'Enabled': True}})
b = tvl2.TVL2Denoise(Sw, lmbda, opt)
sl = b.solve()
sh = Sw - sl

"""
CDL without a spatial mask using :class:`.dictlrn.cbpdndl.ConvBPDNDictLearn`.
"""

lmbda = 0.05
opt1 = cbpdndl.ConvBPDNDictLearn.Options({'Verbose': True,
            'MaxMainIter': 200, 'AccurateDFid': True,
            'CBPDN': {'rho': 50.0*lmbda + 0.5},
            'CCMOD': {'rho': 1e2}}, dmethod='cns')
d1 = cbpdndl.ConvBPDNDictLearn(D0, sh, lmbda, opt1, dmethod='cns')
D1 = d1.solve()

"""
Reconstruct from the CDL solution without a spatial mask.
"""

sr1 = d1.reconstruct().squeeze() + sl

"""
CDL with a spatial mask using :class:`.cbpdndlmd.ConvBPDNMaskDictLearn`. (Note that :class:`.prlcnscdl.ConvBPDNMaskDcplDictLearn_Consensus` solves the same problem, but is substantially faster on a multi-core architecture.)
"""

opt2 = cbpdndlmd.ConvBPDNMaskDictLearn.Options({'Verbose': True,
            'MaxMainIter': 200, 'AccurateDFid': True,
            'CBPDN': {'rho': 20.0*lmbda + 0.5},
            'CCMOD': {'rho': 2e-1}}, dmethod='cns')
d2 = cbpdndlmd.ConvBPDNMaskDictLearn(D0, sh, lmbda, W, opt2, dmethod='cns')
D2 = d2.solve()

"""
Reconstruct from the CDL solution with a spatial mask.
"""

sr2 = d2.reconstruct().squeeze() + sl

"""
Compare dictionaries.
"""

fig = plot.figure(figsize=(14, 7))
plot.subplot(1, 2, 1)
plot.imview(util.tiledict(D1.squeeze()), title='Without Mask Decoupling',
            fig=fig)
plot.subplot(1, 2, 2)
plot.imview(util.tiledict(D2.squeeze()), title='With Mask Decoupling',
            fig=fig)
fig.show()

"""
Display reference and training images.
"""

fig = plot.figure(figsize=(14, 14))
plot.subplot(2, 2, 1)
plot.imview(S[...,0], title='Reference', fig=fig)
plot.subplot(2, 2, 2)
plot.imview(Sw[...,0], title='Test', fig=fig)
plot.subplot(2, 2, 3)
plot.imview(S[...,1], title='Reference', fig=fig)
plot.subplot(2, 2, 4)
plot.imview(Sw[...,1], title='Test', fig=fig)
fig.show()

"""
Compare reconstructed images.
"""

fig = plot.figure(figsize=(14, 14))
plot.subplot(2, 2, 1)
plot.imview(sr1[...,0], title='Without Mask Decoupling', fig=fig)
plot.subplot(2, 2, 2)
plot.imview(sr2[...,0], title='With Mask Decoupling', fig=fig)
plot.subplot(2, 2, 3)
plot.imview(sr1[...,1], title='Without Mask Decoupling', fig=fig)
plot.subplot(2, 2, 4)
plot.imview(sr2[...,1], title='With Mask Decoupling', fig=fig)
fig.show()

# Wait for enter on keyboard
input()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import io
import os
import sys

import click
import subprocess

ISORT_CONFIG = {
    'line_length': 100,
    'multi_line_output': 4,  # 4-vert-grid
}

EXTENSIONS = {'.py', '.cpp', '.c', '.h', '.hpp', '.hxx', '.cxx', '.java', '.js'}

@click.command()
@click.argument('files', nargs=-1, type=click.Path(exists=True, dir_okay=False, writable=True))
@click.option('--check', default=False, is_flag=True, help='check instead of formatting files')
@click.option('--stdin', default=False, is_flag=True, help='read filenames from stdin (1 per line)')
@click.option('-c', '--commit', default=False, is_flag=True, help='use modified files from git')
def main(files, check, stdin, commit):
    """Console script for esss_fix_format"""
    import isort
    if stdin:
        files = [x.strip() for x in click.get_text_stream('stdin').readlines()]
    if commit:
        files = get_files_from_git()
    changed_files = 0
    for filename in files:
        extension = os.path.splitext(filename)[1]
        if extension in EXTENSIONS:
            with io.open(filename, 'rb') as f:
                old_contents = f.read().decode('UTF-8')
            if extension == '.py':
                new_contents = isort.SortImports(file_contents=old_contents, **ISORT_CONFIG).output
            else:
                new_contents = old_contents
            new_contents = fix_whitespace(new_contents)
            changed = new_contents != old_contents
        else:
            click.secho(click.format_filename(filename) + ': Unknown extension', fg='blue')
            continue
        if not check and changed:
            with io.open(filename, 'wb') as f:
                f.write(new_contents.encode('UTF-8'))
        changed_files += int(changed)
        status, color = _get_status_and_color(check, changed)
        click.secho(click.format_filename(filename) + ': ' + status, fg=color)

    if check and changed_files:
        sys.exit(1)

def _get_status_and_color(check, changed):
    """
    Return a pair (status message, color) based if we are checking a file for correct formatting and if
    the file is supposed to be changed or not.
    """
    if check:
        if changed:
            return 'Failed', 'red'
        else:
            return 'OK', 'green'
    else:
        if changed:
            return 'Fixed', 'green'
        else:
            return 'Skipped', 'yellow'

def fix_whitespace(old_contents):
    """
    Fix whitespace issues in the given list of lines.

    :param unicode old_contents:
        List of lines to fix spaces and indentations.

    :return unicode:
        Returns the new contents.
    """
    input_lines = old_contents.split('\n')
    lines = [i.rstrip('\r') for i in input_lines]
    lines = _right_trim_spaces(lines)
    lines = _fix_tabs(lines)
    contents = '\n'.join(lines)
    return contents

def _right_trim_spaces(lines):
    """
    Remove spaces from the right side of each line.

    :param list(unicode) lines:
        Input lines.

    :return list(unicode):
        Modified lines.
    """
    return [i.rstrip(' \t') for i in lines]

def _fix_tabs(lines, tab_width=4):
    """
    Expands tab characters in the given list of lines for spaces..

    :param list(unicode) lines:
        Input lines.

    :return list(unicode):
        Modified lines.
    """
    return [i.expandtabs(tab_width) for i in lines]

def get_files_from_git():
    """Obtain from a list of modified files in the current repository."""
    def get_files(cmd):
        output = subprocess.check_output(cmd, shell=True)
        return output.splitlines()

    result = set()
    result.update(get_files('git diff --name-only --diff-filter=ACM --staged'))
    result.update(get_files('git diff --name-only --diff-filter=ACM'))
    result.update(get_files('git ls-files -o --full-name --exclude-standard'))
    return sorted(result)

if __name__ == "__main__":
    main()
<commit_msg>Use white font when skipping unknown extensions<commit_after># -*- coding: utf-8 -*-
import io
import os
import sys

import click
import subprocess

ISORT_CONFIG = {
    'line_length': 100,
    'multi_line_output': 4,  # 4-vert-grid
}

EXTENSIONS = {'.py', '.cpp', '.c', '.h', '.hpp', '.hxx', '.cxx', '.java', '.js'}

@click.command()
@click.argument('files', nargs=-1, type=click.Path(exists=True, dir_okay=False, writable=True))
@click.option('--check', default=False, is_flag=True, help='check instead of formatting files')
@click.option('--stdin', default=False, is_flag=True, help='read filenames from stdin (1 per line)')
@click.option('-c', '--commit', default=False, is_flag=True, help='use modified files from git')
def main(files, check, stdin, commit):
    """Console script for esss_fix_format"""
    import isort
    if stdin:
        files = [x.strip() for x in click.get_text_stream('stdin').readlines()]
    if commit:
        files = get_files_from_git()
    changed_files = 0
    for filename in files:
        extension = os.path.splitext(filename)[1]
        if extension in EXTENSIONS:
            with io.open(filename, 'rb') as f:
                old_contents = f.read().decode('UTF-8')
            if extension == '.py':
                new_contents = isort.SortImports(file_contents=old_contents, **ISORT_CONFIG).output
            else:
                new_contents = old_contents
            new_contents = fix_whitespace(new_contents)
            changed = new_contents != old_contents
        else:
            click.secho(click.format_filename(filename) + ': Unknown extension', fg='white')
            continue
        if not check and changed:
            with io.open(filename, 'wb') as f:
                f.write(new_contents.encode('UTF-8'))
        changed_files += int(changed)
        status, color = _get_status_and_color(check, changed)
        click.secho(click.format_filename(filename) + ': ' + status, fg=color)

    if check and changed_files:
        sys.exit(1)

def _get_status_and_color(check, changed):
    """
    Return a pair (status message, color) based if we are checking a file for correct formatting and if
    the file is supposed to be changed or not.
    """
    if check:
        if changed:
            return 'Failed', 'red'
        else:
            return 'OK', 'green'
    else:
        if changed:
            return 'Fixed', 'green'
        else:
            return 'Skipped', 'yellow'

def fix_whitespace(old_contents):
    """
    Fix whitespace issues in the given list of lines.

    :param unicode old_contents:
        List of lines to fix spaces and indentations.

    :return unicode:
        Returns the new contents.
    """
    input_lines = old_contents.split('\n')
    lines = [i.rstrip('\r') for i in input_lines]
    lines = _right_trim_spaces(lines)
    lines = _fix_tabs(lines)
    contents = '\n'.join(lines)
    return contents

def _right_trim_spaces(lines):
    """
    Remove spaces from the right side of each line.

    :param list(unicode) lines:
        Input lines.

    :return list(unicode):
        Modified lines.
    """
    return [i.rstrip(' \t') for i in lines]

def _fix_tabs(lines, tab_width=4):
    """
    Expands tab characters in the given list of lines for spaces..

    :param list(unicode) lines:
        Input lines.

    :return list(unicode):
        Modified lines.
    """
    return [i.expandtabs(tab_width) for i in lines]

def get_files_from_git():
    """Obtain from a list of modified files in the current repository."""
    def get_files(cmd):
        output = subprocess.check_output(cmd, shell=True)
        return output.splitlines()

    result = set()
    result.update(get_files('git diff --name-only --diff-filter=ACM --staged'))
    result.update(get_files('git diff --name-only --diff-filter=ACM'))
    result.update(get_files('git ls-files -o --full-name --exclude-standard'))
    return sorted(result)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from __future__ import print_function
import structures
from copy import copy
from copy import deepcopy
from datetime import time

class Week:

    """A particular week of courses, consisting of 5 day objects"""

    def __init__(self, rooms, this_scheduler):
        """Initialize week object with list of room objects"""
        self.schedule = this_scheduler
        self.days = [structures.Day(rooms, day_code, self)
                     for day_code in 'mtwrf']
        self.fitness = 0
        self.valid = True
        #Week's copy of courses
        self.sections = []

    def info(self, query):
        """Goes up the object hierarchy to find object for given week
        Possible queries: schedule
        IN: query string
        OUT: object of query's type for given week"""
        if query not in ["Schedule"]:
            print("Invalid query for Week")
            return
        elif query == "Schedule":
            return self.schedule

    def update_sections(self, courses):
        """Updates list of sections with all details
        IN: list of courses
        OUT: updated section attribute"""
        if len(self.sections) > 0:
            self.sections = []
        for each_course in courses:
            each_slots = self.find_course(each_course)
            each_section = structures.Section(each_course, each_slots)
            self.sections.append(each_section)

    def find_section(self, course_code):
        """IN: course_code as string
        OUT: section object; note that its attr's are shallow copies of structure objects
        Returns None if not found"""
        for each_section in self.sections:
            if each_section.course.code == course_code:
                #found
                return each_section
        #not found
        return None

    def deep_copy(self):
        """Returns a deep copy of week"""
        return deepcopy(self)

    def find_time_slot(self, day, room, time_slot):
        """Returns the time slot for the given time
        IN: day, room, time objects
        OUT: corresponding time slot object"""
        for each_day in self.days:
            # todo: write __eq__ for all classes and change these comparisons
            # accordingly
            if each_day.day_code == day.day_code:
                for each_room in each_day.rooms:
                    if each_room.number == room.number:
                        for each_time_slot in each_room.schedule:
                            if each_time_slot.start_time == time_slot.start_time:
                                return each_time_slot
        # todo: specificity on error; fail fast/gracefully
        print("Time slot not found")
        return

    def find_course(self, course):
        """Returns list of time slot objects for given course object in week
        IN: course object
        OUT: list of time slot objects"""
        time_slots = []
        for each_day in self.days:
            for each_room in each_day.rooms:
                for each_slot in each_room.schedule:
                    # If there is a course
                    if each_slot.course:
                        # If they are the same course
                        if each_slot.course.code == course.code:
                            time_slots.append(each_slot)
        return time_slots

    def __getitem__(self, k):
        if k not in "mtwrf":
            raise ValueError

        for day in self.days:
            if day.day_code == k:
                return day

    def list_time_slots(self):
        """Gives list of all time slot objects in week while indexing them"""
        list_of_slots = []
        # index counters
        day = 0
        room = 0
        slot = 0

        for each_day in self.days:
            for each_room in each_day.rooms:
                for each_slot in each_room.schedule:
                    list_of_slots.append(each_slot)
                    each_slot.set_indices(day, room, slot)
                    slot += 1
                room += 1
                slot = 0
            day += 1
            room = 0
        return list_of_slots

    def find_empty_time_slots(self):
        """Returns a list of empty (no course) time slot objects"""
        empty_slots = []
        for each_day in self.days:
            for each_room in each_day.rooms:
                for each_slot in each_room.schedule:
                    if each_slot.course is None:
                        empty_slots.append(each_slot)
        return empty_slots

    def fill_week(self, courses):
        """Fills the week based on the criteria listed in courses"""
        # check that courses has the correct structure
        #[{code:"CSC130", credit:"3", startTime:"11:00", endTime:"12:00", days:"MWF", room:"CHEK209"}, ...]

        try:
            # not using find_time_slot because this way should be faster
            for each_course in courses:
                start_hour, start_min = map(
                    int, each_course["startTime"].strip().split(':'))
                end_hour, end_min = map(
                    int, each_course["endTime"].strip().split(':'))
                startTime = time(start_hour, start_min)
                endTime = time(end_hour, end_min)
                # loop through days and find timeslot associated with
                # starttime, room, day
                for each_day in self.days:
                    if each_day.day_code in each_course["days"].lower():
                        for each_slot in each_day.get_room(each_course["room"]):
                            if each_slot.start_time == startTime and \
                               each_slot.end_time == endTime:
                                for each_s_course in self.schedule.courses:
                                    if each_s_course.code == each_course["code"]:
                                        each_slot.course = each_s_course
                                        each_slot.instructor = each_s_course.instructor

        except KeyError, AttributeError:
            # error stuff
            print("Unable to fill week because bad input")

    def print_concise(self):
        """Returns a concise list of courses for week in the structure:
            course_code day_code room_number start_time-end_time"""
        courses_dyct = {
        }  # structure of {course_code : (day_code, room_number, start_time, end_time)}
        instructors = []
        for each_slot in self.list_time_slots():
            if each_slot.course != None:
                if courses_dyct.has_key(each_slot.course.code):
                    courses_dyct[each_slot.course.code][0] += each_slot.day
                else:
                    courses_dyct[each_slot.course.code] =  [each_slot.day, each_slot.room.building, \
                                                            each_slot.room.number, each_slot.start_time, \
                                                            each_slot.end_time, each_slot.instructor]
                    if each_slot.instructor not in instructors:
                        instructors.append(each_slot.instructor)

        concise_schedule_str = ""
        for instructor in instructors:
            concise_schedule_str += instructor.name + "\n"
            for key in instructor.courses:
                concise_schedule_str += str(key) + ' ' + courses_dyct[key.code][0] + ' ' + \
                    str(courses_dyct[key.code][1]) + ' ' + str(courses_dyct[key.code][2]) + ' ' + \
                    str(courses_dyct[key.code][3]) + '-' + str(courses_dyct[key.code][4]) + '\n'

        print ("=" * 25)
        print ("Fitness score: ", self.fitness)
        print ("Is Valid: ", self.valid)
        print (concise_schedule_str)
        print ("=" * 25)
        return concise_schedule_str

    def __str__(self):
        """Returns string representation of given week"""
        return "\n".join([str(d) for d in self.days])
<commit_msg>Format start and end time to remove seconds value (13:00:00 --> 13:00)<commit_after>from __future__ import print_function
import structures
from copy import copy
from copy import deepcopy
from datetime import time

class Week:

    """A particular week of courses, consisting of 5 day objects"""

    def __init__(self, rooms, this_scheduler):
        """Initialize week object with list of room objects"""
        self.schedule = this_scheduler
        self.days = [structures.Day(rooms, day_code, self)
                     for day_code in 'mtwrf']
        self.fitness = 0
        self.valid = True
        #Week's copy of courses
        self.sections = []

    def info(self, query):
        """Goes up the object hierarchy to find object for given week
        Possible queries: schedule
        IN: query string
        OUT: object of query's type for given week"""
        if query not in ["Schedule"]:
            print("Invalid query for Week")
            return
        elif query == "Schedule":
            return self.schedule

    def update_sections(self, courses):
        """Updates list of sections with all details
        IN: list of courses
        OUT: updated section attribute"""
        if len(self.sections) > 0:
            self.sections = []
        for each_course in courses:
            each_slots = self.find_course(each_course)
            each_section = structures.Section(each_course, each_slots)
            self.sections.append(each_section)

    def find_section(self, course_code):
        """IN: course_code as string
        OUT: section object; note that its attr's are shallow copies of structure objects
        Returns None if not found"""
        for each_section in self.sections:
            if each_section.course.code == course_code:
                #found
                return each_section
        #not found
        return None

    def deep_copy(self):
        """Returns a deep copy of week"""
        return deepcopy(self)

    def find_time_slot(self, day, room, time_slot):
        """Returns the time slot for the given time
        IN: day, room, time objects
        OUT: corresponding time slot object"""
        for each_day in self.days:
            # todo: write __eq__ for all classes and change these comparisons
            # accordingly
            if each_day.day_code == day.day_code:
                for each_room in each_day.rooms:
                    if each_room.number == room.number:
                        for each_time_slot in each_room.schedule:
                            if each_time_slot.start_time == time_slot.start_time:
                                return each_time_slot
        # todo: specificity on error; fail fast/gracefully
        print("Time slot not found")
        return

    def find_course(self, course):
        """Returns list of time slot objects for given course object in week
        IN: course object
        OUT: list of time slot objects"""
        time_slots = []
        for each_day in self.days:
            for each_room in each_day.rooms:
                for each_slot in each_room.schedule:
                    # If there is a course
                    if each_slot.course:
                        # If they are the same course
                        if each_slot.course.code == course.code:
                            time_slots.append(each_slot)
        return time_slots

    def __getitem__(self, k):
        if k not in "mtwrf":
            raise ValueError

        for day in self.days:
            if day.day_code == k:
                return day

    def list_time_slots(self):
        """Gives list of all time slot objects in week while indexing them"""
        list_of_slots = []
        # index counters
        day = 0
        room = 0
        slot = 0

        for each_day in self.days:
            for each_room in each_day.rooms:
                for each_slot in each_room.schedule:
                    list_of_slots.append(each_slot)
                    each_slot.set_indices(day, room, slot)
                    slot += 1
                room += 1
                slot = 0
            day += 1
            room = 0
        return list_of_slots

    def find_empty_time_slots(self):
        """Returns a list of empty (no course) time slot objects"""
        empty_slots = []
        for each_day in self.days:
            for each_room in each_day.rooms:
                for each_slot in each_room.schedule:
                    if each_slot.course is None:
                        empty_slots.append(each_slot)
        return empty_slots

    def fill_week(self, courses):
        """Fills the week based on the criteria listed in courses"""
        # check that courses has the correct structure
        #[{code:"CSC130", credit:"3", startTime:"11:00", endTime:"12:00", days:"MWF", room:"CHEK209"}, ...]

        try:
            # not using find_time_slot because this way should be faster
            for each_course in courses:
                start_hour, start_min = map(
                    int, each_course["startTime"].strip().split(':'))
                end_hour, end_min = map(
                    int, each_course["endTime"].strip().split(':'))
                startTime = time(start_hour, start_min)
                endTime = time(end_hour, end_min)
                # loop through days and find timeslot associated with
                # starttime, room, day
                for each_day in self.days:
                    if each_day.day_code in each_course["days"].lower():
                        for each_slot in each_day.get_room(each_course["room"]):
                            if each_slot.start_time == startTime and \
                               each_slot.end_time == endTime:
                                for each_s_course in self.schedule.courses:
                                    if each_s_course.code == each_course["code"]:
                                        each_slot.course = each_s_course
                                        each_slot.instructor = each_s_course.instructor

        except KeyError, AttributeError:
            # error stuff
            print("Unable to fill week because bad input")

    def print_concise(self):
        """Returns a concise list of courses for week in the structure:
            course_code day_code room_number start_time-end_time"""
        courses_dyct = {
        }  # structure of {course_code : (day_code, room_number, start_time, end_time)}
        instructors = []
        for each_slot in self.list_time_slots():
            if each_slot.course != None:
                if courses_dyct.has_key(each_slot.course.code):
                    courses_dyct[each_slot.course.code][0] += each_slot.day
                else:
                    courses_dyct[each_slot.course.code] =  [each_slot.day, each_slot.room.building, \
                                                            each_slot.room.number, each_slot.start_time, \
                                                            each_slot.end_time, each_slot.instructor]
                    if each_slot.instructor not in instructors:
                        instructors.append(each_slot.instructor)

        concise_schedule_str = ""
        for instructor in instructors:
            concise_schedule_str += instructor.name + "\n"
            for key in instructor.courses:
                # course / days / building / room number / start time / - / end time
                concise_schedule_str += str(key) + ' ' + courses_dyct[key.code][0] + ' ' + \
                    str(courses_dyct[key.code][1]) + ' ' + str(courses_dyct[key.code][2]) + ' ' + \
                    str(courses_dyct[key.code][3])[:-3] + '-' + str(courses_dyct[key.code][4])[:-3] + '\n'
                    #format start and end time to remove seconds value

        print ("=" * 25)
        print ("Fitness score: ", self.fitness)
        print ("Is Valid: ", self.valid)
        print (concise_schedule_str)
        print ("=" * 25)
        return concise_schedule_str

    def __str__(self):
        """Returns string representation of given week"""
        return "\n".join([str(d) for d in self.days])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os
import re
from .cpickle_cache import cPickleCache
current_dir = os.path.dirname(os.path.realpath(__file__))

from werkzeug.utils import cached_property

class LazyData(object):

    @cached_property
    def en_us_dict(self):
        import enchant
        return enchant.Dict("en_US")

    @cached_property
    def two_length_words(self):
        """
        检查 2个字符 的字符组 是否是正常单词 或 词组
        e.g. `"At" in two_length_words`
        """
        def load__two_length_words__func():
            import nltk
            import marisa_trie
            return marisa_trie.Trie( \
                        sorted( \
                        list( \
                        set(\
                            [w1 for w1 in nltk.corpus.abc.words() \
                                if (len(w1) == 2) and \
                                    re.compile("[a-z]", re.IGNORECASE).match(w1[0]) \
                            ] \
                        ))))

        return cPickleCache(current_dir + "/two_length_words.cPickle", load__two_length_words__func).process()

    @cached_property
    def regular_words(self):
        import nltk
        import marisa_trie
        from .regexp import word_regexp
        return marisa_trie.Trie([unicode(w1) for w1 in nltk.corpus.abc.words() if word_regexp.match(w1)])

    @cached_property
    def lemmatize(self):
        """
        return lemmatize method.

        必须得注明 词形 才行
        >>> lmtzr.lemmatize("having")
        'having'
        >>> lmtzr.lemmatize("having", 'v')
        'have'

        http://stackoverflow.com/questions/771918/how-do-i-do-word-stemming-or-lemmatization
        """
        from nltk.stem.wordnet import WordNetLemmatizer
        lmtzr = WordNetLemmatizer()

        def func(word1, tag1=None):
            if tag1 is None:
                word1 = lmtzr.lemmatize(word1, 'v')
                tag1 = 'n'
            return lmtzr.lemmatize(word1, tag1)
        return func

    @cached_property
    def tagged_words__dict(self):
        import nltk
        return {w1:t1.upper() for w1, t1 in nltk.corpus.brown.tagged_words()}

ld = LazyData()
<commit_msg>Update cache_utils<commit_after># -*- coding: utf-8 -*-

import os
import re
from .cache_utils import cpickle_cache, cached_property
current_dir = os.path.dirname(os.path.realpath(__file__))

class LazyData(object):

    @cached_property
    def en_us_dict(self):
        import enchant
        return enchant.Dict("en_US")

    @cached_property
    def two_length_words(self):
        """
        检查 2个字符 的字符组 是否是正常单词 或 词组
        e.g. `"At" in two_length_words`
        """
        def load__two_length_words__func():
            import nltk
            import marisa_trie
            return marisa_trie.Trie( \
                        sorted( \
                        list( \
                        set(\
                            [w1 for w1 in nltk.corpus.abc.words() \
                                if (len(w1) == 2) and \
                                    re.compile("[a-z]", re.IGNORECASE).match(w1[0]) \
                            ] \
                        ))))

        return cPickleCache(current_dir + "/two_length_words.cPickle", load__two_length_words__func).process()

    @cached_property
    def regular_words(self):
        import nltk
        import marisa_trie
        from .regexp import word_regexp
        return marisa_trie.Trie([unicode(w1) for w1 in nltk.corpus.abc.words() if word_regexp.match(w1)])

    @cached_property
    def lemmatize(self):
        """
        return lemmatize method.

        必须得注明 词形 才行
        >>> lmtzr.lemmatize("having")
        'having'
        >>> lmtzr.lemmatize("having", 'v')
        'have'

        http://stackoverflow.com/questions/771918/how-do-i-do-word-stemming-or-lemmatization
        """
        from nltk.stem.wordnet import WordNetLemmatizer
        lmtzr = WordNetLemmatizer()

        def func(word1, tag1=None):
            if tag1 is None:
                word1 = lmtzr.lemmatize(word1, 'v')
                tag1 = 'n'
            return lmtzr.lemmatize(word1, tag1)
        return func

    @cached_property
    def tagged_words__dict(self):
        import nltk
        return {w1:t1.upper() for w1, t1 in nltk.corpus.brown.tagged_words()}

ld = LazyData()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""provide cached and throttled querying of `NCBI E-utilities
<http://www.ncbi.nlm.nih.gov/books/NBK25499/>`_.

QueryService defaults to returning XML documents only. This behavior
may be controlled upon instantiation by setting default_args.

::

    # create an instance of QueryService
    >>> qs = QueryService()

    # get xml for database info (in this case, a list of available database)
    >>> result = qs.einfo()

    # execute a search using an NCBI query against the gene database
    >>> result = qs.esearch({'db': 'gene', 'term': 'VEGF AND human[organism]'})

    # get xml doc for gene id=7157
    >>> result = qs.efetch({'db': 'gene', 'id': 7157})

"""
from __future__ import absolute_import, division, print_function, unicode_literals

# TODO: Fetch & compare
# TODO: TTL support in cache, request-specific TTLs?
# TODO: optional db -> options map (esp. for rettype & retmode)
# TODO: deal with caching status 200 replies that are bogus (e.g., truncated xml) -- callbacks?
# TODO: provide uncached access
# TODO: support history
# TODO: default args is misplaced -- it should go in client instead

import collections
import datetime
import hashlib
import logging
import os
import time

import lxml.etree
import pytz
import requests

from eutils.sqlitecache import SQLiteCache
from eutils.exceptions import EutilsRequestError
from eutils.compat import pickle

logger = logging.getLogger(__name__)

url_base = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils'
default_default_args = {'retmode': 'xml', 'usehistory': 'y', 'retmax': 250}
default_tool = __package__
default_email = 'biocommons-dev@googlegroups.com'
default_request_interval = 0.333
default_cache_path = os.path.join(os.path.expanduser('~'), '.cache', 'eutils-cache.db')

eastern_tz = pytz.timezone('US/Eastern')

def time_dep_request_interval(utc_dt=None):
    """
    returns a request interval approrpriate for the current time

    From http://www.ncbi.nlm.nih.gov/books/NBK25497/:

      "In order not to overload the E-utility servers, NCBI recommends that
      users post no more than three URL requests per second and limit
      large jobs to either weekends or between 9:00 PM and 5:00 AM Eastern
      time during weekdays."

    Translation: Weekdays 0500-2100 => 0.333s between requests; no throttle otherwise
    """

    if utc_dt is None:
        utc_dt = datetime.datetime.utcnow()
    eastern_dt = eastern_tz.fromutc(utc_dt)
    return default_request_interval if (0 <= eastern_dt.weekday() <= 4 and 5 <= eastern_dt.hour < 21) else 0

class QueryService(object):

    """*provides throttled and cached querying of NCBI E-utilities services*

    QueryService has three functions:

    * construct URLs appropriate for eutils endpoints

    * throttle queries per NCBI guidelines

    * cache results in persistent cache (sqlite)

    QueryService works with any valid query arguments, passed as
    dictionaries.

    Implemented interfaces:

        * esearch
        * efetch
        * elink
        * einfo
        * esummary

    Implementing other query modes should be straightforward.

    See also the NCBI's Entrez Programming Utilities Help:
        http://www.ncbi.nlm.nih.gov/books/NBK25500/

    """

    def __init__(self,
                 email=default_email,
                 cache_path=default_cache_path,
                 default_args=default_default_args,
                 request_interval=time_dep_request_interval,
                 tool=default_tool,
                 ):
        """
        :param str email: email of user (for abuse reports)
        :param str cache_path: full path to sqlite file (created if necessary)
        :param dict default_args: dictionary of query args that should accompany all requests
        :param request_interval: seconds between requests
        :type request_interval: int or a callable returning an int
        :param str tool: name of client
        :rtype: None
        :raises OSError: if sqlite file can't be opened

        """

        self.default_args = default_args
        self.email = email
        self.request_interval = request_interval
        self.tool = tool

        self._last_request_clock = 0
        self._cache = SQLiteCache(cache_path) if cache_path else None
        self._ident_args = {'tool': tool, 'email': email}
        self._request_count = 0

    def efetch(self, args):
        """
        execute a cached, throttled efetch query

        :param dict args: dict of query items
        :returns: content of reply
        :rtype: str
        :raises EutilsRequestError: when NCBI replies, but the request failed (e.g., bogus database name)

        """
        return self._query('/efetch.fcgi', args)

    def einfo(self, args=None):
        """
        execute a NON-cached, throttled einfo query

        einfo.fcgi?db=<database>

        Input: Entrez database (&db) or None (returns info on all Entrez databases)

        Output: XML containing database statistics

        Example: Find database statistics for Entrez Protein.

            QueryService.einfo({'db': 'protein'})

        Equivalent HTTP request:

            http://eutils.ncbi.nlm.nih.gov/entrez/eutils/einfo.fcgi?db=protein

        :param dict args: dict of query items (optional)
        :returns: content of reply
        :rtype: str
        :raises EutilsRequestError: when NCBI replies, but the request failed (e.g., bogus database name)

        """
        if args is None:
            args = {}
        return self._query('/einfo.fcgi', args, skip_cache=True)

    def esearch(self, args):
        """
        execute a cached, throttled esearch query

        :param dict args: dict of query items, containing at least 'db' and 'term' keys
        :returns: content of reply
        :rtype: str
        :raises EutilsRequestError: when NCBI replies, but the request failed (e.g., bogus database name)

        """
        return self._query('/esearch.fcgi', args)

    def elink(self, args):
        """
        execute a cached, throttled elink query

        Input: List of UIDs (&id); Source Entrez database (&dbfrom); Destination Entrez database (&db)

        Output: XML containing linked UIDs from source and destination databases

        Example: Find one set of Gene IDs linked to nuccore GIs 34577062 and 24475906

            QueryService.elink({'dbfrom': 'nuccore', 'db': 'gene', 'id': '34577062,24475906'})

        Equivalent HTTP request:

            http://eutils.ncbi.nlm.nih.gov/entrez/eutils/elink.fcgi?dbfrom=nuccore&db=gene&id=34577062,24475906

        :param dict args: dict of query items containing at least the 'db', 'dbfrom', and 'id' keys.
        :returns: content of reply
        :rtype: str
        :raises EutilsRequestError: when NCBI replies, but the request failed (e.g., bogus database name)

        """
        return self._query('/elink.fcgi', args)

    def esummary(self, args):
        """
        execute a cached, throttled esummary query

        Input: List of UIDs (&id); Entrez database (&db)

        Output: XML document summary for requested ID(s) [comma-separated]

        Example:

            QueryService.esummary({ 'db': 'medgen', 'id': 134 })

        Equivalent HTTP request:

            http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=medgen&id=134

        :param dict args: dict of query items containing at least 'db' and 'id' keys.
        :returns: content of reply
        :rtype: str
        :raises EutilsRequestError: when NCBI replies, but the request failed (e.g., bogus database name)

        """
        return self._query('/esummary.fcgi', args)

    ############################################################################
    ## Internals
    def _query(self, path, args={}, skip_cache=False, skip_sleep=False):
        """return results for a NCBI query, possibly from the cache

        :param: path: relative query path (e.g., 'einfo.fcgi')
        :param: args: dictionary of query args
        :param: skip_cache: whether to bypass the cache on reading
        :param: skip_sleep: whether to bypass query throttling
        :rtype: xml string

        The args are joined with args required by NCBI (tool and email
        address) and with the default args declared when instantiating
        the client.
        """
        # cache key: the key associated with this endpoint and args The
        # key intentionally excludes the identifying args (tool and email)
        # and is independent of the request method (GET/POST) args are
        # sorted for canonicalization

        url = url_base + path

        # next 3 lines converted by 2to3 -nm
        defining_args = dict(list(self.default_args.items()) + list(args.items()))
        full_args = dict(list(self._ident_args.items()) + list(defining_args.items()))
        cache_key = hashlib.md5(pickle.dumps((url, sorted(defining_args.items())))).hexdigest()

        sqas = ';'.join([k + '=' + str(v) for k, v in sorted(args.items())])
        full_args_str = ';'.join([k + '=' + str(v) for k, v in sorted(full_args.items())])

        if not skip_cache and self._cache:
            try:
                v = self._cache[cache_key]
                logger.debug('cache hit for key {cache_key} ({url}, {sqas}) '.format(
                    cache_key=cache_key,
                    url=url,
                    sqas=sqas))
                return v
            except KeyError:
                logger.debug('cache miss for key {cache_key} ({url}, {sqas}) '.format(
                    cache_key=cache_key,
                    url=url,
                    sqas=sqas))
                pass

        if not skip_sleep:
            req_int = self.request_interval() if isinstance(self.request_interval, collections.Callable) else self.request_interval
            sleep_time = req_int - (time.clock() - self._last_request_clock)
            if sleep_time > 0:
                logger.debug('sleeping {sleep_time:.3f}'.format(sleep_time=sleep_time))
                time.sleep(sleep_time)
        r = requests.post(url, full_args)
        self._last_request_clock = time.clock()
        logger.debug('post({url}, {fas}): {r.status_code} {r.reason}, {len})'.format(
            url=url,
            fas=full_args_str,
            r=r,
            len=len(r.text)))

        if not r.ok or any(bad_word in r.text for bad_word in ['<error>', '<ERROR>']):
            # TODO: discriminate between types of errors
            xml = lxml.etree.fromstring(r.text.encode('utf-8'))
            raise EutilsRequestError('{r.reason} ({r.status_code}): {error}'.format(r=r, error=xml.find('ERROR').text))

        if self._cache:
            # N.B. we cache the read even if skip_cache is true
            self._cache[cache_key] = r.content
            logger.info('cached results for key {cache_key} ({url}, {sqas}) '.format(
                cache_key=cache_key,
                url=url,
                sqas=sqas))

        return r.content

if __name__ == '__main__':
    qs = QueryService()
    r = qs.einfo({'db': 'protein'})

# <LICENSE>
# Copyright 2015 eutils Committers
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing
# permissions and limitations under the License.
# </LICENSE>
<commit_msg>fixes #134: don't cache when no-cache requested (and catch blocked access replies)<commit_after># -*- coding: utf-8 -*-

"""provide cached and throttled querying of `NCBI E-utilities
<http://www.ncbi.nlm.nih.gov/books/NBK25499/>`_.

QueryService defaults to returning XML documents only. This behavior
may be controlled upon instantiation by setting default_args.

::

    # create an instance of QueryService
    >>> qs = QueryService()

    # get xml for database info (in this case, a list of available database)
    >>> result = qs.einfo()

    # execute a search using an NCBI query against the gene database
    >>> result = qs.esearch({'db': 'gene', 'term': 'VEGF AND human[organism]'})

    # get xml doc for gene id=7157
    >>> result = qs.efetch({'db': 'gene', 'id': 7157})

"""
from __future__ import absolute_import, division, print_function, unicode_literals

# TODO: Fetch & compare
# TODO: TTL support in cache, request-specific TTLs?
# TODO: optional db -> options map (esp. for rettype & retmode)
# TODO: deal with caching status 200 replies that are bogus (e.g., truncated xml) -- callbacks?
# TODO: provide uncached access
# TODO: support history
# TODO: default args is misplaced -- it should go in client instead

import collections
import datetime
import hashlib
import logging
import os
import time

import lxml.etree
import pytz
import requests

from eutils.sqlitecache import SQLiteCache
from eutils.exceptions import EutilsRequestError
from eutils.compat import pickle

logger = logging.getLogger(__name__)

url_base = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils'
default_default_args = {'retmode': 'xml', 'usehistory': 'y', 'retmax': 250}
default_tool = __package__
default_email = 'biocommons-dev@googlegroups.com'
default_request_interval = 0.333
default_cache_path = os.path.join(os.path.expanduser('~'), '.cache', 'eutils-cache.db')

eastern_tz = pytz.timezone('US/Eastern')

def time_dep_request_interval(utc_dt=None):
    """
    returns a request interval approrpriate for the current time

    From http://www.ncbi.nlm.nih.gov/books/NBK25497/:

      "In order not to overload the E-utility servers, NCBI recommends that
      users post no more than three URL requests per second and limit
      large jobs to either weekends or between 9:00 PM and 5:00 AM Eastern
      time during weekdays."

    Translation: Weekdays 0500-2100 => 0.333s between requests; no throttle otherwise
    """

    if utc_dt is None:
        utc_dt = datetime.datetime.utcnow()
    eastern_dt = eastern_tz.fromutc(utc_dt)
    return default_request_interval if (0 <= eastern_dt.weekday() <= 4 and 5 <= eastern_dt.hour < 21) else 0

class QueryService(object):

    """*provides throttled and cached querying of NCBI E-utilities services*

    QueryService has three functions:

    * construct URLs appropriate for eutils endpoints

    * throttle queries per NCBI guidelines

    * cache results in persistent cache (sqlite)

    QueryService works with any valid query arguments, passed as
    dictionaries.

    Implemented interfaces:

        * esearch
        * efetch
        * elink
        * einfo
        * esummary

    Implementing other query modes should be straightforward.

    See also the NCBI's Entrez Programming Utilities Help:
        http://www.ncbi.nlm.nih.gov/books/NBK25500/

    """

    def __init__(self,
                 email=default_email,
                 cache_path=default_cache_path,
                 default_args=default_default_args,
                 request_interval=time_dep_request_interval,
                 tool=default_tool,
                 ):
        """
        :param str email: email of user (for abuse reports)
        :param str cache_path: full path to sqlite file (created if necessary)
        :param dict default_args: dictionary of query args that should accompany all requests
        :param request_interval: seconds between requests
        :type request_interval: int or a callable returning an int
        :param str tool: name of client
        :rtype: None
        :raises OSError: if sqlite file can't be opened

        """

        self.default_args = default_args
        self.email = email
        self.request_interval = request_interval
        self.tool = tool

        self._last_request_clock = 0
        self._cache = SQLiteCache(cache_path) if cache_path else None
        self._ident_args = {'tool': tool, 'email': email}
        self._request_count = 0

    def efetch(self, args):
        """
        execute a cached, throttled efetch query

        :param dict args: dict of query items
        :returns: content of reply
        :rtype: str
        :raises EutilsRequestError: when NCBI replies, but the request failed (e.g., bogus database name)

        """
        return self._query('/efetch.fcgi', args)

    def einfo(self, args=None):
        """
        execute a NON-cached, throttled einfo query

        einfo.fcgi?db=<database>

        Input: Entrez database (&db) or None (returns info on all Entrez databases)

        Output: XML containing database statistics

        Example: Find database statistics for Entrez Protein.

            QueryService.einfo({'db': 'protein'})

        Equivalent HTTP request:

            http://eutils.ncbi.nlm.nih.gov/entrez/eutils/einfo.fcgi?db=protein

        :param dict args: dict of query items (optional)
        :returns: content of reply
        :rtype: str
        :raises EutilsRequestError: when NCBI replies, but the request failed (e.g., bogus database name)

        """
        if args is None:
            args = {}
        return self._query('/einfo.fcgi', args, skip_cache=True)

    def esearch(self, args):
        """
        execute a cached, throttled esearch query

        :param dict args: dict of query items, containing at least 'db' and 'term' keys
        :returns: content of reply
        :rtype: str
        :raises EutilsRequestError: when NCBI replies, but the request failed (e.g., bogus database name)

        """
        return self._query('/esearch.fcgi', args)

    def elink(self, args):
        """
        execute a cached, throttled elink query

        Input: List of UIDs (&id); Source Entrez database (&dbfrom); Destination Entrez database (&db)

        Output: XML containing linked UIDs from source and destination databases

        Example: Find one set of Gene IDs linked to nuccore GIs 34577062 and 24475906

            QueryService.elink({'dbfrom': 'nuccore', 'db': 'gene', 'id': '34577062,24475906'})

        Equivalent HTTP request:

            http://eutils.ncbi.nlm.nih.gov/entrez/eutils/elink.fcgi?dbfrom=nuccore&db=gene&id=34577062,24475906

        :param dict args: dict of query items containing at least the 'db', 'dbfrom', and 'id' keys.
        :returns: content of reply
        :rtype: str
        :raises EutilsRequestError: when NCBI replies, but the request failed (e.g., bogus database name)

        """
        return self._query('/elink.fcgi', args)

    def esummary(self, args):
        """
        execute a cached, throttled esummary query

        Input: List of UIDs (&id); Entrez database (&db)

        Output: XML document summary for requested ID(s) [comma-separated]

        Example:

            QueryService.esummary({ 'db': 'medgen', 'id': 134 })

        Equivalent HTTP request:

            http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=medgen&id=134

        :param dict args: dict of query items containing at least 'db' and 'id' keys.
        :returns: content of reply
        :rtype: str
        :raises EutilsRequestError: when NCBI replies, but the request failed (e.g., bogus database name)

        """
        return self._query('/esummary.fcgi', args)

    ############################################################################
    ## Internals
    def _query(self, path, args={}, skip_cache=False, skip_sleep=False):
        """return results for a NCBI query, possibly from the cache

        :param: path: relative query path (e.g., 'einfo.fcgi')
        :param: args: dictionary of query args
        :param: skip_cache: whether to bypass the cache on reading
        :param: skip_sleep: whether to bypass query throttling
        :rtype: xml string

        The args are joined with args required by NCBI (tool and email
        address) and with the default args declared when instantiating
        the client.
        """
        def _cacheable(r):
            """return False if r shouldn't be cached (contains a no-cache meta
            line); True otherwise"""
            return not ("no-cache" in r  # obviate parsing, maybe
                        and lxml.etree.XML(r).xpath("//meta/@content='no-cache'"))

        # cache key: the key associated with this endpoint and args The
        # key intentionally excludes the identifying args (tool and email)
        # and is independent of the request method (GET/POST) args are
        # sorted for canonicalization

        url = url_base + path

        # next 3 lines converted by 2to3 -nm
        defining_args = dict(list(self.default_args.items()) + list(args.items()))
        full_args = dict(list(self._ident_args.items()) + list(defining_args.items()))
        cache_key = hashlib.md5(pickle.dumps((url, sorted(defining_args.items())))).hexdigest()

        sqas = ';'.join([k + '=' + str(v) for k, v in sorted(args.items())])
        full_args_str = ';'.join([k + '=' + str(v) for k, v in sorted(full_args.items())])

        logging.debug("CACHE:" + str(skip_cache) + "//" + str(self._cache))
        if not skip_cache and self._cache:
            try:
                v = self._cache[cache_key]
                logger.debug('cache hit for key {cache_key} ({url}, {sqas}) '.format(
                    cache_key=cache_key,
                    url=url,
                    sqas=sqas))
                return v
            except KeyError:
                logger.debug('cache miss for key {cache_key} ({url}, {sqas}) '.format(
                    cache_key=cache_key,
                    url=url,
                    sqas=sqas))
                pass

        if not skip_sleep:
            req_int = self.request_interval() if isinstance(self.request_interval, collections.Callable) else self.request_interval
            sleep_time = req_int - (time.clock() - self._last_request_clock)
            if sleep_time > 0:
                logger.debug('sleeping {sleep_time:.3f}'.format(sleep_time=sleep_time))
                time.sleep(sleep_time)
        r = requests.post(url, full_args)
        self._last_request_clock = time.clock()
        logger.debug('post({url}, {fas}): {r.status_code} {r.reason}, {len})'.format(
            url=url,
            fas=full_args_str,
            r=r,
            len=len(r.text)))

        if not r.ok:
            # TODO: discriminate between types of errors
            xml = lxml.etree.fromstring(r.text.encode('utf-8'))
            raise EutilsRequestError('{r.reason} ({r.status_code}): {error}'.format(r=r, error=xml.find('ERROR').text))

        if any(bad_word in r.text for bad_word in ['<error>', '<ERROR>']):
            xml = lxml.etree.fromstring(r.text.encode('utf-8'))
            raise EutilsRequestError('{r.reason} ({r.status_code}): {error}'.format(r=r, error=xml.find('ERROR').text))

        if '<h1 class="error">Access Denied</h1>' in r.text:
            raise EutilsRequestError('Access Denied: {url}'.format(url=url))

        if self._cache and _cacheable(r.text):
            # N.B. we cache results even when skip_cache (read) is true
            self._cache[cache_key] = r.content
            logger.info('cached results for key {cache_key} ({url}, {sqas}) '.format(
                cache_key=cache_key,
                url=url,
                sqas=sqas))

        return r.content

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    qs = QueryService()
    r = qs.einfo({'db': 'protein'})
    r = qs.efetch({'db': 'protein', 'id': '319655736'})

# <LICENSE>
# Copyright 2015 eutils Committers
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing
# permissions and limitations under the License.
# </LICENSE>
<|endoftext|>