<commit_before>"""
Django settings for pypolo project.

Generated by 'django-admin startproject' using Django 1.9.

For more information on this file, see
https://docs.djangoproject.com/en/1.9/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.9/ref/settings/
"""

import os

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '))iy*8&jyl3s6m58g(z=6xo5yq^hl^-ppk$v-h-pd#@fhn&k@0'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
MAKE_TRADES = True

ALLOWED_HOSTS = ['trader.owocki.com', '45.55.42.224']

# Application definition

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'history',
    'chartit',
]

MIDDLEWARE_CLASSES = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'pypolo.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': ['history/templates/', ],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'pypolo.wsgi.application'

# Password validation
# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.9/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'MST'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.9/howto/static-files/

STATIC_URL = '/static/'
STATICFILES_DIRS = [
    os.path.join(BASE_DIR, "static"),
]

# logfile
LOG_FILE = "/var/log/django.log"

# Default number of threads for workers
NUM_THREADS = 1

TRADER_GRANULARITY_MINS = 10  # TODO: change me when granularity changes (search this string <---)

TRADER_CURRENCY_CONFIG = [
    {'type': 'nn',
     'name': 'ETH / 5',
     'symbol': 'BTC_ETH',
     'weight': 0.1,
     'granularity': TRADER_GRANULARITY_MINS,
     'datasetinputs': 5},
    {'type': 'nn',
     'name': 'ETH / 5',
     'symbol': 'BTC_ETH',
     'weight': 0.1,
     'granularity': TRADER_GRANULARITY_MINS,
     'datasetinputs': 4},
    {'type': 'classifier',
     'symbol': 'USDT_BTC',
     'name': 'AdaBoost',
     'weight': 0.1,
     'granularity': TRADER_GRANULARITY_MINS,
     'datasetinputs': 2,
     'minutes_back': 1000},
    {'type': 'classifier',
     'symbol': 'USDT_BTC',
     'name': 'Naive Bayes',
     'weight': 0.1,
     'granularity': TRADER_GRANULARITY_MINS,
     'datasetinputs': 2,
     'minutes_back': 1000},
    {'type': 'classifier',
     'symbol': 'BTC_ETH',
     'name': 'Naive Bayes',
     'weight': 2,
     'granularity': TRADER_GRANULARITY_MINS * 3,
     'datasetinputs': 2,
     'minutes_back': 1000},
]

TRAINER_CURRENCY_CONFIG = {
    'classifiers': {
        'ticker': ['BTC_ETH', 'USDT_BTC'],
        'min_back': [100, 1000, 24 * 60, 24 * 60 * 2],
        'granularity': [10, 15, 20, 30, 40, 50, 60, 120, 240],
        'datasetinputs': [3],
        'timedelta_back_in_granularity_increments': [10, 30, 60, 100, 1000],
        'name': ["Nearest Neighbors", "Linear SVM", "RBF SVM", "Decision Tree",
                 "Random Forest", "AdaBoost", "Naive Bayes", "Linear Discriminant Analysis",
                 "Quadratic Discriminant Analysis"],
    },
    'supervised_nn': {
        'ticker': ['BTC_ETH', 'USDT_BTC'],
        'hidden_layer': [1, 5, 15, 40],
        # 2/23 -- removed 15, it was barely edged out by 1,5.
        # 2/25 -- added 15, 40 in because of recent bugs
        'min_back': [100, 1000, 24 * 60, 24 * 60 * 2],
        # 2/22 - eliminated 10000
        # 2/25 -- added 24*50, 24*60*2 because "maybe because NN only contains last 1000 data points (1/3 day).
        #   if only selling happened during that time, nn will bias towards selling.  duh!"
        'granularity': [10, 15, 20, 30, 40, 50, 60, 120, 240],
        # 2/23 notes - results so far: 59 (54% correct) 15 (56% correct) 1 (50% correct) 5 (52% correct).
        #   removing 1,5, adding 30 . 2/23 (pt 2) -- added 119, 239
        # 2/24 notes -- removed 120,240, added 20, 40, 45
        # 2/25 notes -- added 10, 50, removed 45
        # 2/25 -- added 120,240 back in to retest in light of recent bugs
        'datasetinput': [1, 2, 3, 4, 5, 6, 15, 10, 20, 40, 100, 200],
        # 2/23 -- removed 3,5,15 -- added 20,40,100
        # 2/24 -- removed 7,10,20,40,100, added 3,4,5
        # 2/25 -- added 3,5,15,10,20,40,100 back in to retest in light of recent bugs
        'epoch': [1000],
        # 2/22 -- eliminated 4000, 100
        'bias': [True],
        # 2/22 -- Eliminated 'False'
        'momentum': [0.1],
        'learningrate': [0.05],
        # 2/22 -- elimated 0.005, 0.01, adding 0.03 and 0.1 today.  #2/23 - 0.1 (54% correct) 0.05 (55% correct) 0.03
        # (54% correct) .  eliminating everything but 0.05 so i can test more #datasetinput_options
        'weightdecay': [0.0],
        # 2/22 -- eliminated 0.1,0.2
        'recurrent': [True],
        # 2/23 notes - 0 (52% correct) 1 (55% correct), removed false
        'timedelta_back_in_granularity_increments': [10, 30, 60, 100, 1000],
    }
}

# Include local settings overrides
try:
    from pypolo.local_settings import *  # NOQA
    INSTALLED_APPS += DEBUG_APPS
except (ImportError, NameError) as exp:
    print 'Failed to load pypolo/local_settings.py because: %s' % exp
<commit_msg>diff test params<commit_after>"""
Django settings for pypolo project.

Generated by 'django-admin startproject' using Django 1.9.

For more information on this file, see
https://docs.djangoproject.com/en/1.9/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.9/ref/settings/
"""

import os

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '))iy*8&jyl3s6m58g(z=6xo5yq^hl^-ppk$v-h-pd#@fhn&k@0'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
MAKE_TRADES = True

ALLOWED_HOSTS = ['trader.owocki.com', '45.55.42.224']

# Application definition

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'history',
    'chartit',
]

MIDDLEWARE_CLASSES = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'pypolo.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': ['history/templates/', ],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'pypolo.wsgi.application'

# Password validation
# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.9/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'MST'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.9/howto/static-files/

STATIC_URL = '/static/'
STATICFILES_DIRS = [
    os.path.join(BASE_DIR, "static"),
]

# logfile
LOG_FILE = "/var/log/django.log"

# Default number of threads for workers
NUM_THREADS = 1

TRADER_GRANULARITY_MINS = 10  # TODO: change me when granularity changes (search this string <---)

TRADER_CURRENCY_CONFIG = [
    {'type': 'nn',
     'name': 'ETH / 5',
     'symbol': 'BTC_ETH',
     'weight': 0.1,
     'granularity': TRADER_GRANULARITY_MINS,
     'datasetinputs': 5},
    {'type': 'nn',
     'name': 'ETH / 5',
     'symbol': 'BTC_ETH',
     'weight': 0.1,
     'granularity': TRADER_GRANULARITY_MINS,
     'datasetinputs': 4},
    {'type': 'classifier',
     'symbol': 'USDT_BTC',
     'name': 'AdaBoost',
     'weight': 0.1,
     'granularity': TRADER_GRANULARITY_MINS,
     'datasetinputs': 2,
     'minutes_back': 1000},
    {'type': 'classifier',
     'symbol': 'USDT_BTC',
     'name': 'Naive Bayes',
     'weight': 0.1,
     'granularity': TRADER_GRANULARITY_MINS,
     'datasetinputs': 2,
     'minutes_back': 1000},
    {'type': 'classifier',
     'symbol': 'BTC_ETH',
     'name': 'Naive Bayes',
     'weight': 2,
     'granularity': TRADER_GRANULARITY_MINS * 3,
     'datasetinputs': 2,
     'minutes_back': 1000},
]

TRAINER_CURRENCY_CONFIG = {
    'classifiers': {
        'ticker': ['BTC_ETH', 'USDT_BTC'],
        'min_back': [100, 1000, 24 * 60, 24 * 60 * 2],
        'granularity': [10, 15, 20, 30, 40, 50, 60, 120, 240],
        'datasetinputs': [2, 3, 5],
        'timedelta_back_in_granularity_increments': [10, 30, 60, 100, 1000],
        'name': ["Nearest Neighbors", "Linear SVM", "RBF SVM", "Decision Tree",
                 "Random Forest", "AdaBoost", "Naive Bayes", "Linear Discriminant Analysis",
                 "Quadratic Discriminant Analysis"],
    },
    'supervised_nn': {
        'ticker': ['BTC_ETH', 'USDT_BTC'],
        'hidden_layer': [1, 5, 15, 40],
        # 2/23 -- removed 15, it was barely edged out by 1,5.
        # 2/25 -- added 15, 40 in because of recent bugs
        'min_back': [100, 1000, 24 * 60, 24 * 60 * 2],
        # 2/22 - eliminated 10000
        # 2/25 -- added 24*50, 24*60*2 because "maybe because NN only contains last 1000 data points (1/3 day).
        #   if only selling happened during that time, nn will bias towards selling.  duh!"
        'granularity': [10, 15, 20, 30, 40, 50, 60, 120, 240],
        # 2/23 notes - results so far: 59 (54% correct) 15 (56% correct) 1 (50% correct) 5 (52% correct).
        #   removing 1,5, adding 30 . 2/23 (pt 2) -- added 119, 239
        # 2/24 notes -- removed 120,240, added 20, 40, 45
        # 2/25 notes -- added 10, 50, removed 45
        # 2/25 -- added 120,240 back in to retest in light of recent bugs
        'datasetinput': [1, 2, 3, 4, 5, 6, 15, 10, 20, 40, 100, 200],
        # 2/23 -- removed 3,5,15 -- added 20,40,100
        # 2/24 -- removed 7,10,20,40,100, added 3,4,5
        # 2/25 -- added 3,5,15,10,20,40,100 back in to retest in light of recent bugs
        'epoch': [1000],
        # 2/22 -- eliminated 4000, 100
        'bias': [True],
        # 2/22 -- Eliminated 'False'
        'momentum': [0.1],
        'learningrate': [0.05],
        # 2/22 -- elimated 0.005, 0.01, adding 0.03 and 0.1 today.  #2/23 - 0.1 (54% correct) 0.05 (55% correct) 0.03
        # (54% correct) .  eliminating everything but 0.05 so i can test more #datasetinput_options
        'weightdecay': [0.0],
        # 2/22 -- eliminated 0.1,0.2
        'recurrent': [True],
        # 2/23 notes - 0 (52% correct) 1 (55% correct), removed false
        'timedelta_back_in_granularity_increments': [10, 30, 60, 100, 1000],
    }
}

# Include local settings overrides
try:
    from pypolo.local_settings import *  # NOQA
    INSTALLED_APPS += DEBUG_APPS
except (ImportError, NameError) as exp:
    print 'Failed to load pypolo/local_settings.py because: %s' % exp
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
util.py

Created by Stephan Hügel on 2016-07-19

This file is part of pypolyline.

The MIT License (MIT)

Copyright (c) 2016 Stephan Hügel

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

"""

import os
from sys import platform, version_info
from ctypes import Structure, POINTER, c_void_p, c_size_t, c_double, c_uint32, c_char_p, cast, cdll
import numpy as np

__author__ = u"Stephan Hügel"
__version__ = "0.1.5"

file_path = os.path.dirname(__file__)

prefix = {'win32': ''}.get(platform, 'lib')
extension = {'darwin': '.dylib', 'win32': '.dll'}.get(platform, '.so')
fpath = {'darwin': '', 'win32': ''}.get(platform, os.path.join(file_path, ".libs"))

# Python 3 check
if (version_info > (3, 0)):
    from subprocess import getoutput as spop
    py3 = True
else:
    from subprocess import check_output as spop
    py3 = False

try:
    lib = cdll.LoadLibrary(os.path.join(file_path, prefix + "polyline_ffi" + extension))
except OSError:
    # the Rust lib's been grafted by manylinux1
    if not py3:
        fname = spop(["ls", fpath]).split()[0]
    else:
        fname = spop(["ls %s" % fpath]).split()[0]
    lib = cdll.LoadLibrary(os.path.join(file_path, ".libs", fname))

class EncodingError(Exception):
    def __init__(self, message):
        super(EncodingError, self).__init__(message)

class DecodingError(Exception):
    def __init__(self, message):
        super(DecodingError, self).__init__(message)

class _FFIArray(Structure):
    """
    Convert sequence of float lists to a C-compatible void array
    example: [[1.0, 2.0], [3.0, 4.0]]

    """
    _fields_ = [("data", c_void_p),
                ("len", c_size_t)]

    @classmethod
    def from_param(cls, seq):
        """  Allow implicit conversions """
        return seq if isinstance(seq, cls) else cls(seq)

    def __init__(self, seq, data_type = c_double):
        self.data = cast(
            np.array(seq, dtype=np.float64).ctypes.data_as(POINTER(data_type)),
            c_void_p
        )
        self.len = len(seq)

class _CoordResult(Structure):
    """ Container for returned FFI coordinate data """
    _fields_ = [("coords", _FFIArray)]

class _PolylineResult(Structure):
    """ Container for returned FFI Polyline data """
    _fields_ = [("line", c_void_p)]

def _void_array_to_nested_list(res, _func, _args):
    """ Dereference the FFI result to a list of coordinates """
    try:
        shape = res.coords.len, 2
        ptr = cast(res.coords.data, POINTER(c_double))
        array = np.ctypeslib.as_array(ptr, shape)
        if np.isnan(np.sum(array)):
            raise DecodingError("Your Polyline was not valid and could not be decoded")
        return array.tolist()
    finally:
        _drop_array(res.coords)

def _void_array_to_string(res, _func, _args):
    """ Dereference the FFI result to a utf8 polyline """
    try:
        result = cast(res.line, c_char_p)
        polyline = bytes(result.value)
        if polyline.startswith(b"Latitude"):
            raise EncodingError("%s. Latitudes must be between -90.0 and 90.0" % polyline)
        elif polyline.startswith(b"Longitude"):
            raise EncodingError("%s. Longitudes must be between -180.0 and 180.0" % polyline)
        return polyline
    finally:
        _drop_cstring(res.line)

decode_polyline = lib.decode_polyline_ffi
decode_polyline.argtypes = (c_char_p, c_uint32)
decode_polyline.restype = _CoordResult
decode_polyline.errcheck = _void_array_to_nested_list
decode_polyline.__doc__ = """
    Decode an encoded Polyline to coordinates.
    Input: a Polyline string, and a precision int (5 for Google, 6 for OSM-derived).
    Output: a list of lat, lon coordinates.

    Example: decode_polyline(_p~iF~ps|U_ulLnnqC_mqNvxq`@, 5)
    Result: [[38.5, -120.2], [40.7, -120.95], [43.252, -126.453]]

    Incorrect Polyline input will throw util.DecodingError

 """

encode_coordinates = lib.encode_coordinates_ffi
encode_coordinates.argtypes = (_FFIArray, c_uint32)
encode_coordinates.restype = _PolylineResult
encode_coordinates.errcheck = _void_array_to_string
encode_coordinates.__doc__ = """
    Encode coordinates as a Polyline.
    Input: a list of lat, lon coordinates, and a precision int (5 for Google, 6 for OSM-derived).
    Output: an encoded Polyline string.

    Example: encode_coordinates([[38.5, -120.2], [40.7, -120.95], [43.252, -126.453]], 5)
    Result: "_p~iF~ps|U_ulLnnqC_mqNvxq`@"

    Incorrect coordinate input will throw util.EncodingError

"""

# Free FFI-allocated memory
_drop_cstring = lib.drop_cstring
_drop_cstring.argtypes = (c_void_p,)
_drop_cstring.restype = None

_drop_array = lib.drop_float_array
_drop_array.argtypes = (_FFIArray,)
_drop_array.restype = None
<commit_msg>Version bump<commit_after># -*- coding: utf-8 -*-
"""
util.py

Created by Stephan Hügel on 2016-07-19

This file is part of pypolyline.

The MIT License (MIT)

Copyright (c) 2016 Stephan Hügel

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

"""

import os
from sys import platform, version_info
from ctypes import Structure, POINTER, c_void_p, c_size_t, c_double, c_uint32, c_char_p, cast, cdll
import numpy as np

__author__ = u"Stephan Hügel"
__version__ = "0.1.7"

file_path = os.path.dirname(__file__)

prefix = {'win32': ''}.get(platform, 'lib')
extension = {'darwin': '.dylib', 'win32': '.dll'}.get(platform, '.so')
fpath = {'darwin': '', 'win32': ''}.get(platform, os.path.join(file_path, ".libs"))

# Python 3 check
if (version_info > (3, 0)):
    from subprocess import getoutput as spop
    py3 = True
else:
    from subprocess import check_output as spop
    py3 = False

try:
    lib = cdll.LoadLibrary(os.path.join(file_path, prefix + "polyline_ffi" + extension))
except OSError:
    # the Rust lib's been grafted by manylinux1
    if not py3:
        fname = spop(["ls", fpath]).split()[0]
    else:
        fname = spop(["ls %s" % fpath]).split()[0]
    lib = cdll.LoadLibrary(os.path.join(file_path, ".libs", fname))

class EncodingError(Exception):
    def __init__(self, message):
        super(EncodingError, self).__init__(message)

class DecodingError(Exception):
    def __init__(self, message):
        super(DecodingError, self).__init__(message)

class _FFIArray(Structure):
    """
    Convert sequence of float lists to a C-compatible void array
    example: [[1.0, 2.0], [3.0, 4.0]]

    """
    _fields_ = [("data", c_void_p),
                ("len", c_size_t)]

    @classmethod
    def from_param(cls, seq):
        """  Allow implicit conversions """
        return seq if isinstance(seq, cls) else cls(seq)

    def __init__(self, seq, data_type = c_double):
        self.data = cast(
            np.array(seq, dtype=np.float64).ctypes.data_as(POINTER(data_type)),
            c_void_p
        )
        self.len = len(seq)

class _CoordResult(Structure):
    """ Container for returned FFI coordinate data """
    _fields_ = [("coords", _FFIArray)]

class _PolylineResult(Structure):
    """ Container for returned FFI Polyline data """
    _fields_ = [("line", c_void_p)]

def _void_array_to_nested_list(res, _func, _args):
    """ Dereference the FFI result to a list of coordinates """
    try:
        shape = res.coords.len, 2
        ptr = cast(res.coords.data, POINTER(c_double))
        array = np.ctypeslib.as_array(ptr, shape)
        if np.isnan(np.sum(array)):
            raise DecodingError("Your Polyline was not valid and could not be decoded")
        return array.tolist()
    finally:
        _drop_array(res.coords)

def _void_array_to_string(res, _func, _args):
    """ Dereference the FFI result to a utf8 polyline """
    try:
        result = cast(res.line, c_char_p)
        polyline = bytes(result.value)
        if polyline.startswith(b"Latitude"):
            raise EncodingError("%s. Latitudes must be between -90.0 and 90.0" % polyline)
        elif polyline.startswith(b"Longitude"):
            raise EncodingError("%s. Longitudes must be between -180.0 and 180.0" % polyline)
        return polyline
    finally:
        _drop_cstring(res.line)

decode_polyline = lib.decode_polyline_ffi
decode_polyline.argtypes = (c_char_p, c_uint32)
decode_polyline.restype = _CoordResult
decode_polyline.errcheck = _void_array_to_nested_list
decode_polyline.__doc__ = """
    Decode an encoded Polyline to coordinates.
    Input: a Polyline string, and a precision int (5 for Google, 6 for OSM-derived).
    Output: a list of lat, lon coordinates.

    Example: decode_polyline(_p~iF~ps|U_ulLnnqC_mqNvxq`@, 5)
    Result: [[38.5, -120.2], [40.7, -120.95], [43.252, -126.453]]

    Incorrect Polyline input will throw util.DecodingError

 """

encode_coordinates = lib.encode_coordinates_ffi
encode_coordinates.argtypes = (_FFIArray, c_uint32)
encode_coordinates.restype = _PolylineResult
encode_coordinates.errcheck = _void_array_to_string
encode_coordinates.__doc__ = """
    Encode coordinates as a Polyline.
    Input: a list of lat, lon coordinates, and a precision int (5 for Google, 6 for OSM-derived).
    Output: an encoded Polyline string.

    Example: encode_coordinates([[38.5, -120.2], [40.7, -120.95], [43.252, -126.453]], 5)
    Result: "_p~iF~ps|U_ulLnnqC_mqNvxq`@"

    Incorrect coordinate input will throw util.EncodingError

"""

# Free FFI-allocated memory
_drop_cstring = lib.drop_cstring
_drop_cstring.argtypes = (c_void_p,)
_drop_cstring.restype = None

_drop_array = lib.drop_float_array
_drop_array.argtypes = (_FFIArray,)
_drop_array.restype = None
<|endoftext|>


<commit_before>import sys
import numpy as np
import cv2
import os
from PyQt5.QtCore import Qt
from PyQt5.QtGui import QPixmap, QImage
from PyQt5.QtWidgets import (QWidget, QLabel, QHBoxLayout, QVBoxLayout,
                             QApplication, QPushButton, QSlider,
                             QFileDialog)

class Image(QWidget):
    """Common base for the images"""

    def __init__(self, name):
        super(Image, self).__init__()

        self.setMinimumSize(500,500)

        # Label for the image
        self.name_lbl = QLabel(name)

        # Label (frame) where the original image will be located, with true scaling and maximum size
        self.frame_lbl = QLabel(self)
        self.frame_lbl.setMinimumSize(400,400)
        self.frame_lbl.setScaledContents(False)

        self.createLayout()

    def createLayout(self):
        self.v_lay = QVBoxLayout()

        self.v_lay.addWidget(self.name_lbl)
        self.v_lay.addWidget(self.frame_lbl)

    def updateImage(self, opencv_rgb_image):

        self.cv_img_rgb = opencv_rgb_image

        height, width, channel = self.cv_img_rgb.shape
        bytesPerLine = 3 * width
        self.q_image = QImage(self.cv_img_rgb.data, width, height, bytesPerLine, QImage.Format_RGB888)

        self.frame_lbl.setPixmap(QPixmap.fromImage(self.q_image))

class Filter(QWidget):
    """Common base class for all filters"""
    defaultK = 3
    filterCount = 0

    def __init__(self, name, minValue, maxValue, init, parent=None):
        super(Filter, self).__init__(parent=parent)

        self.setMaximumHeight(65)

        self.k = init

        # Increase the number of filters created
        Filter.filterCount += 1

        # Variable for the slider/label layout
        self.lay = QHBoxLayout(self)

        # Variable for the constant of the OpenCV filter
        self.k = self.defaultK

        # Label for the slider
        self.k_lbl = QLabel(str(self.k))

        # Name for the slider
        self.name_lbl = QLabel(name)

        # Set default parameters
        self.setParameters(minValue, maxValue)

        # Create delete button
        self.delete_btn = QPushButton('X')

        # Adds the slider and it's label to the layout
        self.createLayout()

        # Function sending the slider signal to the processing function
        self.thresh_sld.valueChanged.connect(self.changeValue)

    def setParameters(self, minValue, maxValue):
        # Creates the slider for the OpenCV filter, with min, max, default and step values
        self.thresh_sld = QSlider(Qt.Horizontal, self)
        self.thresh_sld.setFocusPolicy(Qt.NoFocus)
        self.thresh_sld.setMinimum(minValue)
        self.thresh_sld.setMaximum(maxValue)
        self.thresh_sld.setValue(self.k)
        self.thresh_sld.setSingleStep(2)

    def createLayout(self):
        # Adds the slider and its label to the bottom of the main layout

        self.lay.addWidget(self.name_lbl)
        self.lay.addWidget(self.k_lbl)
        self.lay.addWidget(self.thresh_sld)
        self.lay.addWidget(self.delete_btn)

    def changeValue(self, value):
        # Function for setting the value of k1

        if value % 2 == 1:
            self.k = value
        else:
            self.k = value + 1

        self.thresh_sld.setValue(self.k)
        self.k_lbl.setText(str(self.k))
        self.parent().updateImages()

    def resetValue(self):
        # Resets the K value to it's default
        self.changeValue(self.defaultK)

class MainWindow(QWidget):

    def __init__(self):
        super(MainWindow, self).__init__()

        self.initUI()
        self.filters = [10]

    def initUI(self):

        self.filter1 = Filter("Hue", 31, 225, 31)
        self.filter2 = Filter("Gaussian", 3, 51, 3)
        self.filter3 = Filter("TopHat", 3, 51, 3)
        self.filter4 = Filter("Erosion", 3, 51, 3)
        self.filter5 = Filter("ValueMin", 0, 255, 0)
        self.filter6 = Filter("ValueMax", 0, 255, 0)

        self.original_image = Image('Original')
        self.processed_image = Image('Processed')
        self.setBackground()

        self.createImagesLayout()
        self.createFiltersLayout()
        self.createButtons()
        self.createMainLayout()

    def createNewFilter(self):
        self.filters[0] = Filter('Filtro' + str(Filter.filterCount), 3, 51, 3)
        self.v_filters_lay.addWidget(self.filters[0])
        print Filter.filterCount

    def createImagesLayout(self):
        # Horizontal layout for the two images
        self.h_img_lay = QHBoxLayout()
        self.h_img_lay.addStretch()
        self.h_img_lay.addWidget(self.original_image)
        self.h_img_lay.addWidget(self.processed_image)
        self.h_img_lay.addStretch()

    def createFiltersLayout(self):
        self.v_filters_lay = QVBoxLayout()
        self.v_filters_lay.addWidget(self.filter1)
        self.v_filters_lay.addWidget(self.filter2)
        self.v_filters_lay.addWidget(self.filter3)
        self.v_filters_lay.addWidget(self.filter4)
        self.v_filters_lay.addWidget(self.filter5)
        self.v_filters_lay.addWidget(self.filter6)

    def createMainLayout(self):
        # Creates the main layout (vertical)
        self.v_main_lay = QVBoxLayout()
        # Adds the images horizontal layout to the main layout
        self.v_main_lay.addLayout(self.h_img_lay)
        # Adds the sliders and their labels to the bottom of the main layout
        self.v_main_lay.addLayout(self.v_filters_lay)
        # Adds the buttons horizontal layout to the bottom of the main layout
        self.v_main_lay.addLayout(self.h_btn_lay)
        # Sets the main layout
        self.setLayout(self.v_main_lay)

        # Sets the geometry, position, window title and window default mode
        self.setGeometry(500, 100, 0, 0)
        self.setWindowTitle('Image processing')
        self.show()

    def createButtons(self):

        # Button for adding filters
        self.add_filter_btn = QPushButton('Add filter')
        self.add_filter_btn.clicked.connect(self.createNewFilter)

        # Button for selecting image
        self.select_image_btn = QPushButton('Select image')
        self.select_image_btn.clicked.connect(self.openImage)

        # Button for cleaning image
        self.clean_image_btn = QPushButton('Save image')
        self.clean_image_btn.clicked.connect(self.saveImages)

        # Horizontal layout for the buttons
        self.h_btn_lay = QHBoxLayout()
        self.h_btn_lay.addStretch(1)
        self.h_btn_lay.addWidget(self.add_filter_btn)
        self.h_btn_lay.addWidget(self.select_image_btn)
        self.h_btn_lay.addWidget(self.clean_image_btn)
        self.h_btn_lay.addStretch(1)

    def openImage(self):
        # Function for selecting the original image

        filter = "Images (*.png *.jpg)"
        image_path, _ = QFileDialog.getOpenFileName(self, 'Open image', 'Desktop', filter)
        self.path = image_path

        cv_img_bgr = cv2.imread(self.path)
        cv_img_rgb = cv2.cvtColor(cv_img_bgr, cv2.COLOR_BGR2RGB)

        self.original_image.updateImage(cv_img_rgb)
        self.updateImages()

    def saveImages(self):
        # Function for saving the processed image

        filter = "Images (*.png *.jpg)"

        image_path_orig, _ = QFileDialog.getSaveFileName(self, filter=filter)
        self.cv_img_processed_bgr = cv2.cvtColor(self.cv_img_processed_rgb, cv2.COLOR_RGB2BGR)
        cv2.imwrite(image_path_orig, self.cv_img_processed_bgr)

        image_path_proc, _ = QFileDialog.getSaveFileName(self, filter=filter)
        self.orig_img_calculated_bgr = cv2.cvtColor(self.orig_img_calculated_rgb, cv2.COLOR_RGB2BGR)
        cv2.imwrite(image_path_proc, self.orig_img_calculated_bgr)

    def updateImages(self):
        self.processImage()
        self.calculateOriginal()

    def processImage(self):
        # Function that processes the image using the OpenCV methods

        # Open the image
        self.cv_img_bgr = cv2.imread(self.path)
        self.cv_img_rgb = cv2.cvtColor(self.cv_img_bgr, cv2.COLOR_BGR2RGB)

        # Convert image to HSV
        cv_img_hsv = cv2.cvtColor(self.cv_img_bgr, cv2.COLOR_BGR2HSV)

        # Define range of desired color in HSV
        lower_color = np.array([self.filter1.k - 30, 0, self.filter5.k])
        upper_color = np.array([self.filter1.k + 30, 255, self.filter6.k])

        hsv_mask = cv2.inRange(cv_img_hsv, lower_color, upper_color)

        # Convert image to grayscale
        cv_img_gray = cv2.cvtColor(self.cv_img_bgr, cv2.COLOR_BGR2GRAY)

        # Apply blur
        cv_img_blur = cv2.GaussianBlur(hsv_mask, (self.filter2.k, self.filter2.k), 0)

        # Apply Top Hat
        opening_kernel = np.ones((self.filter3.k, self.filter3.k), np.uint8)
        cv_img_open = cv2.morphologyEx(cv_img_blur, cv2.MORPH_TOPHAT, opening_kernel)

        # Apply erosion
        erosion_kernel = np.ones((self.filter4.k, self.filter4.k), np.uint8)
        cv_img_eros = cv2.morphologyEx(cv_img_open, cv2.MORPH_OPEN, erosion_kernel)

        # Last image phase
        self.cv_img_processed_gray = cv_img_eros

        # Convert image to RGB
        self.cv_img_processed_rgb = cv2.cvtColor(self.cv_img_processed_gray, cv2.COLOR_GRAY2RGB)

        # Updates the processed image frame
        self.processed_image.updateImage(self.cv_img_processed_rgb)

    def calculateOriginal(self):
        # Update original image with the contours

        self.orig_img_calculated_rgb = self.cv_img_rgb

        # Find contours
        img, contours, hierarchy = cv2.findContours(self.cv_img_processed_gray, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)

        # Apply contours
        cv2.drawContours(self.orig_img_calculated_rgb, contours, -1, (0, 0, 255), 3)

        self.original_image.updateImage(self.orig_img_calculated_rgb)

    def setBackground(self):
        script_path = os.path.basename(os.path.dirname(os.path.realpath(__file__)))
        MainWindow.path = os.path.dirname(script_path) + 'default_BG.jpg'

        cv_img_bgr = cv2.imread(self.path)
        cv_img_rgb = cv2.cvtColor(cv_img_bgr, cv2.COLOR_BGR2RGB)

        self.original_image.updateImage(cv_img_rgb)
        self.processed_image.updateImage(cv_img_rgb)

if __name__ == '__main__':
    app = QApplication(sys.argv)
    window = MainWindow()
    sys.exit(app.exec_())<commit_msg>Creating and deleting Filter() instances implemented<commit_after>import sys
import numpy as np
import cv2
import os
from PyQt5.QtCore import Qt
from PyQt5.QtGui import QPixmap, QImage
from PyQt5.QtWidgets import (QWidget, QLabel, QHBoxLayout, QVBoxLayout,
                             QApplication, QPushButton, QSlider,
                             QFileDialog)

class Image(QWidget):
    """Common base for the images"""

    def __init__(self, name):
        super(Image, self).__init__()

        self.setFixedSize(500,500)

        # Label for the image
        self.name_lbl = QLabel(name)

        # Label (frame) where the original image will be located, with true scaling and maximum size
        self.frame_lbl = QLabel(self)
        self.frame_lbl.setMinimumSize(400,400)
        self.frame_lbl.setScaledContents(False)

        self.createLayout()

    def createLayout(self):
        self.v_lay = QVBoxLayout()

        self.v_lay.addWidget(self.name_lbl)
        self.v_lay.addWidget(self.frame_lbl)

    def updateImage(self, opencv_rgb_image):

        self.cv_img_rgb = opencv_rgb_image

        height, width, channel = self.cv_img_rgb.shape
        bytesPerLine = 3 * width
        self.q_image = QImage(self.cv_img_rgb.data, width, height, bytesPerLine, QImage.Format_RGB888)

        self.frame_lbl.setPixmap(QPixmap.fromImage(self.q_image))

class Filter(QWidget):
    """Common base class for all filters"""
    defaultK = 3
    filterCount = 0

    def __init__(self, name, minValue, maxValue, init, parent=None):
        super(Filter, self).__init__(parent=parent)

        self.setMaximumHeight(65)

        self.k = init

        # Increase the number of filters created
        Filter.filterCount += 1

        # Variable for the slider/label layout
        self.lay = QHBoxLayout(self)

        # Variable for the constant of the OpenCV filter
        self.k = self.defaultK

        # Label for the slider
        self.k_lbl = QLabel(str(self.k))

        # Name for the slider
        self.name_lbl = QLabel(name)

        # Set default parameters
        self.setParameters(minValue, maxValue)

        # Adds the slider and it's label to the layout
        self.createLayout()

        # Function sending the slider signal to the processing function
        self.thresh_sld.valueChanged.connect(self.changeValue)

    def setParameters(self, minValue, maxValue):
        # Creates the slider for the OpenCV filter, with min, max, default and step values
        self.thresh_sld = QSlider(Qt.Horizontal, self)
        self.thresh_sld.setFocusPolicy(Qt.NoFocus)
        self.thresh_sld.setMinimum(minValue)
        self.thresh_sld.setMaximum(maxValue)
        self.thresh_sld.setValue(self.k)
        self.thresh_sld.setSingleStep(2)

    def createLayout(self):
        # Adds the slider and its label to the bottom of the main layout

        self.lay.addWidget(self.name_lbl)
        self.lay.addWidget(self.k_lbl)
        self.lay.addWidget(self.thresh_sld)

    def changeValue(self, value):
        # Function for setting the value of k1

        if value % 2 == 1:
            self.k = value
        else:
            self.k = value + 1

        self.thresh_sld.setValue(self.k)
        self.k_lbl.setText(str(self.k))
        self.parent().updateImages()

    def resetValue(self):
        # Resets the K value to it's default
        self.changeValue(self.defaultK)

class MainWindow(QWidget):

    filter_count = 0

    def __init__(self):
        super(MainWindow, self).__init__()

        self.initUI()
        self.filters = [None] * 100

    def initUI(self):

        self.original_image = Image('Original')
        self.processed_image = Image('Processed')
        self.setBackground()

        self.createImagesLayout()
        self.createFiltersLayout()
        self.createButtons()
        self.createMainLayout()

    def createNewFilter(self):

        self.filters[self.filter_count] = Filter('Filtro' + str(self.filter_count) + ': ', 3, 51, 3)
        self.v_filters_lay.addWidget(self.filters[self.filter_count])
        print self.filter_count
        self.filter_count += 1

    def deleteFilter(self):
        if self.filter_count > 0:
            self.filter_count -= 1
            self.v_filters_lay.removeWidget(self.filters[self.filter_count])
            self.filters[self.filter_count].deleteLater()
            self.filters[self.filter_count] = None

            for i in range(0, 10):
                QApplication.processEvents()

            self.resize(self.minimumSizeHint())

    def createImagesLayout(self):
        # Horizontal layout for the two images
        self.h_img_lay = QHBoxLayout()
        self.h_img_lay.addStretch()
        self.h_img_lay.addWidget(self.original_image)
        self.h_img_lay.addWidget(self.processed_image)
        self.h_img_lay.addStretch()

    def createFiltersLayout(self):
        self.v_filters_lay = QVBoxLayout()

    def createMainLayout(self):
        # Creates the main layout (vertical)
        self.v_main_lay = QVBoxLayout()
        # Adds the images horizontal layout to the main layout
        self.v_main_lay.addLayout(self.h_img_lay)
        # Adds the buttons horizontal layout to the bottom of the main layout
        self.v_main_lay.addLayout(self.h_btn_lay)
        # Adds the sliders and their labels to the bottom of the main layout
        self.v_main_lay.addLayout(self.v_filters_lay)
        # Sets the main layout
        self.setLayout(self.v_main_lay)

        # Sets the geometry, position, window title and window default mode
        self.setGeometry(500, 100, 0, 0)
        self.setWindowTitle('Image processing')
        self.show()

    def createButtons(self):

        # Button for adding filters
        self.add_filter_btn = QPushButton('Add filter')
        self.add_filter_btn.clicked.connect(self.createNewFilter)

        # Create delete button
        self.delete_filter_btn = QPushButton('Delete last filter')
        self.delete_filter_btn.clicked.connect(self.deleteFilter)

        # Button for selecting image
        self.select_image_btn = QPushButton('Open image')
        self.select_image_btn.clicked.connect(self.openImage)

        # Button for cleaning image
        self.clean_image_btn = QPushButton('Save images')
        self.clean_image_btn.clicked.connect(self.saveImages)

        # Horizontal layout for the buttons
        self.h_btn_lay = QHBoxLayout()
        self.h_btn_lay.addStretch(1)
        self.h_btn_lay.addWidget(self.add_filter_btn)
        self.h_btn_lay.addWidget(self.delete_filter_btn)
        self.h_btn_lay.addWidget(self.select_image_btn)
        self.h_btn_lay.addWidget(self.clean_image_btn)
        self.h_btn_lay.addStretch(1)

    def openImage(self):
        # Function for selecting the original image

        filter = "Images (*.png *.jpg)"
        image_path, _ = QFileDialog.getOpenFileName(self, 'Open image', 'Desktop', filter)
        self.path = image_path

        cv_img_bgr = cv2.imread(self.path)
        cv_img_rgb = cv2.cvtColor(cv_img_bgr, cv2.COLOR_BGR2RGB)

        self.original_image.updateImage(cv_img_rgb)
        self.updateImages()

    def saveImages(self):
        # Function for saving the processed image

        filter = "Images (*.png *.jpg)"

        image_path_orig, _ = QFileDialog.getSaveFileName(self, filter=filter)
        self.cv_img_processed_bgr = cv2.cvtColor(self.cv_img_processed_rgb, cv2.COLOR_RGB2BGR)
        cv2.imwrite(image_path_orig, self.cv_img_processed_bgr)

        image_path_proc, _ = QFileDialog.getSaveFileName(self, filter=filter)
        self.orig_img_calculated_bgr = cv2.cvtColor(self.orig_img_calculated_rgb, cv2.COLOR_RGB2BGR)
        cv2.imwrite(image_path_proc, self.orig_img_calculated_bgr)

    def updateImages(self):
        self.processImage()
        self.calculateOriginal()

    def processImage(self):
        # Function that processes the image using the OpenCV methods

        # Open the image
        self.cv_img_bgr = cv2.imread(self.path)
        self.cv_img_rgb = cv2.cvtColor(self.cv_img_bgr, cv2.COLOR_BGR2RGB)

        # Convert image to grayscale
        cv_img_gray = cv2.cvtColor(self.cv_img_bgr, cv2.COLOR_BGR2GRAY)

        # Last image phase
        self.cv_img_processed_gray = cv_img_gray

        # Convert image to RGB
        self.cv_img_processed_rgb = cv2.cvtColor(self.cv_img_processed_gray, cv2.COLOR_GRAY2RGB)

        # Updates the processed image frame
        self.processed_image.updateImage(self.cv_img_processed_rgb)

    def calculateOriginal(self):
        # Update original image with the contours

        self.orig_img_calculated_rgb = self.cv_img_rgb

        # Find contours
        img, contours, hierarchy = cv2.findContours(self.cv_img_processed_gray, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)

        # Apply contours
        cv2.drawContours(self.orig_img_calculated_rgb, contours, -1, (0, 0, 255), 3)

        self.original_image.updateImage(self.orig_img_calculated_rgb)

    def setBackground(self):
        script_path = os.path.basename(os.path.dirname(os.path.realpath(__file__)))
        MainWindow.path = os.path.dirname(script_path) + 'default_BG.jpg'

        cv_img_bgr = cv2.imread(self.path)
        cv_img_rgb = cv2.cvtColor(cv_img_bgr, cv2.COLOR_BGR2RGB)

        self.original_image.updateImage(cv_img_rgb)
        self.processed_image.updateImage(cv_img_rgb)

if __name__ == '__main__':
    app = QApplication(sys.argv)
    window = MainWindow()
    sys.exit(app.exec_())<|endoftext|>


<commit_before>"""
A pythonic python-only implementation of Tokyo Tyrant protocol.
Python 2.4+ is needed.

This library takes a "pythonic" approach to make it more clear and easy
to implement.

More information about Tokyo Cabinet:
    http://tokyocabinet.sourceforge.net/

More information about Tokyo Tyrant:
    http://tokyocabinet.sourceforge.net/tyrantdoc/

This is an usage example:

    >>> import pyrant
    >>> t = pyrant.Tyrant(host='127.0.0.1', port=1978)
    >>> if t.dbtype != pyrant.DBTYPETABLE:
    ...     t['key'] = 'foo'
    ...     print t['key']
    ... else:
    ...     t['key'] = {'name': 'foo'}
    ...     print t['key']['name']
    foo
    >>> del t['key']
    >>> print t['key']
    Traceback (most recent call last):
        ...
    KeyError: 'key'

"""

import itertools as _itertools
from protocol import TyrantProtocol, TyrantError

__version__ = '0.0.2'
__all__ = ['Tyrant', 'TyrantError', 'TyrantProtocol', 'Q']

# Constants
DEFAULT_HOST = '127.0.0.1'
DEFAULT_PORT = 1978
MAX_RESULTS = 1000

# Table Types
DBTYPEBTREE = 'B+ tree'
DBTYPETABLE = 'table'
DBTYPEMEMORY = 'on-memory hash'
DBTYPEHASH = 'hash'

def _parse_elem(elem, dbtype, sep=None):
    if dbtype == DBTYPETABLE:
        # Split element by \x00 which is the column separator
        elems = elem.split('\x00')
        if not elems[0]:
            return None

        return dict((elems[i], elems[i + 1]) \
                        for i in xrange(0, len(elems), 2))
    elif sep and sep in elem:
        return elem.split(sep)

    return elem

class Tyrant(dict):
    """Main class of Tyrant implementation. Acts like a python dictionary,
    so you can query object using normal subscript operations.
    """

    def __init__(self, host=DEFAULT_HOST, port=DEFAULT_PORT, separator=None,
                 literal=False):
        """
        Acts like a python dictionary. Params are:

        host: Tyrant Host address
        port: Tyrant port number
        separator: If this parameter is set, you can put and get lists as
        values.
        literal: If is set string is returned instead of unicode
        """
        # We want to make protocol public just in case anyone need any
        # specific option
        self.proto = TyrantProtocol(host, port)
        self.dbtype = self.get_stats()['type']
        self.separator = separator
        self.literal = literal

    def __contains__(self, key):
        try:
            self.proto.vsiz(key)
        except TyrantError:
            return False
        else:
            return True

    def __delitem__(self, key):
        try:
            return self.proto.out(key)
        except TyrantError:
            raise KeyError(key)

    def __getitem__(self, key):
        try:
            return _parse_elem(self.proto.get(key, self.literal), self.dbtype,
                               self.separator)
        except TyrantError:
            raise KeyError(key)

    def __len__(self):
        return self.proto.rnum()

    def __repr__(self):
        return object.__repr__(self)

    def __setitem__(self, key, value):
        if isinstance(value, dict):
            flat = _itertools.chain([key], *value.iteritems())
            self.proto.misc('put', list(flat))

        elif isinstance(value, (list, tuple)):
            assert self.separator, "Separator is not set"

            flat = self.separator.join(value)
            self.proto.put(key, flat)

        else:
            self.proto.put(key, value)

    def call_func(self, func, key, value, record_locking=False,
                  global_locking=False):
        """Call specific function.
        """
        opts = ((record_locking and TyrantProtocol.RDBXOLCKREC) |
                (global_locking and TyrantProtocol.RDBXOLCKGLB))
        return self.proto.ext(func, opts, key, value)

    def clear(self):
        """Used in order to remove all records of a remote database object"""
        self.proto.vanish()

    def concat(self, key, value, width=None):
        """Concatenate columns of the existing record"""
        if width is None:
            self.proto.putcat(key, value)
        else:
            self.proto.putshl(key, value, width)

    def get_size(self, key):
        """Get the size of the value of a record"""
        try:
            return self.proto.vsiz(key)
        except TyrantError:
            raise KeyError(key)

    def get_stats(self):
        """Get the status string of the database.
        The return value is the status message of the database.The message
        format is a dictionary.
        """
        return dict(l.split('\t', 1) \
                        for l in self.proto.stat().splitlines() if l)

    def iterkeys(self):
        """Iterate keys using remote operations"""
        self.proto.iterinit()
        try:
            while True:
                yield self.proto.iternext()
        except TyrantError:
            pass

    def keys(self):
        """Return the list of keys in database"""
        return list(self.iterkeys())

    def update(self, other, **kwargs):
        """Update/Add given objects into database"""
        self.multi_set(other.iteritems())
        if kwargs:
            self.update(kwargs)

    def multi_del(self, keys, no_update_log=False):
        """Remove given records from database"""
        opts = (no_update_log and TyrantProtocol.RDBMONOULOG or 0)
        if not isinstance(keys, (list, tuple)):
            keys = list(keys)

        self.proto.misc("outlist", keys, opts)

    def multi_get(self, keys, no_update_log=False):
        """Returns a list of records that match given keys"""
        opts = (no_update_log and TyrantProtocol.RDBMONOULOG or 0)
        if not isinstance(keys, (list, tuple)):
            keys = list(keys)

        rval = self.proto.misc("getlist", keys, opts)

        if len(rval) <= len(keys):
            # 1.1.10 protocol, may return invalid results
            if len(rval) < len(keys):
                raise KeyError("Missing a result, unusable response in 1.1.10")

            return rval

        # 1.1.11 protocol returns interleaved key, value list
        d = dict((rval[i], _parse_elem(rval[i + 1], self.dbtype,
                                       self.separator)) \
                    for i in xrange(0, len(rval), 2))
        return d

    def multi_set(self, items, no_update_log=False):
        """Store given records into database"""
        opts = (no_update_log and TyrantProtocol.RDBMONOULOG or 0)
        lst = []
        for k, v in items.iteritems():
            if isinstance(v, (list, tuple)):
                assert self.separator, "Separator is not set"

                v = self.separator.join(v)
            lst.extend((k, v))

        self.proto.misc("putlist", lst, opts)

    def get_int(self, key):
        """Get an integer for given key. Must been added by addint"""
        return self.proto.getint(key)

    def get_double(self, key):
        """Get a double for given key. Must been added by adddouble"""
        return self.proto.getdouble(key)

    def prefix_keys(self, prefix, maxkeys=None):
        """Get forward matching keys in a database.
        The return value is a list object of the corresponding keys.
        """
        if maxkeys is None:
            maxkeys = len(self)

        return self.proto.fwmkeys(prefix, maxkeys)

    def sync(self):
        """Synchronize updated content into database"""
        self.proto.sync()

    def _get_query(self):
        return Query(self.proto, self.dbtype)

    query = property(_get_query)

class Q(object):
    """Condition object. You can | this type to ORs conditions,
    but you cannot use operand "&", to do this just add more Q to your filter
    Example:

        >>> t = Tyrant()
        >>> t.clear()
        >>> t.dbtype == pyrant.DBTYPETABLE
        True
        >>> t['i'] = {'name': 'Reflejo', 'test': 0}
        >>> t['you'] = {'name': 'Fulano', 'test': 1}
        >>> res = t.query.filter(Q(name='Reflejo'), Q(test=0))
        >>> res[0]['i']['name']
        u'Reflejo'
        >>> res = t.query.filter(Q(name='Reflejo') | Q(name='Fulano'))
        >>> len(res)
        2

    """

    def __init__(self, **kwargs):
        assert kwargs, "You need to specify at least one condition"

        for kw, val in kwargs.iteritems():
            nameop = kw.split('__')
            self._op = 's' if isinstance(val, (str, unicode)) else 'n'
            self._op += nameop[1] if len(nameop) > 1 else 'eq'
            self.name = nameop[0]
            self.expr = val

    def __or__(self, q):
        import copy
        assert isinstance(q, Q), "Unsupported operand type(s) for |"

        op = '%s_or' % q._op
        if q._op == self._op and op in TyrantProtocol.conditionsmap:
            qcopy = copy.copy(q)
            qcopy._op = op
            qcopy.expr = "%s,%s" % (q.expr , self.expr)

            return qcopy
        else:
            raise TypeError("Unsoported operand for |. You can only do this "\
                            "on contains or eq")

    def _getop(self):
        return TyrantProtocol.conditionsmap[self._op]

    op = property(_getop)

    def __repr__(self):
        return "%s [%s] %s" % (self.name, self.op, self.expr)

class Query(object):
    """Query table operations. This is a lazy object
    that abstract all queries for tyrant protocol.

    Usage:

    # Insert object into database
    >>> t = Tyrant()
    >>> t.dbtype == pyrant.DBTYPETABLE
    True
    >>> t['i'] = {'name': 'Reflejo', 'test': 0}

    # Now let's see if we can reach it querying tyrant
    >>> res = t.query.filter(name='Reflejo', test=0).order('name')
    >>> print res[0]['i']['name']
    Reflejo

    """

    def __init__(self, proto, dbtype):
        self._conditions = []
        self._order = None
        self._order_t = 0
        self._cache = {}
        self._proto = proto
        self._dbtype = dbtype

    def order(self, name):
        """Define result order. name parameter is the column name.
        You can prefix "-" to order desc.
        If "#" is added just before column name, column are ordered as numbers

        Examples:
            order('-name')
            order('-#ranking')
            order('name')

        """
        if name.startswith('-'):
            if name.startswith('-#'):
                order = (name[2:], TyrantProtocol.RDBQONUMDESC)
            else:
                order = (name[1:], TyrantProtocol.RDBQOSTRDESC)
        elif name.startswith('#'):
            order = (name[1:], TyrantProtocol.RDBQONUMASC)
        else:
            order = (name, TyrantProtocol.RDBQOSTRASC)

        if self._order != order[0] or self._order_t != order[1]:
            # Add another rule means that our naive cache should be empty'ed
            self._cache = {}
            self._order = order[0]
            self._order_t = order[1]

        return self

    def filter(self, *args, **kwargs):
        """Add condition to query. This could be done by Q object or by keyword
        arguments. Keys are:
            __eq: Equals (default) to expression
            __lt: Less than expression
            __le: Less or equal to expression
            __gt: Greater than expression
            __ge: Greater or equal to expression

        Example:
            >>> t = Tyrant()
            >>> t.get_stats()['type']
            u'table'
            >>> t['i'] = {'name': 'Reflejo', 'test': 4}
            >>> t['i2'] = {'name': 'Reflejo', 'test': 4}
            >>> t['i3'] = {'name': 'Reflejo', 'test': 4}

            >>> res = t.query.filter(test__gt=0)[:1]
            >>> len(res) == 1
            True
            >>> print res[0]['i']['name']
            Reflejo

        """
        # Add another rule means that our naive cache should be empty'ed
        self._cache = {}

        # Iterate arguments. Should be instances of Q
        for q in args:
            assert isinstance(q, Q), "Arguments must be instances of Q"
            self._conditions.append(q)

        # Generate Q with arguments as needed
        for name, expr in kwargs.iteritems():
            self._conditions.append(Q(**{name: expr}))

        return self

    def __len__(self):
        return len(self[:])

    def __repr__(self):
        # Do the query using getitem
        return str(self[:])

    def __getitem__(self, k):
        # Retrieve an item or slice from the set of results.
        if not isinstance(k, (slice, int, long)):
            raise TypeError("ResultSet indices must be integers")

        # Check slice integrity
        assert (not isinstance(k, slice) and (k >= 0)) \
            or (isinstance(k, slice) and (k.start is None or k.start >= 0) \
            and (k.stop is None or k.stop >= 0)), \
            "Negative indexing is not supported."

        if isinstance(k, slice):
            offset = k.start or 0
            limit = (k.stop - offset) if k.stop is not None else MAX_RESULTS
        else:
            offset = k
            limit = 1

        cache_key = "%s_%s" % (offset, limit)
        if cache_key in self._cache:
            return self._cache[cache_key]

        conditions = [(c.name, c.op, c.expr) for c in self._conditions]

        # Do the search.
        keys = self._proto.search(conditions, limit, offset,
                                  order_type=self._order_t,
                                  order_field=self._order)

        # Since results are keys, we need to query for actual values
        if isinstance(k, slice):
            ret = [{key: _parse_elem(self._proto.get(key, self.literal),
                                     self._dbtype)} \
                        for key in keys]
        else:
            ret = {
                keys[0]: _parse_elem(self._proto.get(keys[0], self.literal),
                                     self._dbtype)
            }

        self._cache[cache_key] = ret
        return ret
<commit_msg>make Tyrant.get() method behave as you'd expect a dict to behave<commit_after>"""
A pythonic python-only implementation of Tokyo Tyrant protocol.
Python 2.4+ is needed.

This library takes a "pythonic" approach to make it more clear and easy
to implement.

More information about Tokyo Cabinet:
    http://tokyocabinet.sourceforge.net/

More information about Tokyo Tyrant:
    http://tokyocabinet.sourceforge.net/tyrantdoc/

This is an usage example:

    >>> import pyrant
    >>> t = pyrant.Tyrant(host='127.0.0.1', port=1978)
    >>> if t.dbtype != pyrant.DBTYPETABLE:
    ...     t['key'] = 'foo'
    ...     print t['key']
    ... else:
    ...     t['key'] = {'name': 'foo'}
    ...     print t['key']['name']
    foo
    >>> del t['key']
    >>> print t['key']
    Traceback (most recent call last):
        ...
    KeyError: 'key'

"""

import itertools as _itertools
from protocol import TyrantProtocol, TyrantError

__version__ = '0.0.2'
__all__ = ['Tyrant', 'TyrantError', 'TyrantProtocol', 'Q']

# Constants
DEFAULT_HOST = '127.0.0.1'
DEFAULT_PORT = 1978
MAX_RESULTS = 1000

# Table Types
DBTYPEBTREE = 'B+ tree'
DBTYPETABLE = 'table'
DBTYPEMEMORY = 'on-memory hash'
DBTYPEHASH = 'hash'

def _parse_elem(elem, dbtype, sep=None):
    if dbtype == DBTYPETABLE:
        # Split element by \x00 which is the column separator
        elems = elem.split('\x00')
        if not elems[0]:
            return None

        return dict((elems[i], elems[i + 1]) \
                        for i in xrange(0, len(elems), 2))
    elif sep and sep in elem:
        return elem.split(sep)

    return elem

class Tyrant(dict):
    """Main class of Tyrant implementation. Acts like a python dictionary,
    so you can query object using normal subscript operations.
    """

    def __init__(self, host=DEFAULT_HOST, port=DEFAULT_PORT, separator=None,
                 literal=False):
        """
        Acts like a python dictionary. Params are:

        host: Tyrant Host address
        port: Tyrant port number
        separator: If this parameter is set, you can put and get lists as
        values.
        literal: If is set string is returned instead of unicode
        """
        # We want to make protocol public just in case anyone need any
        # specific option
        self.proto = TyrantProtocol(host, port)
        self.dbtype = self.get_stats()['type']
        self.separator = separator
        self.literal = literal

    def __contains__(self, key):
        try:
            self.proto.vsiz(key)
        except TyrantError:
            return False
        else:
            return True

    def __delitem__(self, key):
        try:
            return self.proto.out(key)
        except TyrantError:
            raise KeyError(key)

    def __getitem__(self, key):
        try:
            return _parse_elem(self.proto.get(key, self.literal), self.dbtype,
                               self.separator)
        except TyrantError:
            raise KeyError(key)

    def get(self, key, default=None):
        if key in self:
            return self.__getitem__(key)
        else:
            return default

    def __len__(self):
        return self.proto.rnum()

    def __repr__(self):
        return object.__repr__(self)

    def __setitem__(self, key, value):
        if isinstance(value, dict):
            flat = _itertools.chain([key], *value.iteritems())
            self.proto.misc('put', list(flat))

        elif isinstance(value, (list, tuple)):
            assert self.separator, "Separator is not set"

            flat = self.separator.join(value)
            self.proto.put(key, flat)

        else:
            self.proto.put(key, value)

    def call_func(self, func, key, value, record_locking=False,
                  global_locking=False):
        """Call specific function.
        """
        opts = ((record_locking and TyrantProtocol.RDBXOLCKREC) |
                (global_locking and TyrantProtocol.RDBXOLCKGLB))
        return self.proto.ext(func, opts, key, value)

    def clear(self):
        """Used in order to remove all records of a remote database object"""
        self.proto.vanish()

    def concat(self, key, value, width=None):
        """Concatenate columns of the existing record"""
        if width is None:
            self.proto.putcat(key, value)
        else:
            self.proto.putshl(key, value, width)

    def get_size(self, key):
        """Get the size of the value of a record"""
        try:
            return self.proto.vsiz(key)
        except TyrantError:
            raise KeyError(key)

    def get_stats(self):
        """Get the status string of the database.
        The return value is the status message of the database.The message
        format is a dictionary.
        """
        return dict(l.split('\t', 1) \
                        for l in self.proto.stat().splitlines() if l)

    def iterkeys(self):
        """Iterate keys using remote operations"""
        self.proto.iterinit()
        try:
            while True:
                yield self.proto.iternext()
        except TyrantError:
            pass

    def keys(self):
        """Return the list of keys in database"""
        return list(self.iterkeys())

    def update(self, other, **kwargs):
        """Update/Add given objects into database"""
        self.multi_set(other.iteritems())
        if kwargs:
            self.update(kwargs)

    def multi_del(self, keys, no_update_log=False):
        """Remove given records from database"""
        opts = (no_update_log and TyrantProtocol.RDBMONOULOG or 0)
        if not isinstance(keys, (list, tuple)):
            keys = list(keys)

        self.proto.misc("outlist", keys, opts)

    def multi_get(self, keys, no_update_log=False):
        """Returns a list of records that match given keys"""
        opts = (no_update_log and TyrantProtocol.RDBMONOULOG or 0)
        if not isinstance(keys, (list, tuple)):
            keys = list(keys)

        rval = self.proto.misc("getlist", keys, opts)

        if len(rval) <= len(keys):
            # 1.1.10 protocol, may return invalid results
            if len(rval) < len(keys):
                raise KeyError("Missing a result, unusable response in 1.1.10")

            return rval

        # 1.1.11 protocol returns interleaved key, value list
        d = dict((rval[i], _parse_elem(rval[i + 1], self.dbtype,
                                       self.separator)) \
                    for i in xrange(0, len(rval), 2))
        return d

    def multi_set(self, items, no_update_log=False):
        """Store given records into database"""
        opts = (no_update_log and TyrantProtocol.RDBMONOULOG or 0)
        lst = []
        for k, v in items.iteritems():
            if isinstance(v, (list, tuple)):
                assert self.separator, "Separator is not set"

                v = self.separator.join(v)
            lst.extend((k, v))

        self.proto.misc("putlist", lst, opts)

    def get_int(self, key):
        """Get an integer for given key. Must been added by addint"""
        return self.proto.getint(key)

    def get_double(self, key):
        """Get a double for given key. Must been added by adddouble"""
        return self.proto.getdouble(key)

    def prefix_keys(self, prefix, maxkeys=None):
        """Get forward matching keys in a database.
        The return value is a list object of the corresponding keys.
        """
        if maxkeys is None:
            maxkeys = len(self)

        return self.proto.fwmkeys(prefix, maxkeys)

    def sync(self):
        """Synchronize updated content into database"""
        self.proto.sync()

    def _get_query(self):
        return Query(self.proto, self.dbtype)

    query = property(_get_query)

class Q(object):
    """Condition object. You can | this type to ORs conditions,
    but you cannot use operand "&", to do this just add more Q to your filter
    Example:

        >>> t = Tyrant()
        >>> t.clear()
        >>> t.dbtype == pyrant.DBTYPETABLE
        True
        >>> t['i'] = {'name': 'Reflejo', 'test': 0}
        >>> t['you'] = {'name': 'Fulano', 'test': 1}
        >>> res = t.query.filter(Q(name='Reflejo'), Q(test=0))
        >>> res[0]['i']['name']
        u'Reflejo'
        >>> res = t.query.filter(Q(name='Reflejo') | Q(name='Fulano'))
        >>> len(res)
        2

    """

    def __init__(self, **kwargs):
        assert kwargs, "You need to specify at least one condition"

        for kw, val in kwargs.iteritems():
            nameop = kw.split('__')
            self._op = 's' if isinstance(val, (str, unicode)) else 'n'
            self._op += nameop[1] if len(nameop) > 1 else 'eq'
            self.name = nameop[0]
            self.expr = val

    def __or__(self, q):
        import copy
        assert isinstance(q, Q), "Unsupported operand type(s) for |"

        op = '%s_or' % q._op
        if q._op == self._op and op in TyrantProtocol.conditionsmap:
            qcopy = copy.copy(q)
            qcopy._op = op
            qcopy.expr = "%s,%s" % (q.expr , self.expr)

            return qcopy
        else:
            raise TypeError("Unsoported operand for |. You can only do this "\
                            "on contains or eq")

    def _getop(self):
        return TyrantProtocol.conditionsmap[self._op]

    op = property(_getop)

    def __repr__(self):
        return "%s [%s] %s" % (self.name, self.op, self.expr)

class Query(object):
    """Query table operations. This is a lazy object
    that abstract all queries for tyrant protocol.

    Usage:

    # Insert object into database
    >>> t = Tyrant()
    >>> t.dbtype == pyrant.DBTYPETABLE
    True
    >>> t['i'] = {'name': 'Reflejo', 'test': 0}

    # Now let's see if we can reach it querying tyrant
    >>> res = t.query.filter(name='Reflejo', test=0).order('name')
    >>> print res[0]['i']['name']
    Reflejo

    """

    def __init__(self, proto, dbtype):
        self._conditions = []
        self._order = None
        self._order_t = 0
        self._cache = {}
        self._proto = proto
        self._dbtype = dbtype

    def order(self, name):
        """Define result order. name parameter is the column name.
        You can prefix "-" to order desc.
        If "#" is added just before column name, column are ordered as numbers

        Examples:
            order('-name')
            order('-#ranking')
            order('name')

        """
        if name.startswith('-'):
            if name.startswith('-#'):
                order = (name[2:], TyrantProtocol.RDBQONUMDESC)
            else:
                order = (name[1:], TyrantProtocol.RDBQOSTRDESC)
        elif name.startswith('#'):
            order = (name[1:], TyrantProtocol.RDBQONUMASC)
        else:
            order = (name, TyrantProtocol.RDBQOSTRASC)

        if self._order != order[0] or self._order_t != order[1]:
            # Add another rule means that our naive cache should be empty'ed
            self._cache = {}
            self._order = order[0]
            self._order_t = order[1]

        return self

    def filter(self, *args, **kwargs):
        """Add condition to query. This could be done by Q object or by keyword
        arguments. Keys are:
            __eq: Equals (default) to expression
            __lt: Less than expression
            __le: Less or equal to expression
            __gt: Greater than expression
            __ge: Greater or equal to expression

        Example:
            >>> t = Tyrant()
            >>> t.get_stats()['type']
            u'table'
            >>> t['i'] = {'name': 'Reflejo', 'test': 4}
            >>> t['i2'] = {'name': 'Reflejo', 'test': 4}
            >>> t['i3'] = {'name': 'Reflejo', 'test': 4}

            >>> res = t.query.filter(test__gt=0)[:1]
            >>> len(res) == 1
            True
            >>> print res[0]['i']['name']
            Reflejo

        """
        # Add another rule means that our naive cache should be empty'ed
        self._cache = {}

        # Iterate arguments. Should be instances of Q
        for q in args:
            assert isinstance(q, Q), "Arguments must be instances of Q"
            self._conditions.append(q)

        # Generate Q with arguments as needed
        for name, expr in kwargs.iteritems():
            self._conditions.append(Q(**{name: expr}))

        return self

    def __len__(self):
        return len(self[:])

    def __repr__(self):
        # Do the query using getitem
        return str(self[:])

    def __getitem__(self, k):
        # Retrieve an item or slice from the set of results.
        if not isinstance(k, (slice, int, long)):
            raise TypeError("ResultSet indices must be integers")

        # Check slice integrity
        assert (not isinstance(k, slice) and (k >= 0)) \
            or (isinstance(k, slice) and (k.start is None or k.start >= 0) \
            and (k.stop is None or k.stop >= 0)), \
            "Negative indexing is not supported."

        if isinstance(k, slice):
            offset = k.start or 0
            limit = (k.stop - offset) if k.stop is not None else MAX_RESULTS
        else:
            offset = k
            limit = 1

        cache_key = "%s_%s" % (offset, limit)
        if cache_key in self._cache:
            return self._cache[cache_key]

        conditions = [(c.name, c.op, c.expr) for c in self._conditions]

        # Do the search.
        keys = self._proto.search(conditions, limit, offset,
                                  order_type=self._order_t,
                                  order_field=self._order)

        # Since results are keys, we need to query for actual values
        if isinstance(k, slice):
            ret = [{key: _parse_elem(self._proto.get(key, self.literal),
                                     self._dbtype)} \
                        for key in keys]
        else:
            ret = {
                keys[0]: _parse_elem(self._proto.get(keys[0], self.literal),
                                     self._dbtype)
            }

        self._cache[cache_key] = ret
        return ret
<|endoftext|>