<commit_before># Copyright 2015, Pinterest, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Configuration repository stored in Github."""
import base64
import json
import httplib
import time
import urlparse

from pinball.config.pinball_config import PinballConfig
from pinball.config.utils import PinballException
from pinball.repository.repository import Repository

__author__ = 'Pawel Garbacki, Mao Ye'
__copyright__ = 'Copyright 2015, Pinterest, Inc.'
__credits__ = [__author__]
__license__ = 'Apache'
__version__ = '2.0'

class GithubRepository(Repository):
    """Github-backed configuration store."""

    _COMMITTER_NAME = 'Pinball'
    _COMMITTER_EMAIL = PinballConfig.DEFAULT_EMAIL

    def _make_request(self, method, location, body, expected_status):
        """Make a request through the Github API.

        Args:
            method: The request method.
            location: The request path.
            body: The body of the request message.
            expected_status: List of http status codes expected on a successful
                request.
        Returns:
            The API response.
        """
        conn = httplib.HTTPSConnection(PinballConfig.GITHUB_SERVER,
                                       timeout=PinballConfig.GITHUB_HTTP_TIMEOUT_SEC)
        location = urlparse.urljoin(PinballConfig.GITHUB_API_URI, location)
        authorization = base64.b64encode('%s:x-oauth-basic' %
                                         PinballConfig.GITHUB_OAUTH_TOKEN)
        headers = {'Authorization': 'Basic %s' % authorization}
        conn.request(method, location, body, headers)
        response = conn.getresponse()
        if response.status not in expected_status:
            raise PinballException('failed request to %s method %s location '
                                   '%s status %s reason %s content %s' %
                                   (PinballConfig.GITHUB_API_URI, method, location,
                                    response.status, response.reason,
                                    response.read()))
        return response.read()

    def _get_config(self, path):
        location = 'repos/%s/configs/contents%s' % (PinballConfig.USER, path)
        response = self._make_request('GET', location, None, [httplib.OK])
        response_json = json.loads(response)
        encoded_content = response_json['content']
        return base64.b64decode(encoded_content)

    @staticmethod
    def _get_file_type(response):
        """Get the file type described by a response.

        Args:
            response: The parsed response describing a file.
        Returns:
            Type of the file described by the response.
        """
        if type(response) == list:
            return 'dir'
        return response.get('type')

    def _put_config(self, path, content):
        location = 'repos/%s/configs/contents%s' % (PinballConfig.USER, path)
        encoded_contnet = base64.b64encode(content)
        body = {'message': 'updating config',
                'committer': {'name': GithubRepository._COMMITTER_NAME,
                              'email': GithubRepository._COMMITTER_EMAIL},
                'content': encoded_contnet}

        # Check if path exists.
        response = self._make_request('GET', location, None,
                                      [httplib.OK, httplib.NOT_FOUND])
        response_json = json.loads(response)
        file_type = GithubRepository._get_file_type(response_json)
        if file_type:
            # Path exists.
            if file_type != 'file':
                raise PinballException('path %s is not a file but a %s' %
                                       (path, file_type))
            body['sha'] = response_json['sha']

        # Work around a bug in Github.  See
        # http://stackoverflow.com/questions/19576601/\
        # github-api-issue-with-file-upload
        time.sleep(0.5)

        # Create or update the file.
        self._make_request('PUT', location, json.dumps(body),
                           [httplib.CREATED, httplib.OK])

    def _delete_config(self, path):
        location = 'repos/%s/configs/contents%s' % (PinballConfig.USER, path)
        body = {'message': 'updating config',
                'committer': {'name': GithubRepository._COMMITTER_NAME,
                              'email': GithubRepository._COMMITTER_EMAIL}}

        # Get sha of the content.
        response = self._make_request('GET', location, None, [httplib.OK])
        response_json = json.loads(response)
        file_type = GithubRepository._get_file_type(response_json)
        if not file_type or file_type != 'file':
            raise PinballException('path %s is not a file but a %s' %
                                   (path, file_type))
        body['sha'] = response_json['sha']

        # Create or update the file.
        self._make_request('DELETE', location, json.dumps(body),
                           [httplib.CREATED, httplib.OK])

    def _list_directory(self, directory, allow_not_found):
        location = 'repos/%s/configs/contents%s' % (PinballConfig.USER, directory)
        # Remove the trailing slash.
        assert directory[-1] == '/'
        location = location[:-1]
        # Check if path exists.
        expected_status = [httplib.OK]
        if allow_not_found:
            expected_status.append(httplib.NOT_FOUND)
        response_json = self._make_request('GET', location, None,
                                           expected_status)
        response = json.loads(response_json)
        file_type = GithubRepository._get_file_type(response)
        if not file_type:
            assert allow_not_found
            return []
        if file_type != 'dir':
            raise PinballException('path %s is not a dir but a %s' %
                                   (directory, file_type))
        assert type(response) == list
        result = []
        for entry in response:
            file_type = GithubRepository._get_file_type(entry)
            if file_type == 'file':
                result.append(entry['name'])
            elif file_type == 'dir':
                result.append('%s/' % entry['name'])
            else:
                raise PinballException('found content %s of unsupported type '
                                       '%s' % (entry['path'], file_type))
        return result
<commit_msg>Use ssl.create_default_context() to select system's trusted CA certificates<commit_after># Copyright 2015, Pinterest, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Configuration repository stored in Github."""
import base64
import httplib
import json
import ssl
import time
import urlparse

from pinball.config.pinball_config import PinballConfig
from pinball.config.utils import PinballException
from pinball.repository.repository import Repository

__author__ = 'Pawel Garbacki, Mao Ye'
__copyright__ = 'Copyright 2015, Pinterest, Inc.'
__credits__ = [__author__]
__license__ = 'Apache'
__version__ = '2.0'

class GithubRepository(Repository):
    """Github-backed configuration store."""

    _COMMITTER_NAME = 'Pinball'
    _COMMITTER_EMAIL = PinballConfig.DEFAULT_EMAIL

    def _make_request(self, method, location, body, expected_status):
        """Make a request through the Github API.

        Args:
            method: The request method.
            location: The request path.
            body: The body of the request message.
            expected_status: List of http status codes expected on a successful
                request.
        Returns:
            The API response.
        """
        conn = httplib.HTTPSConnection(PinballConfig.GITHUB_SERVER,
                                       timeout=PinballConfig.GITHUB_HTTP_TIMEOUT_SEC,
                                       context=ssl.create_default_context())
        location = urlparse.urljoin(PinballConfig.GITHUB_API_URI, location)
        authorization = base64.b64encode('%s:x-oauth-basic' %
                                         PinballConfig.GITHUB_OAUTH_TOKEN)
        headers = {'Authorization': 'Basic %s' % authorization}
        conn.request(method, location, body, headers)
        response = conn.getresponse()
        if response.status not in expected_status:
            raise PinballException('failed request to %s method %s location '
                                   '%s status %s reason %s content %s' %
                                   (PinballConfig.GITHUB_API_URI, method, location,
                                    response.status, response.reason,
                                    response.read()))
        return response.read()

    def _get_config(self, path):
        location = 'repos/%s/configs/contents%s' % (PinballConfig.USER, path)
        response = self._make_request('GET', location, None, [httplib.OK])
        response_json = json.loads(response)
        encoded_content = response_json['content']
        return base64.b64decode(encoded_content)

    @staticmethod
    def _get_file_type(response):
        """Get the file type described by a response.

        Args:
            response: The parsed response describing a file.
        Returns:
            Type of the file described by the response.
        """
        if type(response) == list:
            return 'dir'
        return response.get('type')

    def _put_config(self, path, content):
        location = 'repos/%s/configs/contents%s' % (PinballConfig.USER, path)
        encoded_contnet = base64.b64encode(content)
        body = {'message': 'updating config',
                'committer': {'name': GithubRepository._COMMITTER_NAME,
                              'email': GithubRepository._COMMITTER_EMAIL},
                'content': encoded_contnet}

        # Check if path exists.
        response = self._make_request('GET', location, None,
                                      [httplib.OK, httplib.NOT_FOUND])
        response_json = json.loads(response)
        file_type = GithubRepository._get_file_type(response_json)
        if file_type:
            # Path exists.
            if file_type != 'file':
                raise PinballException('path %s is not a file but a %s' %
                                       (path, file_type))
            body['sha'] = response_json['sha']

        # Work around a bug in Github.  See
        # http://stackoverflow.com/questions/19576601/\
        # github-api-issue-with-file-upload
        time.sleep(0.5)

        # Create or update the file.
        self._make_request('PUT', location, json.dumps(body),
                           [httplib.CREATED, httplib.OK])

    def _delete_config(self, path):
        location = 'repos/%s/configs/contents%s' % (PinballConfig.USER, path)
        body = {'message': 'updating config',
                'committer': {'name': GithubRepository._COMMITTER_NAME,
                              'email': GithubRepository._COMMITTER_EMAIL}}

        # Get sha of the content.
        response = self._make_request('GET', location, None, [httplib.OK])
        response_json = json.loads(response)
        file_type = GithubRepository._get_file_type(response_json)
        if not file_type or file_type != 'file':
            raise PinballException('path %s is not a file but a %s' %
                                   (path, file_type))
        body['sha'] = response_json['sha']

        # Create or update the file.
        self._make_request('DELETE', location, json.dumps(body),
                           [httplib.CREATED, httplib.OK])

    def _list_directory(self, directory, allow_not_found):
        location = 'repos/%s/configs/contents%s' % (PinballConfig.USER, directory)
        # Remove the trailing slash.
        assert directory[-1] == '/'
        location = location[:-1]
        # Check if path exists.
        expected_status = [httplib.OK]
        if allow_not_found:
            expected_status.append(httplib.NOT_FOUND)
        response_json = self._make_request('GET', location, None,
                                           expected_status)
        response = json.loads(response_json)
        file_type = GithubRepository._get_file_type(response)
        if not file_type:
            assert allow_not_found
            return []
        if file_type != 'dir':
            raise PinballException('path %s is not a dir but a %s' %
                                   (directory, file_type))
        assert type(response) == list
        result = []
        for entry in response:
            file_type = GithubRepository._get_file_type(entry)
            if file_type == 'file':
                result.append(entry['name'])
            elif file_type == 'dir':
                result.append('%s/' % entry['name'])
            else:
                raise PinballException('found content %s of unsupported type '
                                       '%s' % (entry['path'], file_type))
        return result
<|endoftext|>


<commit_before>import os
import pickle

import numpy as np
from numpy.testing import assert_allclose
from scipy.interpolate import RectBivariateSpline

import sncosmo
from sncosmo.salt2utils import BicubicInterpolator, SALT2ColorLaw

# On Python 2 highest protocol is 2.
# Protocols 0 and 1 don't work on the classes here!
TEST_PICKLE_PROTOCOLS = (2, 3, 4)

def test_bicubic_interpolator_vs_snfit():
    datadir = os.path.join(os.path.dirname(__file__), "data")

    # created by running generate script in `misc` directory
    fname_input = os.path.join(datadir, "interpolation_test_input.dat")
    fname_evalx = os.path.join(datadir, "interpolation_test_evalx.dat")
    fname_evaly = os.path.join(datadir, "interpolation_test_evaly.dat")

    # result file was created by running snfit software Grid2DFunction
    fname_result = os.path.join(datadir, "interpolation_test_result.dat")

    # load arrays
    x, y, z = sncosmo.read_griddata_ascii(fname_input)
    xp = np.loadtxt(fname_evalx)
    yp = np.loadtxt(fname_evaly)
    result = np.loadtxt(fname_result)

    f = BicubicInterpolator(x, y, z)
    assert_allclose(f(xp, yp), result, rtol=1e-5)

def test_bicubic_interpolator_shapes():
    """Ensure that input shapes are handled like RectBivariateSpline"""

    x = np.array([1., 2., 3., 4., 5.])
    z = np.ones((len(x), len(x)))

    f = BicubicInterpolator(x, x, z)
    f2 = RectBivariateSpline(x, x, z)

    assert f(0., [1., 2.]).shape == f2(0., [1., 2.]).shape
    assert f([1., 2.], 0.).shape == f2([1., 2.], 0.).shape
    assert f(0., 0.).shape == f2(0., 0.).shape

def test_bicubic_interpolator_pickle():
    x = np.arange(5)
    y = np.arange(10)
    z = np.ones((len(x), len(y)))
    f = BicubicInterpolator(x, y, z)

    for protocol in TEST_PICKLE_PROTOCOLS:
        f2 = pickle.loads(pickle.dumps(f, protocol=protocol))
        assert f2(4., 5.5) == f(4., 5.5)

def test_salt2colorlaw_vs_python():
    """Compare SALT2ColorLaw vs python implementation"""

    B_WAVELENGTH = 4302.57
    V_WAVELENGTH = 5428.55
    colorlaw_coeffs = [-0.504294, 0.787691, -0.461715, 0.0815619]
    colorlaw_range = (2800., 7000.)

    # old python implementation
    def colorlaw_python(wave):
        v_minus_b = V_WAVELENGTH - B_WAVELENGTH

        l = (wave - B_WAVELENGTH) / v_minus_b
        l_lo = (colorlaw_range[0] - B_WAVELENGTH) / v_minus_b
        l_hi = (colorlaw_range[1] - B_WAVELENGTH) / v_minus_b

        alpha = 1. - sum(colorlaw_coeffs)
        coeffs = [0., alpha]
        coeffs.extend(colorlaw_coeffs)
        coeffs = np.array(coeffs)
        prime_coeffs = (np.arange(len(coeffs)) * coeffs)[1:]

        extinction = np.empty_like(wave)

        # Blue side
        idx_lo = l < l_lo
        p_lo = np.polyval(np.flipud(coeffs), l_lo)
        pprime_lo = np.polyval(np.flipud(prime_coeffs), l_lo)
        extinction[idx_lo] = p_lo + pprime_lo * (l[idx_lo] - l_lo)

        # Red side
        idx_hi = l > l_hi
        p_hi = np.polyval(np.flipud(coeffs), l_hi)
        pprime_hi = np.polyval(np.flipud(prime_coeffs), l_hi)
        extinction[idx_hi] = p_hi + pprime_hi * (l[idx_hi] - l_hi)

        # In between
        idx_between = np.invert(idx_lo | idx_hi)
        extinction[idx_between] = np.polyval(np.flipud(coeffs), l[idx_between])

        return -extinction

    colorlaw = SALT2ColorLaw(colorlaw_range, colorlaw_coeffs)

    wave = np.linspace(2000., 9200., 201)
    assert np.all(colorlaw(wave) == colorlaw_python(wave))

def test_salt2colorlaw_pickle():

    colorlaw_coeffs = [-0.504294, 0.787691, -0.461715, 0.0815619]
    colorlaw_range = (2800., 7000.)
    colorlaw = SALT2ColorLaw(colorlaw_range, colorlaw_coeffs)

    for protocol in TEST_PICKLE_PROTOCOLS:
        colorlaw2 = pickle.loads(pickle.dumps(colorlaw, protocol=protocol))
        wave = np.linspace(2000., 9200., 201)
        assert np.all(colorlaw(wave) == colorlaw2(wave))
<commit_msg>Use np.testing.assert_allclose for comparison in test_salt2colorlaw_vs_python (#347)<commit_after>import os
import pickle

import numpy as np
from numpy.testing import assert_allclose
from scipy.interpolate import RectBivariateSpline

import sncosmo
from sncosmo.salt2utils import BicubicInterpolator, SALT2ColorLaw

# On Python 2 highest protocol is 2.
# Protocols 0 and 1 don't work on the classes here!
TEST_PICKLE_PROTOCOLS = (2, 3, 4)

def test_bicubic_interpolator_vs_snfit():
    datadir = os.path.join(os.path.dirname(__file__), "data")

    # created by running generate script in `misc` directory
    fname_input = os.path.join(datadir, "interpolation_test_input.dat")
    fname_evalx = os.path.join(datadir, "interpolation_test_evalx.dat")
    fname_evaly = os.path.join(datadir, "interpolation_test_evaly.dat")

    # result file was created by running snfit software Grid2DFunction
    fname_result = os.path.join(datadir, "interpolation_test_result.dat")

    # load arrays
    x, y, z = sncosmo.read_griddata_ascii(fname_input)
    xp = np.loadtxt(fname_evalx)
    yp = np.loadtxt(fname_evaly)
    result = np.loadtxt(fname_result)

    f = BicubicInterpolator(x, y, z)
    assert_allclose(f(xp, yp), result, rtol=1e-5)

def test_bicubic_interpolator_shapes():
    """Ensure that input shapes are handled like RectBivariateSpline"""

    x = np.array([1., 2., 3., 4., 5.])
    z = np.ones((len(x), len(x)))

    f = BicubicInterpolator(x, x, z)
    f2 = RectBivariateSpline(x, x, z)

    assert f(0., [1., 2.]).shape == f2(0., [1., 2.]).shape
    assert f([1., 2.], 0.).shape == f2([1., 2.], 0.).shape
    assert f(0., 0.).shape == f2(0., 0.).shape

def test_bicubic_interpolator_pickle():
    x = np.arange(5)
    y = np.arange(10)
    z = np.ones((len(x), len(y)))
    f = BicubicInterpolator(x, y, z)

    for protocol in TEST_PICKLE_PROTOCOLS:
        f2 = pickle.loads(pickle.dumps(f, protocol=protocol))
        assert f2(4., 5.5) == f(4., 5.5)

def test_salt2colorlaw_vs_python():
    """Compare SALT2ColorLaw vs python implementation"""

    B_WAVELENGTH = 4302.57
    V_WAVELENGTH = 5428.55
    colorlaw_coeffs = [-0.504294, 0.787691, -0.461715, 0.0815619]
    colorlaw_range = (2800., 7000.)

    # old python implementation
    def colorlaw_python(wave):
        v_minus_b = V_WAVELENGTH - B_WAVELENGTH

        l = (wave - B_WAVELENGTH) / v_minus_b
        l_lo = (colorlaw_range[0] - B_WAVELENGTH) / v_minus_b
        l_hi = (colorlaw_range[1] - B_WAVELENGTH) / v_minus_b

        alpha = 1. - sum(colorlaw_coeffs)
        coeffs = [0., alpha]
        coeffs.extend(colorlaw_coeffs)
        coeffs = np.array(coeffs)
        prime_coeffs = (np.arange(len(coeffs)) * coeffs)[1:]

        extinction = np.empty_like(wave)

        # Blue side
        idx_lo = l < l_lo
        p_lo = np.polyval(np.flipud(coeffs), l_lo)
        pprime_lo = np.polyval(np.flipud(prime_coeffs), l_lo)
        extinction[idx_lo] = p_lo + pprime_lo * (l[idx_lo] - l_lo)

        # Red side
        idx_hi = l > l_hi
        p_hi = np.polyval(np.flipud(coeffs), l_hi)
        pprime_hi = np.polyval(np.flipud(prime_coeffs), l_hi)
        extinction[idx_hi] = p_hi + pprime_hi * (l[idx_hi] - l_hi)

        # In between
        idx_between = np.invert(idx_lo | idx_hi)
        extinction[idx_between] = np.polyval(np.flipud(coeffs), l[idx_between])

        return -extinction

    colorlaw = SALT2ColorLaw(colorlaw_range, colorlaw_coeffs)

    wave = np.linspace(2000., 9200., 201)
    np.testing.assert_allclose(colorlaw(wave), colorlaw_python(wave))

def test_salt2colorlaw_pickle():

    colorlaw_coeffs = [-0.504294, 0.787691, -0.461715, 0.0815619]
    colorlaw_range = (2800., 7000.)
    colorlaw = SALT2ColorLaw(colorlaw_range, colorlaw_coeffs)

    for protocol in TEST_PICKLE_PROTOCOLS:
        colorlaw2 = pickle.loads(pickle.dumps(colorlaw, protocol=protocol))
        wave = np.linspace(2000., 9200., 201)
        assert np.all(colorlaw(wave) == colorlaw2(wave))
<|endoftext|>


<commit_before>from .device import Device
from .. import config as c

class Encoder(Device):

    DEVICE_CODE =   c.devices.encoder.code
    WRITE_CODE =    c.devices.encoder.write_code
    READ_CODE  =    c.devices.encoder.read_code

    def __init__(self, tamproxy, pin_a, pin_b):
        self.pin_a = pin_a
        self.pin_b = pin_b
        self.val = 0
        super(Encoder, self).__init__(tamproxy)
        while self.id is None: pass

    @property
    def add_payload(self):
        return self.DEVICE_CODE + chr(self.pin_a) + chr(self.pin_b)

    def write(self, value):
        value = int(value) & 0xFFFFFFFF
        self.tamp.send_request(self.id,
                               self.WRITE_CODE +
                               chr((int(value) >> 24) & 0xFF) +
                               chr((int(value) >> 16) & 0xFF) +
                               chr((int(value) >> 8) & 0xFF) +
                               chr(int(value) & 0xFF))

    def update(self):
        self.tamp.send_request(self.id, self.READ_CODE, self._handle_update)

    def start_continuous(self, weight=1):
        self.tamp.send_request(self.id, self.READ_CODE, self._handle_update,
                             continuous=True, weight=weight)

    def stop_continuous(self):
        self.tamp.send_request(self.id, self.READ_CODE, self.tamp.empty_callback,
                             continuous=True, weight=1, remove=True)

    def _handle_update(self, request, response):
        self.val = (
            (ord(response[0])<<24) |
            (ord(response[1])<<16) |
            (ord(response[2])<<8) |
            ord(response[3])
        )
<commit_msg>Add continuous mode<commit_after>from .device import Device
from .. import config as c

class Encoder(Device):

    DEVICE_CODE =   c.devices.encoder.code
    WRITE_CODE =    c.devices.encoder.write_code
    READ_CODE  =    c.devices.encoder.read_code

    def __init__(self, tamproxy, pin_a, pin_b, continuous=True):
        self.pin_a = pin_a
        self.pin_b = pin_b
        self.val = 0
        super(Encoder, self).__init__(tamproxy)
        while self.id is None: pass
        if continuous: self.start_continuous()

    @property
    def add_payload(self):
        return self.DEVICE_CODE + chr(self.pin_a) + chr(self.pin_b)

    def write(self, value):
        value = int(value) & 0xFFFFFFFF
        self.tamp.send_request(self.id,
                               self.WRITE_CODE +
                               chr((int(value) >> 24) & 0xFF) +
                               chr((int(value) >> 16) & 0xFF) +
                               chr((int(value) >> 8) & 0xFF) +
                               chr(int(value) & 0xFF))

    def update(self):
        self.tamp.send_request(self.id, self.READ_CODE, self._handle_update)

    def start_continuous(self, weight=1):
        self.tamp.send_request(self.id, self.READ_CODE, self._handle_update,
                             continuous=True, weight=weight)

    def stop_continuous(self):
        self.tamp.send_request(self.id, self.READ_CODE, self.tamp.empty_callback,
                             continuous=True, weight=1, remove=True)

    def _handle_update(self, request, response):
        self.val = (
            (ord(response[0])<<24) |
            (ord(response[1])<<16) |
            (ord(response[2])<<8) |
            ord(response[3])
        )
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Google OAuth 1.0 Client.
#
# Copyright (C) 2011 Yesudeep Mangalapilly <yesudeep@gmail.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from pyoauth.error import SignatureMethodNotSupportedError
from pyoauth.oauth1 import SIGNATURE_METHOD_HMAC_SHA1, SIGNATURE_METHOD_PLAINTEXT

from pyoauth.oauth1.client import Client

class GoogleClient(Client):
    """
    Creates an instance of a Google OAuth 1.0 client.

    :see: http://code.google.com/apis/accounts/docs/OAuth_ref.html
    """
    _OAUTH_TEMPORARY_CREDENTIALS_REQUEST_URI = "https://www.google.com/accounts/OAuthGetRequestToken"
    _OAUTH_RESOURCE_OWNER_AUTHORIZATION_URI = "https://www.google.com/accounts/OAuthAuthorizeToken"
    _OAUTH_TOKEN_CREDENTIALS_REQUEST_URI = "https://www.google.com/accounts/OAuthGetAccessToken"

    def __init__(self,
                 client_credentials,
                 use_authorization_header=True):
        super(GoogleClient, self).__init__(
            client_credentials=client_credentials,
            temporary_credentials_request_uri=self._OAUTH_TEMPORARY_CREDENTIALS_REQUEST_URI,
            resource_owner_authorization_uri=self._OAUTH_RESOURCE_OWNER_AUTHORIZATION_URI,
            token_credentials_request_uri=self._OAUTH_TEMPORARY_CREDENTIALS_REQUEST_URI,
            use_authorization_header=use_authorization_header
        )

    @classmethod
    def _check_signature_method(cls, signature_method):
        if signature_method == SIGNATURE_METHOD_PLAINTEXT:
            raise SignatureMethodNotSupportedError("Google OAuth 1.0 does not support the PLAINTEXT signature method.")

    def build_temporary_credentials_request(self,
                                            scope,
                                            xoauth_displayname=None,
                                            method="POST",
                                            payload_params=None,
                                            headers=None,
                                            realm=None,
                                            oauth_signature_method=SIGNATURE_METHOD_HMAC_SHA1,
                                            oauth_callback="oob",
                                            **extra_oauth_params):
        payload_params = payload_params or {}
        google_params = dict(
            scope=scope,
        )
        if xoauth_displayname:
            google_params["xoauth_displayname"] = xoauth_displayname
        payload_params.update(google_params)

        GoogleClient._check_signature_method(oauth_signature_method)

        super(GoogleClient, self).build_temporary_credentials_request(
            method=method,
            payload_params=payload_params,
            headers=headers,
            realm=realm,
            oauth_signature_method=oauth_signature_method,
            oauth_callback=oauth_callback,
            **extra_oauth_params
        )

    def build_token_credentials_request(self,
                                        temporary_credentials,
                                        oauth_verifier,
                                        method="POST",
                                        payload_params=None,
                                        headers=None,
                                        realm=None,
                                        oauth_signature_method=SIGNATURE_METHOD_HMAC_SHA1,
                                        **extra_oauth_params):

        GoogleClient._check_signature_method(oauth_signature_method)

        super(GoogleClient, self).build_token_credentials_request(
            temporary_credentials=temporary_credentials,
            oauth_verifier=oauth_verifier,
            method=method,
            payload_params=payload_params,
            headers=headers,
            realm=realm,
            oauth_signature_method=oauth_signature_method,
            **extra_oauth_params
        )
<commit_msg>Fixes Google OAuth 1.0 client.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Google OAuth 1.0 Client.
#
# Copyright (C) 2011 Yesudeep Mangalapilly <yesudeep@gmail.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from pyoauth.error import SignatureMethodNotSupportedError
from pyoauth.oauth1 import SIGNATURE_METHOD_HMAC_SHA1, SIGNATURE_METHOD_PLAINTEXT

from pyoauth.oauth1.client import Client

class GoogleClient(Client):
    """
    Creates an instance of a Google OAuth 1.0 client.

    :see: http://code.google.com/apis/accounts/docs/OAuth_ref.html
    """
    _OAUTH_TEMPORARY_CREDENTIALS_REQUEST_URI = "https://www.google.com/accounts/OAuthGetRequestToken"
    _OAUTH_RESOURCE_OWNER_AUTHORIZATION_URI = "https://www.google.com/accounts/OAuthAuthorizeToken"
    _OAUTH_TOKEN_CREDENTIALS_REQUEST_URI = "https://www.google.com/accounts/OAuthGetAccessToken"

    def __init__(self,
                 client_credentials,
                 use_authorization_header=True):
        super(GoogleClient, self).__init__(
            client_credentials=client_credentials,
            temporary_credentials_request_uri=self._OAUTH_TEMPORARY_CREDENTIALS_REQUEST_URI,
            resource_owner_authorization_uri=self._OAUTH_RESOURCE_OWNER_AUTHORIZATION_URI,
            token_credentials_request_uri=self._OAUTH_TEMPORARY_CREDENTIALS_REQUEST_URI,
            use_authorization_header=use_authorization_header
        )

    @classmethod
    def _check_signature_method(cls, signature_method):
        if signature_method == SIGNATURE_METHOD_PLAINTEXT:
            raise SignatureMethodNotSupportedError("Google OAuth 1.0 does not support the PLAINTEXT signature method.")

    def build_temporary_credentials_request(self,
                                            scope,
                                            xoauth_displayname=None,
                                            method="POST",
                                            payload_params=None,
                                            headers=None,
                                            realm=None,
                                            oauth_signature_method=SIGNATURE_METHOD_HMAC_SHA1,
                                            oauth_callback="oob",
                                            **extra_oauth_params):
        payload_params = payload_params or {}
        google_params = dict(
            scope=scope,
        )
        if xoauth_displayname:
            google_params["xoauth_displayname"] = xoauth_displayname
        payload_params.update(google_params)

        GoogleClient._check_signature_method(oauth_signature_method)

        return super(GoogleClient, self).build_temporary_credentials_request(
            method=method,
            payload_params=payload_params,
            headers=headers,
            realm=realm,
            oauth_signature_method=oauth_signature_method,
            oauth_callback=oauth_callback,
            **extra_oauth_params
        )

    def build_token_credentials_request(self,
                                        temporary_credentials,
                                        oauth_verifier,
                                        method="POST",
                                        payload_params=None,
                                        headers=None,
                                        realm=None,
                                        oauth_signature_method=SIGNATURE_METHOD_HMAC_SHA1,
                                        **extra_oauth_params):

        GoogleClient._check_signature_method(oauth_signature_method)

        return super(GoogleClient, self).build_token_credentials_request(
            temporary_credentials=temporary_credentials,
            oauth_verifier=oauth_verifier,
            method=method,
            payload_params=payload_params,
            headers=headers,
            realm=realm,
            oauth_signature_method=oauth_signature_method,
            **extra_oauth_params
        )

    def build_resource_request(self,
                               token_credentials,
                               method,
                               url,
                               payload_params=None,
                               headers=None,
                               realm=None,
                               oauth_signature_method=SIGNATURE_METHOD_HMAC_SHA1,
                               **extra_oauth_params):
        GoogleClient._check_signature_method(oauth_signature_method)

        return super(GoogleClient, self).build_resource_request(
            token_credentials=token_credentials,
            method=method,
            url=url,
            payload_params=payload_params,
            headers=headers,
            realm=realm,
            oauth_signature_method=oauth_signature_method,
            **extra_oauth_params
        )
<|endoftext|>


<commit_before>import os
from django.conf import settings
from django.shortcuts import render_to_response
from django.db.models import get_model

def browse(request, file_type):
    image_model, appname = settings.BROWSER_IMAGE
    file_model, appname = settings.BROWSER_FILE
    files_info = []

    if file_type == 'image':
        model_name, appname = settings.BROWSER_IMAGE
        fieldname = 'image'
    elif file_type == 'file':
        model_name, appname = settings.BROWSER_FILE
        fieldname = 'document'

    model = get_model(appname, model_name)

    objs = model.objects.all()
    for obj in objs:
        field = getattr(obj, fieldname)
        files_info.append((field.name,
                           obj.title))

    return render_to_response('filebrowser/index.html',
                              {'files': files_info,
                               'type': file_type,
                               'title': '%s Browser' % file_type.capitalize()})
<commit_msg>allow override<commit_after>from django.conf import settings
from django.shortcuts import render_to_response
from django.db.models import get_model

def browse(request, file_type):
    files_info = []

    if file_type == 'image':
        model_name, appname = settings.BROWSER_IMAGE
        fieldname = getattr(settings, 'BROWSER_IMAGE_FIELD', 'image')
    elif file_type == 'file':
        model_name, appname = settings.BROWSER_FILE
        fieldname = getattr(settings, 'BROWSER_FILE_FIELD', 'document')

    model = get_model(appname, model_name)

    objs = model.objects.all()
    for obj in objs:
        field = getattr(obj, fieldname)
        files_info.append((field.name,
                           obj.title))

    return render_to_response('filebrowser/index.html',
                              {'files': files_info,
                               'type': file_type,
                               'title': '%s Browser' % file_type.capitalize()})
<|endoftext|>


<commit_before>import logging
import time
from itertools import islice

from django.conf import settings
from django.db import reset_queries

import requests
from elasticutils.contrib.django import get_es, MappingType
from elasticutils.contrib.django import S as BaseS
from elasticsearch.exceptions import ConnectionError, NotFoundError

# Note: This module should not import any Fjord modules. Otherwise we
# get into import recursion issues.

log = logging.getLogger('i.search')

_mapping_types = {}

def es_analyze(text, analyzer=None):
    """Returns analysis of text.

    :arg text: the text to analyze

    :arg analyzer: (optional) the analyzer to use. Defaults to snowball
        which is an English-settings analyzer.

    :returns: list of dicts each describing a token

    """
    es = get_es()
    index = get_index()
    analyzer = analyzer or 'snowball'

    ret = es.indices.analyze(index, body=text, analyzer=analyzer)

    return ret['tokens']

def register_mapping_type(mapping_type):
    """Registers a mapping type.

    This gives us a way to get all the registered mapping types for
    indexing.

    """
    _mapping_types[mapping_type.get_mapping_type_name()] = mapping_type
    # Enable this to be used as a decorator
    return mapping_type

def get_mapping_types(mapping_types=None):
    """Returns a dict of name -> mapping type.

    :arg mapping_types: list of mapping type names to restrict
        the dict to.

    """
    if mapping_types is None:
        return _mapping_types

    return dict((key, val) for key, val in _mapping_types.items()
                if key in mapping_types)

def get_index():
    """Returns the index we're using."""

    # Note: This could probably be defined in utils, but it's defined
    # here because otherwise models imports utils and utils imports
    # models and that turns into a mess.
    return '%s-%s' % (settings.ES_INDEX_PREFIX, settings.ES_INDEXES['default'])

class FjordS(BaseS):
    def process_query_sqs(self, key, val, action):
        """Implements simple_query_string query"""
        return {
            'simple_query_string': {
                'fields': [key],
                'query': val,
                'analyzer': 'snowball',
                'default_operator': 'or',
            }
        }

class FjordMappingType(MappingType):
    """DjangoMappingType with correct index."""
    @classmethod
    def get_index(cls):
        return get_index()

    @classmethod
    def search(cls):
        return FjordS(cls)

def boolean_type():
    return {'type': 'boolean'}

def date_type():
    return {'type': 'date'}

def integer_type():
    return {'type': 'integer'}

def keyword_type():
    return {'type': 'string', 'analyzer': 'keyword'}

def terms_type():
    return {'type': 'string', 'analyzer': 'standard'}

def text_type():
    return {'type': 'string', 'analyzer': 'snowball'}

def format_time(time_to_go):
    """Return minutes and seconds string for given time in seconds.

    :arg time_to_go: Number of seconds to go.

    :returns: string representation of how much time to go.
    """
    if time_to_go < 60:
        return '%ds' % time_to_go
    return '%dm %ds' % (time_to_go / 60, time_to_go % 60)

def create_batch_id():
    """Returns a batch_id"""
    # TODO: This is silly, but it's a good enough way to distinguish
    # between batches by looking at a Record. This is just over the
    # number of seconds in a day.
    return str(int(time.time()))[-6:]

def chunked(iterable, n):
    """Return chunks of n length of iterable.

    If ``len(iterable) % n != 0``, then the last chunk will have
    length less than n.

    Example:

    >>> chunked([1, 2, 3, 4, 5], 2)
    [(1, 2), (3, 4), (5,)]

    :arg iterable: the iterable
    :arg n: the chunk length

    :returns: generator of chunks from the iterable
    """
    iterable = iter(iterable)
    while 1:
        t = tuple(islice(iterable, n))
        if t:
            yield t
        else:
            return

def get_indexes(all_indexes=False):
    """Return list of (name, count) tuples for indexes.

    :arg all_indexes: True if you want to see all indexes and
        False if you want to see only indexes prefexed with
        ``settings.ES_INDEX_PREFIX``.

    :returns: list of (name, count) tuples.

    """
    status = get_es().indices.status()
    indexes = status['indices']

    if not all_indexes:
        indexes = dict((k, v) for k, v in indexes.items()
                       if k.startswith(settings.ES_INDEX_PREFIX))

    indexes = [(k, v['docs']['num_docs']) for k, v in indexes.items()]

    return indexes

def delete_index_if_exists(index):
    """Delete the specified index.

    :arg index: The name of the index to delete.

    """
    try:
        get_es().indices.delete(index)
    except NotFoundError:
        # Can ignore this since it indicates the index doesn't exist
        # and therefore there's nothing to delete.
        pass

def get_index_stats():
    """Return dict of name -> count for documents indexed.

    For example:

    >>> get_index_stats()
    {'response': 122233}

    .. Note::

       This infers the index to use from the registered mapping
       types.

    :returns: mapping type name -> count for documents indexes.

    :throws elasticsearch.exceptions.ConnectionError: if there's a
        connection error
    :throws elasticsearch.exceptions.NotFoundError: if the
        index doesn't exist

    """
    stats = {}
    for name, cls in get_mapping_types().items():
        stats[name] = FjordS(cls).count()

    return stats

def recreate_index(es=None):
    """Delete index if it's there and creates a new one.

    :arg es: ES to use. By default, this creates a new indexing ES.

    """
    if es is None:
        es = get_es()

    mappings = {}
    for name, mt in get_mapping_types().items():
        mapping = mt.get_mapping()
        if mapping is not None:
            mappings[name] = {'properties': mapping}

    index = get_index()

    delete_index_if_exists(index)

    # There should be no mapping-conflict race here since the index
    # doesn't exist. Live indexing should just fail.

    # Simultaneously create the index and the mappings, so live
    # indexing doesn't get a chance to index anything between the two
    # causing ES to infer a possibly bogus mapping (which causes ES to
    # freak out if the inferred mapping is incompatible with the
    # explicit mapping).

    es.indices.create(index, body={'mappings': mappings})

def get_indexable(percent=100, mapping_types=None):
    """Return list of (class, iterable) for all the things to index.

    :arg percent: Defaults to 100.  Allows you to specify how much of
        each doctype you want to index.  This is useful for
        development where doing a full reindex takes an hour.
    :arg mapping_types: List of mapping types to index. Defaults to
        indexing all mapping types.

    :returns: list of (mapping type class, iterable) for all mapping
        types

    """
    to_index = []
    percent = float(percent) / 100

    for name, cls in get_mapping_types(mapping_types).items():
        indexable = cls.get_indexable()
        if percent < 1:
            indexable = indexable[:int(indexable.count() * percent)]
        to_index.append((cls, indexable))

    return to_index

def index_chunk(cls, id_list, es=None):
    """Index a chunk of documents.

    :arg cls: The MappingType class.
    :arg id_list: Iterable of ids of that MappingType to index.
    :arg es: The ES to use. Defaults to creating a new indexing ES.

    """
    if es is None:
        es = get_es()

    for ids in chunked(id_list, 200):
        documents = []
        obj_list = cls.get_model().uncached.filter(id__in=ids)
        documents = [cls.extract_document(obj_id=obj.id, obj=obj)
                     for obj in obj_list]

        if documents:
            cls.bulk_index(documents, id_field='id', es=es)

    if settings.DEBUG:
        # Nix queries so that this doesn't become a complete
        # memory hog and make Will's computer sad when DEBUG=True.
        reset_queries()

def requires_good_connection(fun):
    """Decorator that logs an error on connection issues

    9 out of 10 doctors say that connection errors are usually because
    ES_URLS is set wrong. This catches those errors and helps you out
    with fixing it.

    """
    def _requires_good_connection(*args, **kwargs):
        try:
            return fun(*args, **kwargs)
        except ConnectionError:
            log.error('Either your ElasticSearch process is not quite '
                      'ready to rumble, is not running at all, or ES_URLS '
                      'is set wrong in your settings_local.py file.')
    return _requires_good_connection

@requires_good_connection
def es_reindex_cmd(percent=100, mapping_types=None):
    """Rebuild ElasticSearch indexes.

    :arg percent: 1 to 100--the percentage of the db to index
    :arg mapping_types: list of mapping types to index

    """
    es = get_es()

    log.info('Wiping and recreating %s....', get_index())
    recreate_index(es=es)

    # Shut off auto-refreshing.
    index_settings = es.indices.get_settings(index=get_index())
    old_refresh = (index_settings
                   .get(get_index(), {})
                   .get('settings', {})
                   .get('index.refresh_interval', '1s'))

    es.indices.put_settings(
        index=get_index(), body={'index': {'refresh_interval': '-1'}})

    if mapping_types:
        indexable = get_indexable(percent, mapping_types)
    else:
        indexable = get_indexable(percent)

    start_time = time.time()
    for cls, indexable in indexable:
        cls_start_time = time.time()
        total = len(indexable)

        if total == 0:
            continue

        log.info('Reindex %s. %s to index....',
                 cls.get_mapping_type_name(), total)

        i = 0
        for chunk in chunked(indexable, 1000):
            chunk_start_time = time.time()
            index_chunk(cls, chunk, es=es)

            i += len(chunk)
            time_to_go = (total - i) * ((time.time() - cls_start_time) / i)
            per_1000 = (time.time() - cls_start_time) / (i / 1000.0)
            this_1000 = time.time() - chunk_start_time

            log.info('   %s/%s %s... (%s/1000 avg, %s ETA)',
                     i,
                     total,
                     format_time(this_1000),
                     format_time(per_1000),
                     format_time(time_to_go)
            )

        delta_time = time.time() - cls_start_time
        log.info('   done! (%s total, %s/1000 avg)',
                 format_time(delta_time),
                 format_time(delta_time / (total / 1000.0)))

    es.indices.put_settings(
        index=get_index(), body={'index': {'refresh_interval': old_refresh}})

    delta_time = time.time() - start_time
    log.info('Done! (total time: %s)', format_time(delta_time))

@requires_good_connection
def es_delete_cmd(index):
    """Delete a specified index."""
    indexes = [name for name, count in get_indexes()]

    if index not in indexes:
        log.error('Index "%s" is not a valid index.', index)
        if not indexes:
            log.error('There are no valid indexes.')
        else:
            log.error('Valid indexes: %s', ', '.join(indexes))
        return

    ret = raw_input('Are you sure you want to delete "%s"? (yes/no) ' % index)
    if ret != 'yes':
        return

    log.info('Deleting index "%s"...', index)
    delete_index_if_exists(index)
    log.info('Done!')

@requires_good_connection
def es_status_cmd(checkindex=False, log=log):
    """Show ElasticSearch index status."""
    log.info('Settings:')
    log.info('  ES_URLS               : %s', settings.ES_URLS)
    log.info('  ES_INDEX_PREFIX       : %s', settings.ES_INDEX_PREFIX)
    log.info('  ES_INDEXES            : %s', settings.ES_INDEXES)

    # FIXME - can do this better with elasticsearch API.
    try:
        es_deets = requests.get(settings.ES_URLS[0]).json()
        log.info('  Elasticsearch version : %s', es_deets['version']['number'])
    except requests.exceptions.RequestException:
        log.info('  Could not connect to Elasticsearch')

    log.info('Index (%s) stats:', get_index())

    try:
        mt_stats = get_index_stats()
        log.info('  Index (%s):', get_index())
        for name, count in mt_stats.items():
            log.info('    %-20s: %d', name, count)

    except NotFoundError:
        log.info('  Index does not exist. (%s)', get_index())
<commit_msg>Fix esreindex so it doesn't hose refresh interval<commit_after>import logging
import time
from itertools import islice

from django.conf import settings
from django.db import reset_queries

import requests
from elasticutils.contrib.django import get_es, MappingType
from elasticutils.contrib.django import S as BaseS
from elasticsearch.exceptions import ConnectionError, NotFoundError

# Note: This module should not import any Fjord modules. Otherwise we
# get into import recursion issues.

log = logging.getLogger('i.search')

_mapping_types = {}

def es_analyze(text, analyzer=None):
    """Returns analysis of text.

    :arg text: the text to analyze

    :arg analyzer: (optional) the analyzer to use. Defaults to snowball
        which is an English-settings analyzer.

    :returns: list of dicts each describing a token

    """
    es = get_es()
    index = get_index()
    analyzer = analyzer or 'snowball'

    ret = es.indices.analyze(index, body=text, analyzer=analyzer)

    return ret['tokens']

def register_mapping_type(mapping_type):
    """Registers a mapping type.

    This gives us a way to get all the registered mapping types for
    indexing.

    """
    _mapping_types[mapping_type.get_mapping_type_name()] = mapping_type
    # Enable this to be used as a decorator
    return mapping_type

def get_mapping_types(mapping_types=None):
    """Returns a dict of name -> mapping type.

    :arg mapping_types: list of mapping type names to restrict
        the dict to.

    """
    if mapping_types is None:
        return _mapping_types

    return dict((key, val) for key, val in _mapping_types.items()
                if key in mapping_types)

def get_index():
    """Returns the index we're using."""

    # Note: This could probably be defined in utils, but it's defined
    # here because otherwise models imports utils and utils imports
    # models and that turns into a mess.
    return '%s-%s' % (settings.ES_INDEX_PREFIX, settings.ES_INDEXES['default'])

class FjordS(BaseS):
    def process_query_sqs(self, key, val, action):
        """Implements simple_query_string query"""
        return {
            'simple_query_string': {
                'fields': [key],
                'query': val,
                'analyzer': 'snowball',
                'default_operator': 'or',
            }
        }

class FjordMappingType(MappingType):
    """DjangoMappingType with correct index."""
    @classmethod
    def get_index(cls):
        return get_index()

    @classmethod
    def search(cls):
        return FjordS(cls)

def boolean_type():
    return {'type': 'boolean'}

def date_type():
    return {'type': 'date'}

def integer_type():
    return {'type': 'integer'}

def keyword_type():
    return {'type': 'string', 'analyzer': 'keyword'}

def terms_type():
    return {'type': 'string', 'analyzer': 'standard'}

def text_type():
    return {'type': 'string', 'analyzer': 'snowball'}

def format_time(time_to_go):
    """Return minutes and seconds string for given time in seconds.

    :arg time_to_go: Number of seconds to go.

    :returns: string representation of how much time to go.
    """
    if time_to_go < 60:
        return '%ds' % time_to_go
    return '%dm %ds' % (time_to_go / 60, time_to_go % 60)

def create_batch_id():
    """Returns a batch_id"""
    # TODO: This is silly, but it's a good enough way to distinguish
    # between batches by looking at a Record. This is just over the
    # number of seconds in a day.
    return str(int(time.time()))[-6:]

def chunked(iterable, n):
    """Return chunks of n length of iterable.

    If ``len(iterable) % n != 0``, then the last chunk will have
    length less than n.

    Example:

    >>> chunked([1, 2, 3, 4, 5], 2)
    [(1, 2), (3, 4), (5,)]

    :arg iterable: the iterable
    :arg n: the chunk length

    :returns: generator of chunks from the iterable
    """
    iterable = iter(iterable)
    while 1:
        t = tuple(islice(iterable, n))
        if t:
            yield t
        else:
            return

def get_indexes(all_indexes=False):
    """Return list of (name, count) tuples for indexes.

    :arg all_indexes: True if you want to see all indexes and
        False if you want to see only indexes prefexed with
        ``settings.ES_INDEX_PREFIX``.

    :returns: list of (name, count) tuples.

    """
    status = get_es().indices.status()
    indexes = status['indices']

    if not all_indexes:
        indexes = dict((k, v) for k, v in indexes.items()
                       if k.startswith(settings.ES_INDEX_PREFIX))

    indexes = [(k, v['docs']['num_docs']) for k, v in indexes.items()]

    return indexes

def delete_index_if_exists(index):
    """Delete the specified index.

    :arg index: The name of the index to delete.

    """
    try:
        get_es().indices.delete(index)
    except NotFoundError:
        # Can ignore this since it indicates the index doesn't exist
        # and therefore there's nothing to delete.
        pass

def get_index_stats():
    """Return dict of name -> count for documents indexed.

    For example:

    >>> get_index_stats()
    {'response': 122233}

    .. Note::

       This infers the index to use from the registered mapping
       types.

    :returns: mapping type name -> count for documents indexes.

    :throws elasticsearch.exceptions.ConnectionError: if there's a
        connection error
    :throws elasticsearch.exceptions.NotFoundError: if the
        index doesn't exist

    """
    stats = {}
    for name, cls in get_mapping_types().items():
        stats[name] = FjordS(cls).count()

    return stats

def recreate_index(es=None):
    """Delete index if it's there and creates a new one.

    :arg es: ES to use. By default, this creates a new indexing ES.

    """
    if es is None:
        es = get_es()

    mappings = {}
    for name, mt in get_mapping_types().items():
        mapping = mt.get_mapping()
        if mapping is not None:
            mappings[name] = {'properties': mapping}

    index = get_index()

    delete_index_if_exists(index)

    # There should be no mapping-conflict race here since the index
    # doesn't exist. Live indexing should just fail.

    # Simultaneously create the index and the mappings, so live
    # indexing doesn't get a chance to index anything between the two
    # causing ES to infer a possibly bogus mapping (which causes ES to
    # freak out if the inferred mapping is incompatible with the
    # explicit mapping).

    es.indices.create(index, body={'mappings': mappings})

def get_indexable(percent=100, mapping_types=None):
    """Return list of (class, iterable) for all the things to index.

    :arg percent: Defaults to 100.  Allows you to specify how much of
        each doctype you want to index.  This is useful for
        development where doing a full reindex takes an hour.
    :arg mapping_types: List of mapping types to index. Defaults to
        indexing all mapping types.

    :returns: list of (mapping type class, iterable) for all mapping
        types

    """
    to_index = []
    percent = float(percent) / 100

    for name, cls in get_mapping_types(mapping_types).items():
        indexable = cls.get_indexable()
        if percent < 1:
            indexable = indexable[:int(indexable.count() * percent)]
        to_index.append((cls, indexable))

    return to_index

def index_chunk(cls, id_list, es=None):
    """Index a chunk of documents.

    :arg cls: The MappingType class.
    :arg id_list: Iterable of ids of that MappingType to index.
    :arg es: The ES to use. Defaults to creating a new indexing ES.

    """
    if es is None:
        es = get_es()

    for ids in chunked(id_list, 200):
        documents = []
        obj_list = cls.get_model().uncached.filter(id__in=ids)
        documents = [cls.extract_document(obj_id=obj.id, obj=obj)
                     for obj in obj_list]

        if documents:
            cls.bulk_index(documents, id_field='id', es=es)

    if settings.DEBUG:
        # Nix queries so that this doesn't become a complete
        # memory hog and make Will's computer sad when DEBUG=True.
        reset_queries()

def requires_good_connection(fun):
    """Decorator that logs an error on connection issues

    9 out of 10 doctors say that connection errors are usually because
    ES_URLS is set wrong. This catches those errors and helps you out
    with fixing it.

    """
    def _requires_good_connection(*args, **kwargs):
        try:
            return fun(*args, **kwargs)
        except ConnectionError:
            log.error('Either your ElasticSearch process is not quite '
                      'ready to rumble, is not running at all, or ES_URLS '
                      'is set wrong in your settings_local.py file.')
    return _requires_good_connection

@requires_good_connection
def es_reindex_cmd(percent=100, mapping_types=None):
    """Rebuild ElasticSearch indexes.

    :arg percent: 1 to 100--the percentage of the db to index
    :arg mapping_types: list of mapping types to index

    """
    es = get_es()

    log.info('Wiping and recreating %s....', get_index())
    recreate_index(es=es)

    # Shut off auto-refreshing.
    index_settings = es.indices.get_settings(index=get_index())
    old_refresh = (index_settings
                   .get(get_index(), {})
                   .get('settings', {})
                   .get('index.refresh_interval', '1s'))

    try:
        es.indices.put_settings(
            index=get_index(), body={'index': {'refresh_interval': '-1'}})

        if mapping_types:
            indexable = get_indexable(percent, mapping_types)
        else:
            indexable = get_indexable(percent)

        start_time = time.time()
        for cls, indexable in indexable:
            cls_start_time = time.time()
            total = len(indexable)

            if total == 0:
                continue

            log.info('Reindex %s. %s to index....',
                     cls.get_mapping_type_name(), total)

            i = 0
            for chunk in chunked(indexable, 1000):
                chunk_start_time = time.time()
                index_chunk(cls, chunk, es=es)

                i += len(chunk)
                time_to_go = (total - i) * ((time.time() - cls_start_time) / i)
                per_1000 = (time.time() - cls_start_time) / (i / 1000.0)
                this_1000 = time.time() - chunk_start_time

                log.info('   %s/%s %s... (%s/1000 avg, %s ETA)',
                         i,
                         total,
                         format_time(this_1000),
                         format_time(per_1000),
                         format_time(time_to_go)
                )

            delta_time = time.time() - cls_start_time
            log.info('   done! (%s total, %s/1000 avg)',
                     format_time(delta_time),
                     format_time(delta_time / (total / 1000.0)))

        delta_time = time.time() - start_time
        log.info('Done! (total time: %s)', format_time(delta_time))

    finally:
        es.indices.put_settings(
            index=get_index(),
            body={'index': {'refresh_interval': old_refresh}})

@requires_good_connection
def es_delete_cmd(index):
    """Delete a specified index."""
    indexes = [name for name, count in get_indexes()]

    if index not in indexes:
        log.error('Index "%s" is not a valid index.', index)
        if not indexes:
            log.error('There are no valid indexes.')
        else:
            log.error('Valid indexes: %s', ', '.join(indexes))
        return

    ret = raw_input('Are you sure you want to delete "%s"? (yes/no) ' % index)
    if ret != 'yes':
        return

    log.info('Deleting index "%s"...', index)
    delete_index_if_exists(index)
    log.info('Done!')

@requires_good_connection
def es_status_cmd(checkindex=False, log=log):
    """Show ElasticSearch index status."""
    log.info('Settings:')
    log.info('  ES_URLS               : %s', settings.ES_URLS)
    log.info('  ES_INDEX_PREFIX       : %s', settings.ES_INDEX_PREFIX)
    log.info('  ES_INDEXES            : %s', settings.ES_INDEXES)

    # FIXME - can do this better with elasticsearch API.
    try:
        es_deets = requests.get(settings.ES_URLS[0]).json()
        log.info('  Elasticsearch version : %s', es_deets['version']['number'])
    except requests.exceptions.RequestException:
        log.info('  Could not connect to Elasticsearch')

    log.info('Index (%s) stats:', get_index())

    try:
        mt_stats = get_index_stats()
        log.info('  Index (%s):', get_index())
        for name, count in mt_stats.items():
            log.info('    %-20s: %d', name, count)

    except NotFoundError:
        log.info('  Index does not exist. (%s)', get_index())
<|endoftext|>


<commit_before>from django.conf import settings
from django.shortcuts import get_object_or_404

from rest_framework import viewsets, status
from rest_framework.response import Response
from rest_framework.decorators import action

import snotes20.models as models
import snotes20.serializers as serializers

class PodcastViewSet(viewsets.ViewSet):
    def retrieve(self, request, pk=None):
        try:
            pod = models.Podcast.objects.get(slugs__slug=pk)
        except models.Podcast.DoesNotExist:
            return Response(status=status.HTTP_404_NOT_FOUND)

        data = serializers.PodcastSerializer(pod).data
        return Response(data)

    @action(methods=["GET"])
    def covers(self, request, pk=None):
        podcast = get_object_or_404(models.Podcast, slugs__slug=pk)

        covers = []

        try:
            if podcast.cover is not None:
                covers.append(podcast.cover)
        except models.Cover.DoesNotExist:
            pass

        ep_covers = models.Cover.objects.filter(episodes__podcast=podcast).distinct()

        if len(covers) > 0:
            ep_covers = ep_covers.exclude(id=covers[0].id)

        covers.extend(ep_covers)

        data = serializers.CoverSerializer(covers, many=True).data

        return Response(data)<commit_msg>podcast last numbers API<commit_after>from django.conf import settings
from django.shortcuts import get_object_or_404

from rest_framework import viewsets, status
from rest_framework.response import Response
from rest_framework.decorators import action

import snotes20.models as models
import snotes20.serializers as serializers

class PodcastViewSet(viewsets.ViewSet):
    def retrieve(self, request, pk=None):
        try:
            pod = models.Podcast.objects.get(slugs__slug=pk)
        except models.Podcast.DoesNotExist:
            return Response(status=status.HTTP_404_NOT_FOUND)

        data = serializers.PodcastSerializer(pod).data
        return Response(data)

    @action(methods=["GET"])
    def covers(self, request, pk=None):
        podcast = get_object_or_404(models.Podcast, slugs__slug=pk)

        covers = []

        try:
            if podcast.cover is not None:
                covers.append(podcast.cover)
        except models.Cover.DoesNotExist:
            pass

        ep_covers = models.Cover.objects.filter(episodes__podcast=podcast).distinct()

        if len(covers) > 0:
            ep_covers = ep_covers.exclude(id=covers[0].id)

        covers.extend(ep_covers)

        data = serializers.CoverSerializer(covers, many=True).data

        return Response(data)

    @action(methods=["GET"])
    def numbers(self, request, pk=None):
        podcast = get_object_or_404(models.Podcast, slugs__slug=pk)

        # http://stackoverflow.com/a/8502570/2486196
        episodes = models.Episode.objects.raw(
            "SELECT id, number FROM snotes20_episode "
            "WHERE podcast_id='" + podcast.id + "' AND number IS NOT NULL "
            "ORDER BY NULLIF(regexp_replace(number, E'\\D', '', 'g'), '')::int DESC "
            "LIMIT 3"
        )

        return Response([ep.number for ep in list(episodes)])
<|endoftext|>


<commit_before>import os
import pdb
import rasterio
import gdal
import glob
import gdalconst
import pandas as pd
import numpy as np
import sys

from pygeoprocessing import geoprocessing as gp
from geopy.geocoders import Nominatim
import geocoder
from geopy.exc import GeocoderTimedOut
from shutil import copyfile
from functools32 import lru_cache
import shapefile

# TODO zonal statistics: https://github.com/perrygeo/python-rasterstats
# resize and resample:  http://data.naturalcapitalproject.org/pygeoprocessing/api/latest/api/geoprocessing.html
# temporary_filename
# temporary_folder
# unique_raster_values, unique_raster_values_count
# vectorize_datasets
# assert_datasets_in_same_projection
# calculate_raster_stats_uri
# clip_dataset_uri
# create_rat_uri
# sieve: http://pktools.nongnu.org/html/pksieve.html
# composite/mosaic: http://pktools.nongnu.org/html/pkcomposite.html
# mosaic

def get_dataset_type(path_ds):
    """
    Return dataset type e.g. GeoTiff
    Args:
        path_ds:

    Returns:

    """
    dataset = gdal.Open(path_ds, gdalconst.GA_ReadOnly)
    dataset_type = dataset.GetDriver().LongName

    dataset = None  # Close dataset

    return dataset_type

def get_dataset_datatype(path_ds):
    """
    Return datatype of dataset e.g. GDT_UInt32
    Args:
        path_ds:

    Returns:

    """
    dataset = gdal.Open(path_ds, gdalconst.GA_ReadOnly)

    band = dataset.GetRasterBand(1)
    bandtype = gdal.GetDataTypeName(band.DataType)  # UInt32

    dataset = None  # Close dataset

    if bandtype == 'UInt32':
        return gdalconst.GDT_UInt32
    elif bandtype == 'UInt16':
        return gdalconst.GDT_UInt16
    elif bandtype == 'Float32':
        return gdalconst.GDT_Float32
    elif bandtype == 'Float64':
        return gdalconst.GDT_Float64
    elif bandtype == 'Int16':
        return gdalconst.GDT_Int16
    elif bandtype == 'Int32':
        return gdalconst.GDT_Int32
    elif bandtype == 'Unknown':
        return gdalconst.GDT_Unknown
    else:
        return gdalconst.GDT_UInt32

def get_properties(path_ds, name_property):
    """

    Args:
        path_ds:
        name_property:

    Returns:

    """
    dict_properties = gp.get_raster_properties_uri(path_ds)

    return dict_properties[name_property]

def get_values_rat_column(path_ds, name_col='Value'):
    """

    Args:
        path_ds:
        name_col:

    Returns:

    """
    dict_values = gp.get_rat_as_dictionary_uri(path_ds)

    name_key = [s for s in dict_values.keys() if '.' + name_col in s]

    return dict_values.get(name_key[0], None)

def lookup(path_ds, path_out_ds, from_field='Value', to_field='', overwrite=True):
    """

    Args:
        path_ds:
        path_out_ds:
        from_field:
        to_field:
        overwrite:

    Returns:

    """
    val_from = get_values_rat_column(path_ds, name_col=from_field)
    val_to = get_values_rat_column(path_ds, name_col=to_field)

    dict_reclass = dict(zip(val_from, val_to))
    gp.reclassify_dataset_uri(path_ds, dict_reclass, path_out_ds, out_datatype=get_dataset_datatype(path_ds),
                              out_nodata=gp.get_nodata_from_uri(path_ds))

def convert_raster_to_ascii(path_input_raster, path_ascii_output, overwrite=True):
    """
    Convert input raster to ascii format
    Args:
        path_input_raster:
        path_ascii_output:
        overwrite:

    Returns:

    """
    if overwrite and os.path.isfile(path_ascii_output):
        os.remove(path_ascii_output)

    # Open existing dataset
    path_inp_ds = gdal.Open(path_input_raster)

    # Open output format driver, gdal_translate --formats lists all of them
    format_file = 'AAIGrid'
    driver = gdal.GetDriverByName(format_file)

    # Output to new format
    path_dest_ds = driver.CreateCopy(path_ascii_output, path_inp_ds, 0)

    # Close the datasets to flush to disk
    path_dest_ds = None
    path_inp_ds = None

def get_arr_res(lats, lons):
    """
    Get resolution of array from lat lon data
    Args:
        lats:
        lons:

    Returns:

    """
    # Check if both lat and lon have same shape
    if np.isclose(np.abs(lats[1] - lats[0]),  np.abs(lons[1] - lons[0])):
        return np.abs(lats[1] - lats[0])
    else:
        raise ValueError('lat and lon do not have same shape')

def get_geo_idx(val_dd, array_dd):
    """
    Get the index of the nearest decimal degree in an array of decimal degrees
    Args:
        val_dd:
        array_dd:

    Returns:

    """
    return (np.abs(array_dd - val_dd)).argmin()

@lru_cache(maxsize=1024)
def get_latlon_location(loc):
    """
    Get latitude/longitude of location
    Args:
        loc:

    Returns:

    """
    try:
        # Geopy
        geolocator = Nominatim()
        location = geolocator.geocode(loc, timeout=5)
        lat = location.latitude
        lon = location.longitude
    except Exception, e:
        # TODO Be smarter about catching exceptions here, needs internet connection to work
        # Default behaviour is to exit
        print 'Geolocator not working: ' + loc + ' ' + str(e)
        return 0.0, 0.0  # Equator is default

    return lat, lon

@lru_cache(maxsize=1024)
def get_hemisphere(loc, boundary=0.0):
    """
    Get hemisphere in which a location lies (northern/southern)
    Args:
        loc: Name of country/region to use to get latitude
        boundary: Latitude above which it is N hemisphere

    Returns:

    """
    lat, _ = get_latlon_location(loc)

    if lat >= boundary:
        return 'N'
    else:
        return 'S'

@lru_cache(maxsize=1024)
def is_temperate(loc, n_boundary=23.5, s_boundary=-23.5):
    """

    Args:
        loc:
        n_boundary:
        s_boundary:

    Returns:

    """
    lat, _ = get_latlon_location(loc)

    if lat >= n_boundary or lat <= s_boundary:
        return 'Temperate'
    else:
        return 'Tropical'

# Vector (shapefile data)
def get_att_table_shpfile(path_shpfile):
    """

    Args:
        path_shpfile:

    Returns:

    """
    # Read shapefile data into dataframe
    hndl_shp = shapefile.Reader(path_shpfile)

    fields = hndl_shp.fields[1:]
    field_names = [field[0] for field in fields]

    # construction of a dictionary field_name:value
    df_shp = pd.DataFrame(columns=field_names)
    for rec in hndl_shp.shapeRecords():
        df_shp.loc[len(df_shp)] = rec.record

    return df_shp

def copy_shpfile(path_inp_shp, path_out_shp):
    """

    Args:
        path_inp_shp:
        path_out_shp:

    Returns:

    """
    files_to_copy = glob.glob(os.path.dirname(path_inp_shp) + os.sep +
                              os.path.splitext(os.path.basename(path_inp_shp))[0] + '*')

    name_new_file = os.path.splitext(os.path.basename(path_out_shp))[0]

    path_inp = os.path.dirname(path_inp_shp)
    path_out = os.path.dirname(path_out_shp)

    for fl in files_to_copy:
        ext = os.path.splitext(fl)[1]
        copyfile(fl, path_out + os.sep + name_new_file + ext)

def extract_at_point_from_ras(path_ras, lon, lat):
    """
    Extract value from raster at given longitude and latitude
    East and North are positive, West and South are negative
    Args:
        path_ras:
        lon: 27.8 (E)
        lat: -13.13 (S)

    Returns:

    """
    with rasterio.drivers():
        # Read raster bands directly to numpy arrays
        with rasterio.open(path_ras) as src:
            return np.asarray(src.sample([(lon, lat)]))

def get_grid_cell_area(nrows, ncols):
    """

    :param nrows: Number of rows
    :param ncols: Number of columns
    :return:
    """
    R = 6371.0  # radius of earth
    csize = 180. / nrows

    cell_area = np.zeros(shape=(nrows, ncols))
    lat = np.zeros(shape=(nrows,))

    for i in range(nrows):
        lat[i] = i * csize

    lat = lat * np.pi / 180.

    sarea = np.pi / 180. * csize * np.power(R, 2.) * (np.cos(lat) - np.cos(lat + np.pi * csize / 180))
    cell_area = np.zeros((nrows, ncols))

    for i in range(nrows):
        cell_area[i, :] = sarea[i]

    return cell_area

def get_country_lat_lon_extent(alpha2):
    """

    Args:
        alpha2:

    Returns:

    """
    if alpha2 == 'united_states_of_america':
        return [-175, -50, 22, 70]
    elif alpha2 == 'russian_federation':
        return [22, 179, 35, 83]
    elif alpha2 == 'china':
        return [70, 138, 12, 55]
    elif alpha2 == 'india':
        return [64, 100, 4, 37]
    elif alpha2 == 'argentina':
        return [-80, -50, -16, -64]

if __name__ == '__main__':
    pass
<commit_msg>minor<commit_after>import os
import pdb
import rasterio
import gdal
import glob
import gdalconst
import pandas as pd
import numpy as np
import sys

from pygeoprocessing import geoprocessing as gp
from geopy.geocoders import Nominatim
import geocoder
from geopy.exc import GeocoderTimedOut
from shutil import copyfile
from functools32 import lru_cache
import shapefile

# TODO zonal statistics: https://github.com/perrygeo/python-rasterstats
# resize and resample:  http://data.naturalcapitalproject.org/pygeoprocessing/api/latest/api/geoprocessing.html
# temporary_filename
# temporary_folder
# unique_raster_values, unique_raster_values_count
# vectorize_datasets
# assert_datasets_in_same_projection
# calculate_raster_stats_uri
# clip_dataset_uri
# create_rat_uri
# sieve: http://pktools.nongnu.org/html/pksieve.html
# composite/mosaic: http://pktools.nongnu.org/html/pkcomposite.html
# mosaic

def get_dataset_type(path_ds):
    """
    Return dataset type e.g. GeoTiff
    Args:
        path_ds:

    Returns:

    """
    dataset = gdal.Open(path_ds, gdalconst.GA_ReadOnly)
    dataset_type = dataset.GetDriver().LongName

    dataset = None  # Close dataset

    return dataset_type

def get_dataset_datatype(path_ds):
    """
    Return datatype of dataset e.g. GDT_UInt32
    Args:
        path_ds:

    Returns:

    """
    dataset = gdal.Open(path_ds, gdalconst.GA_ReadOnly)

    band = dataset.GetRasterBand(1)
    bandtype = gdal.GetDataTypeName(band.DataType)  # UInt32

    dataset = None  # Close dataset

    if bandtype == 'UInt32':
        return gdalconst.GDT_UInt32
    elif bandtype == 'UInt16':
        return gdalconst.GDT_UInt16
    elif bandtype == 'Float32':
        return gdalconst.GDT_Float32
    elif bandtype == 'Float64':
        return gdalconst.GDT_Float64
    elif bandtype == 'Int16':
        return gdalconst.GDT_Int16
    elif bandtype == 'Int32':
        return gdalconst.GDT_Int32
    elif bandtype == 'Unknown':
        return gdalconst.GDT_Unknown
    else:
        return gdalconst.GDT_UInt32

def get_properties(path_ds, name_property):
    """

    Args:
        path_ds:
        name_property:

    Returns:

    """
    dict_properties = gp.get_raster_properties_uri(path_ds)

    return dict_properties[name_property]

def get_values_rat_column(path_ds, name_col='Value'):
    """

    Args:
        path_ds:
        name_col:

    Returns:

    """
    dict_values = gp.get_rat_as_dictionary_uri(path_ds)

    name_key = [s for s in dict_values.keys() if '.' + name_col in s]

    return dict_values.get(name_key[0], None)

def lookup(path_ds, path_out_ds, from_field='Value', to_field='', overwrite=True):
    """

    Args:
        path_ds:
        path_out_ds:
        from_field:
        to_field:
        overwrite:

    Returns:

    """
    val_from = get_values_rat_column(path_ds, name_col=from_field)
    val_to = get_values_rat_column(path_ds, name_col=to_field)

    dict_reclass = dict(zip(val_from, val_to))
    gp.reclassify_dataset_uri(path_ds, dict_reclass, path_out_ds, out_datatype=get_dataset_datatype(path_ds),
                              out_nodata=gp.get_nodata_from_uri(path_ds))

def convert_raster_to_ascii(path_input_raster, path_ascii_output, overwrite=True):
    """
    Convert input raster to ascii format
    Args:
        path_input_raster:
        path_ascii_output:
        overwrite:

    Returns:

    """
    if overwrite and os.path.isfile(path_ascii_output):
        os.remove(path_ascii_output)

    # Open existing dataset
    path_inp_ds = gdal.Open(path_input_raster)

    # Open output format driver, gdal_translate --formats lists all of them
    format_file = 'AAIGrid'
    driver = gdal.GetDriverByName(format_file)

    # Output to new format
    path_dest_ds = driver.CreateCopy(path_ascii_output, path_inp_ds, 0)

    # Close the datasets to flush to disk
    path_dest_ds = None
    path_inp_ds = None

def get_arr_res(lats, lons):
    """
    Get resolution of array from lat lon data
    Args:
        lats:
        lons:

    Returns:

    """
    # Check if both lat and lon have same shape
    if np.isclose(np.abs(lats[1] - lats[0]),  np.abs(lons[1] - lons[0])):
        return np.abs(lats[1] - lats[0])
    else:
        raise ValueError('lat and lon do not have same shape')

def get_geo_idx(val_dd, array_dd):
    """
    Get the index of the nearest decimal degree in an array of decimal degrees
    Args:
        val_dd:
        array_dd:

    Returns:

    """
    return (np.abs(array_dd - val_dd)).argmin()

@lru_cache(maxsize=1024)
def get_latlon_location(loc):
    """
    Get latitude/longitude of location
    Args:
        loc:

    Returns:

    """
    try:
        # Geopy
        geolocator = Nominatim()
        location = geolocator.geocode(loc, timeout=5)
        lat = location.latitude
        lon = location.longitude
    except Exception, e:
        # TODO Be smarter about catching exceptions here, needs internet connection to work
        # Default behaviour is to exit
        print 'Geolocator not working: ' + loc + ' ' + str(e)
        return 0.0, 0.0  # Equator is default

    return lat, lon

@lru_cache(maxsize=1024)
def get_hemisphere(loc, boundary=0.0):
    """
    Get hemisphere in which a location lies (northern/southern)
    Args:
        loc: Name of country/region to use to get latitude
        boundary: Latitude above which it is N hemisphere

    Returns:

    """
    lat, _ = get_latlon_location(loc)

    if lat >= boundary:
        return 'N'
    else:
        return 'S'

@lru_cache(maxsize=1024)
def is_temperate(loc, n_boundary=23.5, s_boundary=-23.5):
    """

    Args:
        loc:
        n_boundary:
        s_boundary:

    Returns:

    """
    lat, _ = get_latlon_location(loc)

    if lat >= n_boundary or lat <= s_boundary:
        return 'Temperate'
    else:
        return 'Tropical'

# Vector (shapefile data)
def get_att_table_shpfile(path_shpfile):
    """

    Args:
        path_shpfile:

    Returns:

    """
    # Read shapefile data into dataframe
    hndl_shp = shapefile.Reader(path_shpfile)

    fields = hndl_shp.fields[1:]
    field_names = [field[0] for field in fields]

    # construction of a dictionary field_name:value
    df_shp = pd.DataFrame(columns=field_names)
    for rec in hndl_shp.shapeRecords():
        df_shp.loc[len(df_shp)] = rec.record

    return df_shp

def copy_shpfile(path_inp_shp, path_out_shp):
    """

    Args:
        path_inp_shp:
        path_out_shp:

    Returns:

    """
    files_to_copy = glob.glob(os.path.dirname(path_inp_shp) + os.sep +
                              os.path.splitext(os.path.basename(path_inp_shp))[0] + '*')

    name_new_file = os.path.splitext(os.path.basename(path_out_shp))[0]

    path_inp = os.path.dirname(path_inp_shp)
    path_out = os.path.dirname(path_out_shp)

    for fl in files_to_copy:
        ext = os.path.splitext(fl)[1]
        copyfile(fl, path_out + os.sep + name_new_file + ext)

def extract_at_point_from_ras(path_ras, lon, lat):
    """
    Extract value from raster at given longitude and latitude
    East and North are positive, West and South are negative
    Args:
        path_ras:
        lon: 27.8 (E)
        lat: -13.13 (S)

    Returns:

    """
    with rasterio.drivers():
        # Read raster bands directly to numpy arrays
        with rasterio.open(path_ras) as src:
            return np.asarray(src.sample([(lon, lat)]))

def get_grid_cell_area(nrows, ncols):
    """

    :param nrows: Number of rows
    :param ncols: Number of columns
    :return:
    """
    R = 6371.0  # radius of earth
    csize = 180. / nrows

    cell_area = np.zeros(shape=(nrows, ncols))
    lat = np.zeros(shape=(nrows,))

    for i in range(nrows):
        lat[i] = i * csize

    lat = lat * np.pi / 180.

    sarea = np.pi / 180. * csize * np.power(R, 2.) * (np.cos(lat) - np.cos(lat + np.pi * csize / 180))
    cell_area = np.zeros((nrows, ncols))

    for i in range(nrows):
        cell_area[i, :] = sarea[i]

    return cell_area

def get_country_lat_lon_extent(alpha2):
    """

    Args:
        alpha2:

    Returns:

    """
    if alpha2 == 'united_states_of_america':
        return [-130, -60, 25, 55]
    elif alpha2 == 'russian_federation':
        return [22, 179, 35, 83]
    elif alpha2 == 'china':
        return [70, 138, 12, 55]
    elif alpha2 == 'india':
        return [64, 100, 4, 37]
    elif alpha2 == 'argentina':
        return [-80, -50, -16, -64]

if __name__ == '__main__':
    pass
<|endoftext|>