<commit_before>import os
import re
import logging

import requests
from bs4 import BeautifulSoup
from PIL import Image
from django.conf import settings
from django.utils.text import slugify

from games.models import Company
from platforms.models import Platform

LOGGER = logging.getLogger(__name__)
API_URL = "http://thegamesdb.net/api/"
UNSUPPORTED_PLATFORMS = (
    'Microsoft Xbox',
    'Microsoft Xbox 360',
    'Microsoft Xbox One',
    'Sony Playstation 3',
    'Sony Playstation 4',
    'Sony Playstation Vita',
    'Android',
    'iOS',
    'Nintendo 3DS',
    'N-Gage',
    'Mac OS',
)

PLATFORM_MAP = {
    'Arcade': 'arcade',
    'Amiga': 'amiga',
    'Atari 2600': 'atari-2600',
    'Atari 5200': 'atari-5200',
    'Atari ST': 'atari-st',
    'PC': 'windows',
    'Sega Genesis': 'sega-genesis',
    'Sega Mega Drive': 'sega-genesis',
    'Sega Master System': 'sega-master-system',
    'Sega Saturn': 'sega-saturn',
    'Sega Game Gear': 'sega-game-gear',
    'Sega Dreamcast': 'sega-dreamcast',
    'Atari Jaguar': 'atari-jaguar',
    'Sony Playstation': 'sony-playstation',
    'Sony Playstation 2': 'sony-playstation-2',
    'Sony PSP': 'sony-psp',
    'Nintendo Entertainment System (NES)': 'nes',
    'Super Nintendo (SNES)': 'super-nintendo',
    'Neo Geo': 'neo-geo',
    'Neo Geo CD': 'neo-geo-cd',
    'Nintendo Game Boy': 'game-boy',
    'Nintendo Game Boy Color': 'nintendo-game-boy-color',
    'Nintendo Game Boy Advance': 'nintendo-game-boy-advance',
    'Nintendo 64': 'nintendo-64',
    'Nintendo GameCube': 'gamecube',
    'Nintendo DS': 'nintendo-ds',
    'Nintendo Wii': 'nintendo-wii',
    'NeoGeo': 'neo-geo',
    'TurboGrafx 16': 'turbografx-16',
    'Vectrex': 'vectrex',
}

def download_image(media_type, url):
    """Download an image from TheGamesDB (is necessary) and return its local path"""
    media_type_map = {
        'clearlogo': settings.TGD_CLEAR_LOGO_PATH,
        'banner': settings.TGD_BANNER_PATH,
        'screenshot': settings.TGD_SCREENSHOT_PATH,
        'fanart': settings.TGD_FANART_PATH
    }
    dest_path = os.path.join(media_type_map[media_type], os.path.basename(url))
    if not os.path.exists(dest_path):
        response = requests.get(url)
        with open(dest_path, 'w') as dest_file:
            dest_file.write(response.content)
    return dest_path

def convert_clearlogo_to_banner(logo_path):
    base_width, base_height = settings.BANNER_SIZE.split('x')
    base_ratio = float(base_width) / float(base_height)

    try:
        logo = Image.open(logo_path)
    except IOError:
        return
    logo_width = float(logo.width)
    logo_height = float(logo.height)
    logo_ratio = logo_width / logo_height

    margin_factor = 0.1
    if logo_ratio < base_ratio:
        new_h = logo_height + (logo_height * margin_factor)
        new_w = new_h * base_ratio
    else:
        new_w = logo_width + (logo_width * margin_factor)
        new_h = new_w / base_ratio

    new_image = Image.new('RGBA', (int(new_w), int(new_h)), (0, 0, 0, 255))
    if logo.mode == 'RGBA':
        mask = logo
    else:
        mask = None
    offset = (int((new_w - logo_width) / 2), int((new_h - logo_height) / 2))
    new_image.paste(logo, offset, mask)
    dest_path = os.path.join(settings.TGD_LUTRIS_BANNER_PATH,
                             os.path.basename(logo_path))
    new_image.save(dest_path)
    return dest_path

def api_request(url):
    response = requests.get(API_URL + url)
    return BeautifulSoup(response.content, 'xml')

def get_value(soup, field):
    if not soup:
        LOGGER.warn("No soup provided")
        return
    tag = soup.find(field)
    if tag:
        return tag.text

def get_similar(game_data):
    similar = game_data.find('Similar')
    if similar:
        return [
            {
                'id': get_value(game, 'id'),
                'platform_id': get_value(game, 'PlatformId')
            } for game in similar.find_all('Game')
        ]

def get_pics(game_data, tag_name):
    fanarts = game_data.find_all(tag_name)
    results = []
    for fanart in fanarts:
        original = fanart.find('original')
        original_meta = original.attrs
        original_meta['url'] = original.get_text()
        results.append({
            'original': original_meta,
            'thumb': get_value(fanart, 'thumb'),
        })
    return results

def get_tags_with_attrs(soup, tag_name, value_name='value'):
    tags = soup.find_all(tag_name)
    results = []
    for tag in tags:
        attrs = tag.attrs
        if attrs:
            attrs[value_name] = tag.get_text()
        else:
            attrs = tag.get_text()
        results.append(attrs)
    if len(results) == 1:
        return results[0]
    else:
        return results

def get_games_list(query, remove_unsupported=True):
    soup = api_request('GetGamesList.php?name=' + query)
    game_list = []
    games = soup.find_all('Game')
    for game in games:
        platform = get_value(game, 'Platform')
        if remove_unsupported and platform in UNSUPPORTED_PLATFORMS:
            continue
        game_list.append({
            'id': get_value(game, 'id'),
            'game_title': get_value(game, 'GameTitle'),
            'release_date': get_value(game, 'ReleaseDate'),
            'platform': get_value(game, 'Platform')
        })
    return game_list

def get_game(game_id):
    soup = api_request('GetGame.php?id={}'.format(game_id))
    game_data = soup.find('Game')
    game_info = {
        'id': get_value(game_data, 'id'),
        'base_img_url': get_value(soup, 'baseImgUrl'),
        'game_title': get_value(game_data, 'GameTitle'),
        'platform': get_value(game_data, 'Platform'),
        'release_date': get_value(game_data, 'ReleaseDate'),
        'overview': get_value(game_data, 'Overview'),
        'esrb': get_value(game_data, 'ESRB'),
        'publisher': get_value(game_data, 'Publisher'),
        'developer': get_value(game_data, 'Developer'),
        'rating': get_value(game_data, 'Rating'),
        'similar': get_similar(game_data),
        'fanarts': get_pics(game_data, 'fanart'),
        'boxarts': get_tags_with_attrs(game_data, 'boxart'),
        'banners': get_tags_with_attrs(game_data, 'banner'),
        'screenshots': get_pics(game_data, 'screenshot'),
        'clearlogo': get_tags_with_attrs(game_data, 'clearlogo'),
    }
    return game_info

def get_lutris_platform(platform_name):
    try:
        slug = PLATFORM_MAP[platform_name]
    except KeyError:
        LOGGER.error('No map for %s', platform_name)
        return
    return Platform.objects.get(slug=slug)

def to_lutris(game):
    lutris_game = {}
    lutris_game['name'] = game['game_title']
    if game['release_date']:
        match = re.search(r'\d{4}', game['release_date'])
        lutris_game['year'] = match.group(0)

    publisher = get_or_create_company(game['publisher'])
    if publisher:
        lutris_game['publisher'] = publisher.id

    if game['developer'] == game['publisher']:
        developer = publisher
    else:
        developer = get_or_create_company(game['developer'])
    if developer:
        lutris_game['developer'] = developer.id

    platform = get_lutris_platform(game['platform'])
    if platform:
        lutris_game['platforms'] = [platform.id]
    if game['clearlogo']:
        clearlogo_url = game['base_img_url'] + game['clearlogo']['value']
        logo_path = download_image('clearlogo', clearlogo_url)
        banner_path = convert_clearlogo_to_banner(logo_path)
        if banner_path:
            banner_url = banner_path[len(settings.MEDIA_ROOT):].strip('/')
            lutris_game['banner'] = settings.MEDIA_URL + banner_url
    return lutris_game

def get_or_create_company(name):
    if not name:
        return
    slug = slugify(name)[:50]
    try:
        company = Company.objects.get(slug=slug)
    except Company.DoesNotExist:
        company = Company.objects.create(
            slug=slug,
            name=name
        )
    return company
<commit_msg>Fix for thegamesdb.get_similar when not receiving any data<commit_after>import os
import re
import logging

import requests
from bs4 import BeautifulSoup
from PIL import Image
from django.conf import settings
from django.utils.text import slugify

from games.models import Company
from platforms.models import Platform

LOGGER = logging.getLogger(__name__)
API_URL = "http://thegamesdb.net/api/"
UNSUPPORTED_PLATFORMS = (
    'Microsoft Xbox',
    'Microsoft Xbox 360',
    'Microsoft Xbox One',
    'Sony Playstation 3',
    'Sony Playstation 4',
    'Sony Playstation Vita',
    'Android',
    'iOS',
    'Nintendo 3DS',
    'N-Gage',
    'Mac OS',
)

PLATFORM_MAP = {
    'Arcade': 'arcade',
    'Amiga': 'amiga',
    'Atari 2600': 'atari-2600',
    'Atari 5200': 'atari-5200',
    'Atari ST': 'atari-st',
    'PC': 'windows',
    'Sega Genesis': 'sega-genesis',
    'Sega Mega Drive': 'sega-genesis',
    'Sega Master System': 'sega-master-system',
    'Sega Saturn': 'sega-saturn',
    'Sega Game Gear': 'sega-game-gear',
    'Sega Dreamcast': 'sega-dreamcast',
    'Atari Jaguar': 'atari-jaguar',
    'Sony Playstation': 'sony-playstation',
    'Sony Playstation 2': 'sony-playstation-2',
    'Sony PSP': 'sony-psp',
    'Nintendo Entertainment System (NES)': 'nes',
    'Super Nintendo (SNES)': 'super-nintendo',
    'Neo Geo': 'neo-geo',
    'Neo Geo CD': 'neo-geo-cd',
    'Nintendo Game Boy': 'game-boy',
    'Nintendo Game Boy Color': 'nintendo-game-boy-color',
    'Nintendo Game Boy Advance': 'nintendo-game-boy-advance',
    'Nintendo 64': 'nintendo-64',
    'Nintendo GameCube': 'gamecube',
    'Nintendo DS': 'nintendo-ds',
    'Nintendo Wii': 'nintendo-wii',
    'NeoGeo': 'neo-geo',
    'TurboGrafx 16': 'turbografx-16',
    'Vectrex': 'vectrex',
}

def download_image(media_type, url):
    """Download an image from TheGamesDB (is necessary) and return its local path"""
    media_type_map = {
        'clearlogo': settings.TGD_CLEAR_LOGO_PATH,
        'banner': settings.TGD_BANNER_PATH,
        'screenshot': settings.TGD_SCREENSHOT_PATH,
        'fanart': settings.TGD_FANART_PATH
    }
    dest_path = os.path.join(media_type_map[media_type], os.path.basename(url))
    if not os.path.exists(dest_path):
        response = requests.get(url)
        with open(dest_path, 'w') as dest_file:
            dest_file.write(response.content)
    return dest_path

def convert_clearlogo_to_banner(logo_path):
    base_width, base_height = settings.BANNER_SIZE.split('x')
    base_ratio = float(base_width) / float(base_height)

    try:
        logo = Image.open(logo_path)
    except IOError:
        return
    logo_width = float(logo.width)
    logo_height = float(logo.height)
    logo_ratio = logo_width / logo_height

    margin_factor = 0.1
    if logo_ratio < base_ratio:
        new_h = logo_height + (logo_height * margin_factor)
        new_w = new_h * base_ratio
    else:
        new_w = logo_width + (logo_width * margin_factor)
        new_h = new_w / base_ratio

    new_image = Image.new('RGBA', (int(new_w), int(new_h)), (0, 0, 0, 255))
    if logo.mode == 'RGBA':
        mask = logo
    else:
        mask = None
    offset = (int((new_w - logo_width) / 2), int((new_h - logo_height) / 2))
    new_image.paste(logo, offset, mask)
    dest_path = os.path.join(settings.TGD_LUTRIS_BANNER_PATH,
                             os.path.basename(logo_path))
    new_image.save(dest_path)
    return dest_path

def api_request(url):
    response = requests.get(API_URL + url)
    return BeautifulSoup(response.content, 'xml')

def get_value(soup, field):
    if not soup:
        LOGGER.warn("No soup provided")
        return
    tag = soup.find(field)
    if tag:
        return tag.text

def get_similar(game_data):
    if not game_data:
        return
    similar = game_data.find('Similar')
    if similar:
        return [
            {
                'id': get_value(game, 'id'),
                'platform_id': get_value(game, 'PlatformId')
            } for game in similar.find_all('Game')
        ]

def get_pics(game_data, tag_name):
    fanarts = game_data.find_all(tag_name)
    results = []
    for fanart in fanarts:
        original = fanart.find('original')
        original_meta = original.attrs
        original_meta['url'] = original.get_text()
        results.append({
            'original': original_meta,
            'thumb': get_value(fanart, 'thumb'),
        })
    return results

def get_tags_with_attrs(soup, tag_name, value_name='value'):
    tags = soup.find_all(tag_name)
    results = []
    for tag in tags:
        attrs = tag.attrs
        if attrs:
            attrs[value_name] = tag.get_text()
        else:
            attrs = tag.get_text()
        results.append(attrs)
    if len(results) == 1:
        return results[0]
    else:
        return results

def get_games_list(query, remove_unsupported=True):
    soup = api_request('GetGamesList.php?name=' + query)
    game_list = []
    games = soup.find_all('Game')
    for game in games:
        platform = get_value(game, 'Platform')
        if remove_unsupported and platform in UNSUPPORTED_PLATFORMS:
            continue
        game_list.append({
            'id': get_value(game, 'id'),
            'game_title': get_value(game, 'GameTitle'),
            'release_date': get_value(game, 'ReleaseDate'),
            'platform': get_value(game, 'Platform')
        })
    return game_list

def get_game(game_id):
    soup = api_request('GetGame.php?id={}'.format(game_id))
    game_data = soup.find('Game')
    game_info = {
        'id': get_value(game_data, 'id'),
        'base_img_url': get_value(soup, 'baseImgUrl'),
        'game_title': get_value(game_data, 'GameTitle'),
        'platform': get_value(game_data, 'Platform'),
        'release_date': get_value(game_data, 'ReleaseDate'),
        'overview': get_value(game_data, 'Overview'),
        'esrb': get_value(game_data, 'ESRB'),
        'publisher': get_value(game_data, 'Publisher'),
        'developer': get_value(game_data, 'Developer'),
        'rating': get_value(game_data, 'Rating'),
        'similar': get_similar(game_data),
        'fanarts': get_pics(game_data, 'fanart'),
        'boxarts': get_tags_with_attrs(game_data, 'boxart'),
        'banners': get_tags_with_attrs(game_data, 'banner'),
        'screenshots': get_pics(game_data, 'screenshot'),
        'clearlogo': get_tags_with_attrs(game_data, 'clearlogo'),
    }
    return game_info

def get_lutris_platform(platform_name):
    try:
        slug = PLATFORM_MAP[platform_name]
    except KeyError:
        LOGGER.error('No map for %s', platform_name)
        return
    return Platform.objects.get(slug=slug)

def to_lutris(game):
    lutris_game = {}
    lutris_game['name'] = game['game_title']
    if game['release_date']:
        match = re.search(r'\d{4}', game['release_date'])
        lutris_game['year'] = match.group(0)

    publisher = get_or_create_company(game['publisher'])
    if publisher:
        lutris_game['publisher'] = publisher.id

    if game['developer'] == game['publisher']:
        developer = publisher
    else:
        developer = get_or_create_company(game['developer'])
    if developer:
        lutris_game['developer'] = developer.id

    platform = get_lutris_platform(game['platform'])
    if platform:
        lutris_game['platforms'] = [platform.id]
    if game['clearlogo']:
        clearlogo_url = game['base_img_url'] + game['clearlogo']['value']
        logo_path = download_image('clearlogo', clearlogo_url)
        banner_path = convert_clearlogo_to_banner(logo_path)
        if banner_path:
            banner_url = banner_path[len(settings.MEDIA_ROOT):].strip('/')
            lutris_game['banner'] = settings.MEDIA_URL + banner_url
    return lutris_game

def get_or_create_company(name):
    if not name:
        return
    slug = slugify(name)[:50]
    try:
        company = Company.objects.get(slug=slug)
    except Company.DoesNotExist:
        company = Company.objects.create(
            slug=slug,
            name=name
        )
    return company
<|endoftext|>


<commit_before>"""Miscellaneous utilities"""

# Standard library
import inspect
import contextlib
from tempfile import NamedTemporaryFile

# Third-party
import astropy.units as u
from astropy.table.meta import get_header_from_yaml
from astropy.io.misc.hdf5 import meta_path
from astropy.utils.decorators import wraps
import numpy as np
import tables as tb

# Package
from .samples import JokerSamples

__all__ = ['batch_tasks', 'table_header_to_units', 'read_batch',
           'tempfile_decorator']

def batch_tasks(n_tasks, n_batches, arr=None, args=None, start_idx=0):
    """Split the tasks into some number of batches to sent out to MPI workers.

    Parameters
    ----------
    n_tasks : int
        The total number of tasks to divide.
    n_batches : int
        The number of batches to split the tasks into. Often, you may want to do
        ``n_batches=pool.size`` for equal sharing amongst MPI workers.
    arr : iterable (optional)
        Instead of returning indices that specify the batches, you can also
        directly split an array into batches.
    args : iterable (optional)
        Other arguments to add to each task.
    start_idx : int (optional)
        What index in the tasks to start from?

    """
    if args is None:
        args = []
    args = list(args)

    tasks = []
    if n_batches > 0 and n_tasks >= n_batches:
        # chunk by the number of batches, often the pool size
        base_batch_size = n_tasks // n_batches
        rmdr = n_tasks % n_batches

        i1 = start_idx
        for i in range(n_batches):
            i2 = i1 + base_batch_size
            if i < rmdr:
                i2 += 1

            if arr is None:  # store indices
                tasks.append([(i1, i2), i1] + args)

            else:  # store sliced array
                tasks.append([arr[i1:i2], i1] + args)

            i1 = i2

    else:
        if arr is None:  # store indices
            tasks.append([(start_idx, n_tasks+start_idx), start_idx] + args)

        else:  # store sliced array
            tasks.append([arr[start_idx:n_tasks+start_idx], start_idx] + args)

    return tasks

def table_header_to_units(header_dataset):
    """
    Convert a YAML-ized astropy.table header into a dictionary that maps from
    column name to unit.
    """

    header = get_header_from_yaml(h.decode('utf-8')
                                  for h in header_dataset)

    units = dict()
    for row in header['datatype']:
        units[row['name']] = u.Unit(row.get('unit', u.one))

    return units

def table_contains_column(root, column):
    from .samples import JokerSamples

    path = meta_path(JokerSamples._hdf5_path)
    header = get_header_from_yaml(h.decode('utf-8') for h in root[path])

    columns = []
    for row in header['datatype']:
        columns.append(row['name'])

    return column in columns

def read_batch(prior_samples_file, columns, slice_or_idx, units=None,
               random_state=None):
    """
    Single-point interface to all read_batch functions below that infers the
    type of read from the ``slice_or_idx`` argument.

    Parameters
    ----------
    prior_samples_file : `str`
        Path to an HDF5 file containing prior samples from The Joker.
    columns : `list`, iterable
        A list of string column names to read in to the batch.
    slice_or_idx : `slice`, `int`, `numpy.ndarray`
        This determines what rows are read in to the batch from the prior
        samples file. If a slice or tuple, this is interpreted as a range of
        contiguous row indices to read. If an integer, this is interpreted as
        the size used to generate random row indices. If a numpy array, this is
        interpreted as the row indices to read.
    units : dict (optional)
        The desired output units to convert the prior samples to.
    random_state : `numpy.random.RandomState`
        Used to generate random row indices if passing an integer (size) in to
        ``slice_or_idx``. This argument is ignored for other read methods.

    Returns
    -------
    batch : `numpy.ndarray`
        The batch of prior samples, loaded into a numpy array and stripped of
        units (converted to the specified ``units``). This array will have
        shape: ``(n_samples, len(columns))``.
    """
    if isinstance(slice_or_idx, tuple):
        # read a contiguous batch of prior samples
        batch = read_batch(prior_samples_file, columns,
                           slice(*slice_or_idx), units=units)

    elif isinstance(slice_or_idx, slice):
        # read a contiguous batch of prior samples
        batch = read_batch_slice(prior_samples_file, columns,
                                 slice_or_idx, units=units)

    elif isinstance(slice_or_idx, int):
        # read a random batch of samples of size "slice_or_idx"
        batch = read_random_batch(prior_samples_file, columns,
                                  slice_or_idx, units=units,
                                  random_state=random_state)

    elif isinstance(slice_or_idx, np.ndarray):
        # read a random batch of samples of size "slice_or_idx"
        batch = read_batch_idx(prior_samples_file, columns,
                               slice_or_idx, units=units)

    else:
        raise ValueError("Invalid input for slice_or_idx: must be a slice, "
                         "int, or numpy array.")

    return batch

def read_batch_slice(prior_samples_file, columns, slice, units=None):
    """
    Read a batch (row block) of prior samples into a plain numpy array,
    converting units where necessary.
    """

    path = JokerSamples._hdf5_path

    batch = None
    with tb.open_file(prior_samples_file, mode='r') as f:

        for i, name in enumerate(columns):
            arr = f.root[path].read(slice.start, slice.stop, slice.step,
                                    field=name)
            if batch is None:
                batch = np.zeros((len(arr), len(columns)), dtype=arr.dtype)
            batch[:, i] = arr

        if units is not None:
            table_units = table_header_to_units(f.root[meta_path(path)])
            for i, name in enumerate(columns):
                if name in units:
                    batch[:, i] *= table_units[name].to(units[name])

    return batch

def read_batch_idx(prior_samples_file, columns, idx, units=None):
    """
    Read a batch (row block) of prior samples specified by the input index
    array, ``idx``, into a plain numpy array, converting units where necessary.
    """
    path = JokerSamples._hdf5_path

    batch = np.zeros((len(idx), len(columns)))
    with tb.open_file(prior_samples_file, mode='r') as f:
        for i, name in enumerate(columns):
            batch[:, i] = f.root[path].read_coordinates(idx, field=name)

        if units is not None:
            table_units = table_header_to_units(f.root[meta_path(path)])
            for i, name in enumerate(columns):
                if name in units:
                    batch[:, i] *= table_units[name].to(units[name])

    return batch

def read_random_batch(prior_samples_file, columns, size, units=None,
                      random_state=None):
    """
    Read a random batch (row block) of prior samples into a plain numpy array,
    converting units where necessary.
    """

    if random_state is None:
        random_state = np.random.default_rng()

    path = JokerSamples._hdf5_path
    with tb.open_file(prior_samples_file, mode='r') as f:
        idx = random_state.integers(0, f.root[path].shape[0], size=size)

    return read_batch_idx(prior_samples_file, columns, idx=idx, units=units)

def tempfile_decorator(func):
    wrapped_signature = inspect.signature(func)
    func_args = list(wrapped_signature.parameters.keys())
    if 'prior_samples_file' not in func_args:
        raise ValueError("Cant decorate function because it doesn't contain an "
                         "argument called 'prior_samples_file'")

    @wraps(func)
    def wrapper(*args, **kwargs):
        args = list(args)
        if 'prior_samples_file' in kwargs:
            prior_samples = kwargs['prior_samples_file']
        else:
            prior_samples = args.pop(func_args.index('prior_samples_file'))

        in_memory = kwargs.get('in_memory', False)

        if not isinstance(prior_samples, str) and not in_memory:
            if not isinstance(prior_samples, JokerSamples):
                raise TypeError("prior_samples_file must either be a string "
                                "filename specifying a cache file contining "
                                "prior samples, or must be a JokerSamples "
                                f"instance, not: {type(prior_samples)}")

            with NamedTemporaryFile(mode='r+', suffix='.hdf5') as f:
                # write samples to tempfile and recursively call this method
                prior_samples.write(f.name, overwrite=True)
                kwargs['prior_samples_file'] = f.name
                func_return = func(*args, **kwargs)

        else:
            # FIXME: it's a string, so it's probably a filename, but we should
            # validate the contents of the file!
            kwargs['prior_samples_file'] = prior_samples
            func_return = func(*args, **kwargs)

        return func_return

    return wrapper

@contextlib.contextmanager
def random_state_context(random_state):
    state = np.random.get_state()
    if random_state is not None:
        np.random.seed(random_state.randint(2**32-1))  # HACK
    try:
        yield
    finally:
        if random_state is not None:
            np.random.set_state(state)
<commit_msg>use integers not randint<commit_after>"""Miscellaneous utilities"""

# Standard library
import inspect
import contextlib
from tempfile import NamedTemporaryFile

# Third-party
import astropy.units as u
from astropy.table.meta import get_header_from_yaml
from astropy.io.misc.hdf5 import meta_path
from astropy.utils.decorators import wraps
import numpy as np
import tables as tb

# Package
from .samples import JokerSamples

__all__ = ['batch_tasks', 'table_header_to_units', 'read_batch',
           'tempfile_decorator']

def batch_tasks(n_tasks, n_batches, arr=None, args=None, start_idx=0):
    """Split the tasks into some number of batches to sent out to MPI workers.

    Parameters
    ----------
    n_tasks : int
        The total number of tasks to divide.
    n_batches : int
        The number of batches to split the tasks into. Often, you may want to do
        ``n_batches=pool.size`` for equal sharing amongst MPI workers.
    arr : iterable (optional)
        Instead of returning indices that specify the batches, you can also
        directly split an array into batches.
    args : iterable (optional)
        Other arguments to add to each task.
    start_idx : int (optional)
        What index in the tasks to start from?

    """
    if args is None:
        args = []
    args = list(args)

    tasks = []
    if n_batches > 0 and n_tasks >= n_batches:
        # chunk by the number of batches, often the pool size
        base_batch_size = n_tasks // n_batches
        rmdr = n_tasks % n_batches

        i1 = start_idx
        for i in range(n_batches):
            i2 = i1 + base_batch_size
            if i < rmdr:
                i2 += 1

            if arr is None:  # store indices
                tasks.append([(i1, i2), i1] + args)

            else:  # store sliced array
                tasks.append([arr[i1:i2], i1] + args)

            i1 = i2

    else:
        if arr is None:  # store indices
            tasks.append([(start_idx, n_tasks+start_idx), start_idx] + args)

        else:  # store sliced array
            tasks.append([arr[start_idx:n_tasks+start_idx], start_idx] + args)

    return tasks

def table_header_to_units(header_dataset):
    """
    Convert a YAML-ized astropy.table header into a dictionary that maps from
    column name to unit.
    """

    header = get_header_from_yaml(h.decode('utf-8')
                                  for h in header_dataset)

    units = dict()
    for row in header['datatype']:
        units[row['name']] = u.Unit(row.get('unit', u.one))

    return units

def table_contains_column(root, column):
    from .samples import JokerSamples

    path = meta_path(JokerSamples._hdf5_path)
    header = get_header_from_yaml(h.decode('utf-8') for h in root[path])

    columns = []
    for row in header['datatype']:
        columns.append(row['name'])

    return column in columns

def read_batch(prior_samples_file, columns, slice_or_idx, units=None,
               random_state=None):
    """
    Single-point interface to all read_batch functions below that infers the
    type of read from the ``slice_or_idx`` argument.

    Parameters
    ----------
    prior_samples_file : `str`
        Path to an HDF5 file containing prior samples from The Joker.
    columns : `list`, iterable
        A list of string column names to read in to the batch.
    slice_or_idx : `slice`, `int`, `numpy.ndarray`
        This determines what rows are read in to the batch from the prior
        samples file. If a slice or tuple, this is interpreted as a range of
        contiguous row indices to read. If an integer, this is interpreted as
        the size used to generate random row indices. If a numpy array, this is
        interpreted as the row indices to read.
    units : dict (optional)
        The desired output units to convert the prior samples to.
    random_state : `numpy.random.RandomState`
        Used to generate random row indices if passing an integer (size) in to
        ``slice_or_idx``. This argument is ignored for other read methods.

    Returns
    -------
    batch : `numpy.ndarray`
        The batch of prior samples, loaded into a numpy array and stripped of
        units (converted to the specified ``units``). This array will have
        shape: ``(n_samples, len(columns))``.
    """
    if isinstance(slice_or_idx, tuple):
        # read a contiguous batch of prior samples
        batch = read_batch(prior_samples_file, columns,
                           slice(*slice_or_idx), units=units)

    elif isinstance(slice_or_idx, slice):
        # read a contiguous batch of prior samples
        batch = read_batch_slice(prior_samples_file, columns,
                                 slice_or_idx, units=units)

    elif isinstance(slice_or_idx, int):
        # read a random batch of samples of size "slice_or_idx"
        batch = read_random_batch(prior_samples_file, columns,
                                  slice_or_idx, units=units,
                                  random_state=random_state)

    elif isinstance(slice_or_idx, np.ndarray):
        # read a random batch of samples of size "slice_or_idx"
        batch = read_batch_idx(prior_samples_file, columns,
                               slice_or_idx, units=units)

    else:
        raise ValueError("Invalid input for slice_or_idx: must be a slice, "
                         "int, or numpy array.")

    return batch

def read_batch_slice(prior_samples_file, columns, slice, units=None):
    """
    Read a batch (row block) of prior samples into a plain numpy array,
    converting units where necessary.
    """

    path = JokerSamples._hdf5_path

    batch = None
    with tb.open_file(prior_samples_file, mode='r') as f:

        for i, name in enumerate(columns):
            arr = f.root[path].read(slice.start, slice.stop, slice.step,
                                    field=name)
            if batch is None:
                batch = np.zeros((len(arr), len(columns)), dtype=arr.dtype)
            batch[:, i] = arr

        if units is not None:
            table_units = table_header_to_units(f.root[meta_path(path)])
            for i, name in enumerate(columns):
                if name in units:
                    batch[:, i] *= table_units[name].to(units[name])

    return batch

def read_batch_idx(prior_samples_file, columns, idx, units=None):
    """
    Read a batch (row block) of prior samples specified by the input index
    array, ``idx``, into a plain numpy array, converting units where necessary.
    """
    path = JokerSamples._hdf5_path

    batch = np.zeros((len(idx), len(columns)))
    with tb.open_file(prior_samples_file, mode='r') as f:
        for i, name in enumerate(columns):
            batch[:, i] = f.root[path].read_coordinates(idx, field=name)

        if units is not None:
            table_units = table_header_to_units(f.root[meta_path(path)])
            for i, name in enumerate(columns):
                if name in units:
                    batch[:, i] *= table_units[name].to(units[name])

    return batch

def read_random_batch(prior_samples_file, columns, size, units=None,
                      random_state=None):
    """
    Read a random batch (row block) of prior samples into a plain numpy array,
    converting units where necessary.
    """

    if random_state is None:
        random_state = np.random.default_rng()

    path = JokerSamples._hdf5_path
    with tb.open_file(prior_samples_file, mode='r') as f:
        idx = random_state.integers(0, f.root[path].shape[0], size=size)

    return read_batch_idx(prior_samples_file, columns, idx=idx, units=units)

def tempfile_decorator(func):
    wrapped_signature = inspect.signature(func)
    func_args = list(wrapped_signature.parameters.keys())
    if 'prior_samples_file' not in func_args:
        raise ValueError("Cant decorate function because it doesn't contain an "
                         "argument called 'prior_samples_file'")

    @wraps(func)
    def wrapper(*args, **kwargs):
        args = list(args)
        if 'prior_samples_file' in kwargs:
            prior_samples = kwargs['prior_samples_file']
        else:
            prior_samples = args.pop(func_args.index('prior_samples_file'))

        in_memory = kwargs.get('in_memory', False)

        if not isinstance(prior_samples, str) and not in_memory:
            if not isinstance(prior_samples, JokerSamples):
                raise TypeError("prior_samples_file must either be a string "
                                "filename specifying a cache file contining "
                                "prior samples, or must be a JokerSamples "
                                f"instance, not: {type(prior_samples)}")

            with NamedTemporaryFile(mode='r+', suffix='.hdf5') as f:
                # write samples to tempfile and recursively call this method
                prior_samples.write(f.name, overwrite=True)
                kwargs['prior_samples_file'] = f.name
                func_return = func(*args, **kwargs)

        else:
            # FIXME: it's a string, so it's probably a filename, but we should
            # validate the contents of the file!
            kwargs['prior_samples_file'] = prior_samples
            func_return = func(*args, **kwargs)

        return func_return

    return wrapper

@contextlib.contextmanager
def random_state_context(random_state):
    state = np.random.get_state()
    if random_state is not None:
        np.random.seed(random_state.integers(2**32-1))  # HACK
    try:
        yield
    finally:
        if random_state is not None:
            np.random.set_state(state)
<|endoftext|>


<commit_before>__author__ = 'kmadac'

import requests
import time
import hmac
import hashlib

class public():
    def __init__(self, proxydict=None):
        self.proxydict = proxydict

    def ticker(self):
        """
        Return dictionary
        """
        r = requests.get("https://www.bitstamp.net/api/ticker/", proxies=self.proxydict)
        if r.status_code == 200:
            return r.json()
        else:
            r.raise_for_status()

    def order_book(self, group=True):
        """
        Returns JSON dictionary with "bids" and "asks".
        Each is a list of open orders and each order is represented as a list of price and amount.
        """
        params = {'group': group}

        r = requests.get("https://www.bitstamp.net/api/order_book/", params=params, proxies=self.proxydict)
        if r.status_code == 200:
            return r.json()
        else:
            r.raise_for_status()

    def transactions(self, timedelta_secs=86400):
        """
        Returns transactions for the last 'timedelta' seconds
        """
        params = {'timedelta': timedelta_secs}

        r = requests.get("https://www.bitstamp.net/api/transactions/", params=params, proxies=self.proxydict)
        if r.status_code == 200:
            return r.json()
        else:
            r.raise_for_status()

    def bitinstant_reserves(self):
        """
        Returns simple dictionary {'usd': 'Bitinstant USD reserves'}
        """
        r = requests.get("https://www.bitstamp.net/api/bitinstant/", proxies=self.proxydict)
        if r.status_code == 200:
            return r.json()
        else:
            r.raise_for_status()

    def conversion_rate_usd_eur(self):
        """
        Returns simple dictionary
        {'buy': 'buy conversion rate', 'sell': 'sell conversion rate'}
        """
        r = requests.get("https://www.bitstamp.net/api/eur_usd/", proxies=self.proxydict)
        if r.status_code == 200:
            return r.json()
        else:
            r.raise_for_status()

class trading():
    def __init__(self, username, key, secret, proxydict=None):
        self.proxydict = proxydict
        self.username = username
        self.key = key
        self.secret = secret
        self.nonce = int(time.time())

    def get_params(self):
        params = {}
        params['key'] = self.key
        msg = str(self.nonce) + self.username + self.key
        signature = hmac.new(self.secret, msg=msg, digestmod=hashlib.sha256).hexdigest().upper()
        params['signature'] = signature
        params['nonce'] = self.nonce
        self.nonce += 1
        return params

    def account_balance(self):
        """
        Returns dictionary:
        {u'btc_reserved': u'0',
         u'fee': u'0.5000',
         u'btc_available': u'2.30856098',
         u'usd_reserved': u'0',
         u'btc_balance': u'2.30856098',
         u'usd_balance': u'114.64',
         u'usd_available': u'114.64'}
        """
        params = self.get_params()
        r = requests.post("https://www.bitstamp.net/api/balance/", data=params, proxies=self.proxydict)
        if r.status_code == 200:
            if 'error' in r.json():
                return False, r.json()['error']
            else:
                return r.json()
        else:
            r.raise_for_status()

    def user_transactions(self, offset=0, limit=100, descending=True):
        """
        Returns descending list of transactions. Every transaction (dictionary) contains
        {u'usd': u'-39.25',
         u'datetime': u'2013-03-26 18:49:13',
         u'fee': u'0.20', u'btc': u'0.50000000',
         u'type': 2,
         u'id': 213642}
        """
        params = self.get_params()
        params['offset'] = offset
        params['limit'] = limit
        if descending:
            params['sort'] = "desc"
        else:
            params['sort'] = "asc"

        r = requests.post("https://www.bitstamp.net/api/user_transactions/", data=params, proxies=self.proxydict)
        if r.status_code == 200:
            if 'error' in r.json():
                return False, r.json()['error']
            else:
                return r.json()
        else:
            r.raise_for_status()

    def open_orders(self):
        """
        Returns JSON list of open orders. Each order is represented as dictionary:
        """
        params = self.get_params()
        r = requests.post("https://www.bitstamp.net/api/open_orders/", data=params, proxies=self.proxydict)
        if r.status_code == 200:
            if 'error' in r.json():
                return False, r.json()['error']
            else:
                return r.json()
        else:
            r.raise_for_status()

    def cancel_order(self, order_id):
        """
        Cancel the order specified by order_id
        Returns True if order was successfully canceled,
        otherwise tuple (False, msg) like (False, u'Order not found')
        """
        params = self.get_params()
        params['id'] = order_id
        r = requests.post("https://www.bitstamp.net/api/cancel_order/", data=params, proxies=self.proxydict)
        if r.status_code == 200:
            if r.text == u'true':
                return True
            else:
                return False, r.json()['error']
        else:
            r.raise_for_status()

    def buy_limit_order(self, amount, price):
        """
        Order to buy amount of bitcoins for specified price
        """
        params = self.get_params()
        params['amount'] = amount
        params['price'] = price

        r = requests.post("https://www.bitstamp.net/api/buy/", data=params, proxies=self.proxydict)
        if r.status_code == 200:
            if 'error' in r.json():
                return False, r.json()['error']
            else:
                return r.json()
        else:
            r.raise_for_status()

    def sell_limit_order(self, amount, price):
        """
        Order to buy amount of bitcoins for specified price
        """
        params = self.get_params()
        params['amount'] = amount
        params['price'] = price

        r = requests.post("https://www.bitstamp.net/api/sell/", data=params, proxies=self.proxydict)
        if r.status_code == 200:
            if 'error' in r.json():
                return False, r.json()['error']
            else:
                return r.json()
        else:
            r.raise_for_status()

    def check_bitstamp_code(self, code):
        """
        Returns JSON dictionary containing USD and BTC amount included in given bitstamp code.
        """
        params = self.get_params()
        params['code'] = code
        r = requests.post("https://www.bitstamp.net/api/check_code/", data=params,
                          proxies=self.proxydict)
        if r.status_code == 200:
            if 'error' in r.json():
                return False, r.json()['error']
            else:
                return r.json()
        else:
            r.raise_for_status()

    def redeem_bitstamp_code(self, code):
        """
        Returns JSON dictionary containing USD and BTC amount added to user's account.
        """
        params = self.get_params()
        params['code'] = code
        r = requests.post("https://www.bitstamp.net/api/redeem_code/", data=params,
                          proxies=self.proxydict)
        if r.status_code == 200:
            if 'error' in r.json():
                return False, r.json()['error']
            else:
                return r.json()
        else:
            r.raise_for_status()

    def withdrawal_requests(self):
        """
        Returns list of withdrawal requests. Each request is represented as dictionary
        """
        params = self.get_params()
        r = requests.post("https://www.bitstamp.net/api/withdrawal_requests/", data=params,
                          proxies=self.proxydict)
        if r.status_code == 200:
            if 'error' in r.json():
                return False, r.json()['error']
            else:
                return r.json()
        else:
            r.raise_for_status()

    def bitcoin_withdrawal(self, amount, address):
        """
        Send bitcoins to another bitcoin wallet specified by address
        """
        params = self.get_params()
        params['amount'] = amount
        params['address'] = address

        r = requests.post("https://www.bitstamp.net/api/bitcoin_withdrawal/", data=params, proxies=self.proxydict)
        if r.status_code == 200:
            if r.text == u'true':
                return True
            else:
                return False, r.json()['error']
        else:
            r.raise_for_status()

    def bitcoin_deposit_address(self):
        """
        Returns bitcoin deposit address as unicode string
        """
        params = self.get_params()
        r = requests.post("https://www.bitstamp.net/api/bitcoin_deposit_address/", data=params,
                          proxies=self.proxydict)
        if r.status_code == 200:
            if 'error' in r.json():
                return False, r.json()['error']
            else:
                return r.text
        else:
            r.raise_for_status()

    def unconfirmed_bitcoin_deposits(self):
        """
        Returns JSON list of unconfirmed bitcoin transactions. Each transaction is represented as dictionary:
        amount - bitcoin amount
        address - deposit address used
        confirmations - number of confirmations
        """
        params = self.get_params()
        r = requests.post("https://www.bitstamp.net/api/unconfirmed_btc/", data=params,
                          proxies=self.proxydict)
        if r.status_code == 200:
            if 'error' in r.json():
                return False, r.json()['error']
            else:
                return r.json()
        else:
            r.raise_for_status()

    def ripple_withdrawal(self, amount, address, currency):
        """
        Returns true if successful
        """
        params = self.get_params()
        params['amount'] = amount
        params['address'] = address
        params['currency'] = currency

        r = requests.post("https://www.bitstamp.net/api/ripple_withdrawal/", data=params, proxies=self.proxydict)
        if r.status_code == 200:
            if r.text == u'true':
                return True
            else:
                return False, r.json()['error']
        else:
            r.raise_for_status()

    def ripple_deposit_address(self):
        """
        Returns ripple deposit address as unicode string
        """
        params = self.get_params()
        r = requests.post("https://www.bitstamp.net/api/ripple_address/", data=params,
                          proxies=self.proxydict)
        if r.status_code == 200:
            if 'error' in r.json():
                return False, r.json()['error']
            else:
                return r.text
        else:
            r.raise_for_status()
<commit_msg>trading part compatible with python 3<commit_after>__author__ = 'kmadac'

import requests
import time
import hmac
import hashlib
import sys

class public():
    def __init__(self, proxydict=None):
        self.proxydict = proxydict

    def ticker(self):
        """
        Return dictionary
        """
        r = requests.get("https://www.bitstamp.net/api/ticker/", proxies=self.proxydict)
        if r.status_code == 200:
            return r.json()
        else:
            r.raise_for_status()

    def order_book(self, group=True):
        """
        Returns JSON dictionary with "bids" and "asks".
        Each is a list of open orders and each order is represented as a list of price and amount.
        """
        params = {'group': group}

        r = requests.get("https://www.bitstamp.net/api/order_book/", params=params, proxies=self.proxydict)
        if r.status_code == 200:
            return r.json()
        else:
            r.raise_for_status()

    def transactions(self, timedelta_secs=86400):
        """
        Returns transactions for the last 'timedelta' seconds
        """
        params = {'timedelta': timedelta_secs}

        r = requests.get("https://www.bitstamp.net/api/transactions/", params=params, proxies=self.proxydict)
        if r.status_code == 200:
            return r.json()
        else:
            r.raise_for_status()

    def bitinstant_reserves(self):
        """
        Returns simple dictionary {'usd': 'Bitinstant USD reserves'}
        """
        r = requests.get("https://www.bitstamp.net/api/bitinstant/", proxies=self.proxydict)
        if r.status_code == 200:
            return r.json()
        else:
            r.raise_for_status()

    def conversion_rate_usd_eur(self):
        """
        Returns simple dictionary
        {'buy': 'buy conversion rate', 'sell': 'sell conversion rate'}
        """
        r = requests.get("https://www.bitstamp.net/api/eur_usd/", proxies=self.proxydict)
        if r.status_code == 200:
            return r.json()
        else:
            r.raise_for_status()

class trading():
    def __init__(self, username, key, secret, proxydict=None):
        self.proxydict = proxydict
        self.username = username
        self.key = key
        self.secret = secret
        self.nonce = int(time.time())

    def get_params(self):
        params = {}
        params['key'] = self.key
        msg = str(self.nonce) + self.username + self.key

        if sys.version_info.major == 2:
            signature = hmac.new(self.secret, msg=msg, digestmod=hashlib.sha256).hexdigest().upper()
        else:
            signature = hmac.new(str.encode(self.secret), msg=str.encode(msg), digestmod=hashlib.sha256).hexdigest()\
                .upper()
        params['signature'] = signature
        params['nonce'] = self.nonce
        self.nonce += 1
        return params

    def account_balance(self):
        """
        Returns dictionary:
        {u'btc_reserved': u'0',
         u'fee': u'0.5000',
         u'btc_available': u'2.30856098',
         u'usd_reserved': u'0',
         u'btc_balance': u'2.30856098',
         u'usd_balance': u'114.64',
         u'usd_available': u'114.64'}
        """
        params = self.get_params()
        r = requests.post("https://www.bitstamp.net/api/balance/", data=params, proxies=self.proxydict)
        if r.status_code == 200:
            if 'error' in r.json():
                return False, r.json()['error']
            else:
                return r.json()
        else:
            r.raise_for_status()

    def user_transactions(self, offset=0, limit=100, descending=True):
        """
        Returns descending list of transactions. Every transaction (dictionary) contains
        {u'usd': u'-39.25',
         u'datetime': u'2013-03-26 18:49:13',
         u'fee': u'0.20', u'btc': u'0.50000000',
         u'type': 2,
         u'id': 213642}
        """
        params = self.get_params()
        params['offset'] = offset
        params['limit'] = limit
        if descending:
            params['sort'] = "desc"
        else:
            params['sort'] = "asc"

        r = requests.post("https://www.bitstamp.net/api/user_transactions/", data=params, proxies=self.proxydict)
        if r.status_code == 200:
            if 'error' in r.json():
                return False, r.json()['error']
            else:
                return r.json()
        else:
            r.raise_for_status()

    def open_orders(self):
        """
        Returns JSON list of open orders. Each order is represented as dictionary:
        """
        params = self.get_params()
        r = requests.post("https://www.bitstamp.net/api/open_orders/", data=params, proxies=self.proxydict)
        if r.status_code == 200:
            if 'error' in r.json():
                return False, r.json()['error']
            else:
                return r.json()
        else:
            r.raise_for_status()

    def cancel_order(self, order_id):
        """
        Cancel the order specified by order_id
        Returns True if order was successfully canceled,
        otherwise tuple (False, msg) like (False, u'Order not found')
        """
        params = self.get_params()
        params['id'] = order_id
        r = requests.post("https://www.bitstamp.net/api/cancel_order/", data=params, proxies=self.proxydict)
        if r.status_code == 200:
            if r.text == u'true':
                return True
            else:
                return False, r.json()['error']
        else:
            r.raise_for_status()

    def buy_limit_order(self, amount, price):
        """
        Order to buy amount of bitcoins for specified price
        """
        params = self.get_params()
        params['amount'] = amount
        params['price'] = price

        r = requests.post("https://www.bitstamp.net/api/buy/", data=params, proxies=self.proxydict)
        if r.status_code == 200:
            if 'error' in r.json():
                return False, r.json()['error']
            else:
                return r.json()
        else:
            r.raise_for_status()

    def sell_limit_order(self, amount, price):
        """
        Order to buy amount of bitcoins for specified price
        """
        params = self.get_params()
        params['amount'] = amount
        params['price'] = price

        r = requests.post("https://www.bitstamp.net/api/sell/", data=params, proxies=self.proxydict)
        if r.status_code == 200:
            if 'error' in r.json():
                return False, r.json()['error']
            else:
                return r.json()
        else:
            r.raise_for_status()

    def check_bitstamp_code(self, code):
        """
        Returns JSON dictionary containing USD and BTC amount included in given bitstamp code.
        """
        params = self.get_params()
        params['code'] = code
        r = requests.post("https://www.bitstamp.net/api/check_code/", data=params,
                          proxies=self.proxydict)
        if r.status_code == 200:
            if 'error' in r.json():
                return False, r.json()['error']
            else:
                return r.json()
        else:
            r.raise_for_status()

    def redeem_bitstamp_code(self, code):
        """
        Returns JSON dictionary containing USD and BTC amount added to user's account.
        """
        params = self.get_params()
        params['code'] = code
        r = requests.post("https://www.bitstamp.net/api/redeem_code/", data=params,
                          proxies=self.proxydict)
        if r.status_code == 200:
            if 'error' in r.json():
                return False, r.json()['error']
            else:
                return r.json()
        else:
            r.raise_for_status()

    def withdrawal_requests(self):
        """
        Returns list of withdrawal requests. Each request is represented as dictionary
        """
        params = self.get_params()
        r = requests.post("https://www.bitstamp.net/api/withdrawal_requests/", data=params,
                          proxies=self.proxydict)
        if r.status_code == 200:
            if 'error' in r.json():
                return False, r.json()['error']
            else:
                return r.json()
        else:
            r.raise_for_status()

    def bitcoin_withdrawal(self, amount, address):
        """
        Send bitcoins to another bitcoin wallet specified by address
        """
        params = self.get_params()
        params['amount'] = amount
        params['address'] = address

        r = requests.post("https://www.bitstamp.net/api/bitcoin_withdrawal/", data=params, proxies=self.proxydict)
        if r.status_code == 200:
            if r.text == u'true':
                return True
            else:
                return False, r.json()['error']
        else:
            r.raise_for_status()

    def bitcoin_deposit_address(self):
        """
        Returns bitcoin deposit address as unicode string
        """
        params = self.get_params()
        r = requests.post("https://www.bitstamp.net/api/bitcoin_deposit_address/", data=params,
                          proxies=self.proxydict)
        if r.status_code == 200:
            if 'error' in r.json():
                return False, r.json()['error']
            else:
                return r.text
        else:
            r.raise_for_status()

    def unconfirmed_bitcoin_deposits(self):
        """
        Returns JSON list of unconfirmed bitcoin transactions. Each transaction is represented as dictionary:
        amount - bitcoin amount
        address - deposit address used
        confirmations - number of confirmations
        """
        params = self.get_params()
        r = requests.post("https://www.bitstamp.net/api/unconfirmed_btc/", data=params,
                          proxies=self.proxydict)
        if r.status_code == 200:
            if 'error' in r.json():
                return False, r.json()['error']
            else:
                return r.json()
        else:
            r.raise_for_status()

    def ripple_withdrawal(self, amount, address, currency):
        """
        Returns true if successful
        """
        params = self.get_params()
        params['amount'] = amount
        params['address'] = address
        params['currency'] = currency

        r = requests.post("https://www.bitstamp.net/api/ripple_withdrawal/", data=params, proxies=self.proxydict)
        if r.status_code == 200:
            if r.text == u'true':
                return True
            else:
                return False, r.json()['error']
        else:
            r.raise_for_status()

    def ripple_deposit_address(self):
        """
        Returns ripple deposit address as unicode string
        """
        params = self.get_params()
        r = requests.post("https://www.bitstamp.net/api/ripple_address/", data=params,
                          proxies=self.proxydict)
        if r.status_code == 200:
            if 'error' in r.json():
                return False, r.json()['error']
            else:
                return r.text
        else:
            r.raise_for_status()
<|endoftext|>


<commit_before>"""
   See https://bittrex.com/Home/Api
"""

import time
import hmac
import hashlib
try:
    from urllib import urlencode
    from urlparse import urljoin
except ImportError:
    from urllib.parse import urlencode
    from urllib.parse import urljoin
import requests

BUY_ORDERBOOK = 'buy'
SELL_ORDERBOOK = 'sell'
BOTH_ORDERBOOK = 'both'

BASE_URL = 'https://bittrex.com/api/v1.1/%s/'

MARKET_SET = {'getopenorders', 'cancel', 'sellmarket', 'selllimit', 'buymarket', 'buylimit'}

ACCOUNT_SET = {'getbalances', 'getbalance', 'getdepositaddress', 'withdraw', 'getorderhistory', 'getorder'}

class Bittrex(object):
    """
    Used for requesting Bittrex with API key and API secret
    """
    def __init__(self, api_key, api_secret):
        self.api_key = str(api_key) if api_key is not None else ''
        self.api_secret = str(api_secret) if api_secret is not None else ''

    def api_query(self, method, options=None):
        """
        Queries Bittrex with given method and options

        :param method: Query method for getting info
        :type method: str

        :param options: Extra options for query
        :type options: dict

        :return: JSON response from Bittrex
        :rtype : dict
        """
        if not options:
            options = {}
        nonce = str(int(time.time() * 1000))
        method_set = 'public'

        if method in MARKET_SET:
            method_set = 'market'
        elif method in ACCOUNT_SET:
            method_set = 'account'

        request_url = (BASE_URL % method_set) + method + '?'

        if method_set != 'public':
            request_url += 'apikey=' + self.api_key + "&nonce=" + nonce + '&'

        request_url += urlencode(options)

        return requests.get(
            request_url,
            headers={"apisign": hmac.new(self.api_secret.encode(), request_url.encode(), hashlib.sha512).hexdigest()}
        ).json()

    def get_markets(self):
        """
        Used to get the open and available trading markets
        at Bittrex along with other meta data.

        :return: Available market info in JSON
        :rtype : dict
        """
        return self.api_query('getmarkets')

    def get_currencies(self):
        """
        Used to get all supported currencies at Bittrex
        along with other meta data.

        :return: Supported currencies info in JSON
        :rtype : dict
        """
        return self.api_query('getcurrencies')

    def get_ticker(self, market):
        """
        Used to get the current tick values for a market.

        :param market: String literal for the market (ex: BTC-LTC)
        :type market: str

        :return: Current values for given market in JSON
        :rtype : dict
        """
        return self.api_query('getticker', {'market': market})

    def get_market_summaries(self):
        """
        Used to get the last 24 hour summary of all active exchanges

        :return: Summaries of active exchanges in JSON
        :rtype : dict
        """
        return self.api_query('getmarketsummaries')

    def get_orderbook(self, market, depth_type, depth=20):
        """
        Used to get retrieve the orderbook for a given market

        :param market: String literal for the market (ex: BTC-LTC)
        :type market: str

        :param depth_type: buy, sell or both to identify the type of orderbook to return.
            Use constants BUY_ORDERBOOK, SELL_ORDERBOOK, BOTH_ORDERBOOK
        :type depth_type: str

        :param depth: how deep of an order book to retrieve. Max is 100, default is 20
        :type depth: int

        :return: Orderbook of market in JSON
        :rtype : dict
        """
        return self.api_query('getorderbook', {'market': market, 'type': depth_type, 'depth': depth})

    def get_market_history(self, market, count):
        """
        Used to retrieve the latest trades that have occurred for a
        specific market.

        /market/getmarkethistory

        :param market: String literal for the market (ex: BTC-LTC)
        :type market: str

        :param count: Number between 1-100 for the number of entries to return (default = 20)
        :type count: int

        :return: Market history in JSON
        :rtype : dict
        """
        return self.api_query('getmarkethistory', {'market': market, 'count': count})

    def buy_market(self, market, quantity):
        """
        Used to place a buy order in a specific market. Use buymarket to
        place market orders. Make sure you have the proper permissions
        set on your API keys for this call to work

        /market/buymarket

        :param market: String literal for the market (ex: BTC-LTC)
        :type market: str

        :param quantity: The amount to purchase
        :type quantity: float

        :param rate: The rate at which to place the order.
            This is not needed for market orders
        :type rate: float

        :return:
        :rtype : dict
        """
        return self.api_query('buymarket', {'market': market, 'quantity': quantity})

    def buy_limit(self, market, quantity, rate):
        """
        Used to place a buy order in a specific market. Use buylimit to place
        limit orders Make sure you have the proper permissions set on your
        API keys for this call to work

        /market/buylimit

        :param market: String literal for the market (ex: BTC-LTC)
        :type market: str

        :param quantity: The amount to purchase
        :type quantity: float

        :param rate: The rate at which to place the order.
            This is not needed for market orders
        :type rate: float

        :return:
        :rtype : dict
        """
        return self.api_query('buylimit', {'market': market, 'quantity': quantity, 'rate': rate})

    def sell_market(self, market, quantity):
        """
        Used to place a sell order in a specific market. Use sellmarket to place
        market orders. Make sure you have the proper permissions set on your
        API keys for this call to work

        /market/sellmarket

        :param market: String literal for the market (ex: BTC-LTC)
        :type market: str

        :param quantity: The amount to purchase
        :type quantity: float

        :param rate: The rate at which to place the order.
            This is not needed for market orders
        :type rate: float

        :return:
        :rtype : dict
        """
        return self.api_query('sellmarket', {'market': market, 'quantity': quantity})

    def sell_limit(self, market, quantity, rate):
        """
        Used to place a sell order in a specific market. Use selllimit to place
        limit orders Make sure you have the proper permissions set on your
        API keys for this call to work

        /market/selllimit

        :param market: String literal for the market (ex: BTC-LTC)
        :type market: str

        :param quantity: The amount to purchase
        :type quantity: float

        :param rate: The rate at which to place the order.
            This is not needed for market orders
        :type rate: float

        :return:
        :rtype : dict
        """
        return self.api_query('selllimit', {'market': market, 'quantity': quantity, 'rate': rate})

    def cancel(self, uuid):
        """
        Used to cancel a buy or sell order

        /market/cancel

        :param uuid: uuid of buy or sell order
        :type uuid: str

        :return:
        :rtype : dict
        """
        return self.api_query('cancel', {'uuid': uuid})

    def get_open_orders(self, market):
        """
        Get all orders that you currently have opened. A specific market can be requested

        /market/getopenorders

        :param market: String literal for the market (ie. BTC-LTC)
        :type market: str

        :return: Open orders info in JSON
        :rtype : dict
        """
        return self.api_query('getopenorders', {'market': market})

    def get_balances(self):
        """
        Used to retrieve all balances from your account

        /account/getbalances

        :return: Balances info in JSON
        :rtype : dict
        """
        return self.api_query('getbalances', {})

    def get_balance(self, currency):
        """
        Used to retrieve the balance from your account for a specific currency

        /account/getbalance

        :param currency: String literal for the currency (ex: LTC)
        :type currency: str

        :return: Balance info in JSON
        :rtype : dict
        """
        return self.api_query('getbalance', {'currency': currency})

    def get_deposit_address(self, currency):
        """
        Used to generate or retrieve an address for a specific currency

        /account/getdepositaddress

        :param currency: String literal for the currency (ie. BTC)
        :type currency: str

        :return: Address info in JSON
        :rtype : dict
        """
        return self.api_query('getdepositaddress', {'currency': currency})

    def withdraw(self, currency, quantity, address):
        """
        Used to withdraw funds from your account

        /account/withdraw

        :param currency: String literal for the currency (ie. BTC)
        :type currency: str

        :param quantity: The quantity of coins to withdraw
        :type quantity: float

        :param address: The address where to send the funds.
        :type address: str

        :return:
        :rtype : dict
        """
        return self.api_query('withdraw', {'currency': currency, 'quantity': quantity, 'address': address})

    def get_order_history(self, market, count):
        """
        Used to reterieve order trade history of account

        /account/getorderhistory

        :param market: optional a string literal for the market (ie. BTC-LTC). If ommited, will return for all markets
        :type market: str

        :param count: optional 	the number of records to return
        :type count: int

        :return: order history in JSON
        :rtype : dict

        """
        return self.api_query('getorderhistory', {'market':market, 'count': count})

    def get_order_history(self, uuid):
        """
        Used to get details of buy or sell order

        /account/getorder

        :param uuid: uuid of buy or sell order
        :type uuid: str

        :return:
        :rtype : dict
        """
        return self.api_query('getorder', {'uuid': uuid})
<commit_msg>Added get_order to retrieve details of buy/sell order<commit_after>"""
   See https://bittrex.com/Home/Api
"""

import time
import hmac
import hashlib
try:
    from urllib import urlencode
    from urlparse import urljoin
except ImportError:
    from urllib.parse import urlencode
    from urllib.parse import urljoin
import requests

BUY_ORDERBOOK = 'buy'
SELL_ORDERBOOK = 'sell'
BOTH_ORDERBOOK = 'both'

BASE_URL = 'https://bittrex.com/api/v1.1/%s/'

MARKET_SET = {'getopenorders', 'cancel', 'sellmarket', 'selllimit', 'buymarket', 'buylimit'}

ACCOUNT_SET = {'getbalances', 'getbalance', 'getdepositaddress', 'withdraw', 'getorderhistory', 'getorder'}

class Bittrex(object):
    """
    Used for requesting Bittrex with API key and API secret
    """
    def __init__(self, api_key, api_secret):
        self.api_key = str(api_key) if api_key is not None else ''
        self.api_secret = str(api_secret) if api_secret is not None else ''

    def api_query(self, method, options=None):
        """
        Queries Bittrex with given method and options

        :param method: Query method for getting info
        :type method: str

        :param options: Extra options for query
        :type options: dict

        :return: JSON response from Bittrex
        :rtype : dict
        """
        if not options:
            options = {}
        nonce = str(int(time.time() * 1000))
        method_set = 'public'

        if method in MARKET_SET:
            method_set = 'market'
        elif method in ACCOUNT_SET:
            method_set = 'account'

        request_url = (BASE_URL % method_set) + method + '?'

        if method_set != 'public':
            request_url += 'apikey=' + self.api_key + "&nonce=" + nonce + '&'

        request_url += urlencode(options)

        return requests.get(
            request_url,
            headers={"apisign": hmac.new(self.api_secret.encode(), request_url.encode(), hashlib.sha512).hexdigest()}
        ).json()

    def get_markets(self):
        """
        Used to get the open and available trading markets
        at Bittrex along with other meta data.

        :return: Available market info in JSON
        :rtype : dict
        """
        return self.api_query('getmarkets')

    def get_currencies(self):
        """
        Used to get all supported currencies at Bittrex
        along with other meta data.

        :return: Supported currencies info in JSON
        :rtype : dict
        """
        return self.api_query('getcurrencies')

    def get_ticker(self, market):
        """
        Used to get the current tick values for a market.

        :param market: String literal for the market (ex: BTC-LTC)
        :type market: str

        :return: Current values for given market in JSON
        :rtype : dict
        """
        return self.api_query('getticker', {'market': market})

    def get_market_summaries(self):
        """
        Used to get the last 24 hour summary of all active exchanges

        :return: Summaries of active exchanges in JSON
        :rtype : dict
        """
        return self.api_query('getmarketsummaries')

    def get_orderbook(self, market, depth_type, depth=20):
        """
        Used to get retrieve the orderbook for a given market

        :param market: String literal for the market (ex: BTC-LTC)
        :type market: str

        :param depth_type: buy, sell or both to identify the type of orderbook to return.
            Use constants BUY_ORDERBOOK, SELL_ORDERBOOK, BOTH_ORDERBOOK
        :type depth_type: str

        :param depth: how deep of an order book to retrieve. Max is 100, default is 20
        :type depth: int

        :return: Orderbook of market in JSON
        :rtype : dict
        """
        return self.api_query('getorderbook', {'market': market, 'type': depth_type, 'depth': depth})

    def get_market_history(self, market, count):
        """
        Used to retrieve the latest trades that have occurred for a
        specific market.

        /market/getmarkethistory

        :param market: String literal for the market (ex: BTC-LTC)
        :type market: str

        :param count: Number between 1-100 for the number of entries to return (default = 20)
        :type count: int

        :return: Market history in JSON
        :rtype : dict
        """
        return self.api_query('getmarkethistory', {'market': market, 'count': count})

    def buy_market(self, market, quantity):
        """
        Used to place a buy order in a specific market. Use buymarket to
        place market orders. Make sure you have the proper permissions
        set on your API keys for this call to work

        /market/buymarket

        :param market: String literal for the market (ex: BTC-LTC)
        :type market: str

        :param quantity: The amount to purchase
        :type quantity: float

        :param rate: The rate at which to place the order.
            This is not needed for market orders
        :type rate: float

        :return:
        :rtype : dict
        """
        return self.api_query('buymarket', {'market': market, 'quantity': quantity})

    def buy_limit(self, market, quantity, rate):
        """
        Used to place a buy order in a specific market. Use buylimit to place
        limit orders Make sure you have the proper permissions set on your
        API keys for this call to work

        /market/buylimit

        :param market: String literal for the market (ex: BTC-LTC)
        :type market: str

        :param quantity: The amount to purchase
        :type quantity: float

        :param rate: The rate at which to place the order.
            This is not needed for market orders
        :type rate: float

        :return:
        :rtype : dict
        """
        return self.api_query('buylimit', {'market': market, 'quantity': quantity, 'rate': rate})

    def sell_market(self, market, quantity):
        """
        Used to place a sell order in a specific market. Use sellmarket to place
        market orders. Make sure you have the proper permissions set on your
        API keys for this call to work

        /market/sellmarket

        :param market: String literal for the market (ex: BTC-LTC)
        :type market: str

        :param quantity: The amount to purchase
        :type quantity: float

        :param rate: The rate at which to place the order.
            This is not needed for market orders
        :type rate: float

        :return:
        :rtype : dict
        """
        return self.api_query('sellmarket', {'market': market, 'quantity': quantity})

    def sell_limit(self, market, quantity, rate):
        """
        Used to place a sell order in a specific market. Use selllimit to place
        limit orders Make sure you have the proper permissions set on your
        API keys for this call to work

        /market/selllimit

        :param market: String literal for the market (ex: BTC-LTC)
        :type market: str

        :param quantity: The amount to purchase
        :type quantity: float

        :param rate: The rate at which to place the order.
            This is not needed for market orders
        :type rate: float

        :return:
        :rtype : dict
        """
        return self.api_query('selllimit', {'market': market, 'quantity': quantity, 'rate': rate})

    def cancel(self, uuid):
        """
        Used to cancel a buy or sell order

        /market/cancel

        :param uuid: uuid of buy or sell order
        :type uuid: str

        :return:
        :rtype : dict
        """
        return self.api_query('cancel', {'uuid': uuid})

    def get_open_orders(self, market):
        """
        Get all orders that you currently have opened. A specific market can be requested

        /market/getopenorders

        :param market: String literal for the market (ie. BTC-LTC)
        :type market: str

        :return: Open orders info in JSON
        :rtype : dict
        """
        return self.api_query('getopenorders', {'market': market})

    def get_balances(self):
        """
        Used to retrieve all balances from your account

        /account/getbalances

        :return: Balances info in JSON
        :rtype : dict
        """
        return self.api_query('getbalances', {})

    def get_balance(self, currency):
        """
        Used to retrieve the balance from your account for a specific currency

        /account/getbalance

        :param currency: String literal for the currency (ex: LTC)
        :type currency: str

        :return: Balance info in JSON
        :rtype : dict
        """
        return self.api_query('getbalance', {'currency': currency})

    def get_deposit_address(self, currency):
        """
        Used to generate or retrieve an address for a specific currency

        /account/getdepositaddress

        :param currency: String literal for the currency (ie. BTC)
        :type currency: str

        :return: Address info in JSON
        :rtype : dict
        """
        return self.api_query('getdepositaddress', {'currency': currency})

    def withdraw(self, currency, quantity, address):
        """
        Used to withdraw funds from your account

        /account/withdraw

        :param currency: String literal for the currency (ie. BTC)
        :type currency: str

        :param quantity: The quantity of coins to withdraw
        :type quantity: float

        :param address: The address where to send the funds.
        :type address: str

        :return:
        :rtype : dict
        """
        return self.api_query('withdraw', {'currency': currency, 'quantity': quantity, 'address': address})

    def get_order_history(self, market, count):
        """
        Used to reterieve order trade history of account

        /account/getorderhistory

        :param market: optional a string literal for the market (ie. BTC-LTC). If ommited, will return for all markets
        :type market: str

        :param count: optional 	the number of records to return
        :type count: int

        :return: order history in JSON
        :rtype : dict

        """
        return self.api_query('getorderhistory', {'market':market, 'count': count})

    def get_order(self, uuid):
        """
        Used to get details of buy or sell order

        /account/getorder

        :param uuid: uuid of buy or sell order
        :type uuid: str

        :return:
        :rtype : dict
        """
        return self.api_query('getorder', {'uuid': uuid})
<|endoftext|>


<commit_before>import os
import time
import subprocess

def setup_package():
    print("Starting cluster")
    os.system("ipcluster start -n 4 &")
    proc = subprocess.Popen(["ipcluster start -n 4 &"], shell=True, executable="/bin/bash")
    print("Sleeping to allow for slow start")
    time.sleep(10)

def teardown_package():
    print("Terminating cluster")
    os.system("ipcluster stop")
    time.sleep(3)

if __name__ == "__main__" :

    setup_package()
    print("Done with setup")

    teardown_package()
    print("End of setup")
<commit_msg>Removing the setup/teardown functions and adding them to .travis.yml<commit_after>import os
import time
import subprocess

def setup():
    print("Starting cluster")
    #proc = subprocess.Popen(["ipcluster start -n 4 --daemon"], shell=True, executable="/bin/bash")
    #print("Sleeping to allow for slow start")
    #time.sleep(20)

def teardown():
    print("Terminating cluster")
    #os.system("ipcluster stop")
    #time.sleep(3)

if __name__ == "__main__" :

    setup_package()
    print("Done with setup")

    teardown_package()
    print("End of setup")
<|endoftext|>


<commit_before><commit_msg>REF: Remove Path.process_dir which is used nowhere<commit_after><|endoftext|>


<commit_before>__version__ = "0.3.1"
<commit_msg>version bump<commit_after>__version__ = "0.3.2"
<|endoftext|>


<commit_before><commit_msg>test_event_str_and_repr: use timezone.now for consistency<commit_after><|endoftext|>


<commit_before><commit_msg>label line not covered until skip-pkg-install tested<commit_after><|endoftext|>