<commit_before># encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <gogogo.vm@gmail.com>
"""

import datetime
from decimal import Decimal

import pytest
import pytz
import six
from typepy import StrictLevel
import typepy

class Test_TypeClass_repr(object):

    @pytest.mark.parametrize(["type_class", "value", "strict_level"], [
        [typepy.type.Integer, -six.MAXSIZE, StrictLevel.MIN],
        [typepy.type.Integer, six.MAXSIZE, StrictLevel.MAX],
        [typepy.type.RealNumber, -0.1, StrictLevel.MIN],
        [typepy.type.RealNumber, 0.1, StrictLevel.MAX],
        [typepy.type.DateTime, 1485685623, StrictLevel.MIN],
    ])
    def test_smoke(self, type_class, value, strict_level):
        type_object = type_class(value, strict_level)

        assert str(type_object)

class Test_RealNumber(object):

    @pytest.mark.parametrize(["value", "float_type", "expected"], [
        [
            "0.1",
            float,
            0.1
        ],
        [
            "0.1",
            Decimal,
            Decimal("0.1")
        ],
    ])
    def test_normal_float_type(self, value, float_type, expected):
        result = typepy.type.RealNumber(
            value, strict_level=StrictLevel.MIN, float_type=float_type).convert()

        assert result == expected

class Test_DateTime(object):

    @pytest.mark.parametrize(["value", "timezone", "expected"], [
        [
            datetime.datetime(2017, 1, 29, 10, 27, 3),
            pytz.utc,
            datetime.datetime(2017, 1, 29, 10, 27, 3)
        ],
        [
            datetime.datetime(2017, 1, 29, 10, 27, 3),
            pytz.timezone("Asia/Tokyo"),
            datetime.datetime(2017, 1, 29, 10, 27, 3)
        ],
        [
            "2017-01-29 19:27:03",
            pytz.utc,
            datetime.datetime(2017, 1, 29, 19, 27, 3)
        ],
        [
            "2017-01-29 19:27:03",
            pytz.timezone("Asia/Tokyo"),
            datetime.datetime(2017, 1, 29, 19, 27, 3)
        ],
        [
            1485685623,
            pytz.utc,
            datetime.datetime(2017, 1, 29, 10, 27, 3)
        ],
        [
            1485685623,
            pytz.timezone("Asia/Tokyo"),
            datetime.datetime(2017, 1, 29, 19, 27, 3),
        ],
    ])
    def test_normal_timezone(self, value, timezone, expected):
        result = typepy.type.DateTime(
            value, strict_level=StrictLevel.MIN, timezone=timezone).convert()

        assert result == timezone.localize(expected)
<commit_msg>pep8 cleanup<commit_after># encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <gogogo.vm@gmail.com>
"""

import datetime
from decimal import Decimal

import pytest
import pytz
import six
from typepy import StrictLevel
import typepy

class Test_TypeClass_repr(object):

    @pytest.mark.parametrize(["type_class", "value", "strict_level"], [
        [typepy.type.Integer, -six.MAXSIZE, StrictLevel.MIN],
        [typepy.type.Integer, six.MAXSIZE, StrictLevel.MAX],
        [typepy.type.RealNumber, -0.1, StrictLevel.MIN],
        [typepy.type.RealNumber, 0.1, StrictLevel.MAX],
        [typepy.type.DateTime, 1485685623, StrictLevel.MIN],
    ])
    def test_smoke(self, type_class, value, strict_level):
        type_object = type_class(value, strict_level)

        assert str(type_object)

class Test_RealNumber(object):

    @pytest.mark.parametrize(["value", "float_type", "expected"], [
        [
            "0.1",
            float,
            0.1
        ],
        [
            "0.1",
            Decimal,
            Decimal("0.1")
        ],
    ])
    def test_normal_float_type(self, value, float_type, expected):
        result = typepy.type.RealNumber(
            value, strict_level=StrictLevel.MIN, float_type=float_type
        ).convert()

        assert result == expected

class Test_DateTime(object):

    @pytest.mark.parametrize(["value", "timezone", "expected"], [
        [
            datetime.datetime(2017, 1, 29, 10, 27, 3),
            pytz.utc,
            datetime.datetime(2017, 1, 29, 10, 27, 3)
        ],
        [
            datetime.datetime(2017, 1, 29, 10, 27, 3),
            pytz.timezone("Asia/Tokyo"),
            datetime.datetime(2017, 1, 29, 10, 27, 3)
        ],
        [
            "2017-01-29 19:27:03",
            pytz.utc,
            datetime.datetime(2017, 1, 29, 19, 27, 3)
        ],
        [
            "2017-01-29 19:27:03",
            pytz.timezone("Asia/Tokyo"),
            datetime.datetime(2017, 1, 29, 19, 27, 3)
        ],
        [
            1485685623,
            pytz.utc,
            datetime.datetime(2017, 1, 29, 10, 27, 3)
        ],
        [
            1485685623,
            pytz.timezone("Asia/Tokyo"),
            datetime.datetime(2017, 1, 29, 19, 27, 3),
        ],
    ])
    def test_normal_timezone(self, value, timezone, expected):
        result = typepy.type.DateTime(
            value, strict_level=StrictLevel.MIN, timezone=timezone).convert()

        assert result == timezone.localize(expected)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import numpy as np

import climin
from climin.util import optimizer, OptimizerDistribution

from nose.plugins.skip import SkipTest

def test_optimizer():
    pairs = [('gd', climin.GradientDescent),
             ('lbfgs', climin.Lbfgs),
             ('ncg', climin.NonlinearConjugateGradient),
             ('rprop', climin.Rprop),
             ('rmsprop', climin.RmsProp),
             ]

    for ident, klass in pairs:
        wrt = np.zeros(10)
        opt = optimizer(ident, wrt, f=None, fprime=None, f_Hp=None, steprate=0.1)
        assert isinstance(opt, klass), 'wrong class for %s: %s' % (ident, type(opt))

def test_optimizer_distribution():
    try:
        import sklearn
    except ImportError:
        raise SkipTest()
    rv = OptimizerDistribution(gd={'steprate': [.1, .2],
                                   'momentum': [.9, .99]})
    opt = rv.rvs()
    assert opt[0] == 'gd'
    assert opt[1]['steprate'] in [.1, .2]
    assert opt[1]['momentum'] in [.9, .99]

    rv = OptimizerDistribution(gd={'steprate': [.1, .2],
                                   'momentum': [.9, .99]},
                               lbfgs={'n_factors': [10, 25]})
    opt = rv.rvs()

    assert opt[0] in ('lbfgs', 'gd')
    if opt[0] == 'lbfgs':
        assert opt[1]['n_factors'] in [10, 25]
    else:
        assert opt[1]['steprate'] in [.1, .2]
        assert opt[1]['momentum'] in [.9, .99]
<commit_msg>added tests for minibatches<commit_after># -*- coding: utf-8 -*-

import numpy as np

import climin
from climin.util import optimizer, OptimizerDistribution, minibatches

from nose.plugins.skip import SkipTest

def test_minibatches():
    """Test if minibatches are correctly generated if given a size."""
    D = np.random.random((13, 5))
    batches = minibatches(D, batch_size=5)
    assert batches[0].shape[0] == 5
    assert batches[1].shape[0] == 5
    assert batches[2].shape[0] == 3

def test_optimizer():
    pairs = [('gd', climin.GradientDescent),
             ('lbfgs', climin.Lbfgs),
             ('ncg', climin.NonlinearConjugateGradient),
             ('rprop', climin.Rprop),
             ('rmsprop', climin.RmsProp),
             ]

    for ident, klass in pairs:
        wrt = np.zeros(10)
        opt = optimizer(ident, wrt, f=None, fprime=None, f_Hp=None, steprate=0.1)
        assert isinstance(opt, klass), 'wrong class for %s: %s' % (ident, type(opt))

def test_optimizer_distribution():
    try:
        import sklearn
    except ImportError:
        raise SkipTest()
    rv = OptimizerDistribution(gd={'steprate': [.1, .2],
                                   'momentum': [.9, .99]})
    opt = rv.rvs()
    assert opt[0] == 'gd'
    assert opt[1]['steprate'] in [.1, .2]
    assert opt[1]['momentum'] in [.9, .99]

    rv = OptimizerDistribution(gd={'steprate': [.1, .2],
                                   'momentum': [.9, .99]},
                               lbfgs={'n_factors': [10, 25]})
    opt = rv.rvs()

    assert opt[0] in ('lbfgs', 'gd')
    if opt[0] == 'lbfgs':
        assert opt[1]['n_factors'] in [10, 25]
    else:
        assert opt[1]['steprate'] in [.1, .2]
        assert opt[1]['momentum'] in [.9, .99]
<|endoftext|>


<commit_before>from charms.reactive import (
    hook,
    when,
    only_once,
    is_state
)

import os.path as path
from charmhelpers.core import hookenv
from charmhelpers.core.host import service_restart, lsb_release
from charmhelpers.core.templating import render
from subprocess import call
from charms.layer import nginx, dokuwiki, php

config = hookenv.config()

# HOOKS -----------------------------------------------------------------------
@hook('config-changed')
def config_changed():

    if not is_state('nginx.available'):
        return

    # Define user
    app_path = nginx.get_app_path()
    render(source='users.auth.php',
           target=path.join(app_path, 'conf/users.auth.php'),
           context=config, perms=0o664)
    call('chown www-data:www-data -R {}'.format(app_path), shell=True)
    call('chmod 775 -R {}/conf'.format(app_path), shell=True)
    call('mkdir -p {app_path}/data && chmod 775 -R {app_path}/data'.format(
        app_path=app_path), shell=True)

    php.restart()
    service_restart('nginx')
    hookenv.status_set('active', 'Ready')

# REACTORS --------------------------------------------------------------------
@when('nginx.available')
@when('php.ready')
@only_once
def install_app():
    """ Performs application installation
    """

    hookenv.log('Installing Dokuwiki', 'info')

    # Configure NGINX vhost
    nginx.configure_site('default', 'vhost.conf',
                         listen_address=php.socket())

    # Update application
    dokuwiki.download_archive()

    # Needs to set dokuwiki directory permissions for installation
    app_path = nginx.get_app_path()

    render(source='local.php',
           target=path.join(app_path, 'conf/local.php'),
           context=config, perms=0o644)

    render(source='acl.auth.php',
           target=path.join(app_path, 'conf/acl.auth.php'),
           context=config, perms=0o644)

    render(source='plugins.local.php',
           target=path.join(app_path, 'conf/plugins.local.php'),
           context=config, perms=0o644)

    # Clean up install.php as we don't need it
    call("rm -f {}/conf/install.php", shell=True)

    php.restart()
    service_restart('nginx')
    hookenv.status_set('active', 'Dokuwiki is installed!')
<commit_msg>fix lint<commit_after>from charms.reactive import (
    hook,
    when,
    only_once,
    is_state
)

import os.path as path
from charmhelpers.core import hookenv
from charmhelpers.core.host import service_restart
from charmhelpers.core.templating import render
from subprocess import call
from charms.layer import nginx, dokuwiki, php

config = hookenv.config()

# HOOKS -----------------------------------------------------------------------
@hook('config-changed')
def config_changed():

    if not is_state('nginx.available'):
        return

    # Define user
    app_path = nginx.get_app_path()
    render(source='users.auth.php',
           target=path.join(app_path, 'conf/users.auth.php'),
           context=config, perms=0o664)
    call('chown www-data:www-data -R {}'.format(app_path), shell=True)
    call('chmod 775 -R {}/conf'.format(app_path), shell=True)
    call('mkdir -p {app_path}/data && chmod 775 -R {app_path}/data'.format(
        app_path=app_path), shell=True)

    php.restart()
    service_restart('nginx')
    hookenv.status_set('active', 'Ready')

# REACTORS --------------------------------------------------------------------
@when('nginx.available')
@when('php.ready')
@only_once
def install_app():
    """ Performs application installation
    """

    hookenv.log('Installing Dokuwiki', 'info')

    # Configure NGINX vhost
    nginx.configure_site('default', 'vhost.conf',
                         listen_address=php.socket())

    # Update application
    dokuwiki.download_archive()

    # Needs to set dokuwiki directory permissions for installation
    app_path = nginx.get_app_path()

    render(source='local.php',
           target=path.join(app_path, 'conf/local.php'),
           context=config, perms=0o644)

    render(source='acl.auth.php',
           target=path.join(app_path, 'conf/acl.auth.php'),
           context=config, perms=0o644)

    render(source='plugins.local.php',
           target=path.join(app_path, 'conf/plugins.local.php'),
           context=config, perms=0o644)

    # Clean up install.php as we don't need it
    call("rm -f {}/conf/install.php", shell=True)

    php.restart()
    service_restart('nginx')
    hookenv.status_set('active', 'Dokuwiki is installed!')
<|endoftext|>


<commit_before>import contextlib
import os
import os.path
import cStringIO as StringIO

import mock
import requests

def get_test_file_path(file_path):
    """translates a file path to be relative to the test files directory"""
    return os.path.join(os.path.dirname(__file__), 'files', file_path)

@contextlib.contextmanager
def mocked_requests(mocked_urls):
    """mocks the requests library to return a given file's content"""
    # if environment variable is set, then don't mock the tests just grab files
    # over the network. Example:
    #    env ULMO_DONT_MOCK_TESTS=1 py.test
    if os.environ.get('ULMO_DONT_MOCK_TESTS', False):
        yield

    else:
        if isinstance(mocked_urls, basestring):
            file_paths = [mocked_urls]
        else:
            file_paths = mocked_urls.values()

        with _open_multiple(file_paths) as open_files:
            if isinstance(mocked_urls, basestring):
                url_files = open_files.values()[0]
            else:
                url_files = {
                    url: open_files.get(file_path)
                    for url, file_path in mocked_urls.iteritems()
                }
            side_effect = _mock_request_side_effect(url_files)

            with mock.patch('requests.get', side_effect=side_effect):
                yield

@contextlib.contextmanager
def mocked_suds_client(waterml_version, mocked_service_calls):
    """mocks the suds library to return a given file's content"""
    # if environment variable is set, then don't mock the tests just grab files
    # over the network. Example:
    #    env ULMO_DONT_MOCK_TESTS=1 py.test
    if os.environ.get('ULMO_DONT_MOCK_TESTS', False):
        yield

    else:
        tns_str = 'http://www.cuahsi.org/his/%s/ws/' % waterml_version
        with _open_multiple(mocked_service_calls.values()) as open_files:
            client = mock.MagicMock()
            client.wsdl.tns = ('tns', tns_str)

            for service_call, filename in mocked_service_calls.iteritems():
                open_file = open_files[filename]
                def _func(*args, **kwargs):
                    return open_file.read()
                setattr(client.service, service_call, _func)

            with mock.patch('suds.client.Client', return_value=client):
                yield

def _mock_request_side_effect(url_files):
    def _side_effect(url, *args, **kwargs):
        mock_response = requests.Response()
        mock_response.request = requests.Request(url, *args, **kwargs)
        mock_response.status_code = 200
        mock_response.url = url
        mock_response.mocked = True
        if isinstance(url_files, dict):
            mock_response.raw = url_files.get(mock_response.request.full_url)
        elif hasattr(url_files, 'read'):
            mock_response.raw = url_files

        # seek to beginning in case this handle has been used before
        mock_response.raw.seek(0)

        return mock_response
    return _side_effect

@contextlib.contextmanager
def _open_multiple(paths, handlers=None, new_paths=None):
    """context manager that yields a dict containing file paths as keys mapped
    to their corresponding open file handles
    """
    # note: recursion is the pattern here
    if not handlers and not new_paths:
        with _open_multiple([], [], paths) as return_dict:
            yield return_dict
    elif len(new_paths) == 0:
        yield dict(zip(paths, handlers))
    else:
        next_path = new_paths.pop(0)
        test_path = get_test_file_path(next_path)
        with open(test_path, 'rb') as f:
            paths.append(next_path)
            handlers.append(f)
            with _open_multiple(paths, handlers, new_paths) as return_dict:
                yield return_dict
<commit_msg>also patch head requests for offline testing<commit_after>import contextlib
import os
import os.path
import cStringIO as StringIO

import mock
import requests

def get_test_file_path(file_path):
    """translates a file path to be relative to the test files directory"""
    return os.path.join(os.path.dirname(__file__), 'files', file_path)

@contextlib.contextmanager
def mocked_requests(mocked_urls):
    """mocks the requests library to return a given file's content"""
    # if environment variable is set, then don't mock the tests just grab files
    # over the network. Example:
    #    env ULMO_DONT_MOCK_TESTS=1 py.test
    if os.environ.get('ULMO_DONT_MOCK_TESTS', False):
        yield

    else:
        if isinstance(mocked_urls, basestring):
            file_paths = [mocked_urls]
        else:
            file_paths = mocked_urls.values()

        with _open_multiple(file_paths) as open_files:
            if isinstance(mocked_urls, basestring):
                url_files = open_files.values()[0]
            else:
                url_files = {
                    url: open_files.get(file_path)
                    for url, file_path in mocked_urls.iteritems()
                }
            side_effect = _mock_request_side_effect(url_files)
            with contextlib.nested(
                    mock.patch('requests.get', side_effect=side_effect),
                    mock.patch('requests.head', side_effect=side_effect),
                    ):
                yield

@contextlib.contextmanager
def mocked_suds_client(waterml_version, mocked_service_calls):
    """mocks the suds library to return a given file's content"""
    # if environment variable is set, then don't mock the tests just grab files
    # over the network. Example:
    #    env ULMO_DONT_MOCK_TESTS=1 py.test
    if os.environ.get('ULMO_DONT_MOCK_TESTS', False):
        yield

    else:
        tns_str = 'http://www.cuahsi.org/his/%s/ws/' % waterml_version
        with _open_multiple(mocked_service_calls.values()) as open_files:
            client = mock.MagicMock()
            client.wsdl.tns = ('tns', tns_str)

            for service_call, filename in mocked_service_calls.iteritems():
                open_file = open_files[filename]
                def _func(*args, **kwargs):
                    return open_file.read()
                setattr(client.service, service_call, _func)

            with mock.patch('suds.client.Client', return_value=client):
                yield

def _mock_request_side_effect(url_files):
    def _side_effect(url, *args, **kwargs):
        mock_response = requests.Response()
        mock_response.request = requests.Request(url, *args, **kwargs)
        mock_response.status_code = 200
        mock_response.url = url
        mock_response.mocked = True
        if isinstance(url_files, dict):
            mock_response.raw = url_files.get(mock_response.request.full_url)
        elif hasattr(url_files, 'read'):
            mock_response.raw = url_files

        # seek to beginning in case this handle has been used before
        mock_response.raw.seek(0)

        return mock_response
    return _side_effect

@contextlib.contextmanager
def _open_multiple(paths, handlers=None, new_paths=None):
    """context manager that yields a dict containing file paths as keys mapped
    to their corresponding open file handles
    """
    # note: recursion is the pattern here
    if not handlers and not new_paths:
        with _open_multiple([], [], paths) as return_dict:
            yield return_dict
    elif len(new_paths) == 0:
        yield dict(zip(paths, handlers))
    else:
        next_path = new_paths.pop(0)
        test_path = get_test_file_path(next_path)
        with open(test_path, 'rb') as f:
            paths.append(next_path)
            handlers.append(f)
            with _open_multiple(paths, handlers, new_paths) as return_dict:
                yield return_dict
<|endoftext|>


<commit_before>#!/usr/bin/env python

import tempfile
from subprocess import check_output
from glob import glob
import re, os
import hashlib
import xml.etree.ElementTree as ET
import sys

envelope = r'''%% processed with readme2tex
\documentclass{article}
%s
\pagestyle{empty}
\begin{document}
%s%s
\end{document}
'''

def rendertex(engine, string, packages, temp_dir, block):
    if engine != 'latex': raise Exception("Not Implemented")
    source = envelope % ('\n'.join(r'\usepackage{%s}' % ''.join(package) for package in packages), 'a' if not block else '', string)
    name = hashlib.md5(string.encode('utf-8')).hexdigest()
    source_file = os.path.join(temp_dir, name + '.tex')
    with open(source_file, 'w') as file:
        file.write(source)

    try:
        check_output(
            [engine, '-output-directory=' + temp_dir, '-interaction', 'nonstopmode', source_file],
            stderr=sys.stdout)
    except:
        print("'%s' has warnings during compilation. See %s/%s" % (string, temp_dir, name))
    dvi = os.path.join(temp_dir, name + '.dvi')
    svg = check_output(
        ['dvisvgm', '-v0', '-a', '-n', '-s', dvi])
    return svg, dvi, name

def extract_equations(content):
    next = lambda n: (content.find('$', n), content.find(r'\begin', n))
    cursor = 0
    lines = [line for line in content.splitlines()]
    while True:
        dollar, begin = next(cursor)
        if dollar is -1: dollar = '-1'
        if begin is -1: begin = '-1'
        if dollar == '-1' and begin == '-1': break
        if dollar != '-1' and (begin == '-1' or dollar < begin):
            # found a $, see if it's $$
            if dollar > 0 and content[dollar - 1] == '\\':
                cursor = dollar + 1
                continue
            # get this line
            cummulative = 0
            line = 0
            for line, string in enumerate(lines):
                cummulative += len(string) + 1
                if dollar < cummulative: break
            if lines[line].startswith('  '):
                cursor = dollar + 2
                continue
            if len(content) > dollar and content[dollar + 1] == '$':
                ## find the next $$
                cursor = content.find('$$', dollar + 2) + 2
                if cursor == 1:
                    cursor = dollar + 1
                    continue
                yield content[dollar: cursor], dollar, cursor, True
            else:
                cursor = content.find('$', dollar + 1) + 1
                if cursor == 0:
                    cursor = dollar + 1
                    continue
                yield content[dollar: cursor], dollar, cursor, False
        else:
            cummulative = 0
            line = 0
            for line, string in enumerate(lines):
                cummulative += len(string) + 1
                if begin < cummulative: break
            if lines[line].startswith('  '):
                cursor = begin + 6
                continue
            leftover = content[begin + 6:]
            if not leftover: break
            match = re.match(r"\{.+?\}", leftover)
            if not match:
                cursor = begin + 6
                continue
            end_marker = '\\end' + match.group()
            end = content.find(end_marker, begin)
            if end is -1:
                cursor = begin + 6
                continue
            cursor = end + len(end_marker)
            yield content[begin: cursor], begin, cursor, True

def render(
        readme=None,
        output='README_GH.md',
        engine='latex',
        packages=('amsmath', 'amssymb'),
        svgdir='svgs',
        branch=None,
        user=None,
        project=None,
        nocdn=False,
        htmlize=False,
        use_valign=False):
    # look for $.$ or $$.$$
    if htmlize:
        nocdn = True
        branch = None
    temp_dir = tempfile.mkdtemp('', 'readme2tex-')
    if not readme or not open(readme):
        md_files = [file for file in glob("*.md") if file.lower() == 'readother.md']
        if not md_files: raise Exception("Must either pass in a --readme file or have a READOTHER.md file.")
        readme = md_files[0]

    with open(readme) as readme_file:
        content = readme_file.read()
    if not content: raise Exception("Cannot read file.")
    content = content.replace('\r', '')

    equations = list(extract_equations(content))
    equation_map = {}
    seen = {}
    for equation, start, end, block in equations:
        if equation in seen:
            equation_map[(start, end)] = equation_map[seen[equation]]
            continue
        seen[equation] = (start, end)
        svg, dvi, name = rendertex(engine, equation, packages, temp_dir, block)
        svg = svg.decode('utf-8')

        xml = (ET.fromstring(svg))
        attributes = xml.attrib
        gfill = xml.find('{http://www.w3.org/2000/svg}g')
        gfill.set('fill-opacity', '0.9')
        if not block:
            uses = gfill.findall('{http://www.w3.org/2000/svg}use')
            use = uses[0]
            # compute baseline off of this dummy element
            x = use.attrib['x']
            y = float(use.attrib['y'])
            viewBox = [float(a) for a in attributes['viewBox'].split()] # min-x, min-y, width, height
            baseline_offset = viewBox[-1] - (y - viewBox[1])
            newViewBox = list(viewBox)

            newViewBox[0] = min(list(float(next.attrib['x']) for next in uses if next.attrib['x'] != x) or [float(x)])
            newViewBox[-2] -= abs(newViewBox[0] - viewBox[0])
            xml.set('viewBox', ' '.join(map(str, newViewBox)))
            xml.set('width', str(newViewBox[-2]) + 'pt')
            gfill.remove(use)
            top = y - newViewBox[1]
            bottom = baseline_offset
            if not use_valign:
                if top > bottom:
                    # extend the bottom
                    height = 2 * top
                    xml.set('height', '%spt' % (height))
                    newViewBox[-1] = height
                    xml.set('viewBox', ' '.join(map(str, newViewBox)))
                else:
                    # extend the top
                    height = 2 * bottom
                    xml.set('height', '%spt' % (height))
                    newViewBox[-1] = height
                    newViewBox[1] -= (height - bottom - top)
                    xml.set('viewBox', ' '.join(map(str, newViewBox)))
                    pass
        else:
            baseline_offset = 0

        svg = ET.tostring(xml).decode('utf-8')

        equation_map[(start, end)] = (svg, name, dvi, baseline_offset)

    # git rev-parse --abbrev-ref HEAD
    try:
        old_branch = check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).decode('utf-8').strip()
    except:
        if not nocdn:
            print("Not in a git repository, please enable --nocdn")
        old_branch = "NONE"

    if not branch or branch == old_branch:
        branch = old_branch
        if not os.path.exists(svgdir):
            os.makedirs(svgdir)
        for equation, start, end, _ in equations:
            svg, name, dvi, off = equation_map[(start, end)]
            with open(os.path.join(svgdir, name + '.svg'), 'w') as file:
                file.write(svg)
    else:
        # git stash -q --keep-index
        if check_output(['git', 'status', '-s']).decode('utf-8').strip():
            raise Exception("Cannot switch to a different branch while there are items for staging.")
        try:
            print("Checking out %s" % branch)
            check_output(['git', 'checkout', branch])

            if not os.path.exists(svgdir):
                os.makedirs(svgdir)
            for equation, start, end, _ in equations:
                svg, name, dvi, off = equation_map[(start, end)]
                with open(os.path.join(svgdir, name + '.svg'), 'w') as file:
                    file.write(svg)

            print("Committing changes...")
            check_output(['git', 'add', svgdir])
            check_output(['git', 'commit', '-m', 'readme2latex render'])

            print("Switching back to the original branch")
            check_output(['git', 'checkout', old_branch])
        except Exception as e:
            print(e)
            print("Cleaning up.")
            check_output(['git', 'checkout', '--', '.'])
            check_output(['git', 'clean', '-df'])
            check_output(['git', 'checkout', old_branch])

    # Make replacements
    if not user or not project:
        try:
            # git remote get-url origin
            giturl = check_output(['git', 'remote', '-v']).strip().decode('utf-8').splitlines()[0]
            start = giturl.find('.com/') + 5
            userproj = giturl[start:]
            end = userproj.find('.git')
            user, project = userproj[:end].split('/')
        except:
            raise Exception("Please specify your github --username and --project.")

    if nocdn:
        svg_url = "{svgdir}/{name}.svg"
    else:
        svg_url = "https://rawgit.com/{user}/{project}/{branch}/{svgdir}/{name}.svg"
    equations = sorted(equations, key=lambda x: (x[1], x[2]))[::-1]
    new = content
    for equation, start, end, block in equations:
        svg, name, dvi, off = equation_map[(start, end)]
        if abs(off) < 1e-2: off = 0
        xml = (ET.fromstring(svg))
        attributes = xml.attrib

        needs_inversion = not (equation.count('tikzpicture') and equation.count('fill='))

        scale = 1.65
        height = float(attributes['height'][:-2]) * scale
        width = float(attributes['width'][:-2]) * scale
        url = svg_url.format(user=user, project=project, branch=branch, svgdir=svgdir, name=name)
        img = '' % (
            url,
            '?invert_in_darkmode' if needs_inversion else '',
            ('valign=%spx'%(-off * scale) if use_valign else 'align=middle'),
            width,
            height)
        if block: img = '
%s
' % img
        new = new[:start] + img + new[end:]
    with open(output, 'w') as outfile:
        outfile.write(new)

    if htmlize:
        try:
            import markdown
        except:
            print("Cannot render markdown, make sure that the markdown package is installed.")
            return
        with open(output+".html", 'w') as outfile:
            outfile.write(markdown.markdown(new))
<commit_msg>Do not commit if the script does not generate images on this run.<commit_after>#!/usr/bin/env python

import tempfile
from subprocess import check_output
from glob import glob
import re, os
import hashlib
import xml.etree.ElementTree as ET
import sys

envelope = r'''%% processed with readme2tex
\documentclass{article}
%s
\pagestyle{empty}
\begin{document}
%s%s
\end{document}
'''

def rendertex(engine, string, packages, temp_dir, block):
    if engine != 'latex': raise Exception("Not Implemented")
    source = envelope % ('\n'.join(r'\usepackage{%s}' % ''.join(package) for package in packages), 'a' if not block else '', string)
    name = hashlib.md5(string.encode('utf-8')).hexdigest()
    source_file = os.path.join(temp_dir, name + '.tex')
    with open(source_file, 'w') as file:
        file.write(source)

    try:
        check_output(
            [engine, '-output-directory=' + temp_dir, '-interaction', 'nonstopmode', source_file],
            stderr=sys.stdout)
    except:
        print("'%s' has warnings during compilation. See %s/%s" % (string, temp_dir, name))
    dvi = os.path.join(temp_dir, name + '.dvi')
    svg = check_output(
        ['dvisvgm', '-v0', '-a', '-n', '-s', dvi])
    return svg, dvi, name

def extract_equations(content):
    next = lambda n: (content.find('$', n), content.find(r'\begin', n))
    cursor = 0
    lines = [line for line in content.splitlines()]
    while True:
        dollar, begin = next(cursor)
        if dollar is -1: dollar = '-1'
        if begin is -1: begin = '-1'
        if dollar == '-1' and begin == '-1': break
        if dollar != '-1' and (begin == '-1' or dollar < begin):
            # found a $, see if it's $$
            if dollar > 0 and content[dollar - 1] == '\\':
                cursor = dollar + 1
                continue
            # get this line
            cummulative = 0
            line = 0
            for line, string in enumerate(lines):
                cummulative += len(string) + 1
                if dollar < cummulative: break
            if lines[line].startswith('  '):
                cursor = dollar + 2
                continue
            if len(content) > dollar and content[dollar + 1] == '$':
                ## find the next $$
                cursor = content.find('$$', dollar + 2) + 2
                if cursor == 1:
                    cursor = dollar + 1
                    continue
                yield content[dollar: cursor], dollar, cursor, True
            else:
                cursor = content.find('$', dollar + 1) + 1
                if cursor == 0:
                    cursor = dollar + 1
                    continue
                yield content[dollar: cursor], dollar, cursor, False
        else:
            cummulative = 0
            line = 0
            for line, string in enumerate(lines):
                cummulative += len(string) + 1
                if begin < cummulative: break
            if lines[line].startswith('  '):
                cursor = begin + 6
                continue
            leftover = content[begin + 6:]
            if not leftover: break
            match = re.match(r"\{.+?\}", leftover)
            if not match:
                cursor = begin + 6
                continue
            end_marker = '\\end' + match.group()
            end = content.find(end_marker, begin)
            if end is -1:
                cursor = begin + 6
                continue
            cursor = end + len(end_marker)
            yield content[begin: cursor], begin, cursor, True

def render(
        readme=None,
        output='README_GH.md',
        engine='latex',
        packages=('amsmath', 'amssymb'),
        svgdir='svgs',
        branch=None,
        user=None,
        project=None,
        nocdn=False,
        htmlize=False,
        use_valign=False):
    # look for $.$ or $$.$$
    if htmlize:
        nocdn = True
        branch = None
    temp_dir = tempfile.mkdtemp('', 'readme2tex-')
    if not readme or not open(readme):
        md_files = [file for file in glob("*.md") if file.lower() == 'readother.md']
        if not md_files: raise Exception("Must either pass in a --readme file or have a READOTHER.md file.")
        readme = md_files[0]

    with open(readme) as readme_file:
        content = readme_file.read()
    if not content: raise Exception("Cannot read file.")
    content = content.replace('\r', '')

    equations = list(extract_equations(content))
    equation_map = {}
    seen = {}
    for equation, start, end, block in equations:
        if equation in seen:
            equation_map[(start, end)] = equation_map[seen[equation]]
            continue
        seen[equation] = (start, end)
        svg, dvi, name = rendertex(engine, equation, packages, temp_dir, block)
        svg = svg.decode('utf-8')

        xml = (ET.fromstring(svg))
        attributes = xml.attrib
        gfill = xml.find('{http://www.w3.org/2000/svg}g')
        gfill.set('fill-opacity', '0.9')
        if not block:
            uses = gfill.findall('{http://www.w3.org/2000/svg}use')
            use = uses[0]
            # compute baseline off of this dummy element
            x = use.attrib['x']
            y = float(use.attrib['y'])
            viewBox = [float(a) for a in attributes['viewBox'].split()] # min-x, min-y, width, height
            baseline_offset = viewBox[-1] - (y - viewBox[1])
            newViewBox = list(viewBox)

            newViewBox[0] = min(list(float(next.attrib['x']) for next in uses if next.attrib['x'] != x) or [float(x)])
            newViewBox[-2] -= abs(newViewBox[0] - viewBox[0])
            xml.set('viewBox', ' '.join(map(str, newViewBox)))
            xml.set('width', str(newViewBox[-2]) + 'pt')
            gfill.remove(use)
            top = y - newViewBox[1]
            bottom = baseline_offset
            if not use_valign:
                if top > bottom:
                    # extend the bottom
                    height = 2 * top
                    xml.set('height', '%spt' % (height))
                    newViewBox[-1] = height
                    xml.set('viewBox', ' '.join(map(str, newViewBox)))
                else:
                    # extend the top
                    height = 2 * bottom
                    xml.set('height', '%spt' % (height))
                    newViewBox[-1] = height
                    newViewBox[1] -= (height - bottom - top)
                    xml.set('viewBox', ' '.join(map(str, newViewBox)))
                    pass
        else:
            baseline_offset = 0

        svg = ET.tostring(xml).decode('utf-8')

        equation_map[(start, end)] = (svg, name, dvi, baseline_offset)

    # git rev-parse --abbrev-ref HEAD
    try:
        old_branch = check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).decode('utf-8').strip()
    except:
        if not nocdn:
            print("Not in a git repository, please enable --nocdn")
        old_branch = "NONE"

    if not branch or branch == old_branch:
        branch = old_branch
        if not os.path.exists(svgdir):
            os.makedirs(svgdir)
        for equation, start, end, _ in equations:
            svg, name, dvi, off = equation_map[(start, end)]
            with open(os.path.join(svgdir, name + '.svg'), 'w') as file:
                file.write(svg)
    else:
        # git stash -q --keep-index
        if check_output(['git', 'status', '-s']).decode('utf-8').strip():
            raise Exception("Cannot switch to a different branch while there are items for staging.")
        try:
            print("Checking out %s" % branch)
            check_output(['git', 'checkout', branch])

            if not os.path.exists(svgdir):
                os.makedirs(svgdir)
            for equation, start, end, _ in equations:
                svg, name, dvi, off = equation_map[(start, end)]
                with open(os.path.join(svgdir, name + '.svg'), 'w') as file:
                    file.write(svg)

            if check_output(['git', 'status', '-s']).decode('utf-8').strip():
                print("Committing changes...")
                check_output(['git', 'add', svgdir])
                check_output(['git', 'commit', '-m', 'readme2latex render'])
            else:
                print("No changes were made.")

            print("Switching back to the original branch")
            check_output(['git', 'checkout', old_branch])
        except Exception as e:
            print(e)
            print("Cleaning up.")
            check_output(['git', 'checkout', '--', '.'])
            check_output(['git', 'clean', '-df'])
            check_output(['git', 'checkout', old_branch])

    # Make replacements
    if not user or not project:
        try:
            # git remote get-url origin
            giturl = check_output(['git', 'remote', '-v']).strip().decode('utf-8').splitlines()[0]
            start = giturl.find('.com/') + 5
            userproj = giturl[start:]
            end = userproj.find('.git')
            user, project = userproj[:end].split('/')
        except:
            raise Exception("Please specify your github --username and --project.")

    if nocdn:
        svg_url = "{svgdir}/{name}.svg"
    else:
        svg_url = "https://rawgit.com/{user}/{project}/{branch}/{svgdir}/{name}.svg"
    equations = sorted(equations, key=lambda x: (x[1], x[2]))[::-1]
    new = content
    for equation, start, end, block in equations:
        svg, name, dvi, off = equation_map[(start, end)]
        if abs(off) < 1e-2: off = 0
        xml = (ET.fromstring(svg))
        attributes = xml.attrib

        needs_inversion = not (equation.count('tikzpicture') and equation.count('fill='))

        scale = 1.65
        height = float(attributes['height'][:-2]) * scale
        width = float(attributes['width'][:-2]) * scale
        url = svg_url.format(user=user, project=project, branch=branch, svgdir=svgdir, name=name)
        img = '' % (
            url,
            '?invert_in_darkmode' if needs_inversion else '',
            ('valign=%spx'%(-off * scale) if use_valign else 'align=middle'),
            width,
            height)
        if block: img = '
%s
' % img
        new = new[:start] + img + new[end:]
    with open(output, 'w') as outfile:
        outfile.write(new)

    if htmlize:
        try:
            import markdown
        except:
            print("Cannot render markdown, make sure that the markdown package is installed.")
            return
        with open(output+".html", 'w') as outfile:
            outfile.write(markdown.markdown(new))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright (C) 2014:
#    Gabes Jean, naparuba@gmail.com

from opsbro_test import *

import os
import sys
from opsbro.yamlmgr import yamler

'''
import opsbro.misc

p = os.path.join(os.path.dirname(opsbro.misc.__file__), 'internalyaml')
sys.path.insert(0, p)

import ruamel.yaml as yaml
'''

class TestYaml(OpsBroTest):
    def setUp(self):
        pass

    def test_yaml_load(self):
        s = '''# Pre comment
# on two lines
- super string # is this a valid comment
- k1: blabla
  k2: 36.000
  k3:
              # is this a valid comment
              - sub 1
              - sub 2
# ending comment
'''

        data = yamler.loads(s)

        self.assert_(data[0] == 'super string')
        self.assert_(data[1]['k1'] == 'blabla')
        self.assert_(data[1]['k2'] == 36.00)

        buf = yamler.dumps(data)
        print "BUF", buf

    def test_yaml_comments(self):
        s = '''# document comment line1
# document comment line2
key1: blabla  # key1 comment

# Key2 line before comment, part1
# Key2 line before comment, part2
key2: 36.000  # key2 same line comment
key3:  # key3 level comment
          # K3 sub list comment
          - entry key3.1  #key3.entry1 comment
          - sub 2
key4: 36
# ending comment
# ending comment bis
        '''
        data = yamler.loads(s)
        print "DATA", data, type(data), dir(data)
        #print 'DICT', data.__dict__
        #print data._yaml_format.__dict__
        #print data._yaml_line_col.__dict__

        print "CA", data.ca
        print "CA internals", data.ca.__dict__
        print "CA dir", dir(data.ca)
        print "CA.attrib", data.ca.attrib
        print "CA comment", data.ca.comment

        print "CA items", data.ca.items

        whole_data_comment = yamler.get_document_comment(data)
        print "Whole data comment", whole_data_comment
        whole_data_comment_OK = '''# document comment line1
# document comment line2'''
        self.assert_(whole_data_comment == whole_data_comment_OK)

        # Now check key1 comment
        key1_comment = yamler.get_key_comment(data, 'key1')
        print "KEY1 comment:", key1_comment
        key1_comment_OK = '# key1 comment'
        #self.assert_(key1_comment == key1_comment_OK)
        self.assertEqual(key1_comment, key1_comment_OK)

        # Key2 is a bit harder: got both lines before and same line comments
        key2_comment = yamler.get_key_comment(data, 'key2')
        print "KEY2 comment:", key2_comment
        key2_comment_OK = '''# Key2 line before comment, part1
# Key2 line before comment, part2
# key2 same line comment'''
        self.assertEqual(key2_comment, key2_comment_OK)

        # Key4: nothing, should be None
        key4_comment = yamler.get_key_comment(data, 'key4')
        print "KEY4 comment:", key4_comment
        self.assertIsNone(key4_comment)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Fix: TEST under python 2.6, assertIsNone do not exist<commit_after>#!/usr/bin/env python
# Copyright (C) 2014:
#    Gabes Jean, naparuba@gmail.com

from opsbro_test import *

import os
import sys
from opsbro.yamlmgr import yamler

'''
import opsbro.misc

p = os.path.join(os.path.dirname(opsbro.misc.__file__), 'internalyaml')
sys.path.insert(0, p)

import ruamel.yaml as yaml
'''

class TestYaml(OpsBroTest):
    def setUp(self):
        pass

    def test_yaml_load(self):
        s = '''# Pre comment
# on two lines
- super string # is this a valid comment
- k1: blabla
  k2: 36.000
  k3:
              # is this a valid comment
              - sub 1
              - sub 2
# ending comment
'''

        data = yamler.loads(s)

        self.assert_(data[0] == 'super string')
        self.assert_(data[1]['k1'] == 'blabla')
        self.assert_(data[1]['k2'] == 36.00)

        buf = yamler.dumps(data)
        print "BUF", buf

    def test_yaml_comments(self):
        s = '''# document comment line1
# document comment line2
key1: blabla  # key1 comment

# Key2 line before comment, part1
# Key2 line before comment, part2
key2: 36.000  # key2 same line comment
key3:  # key3 level comment
          # K3 sub list comment
          - entry key3.1  #key3.entry1 comment
          - sub 2
key4: 36
# ending comment
# ending comment bis
        '''
        data = yamler.loads(s)
        print "DATA", data, type(data), dir(data)
        #print 'DICT', data.__dict__
        #print data._yaml_format.__dict__
        #print data._yaml_line_col.__dict__

        print "CA", data.ca
        print "CA internals", data.ca.__dict__
        print "CA dir", dir(data.ca)
        print "CA.attrib", data.ca.attrib
        print "CA comment", data.ca.comment

        print "CA items", data.ca.items

        whole_data_comment = yamler.get_document_comment(data)
        print "Whole data comment", whole_data_comment
        whole_data_comment_OK = '''# document comment line1
# document comment line2'''
        self.assert_(whole_data_comment == whole_data_comment_OK)

        # Now check key1 comment
        key1_comment = yamler.get_key_comment(data, 'key1')
        print "KEY1 comment:", key1_comment
        key1_comment_OK = '# key1 comment'
        #self.assert_(key1_comment == key1_comment_OK)
        self.assertEqual(key1_comment, key1_comment_OK)

        # Key2 is a bit harder: got both lines before and same line comments
        key2_comment = yamler.get_key_comment(data, 'key2')
        print "KEY2 comment:", key2_comment
        key2_comment_OK = '''# Key2 line before comment, part1
# Key2 line before comment, part2
# key2 same line comment'''
        self.assertEqual(key2_comment, key2_comment_OK)

        # Key4: nothing, should be None
        key4_comment = yamler.get_key_comment(data, 'key4')
        print "KEY4 comment:", key4_comment
        self.assertEqual(key4_comment, None)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>from functools import partial
from concurrent.futures import ThreadPoolExecutor
from BCBio import GFF
from Bio.Seq import Seq
from Bio.SeqRecord import SeqRecord
from Bio.SeqFeature import SeqFeature, FeatureLocation

from .hits import BlastProcessor

def write_cluster(clusters, header, output_path, reference_fasta=None, blastdb=None, sample='sample', threads=1):
    """Write clusters as GFF entries."""
    with open(output_path, "w") as out_handle:
        if blastdb or reference_fasta:
            blast = BlastProcessor(reference_fasta=reference_fasta, blastdb=blastdb)
        else:
            blast = None
        tp = ThreadPoolExecutor(threads)
        futures = []
        records = {tid: SeqRecord(Seq(""), header['SQ'][tid]['SN']) for tid in range(len(header['SQ']))}
        for i, cluster in enumerate(clusters):
            func = partial(get_feature, cluster, sample, i, blast)
            futures.append(tp.submit(func))
        for future in futures:
            tid, feature = future.result()
            records[tid].features.append(feature)
        GFF.write(records.values(), out_handle)
        tp.shutdown(wait=True)

def get_feature(cluster, sample, i, blast=None):
    """Turn a cluster into a biopython SeqFeature."""
    qualifiers = {"source": "findcluster",
                  "score": cluster.score,
                  "left_support": cluster.left_support,
                  "right_support": cluster.right_support,
                  "left_insert": cluster.left_contigs,
                  "right_insert": cluster.right_contigs,
                  "ID": "%s_%d" % (sample, i),
                  "valid_TSD": cluster.valid_tsd}
    feature = SeqFeature(FeatureLocation(cluster.start, cluster.end), type="TE", strand=1, qualifiers=qualifiers)
    if blast:
        seqfeature_args = blast.blast_cluster(cluster).to_feature_args()
        base_args = {'location': FeatureLocation(cluster.start, cluster.end), 'strand': 1}
        subfeatures = []
        for feature_args in seqfeature_args:
            args = base_args.copy()
            args.update(feature_args)
            subfeatures.append(SeqFeature(**args))
        feature.sub_features = subfeatures
    return cluster.tid, feature
<commit_msg>Cleanup BlastProcessor object at the end of a run<commit_after>from functools import partial
from concurrent.futures import ThreadPoolExecutor
from BCBio import GFF
from Bio.Seq import Seq
from Bio.SeqRecord import SeqRecord
from Bio.SeqFeature import SeqFeature, FeatureLocation

from .hits import BlastProcessor

def write_cluster(clusters, header, output_path, reference_fasta=None, blastdb=None, sample='sample', threads=1):
    """Write clusters as GFF entries."""
    with open(output_path, "w") as out_handle:
        try:
            if blastdb or reference_fasta:
                blast = BlastProcessor(reference_fasta=reference_fasta, blastdb=blastdb)
            else:
                blast = None
            tp = ThreadPoolExecutor(threads)
            futures = []
            records = {tid: SeqRecord(Seq(""), header['SQ'][tid]['SN']) for tid in range(len(header['SQ']))}
            for i, cluster in enumerate(clusters):
                func = partial(get_feature, cluster, sample, i, blast)
                futures.append(tp.submit(func))
            for future in futures:
                tid, feature = future.result()
                records[tid].features.append(feature)
            GFF.write(records.values(), out_handle)
            tp.shutdown(wait=True)
        finally:
            if blast:
                blast.close()

def get_feature(cluster, sample, i, blast=None):
    """Turn a cluster into a biopython SeqFeature."""
    qualifiers = {"source": "findcluster",
                  "score": cluster.score,
                  "left_support": cluster.left_support,
                  "right_support": cluster.right_support,
                  "left_insert": cluster.left_contigs,
                  "right_insert": cluster.right_contigs,
                  "ID": "%s_%d" % (sample, i),
                  "valid_TSD": cluster.valid_tsd}
    feature = SeqFeature(FeatureLocation(cluster.start, cluster.end), type="TE", strand=1, qualifiers=qualifiers)
    if blast:
        seqfeature_args = blast.blast_cluster(cluster).to_feature_args()
        base_args = {'location': FeatureLocation(cluster.start, cluster.end), 'strand': 1}
        subfeatures = []
        for feature_args in seqfeature_args:
            args = base_args.copy()
            args.update(feature_args)
            subfeatures.append(SeqFeature(**args))
        feature.sub_features = subfeatures
    return cluster.tid, feature
<|endoftext|>


<commit_before>from __future__ import print_function, division, absolute_import#, unicode_literals # not casa compatible
from builtins import bytes, dict, object, range, map, input#, str # not casa compatible
from future.utils import itervalues, viewitems, iteritems, listvalues, listitems
from io import open

import distributed
from dask import array, delayed
from rfpipe import source, search, util, candidates
from dask.base import tokenize
import numpy as np

import logging
logger = logging.getLogger(__name__)
vys_timeout_default = 10

def pipeline_scan(st, segments=None, cl=None, host=None, cfile=None,
                  vys_timeout=vys_timeout_default):
    """ Given rfpipe state and dask distributed client, run search pipline.
    throttle option will submit only if worker state allows for it.
    parameters of throttling:
      read_overhead scales vismem to peak usage during read.
      read_totfrac is total memory usage allowed in bytes at submission time.
    """

    if cl is None:
        if host is None:
            cl = distributed.Client(n_workers=1, threads_per_worker=16,
                                    resources={"READER": 1, "MEMORY": 24,
                                               "CORES": 16},
                                    local_dir="/lustre/evla/test/realfast/scratch")
        else:
            cl = distributed.Client('{0}:{1}'.format(host, '8786'))

    if not isinstance(segments, list):
        segments = range(st.nsegment)

    futures = []
    for segment in segments:
        futures.append(pipeline_seg(st, segment, cl=cl, cfile=cfile,
                                    vys_timeout=vys_timeout))

    return futures  # list of dicts

def pipeline_seg(st, segment, cl, cfile=None,
                 vys_timeout=vys_timeout_default):
    """ Submit pipeline processing of a single segment to scheduler.
    Can use distributed client or compute locally.

    Uses distributed resources parameter to control scheduling of GPUs.
    Pipeline produces jobs per DM/dt.
    Returns a dict with values as futures of certain jobs (data, collection).
    """

    logger.info('Building dask for observation {0}, scan {1}, segment {2}.'
                .format(st.metadata.datasetId, st.metadata.scan, segment))

    futures = {}

# new style read *TODO: note hack on resources*
    data = lazy_read_segment(st, segment, cfile, vys_timeout)
    data = cl.compute(data, resources={tuple(data.__dask_keys__()[0][0][0]):
                                       {'READER': 1}})  # get future
# old style read
#    data = cl.submit(source.read_segment, st, segment, cfile, vys_timeout,
#                     resources={'READER': 1})

    futures['data'] = data  # save future

    # TODO: put this on READER worker?
    data_prep = cl.submit(source.data_prep, st, segment, data,
                          resources={'CORES': st.prefs.nthread},
                          priority=1)

    saved = []
    if st.fftmode == "fftw":
        searchresources = {'MEMORY': 2*st.immem+2*st.vismem,
                           'CORES': st.prefs.nthread}
        imgranges = [[(min(st.get_search_ints(segment, dmind, dtind)),
                     max(st.get_search_ints(segment, dmind, dtind)))
                      for dtind in range(len(st.dtarr))]
                     for dmind in range(len(st.dmarr))]
        wisdom = cl.submit(search.set_wisdom, st.npixx, st.npixy,
                           resources={'CORES': 1})

        for dmind in range(len(st.dmarr)):
            delay = cl.submit(util.calc_delay, st.freq, st.freq.max(),
                              st.dmarr[dmind], st.inttime,
                              resources={'CORES': 1})
            for dtind in range(len(st.dtarr)):
                data_corr = cl.submit(search.dedisperseresample, data_prep,
                                      delay, st.dtarr[dtind],
                                      parallel=st.prefs.nthread > 1,
                                      resources={'MEMORY': 2*st.vismem,
                                                 'CORES': st.prefs.nthread})

                im0, im1 = imgranges[dmind][dtind]
                integrationlist = [list(range(im0, im1)[i:i+st.chunksize])
                                   for i in range(0, im1-im0, st.chunksize)]
                for integrations in integrationlist:
                    saved.append(cl.submit(search.search_thresh_fftw, st,
                                           segment, data_corr, dmind, dtind,
                                           integrations=integrations,
                                           wisdom=wisdom,
                                           resources=searchresources))

    elif st.fftmode == "cuda":
        for dmind in range(len(st.dmarr)):
            saved.append(cl.submit(search.dedisperse_image_cuda, st, segment,
                                   data_prep, dmind,
                                   resources={'GPU': 1,
                                              'CORES': st.prefs.nthread},
                                   priority=2))

    # TODO: put these on dedicated worker to ensure quick processing?
    canddatalist = cl.submit(mergelists, saved,
                             resources={'CORES': 1},
                             priority=3)
    candcollection = cl.submit(candidates.calc_features, canddatalist,
                               resources={'CORES': 1}, priority=4)
    futures['candcollection'] = candcollection

    return futures

def pipeline_scan_delayed(st, segments=None, cl=None, host=None, cfile=None,
                          vys_timeout=vys_timeout_default):
    """ Submit pipeline processing of a single segment to scheduler.
    Uses delayed function and client.compute to schedule.

    Returns a list of dicts with futures of data, collection jobs.
    """

    if cl is None:
        if host is not None:
            cl = distributed.Client('{0}:{1}'.format(host, '8786'))

    if not isinstance(segments, list):
        segments = range(st.nsegment)

    futures = []
    for segment in segments:
        future = {}
        resources = {}

        logger.info('Building dask for observation {0}, scan {1}, segment {2}.'
                    .format(st.metadata.datasetId, st.metadata.scan, segment))

        data = delayed(source.read_segment)(st, segment, cfile, vys_timeout)
        resources[tuple(data.__dask_keys__())] = {'READER': 1}
        if cl is not None:
            future['data'] = cl.compute(data, resources=resources)

        data_prep = delayed(source.data_prep)(st, segment, data)

        assert st.fftmode == "cuda", "only cuda fftmode supported"
        canddatalist = delayed(search.dedisperse_image_cuda)(st, segment,
                                                             data_prep)
        resources[tuple(canddatalist.__dask_keys__())] = {'GPU': 1}

        candcollection = delayed(candidates.calc_features)(canddatalist)

        if cl is not None:
            future['candcollection'] = cl.compute(candcollection,
                                                  resources=resources,
                                                  priority={canddatalist: 3,   # TODO test if this actually does anything
                                                            candcollection: 2,
                                                            data_prep: 1})

            futures.append(future)
        else:
            futures.append(candcollection)

    return futures

def mergelists(futlists):
    """ Take list of lists and return single list
    ** TODO: could put logic here to find islands, peaks, etc?
    """

    return [fut for futlist in futlists for fut in futlist]

def lazy_read_segment(st, segment, cfile=None,
                      timeout=vys_timeout_default):
    """ rfpipe read_segment as a dask array.
    equivalent to making delayed version of function and then:
    arr = dask.array.from_delayed(dd, st.datashape, np.complex64).
    """

    shape = st.datashape
    chunks = ((shape[0],), (shape[1],), (shape[2],), (shape[3],))

    name = 'read_segment-' + tokenize([st, segment])
    dask = {(name, 0, 0, 0, 0): (source.read_segment, st, segment,
                                 cfile, timeout)}

    return array.Array(dask=dask, name=name, chunks=chunks, dtype=np.complex64)
<commit_msg>bundling functions to improve memory usage<commit_after>from __future__ import print_function, division, absolute_import#, unicode_literals # not casa compatible
from builtins import bytes, dict, object, range, map, input#, str # not casa compatible
from future.utils import itervalues, viewitems, iteritems, listvalues, listitems
from io import open

import distributed
from dask import array, delayed
from rfpipe import source, search, util, candidates
from dask.base import tokenize
import numpy as np

import logging
logger = logging.getLogger(__name__)
vys_timeout_default = 10

def pipeline_scan(st, segments=None, cl=None, host=None, cfile=None,
                  vys_timeout=vys_timeout_default):
    """ Given rfpipe state and dask distributed client, run search pipline.
    throttle option will submit only if worker state allows for it.
    parameters of throttling:
      read_overhead scales vismem to peak usage during read.
      read_totfrac is total memory usage allowed in bytes at submission time.
    """

    if cl is None:
        if host is None:
            cl = distributed.Client(n_workers=1, threads_per_worker=16,
                                    resources={"READER": 1, "MEMORY": 24,
                                               "CORES": 16},
                                    local_dir="/lustre/evla/test/realfast/scratch")
        else:
            cl = distributed.Client('{0}:{1}'.format(host, '8786'))

    if not isinstance(segments, list):
        segments = range(st.nsegment)

    futures = []
    for segment in segments:
        futures.append(pipeline_seg(st, segment, cl=cl, cfile=cfile,
                                    vys_timeout=vys_timeout))

    return futures  # list of dicts

def pipeline_seg(st, segment, cl, cfile=None,
                 vys_timeout=vys_timeout_default):
    """ Submit pipeline processing of a single segment to scheduler.
    Can use distributed client or compute locally.

    Uses distributed resources parameter to control scheduling of GPUs.
    Pipeline produces jobs per DM/dt.
    Returns a dict with values as futures of certain jobs (data, collection).
    """

    logger.info('Building dask for observation {0}, scan {1}, segment {2}.'
                .format(st.metadata.datasetId, st.metadata.scan, segment))

    futures = {}

# new style read *TODO: note hack on resources*
    data = lazy_read_segment(st, segment, cfile, vys_timeout)
    data = cl.compute(data, resources={tuple(data.__dask_keys__()[0][0][0]):
                                       {'READER': 1}})  # get future
# old style read
#    data = cl.submit(source.read_segment, st, segment, cfile, vys_timeout,
#                     resources={'READER': 1})

    futures['data'] = data  # save future

    # TODO: put this on READER worker?
    data_prep = cl.submit(source.data_prep, st, segment, data,
                          resources={'CORES': st.prefs.nthread},
                          priority=1)

    saved = []
    if st.fftmode == "fftw":
        searchresources = {'MEMORY': 2*st.immem+2*st.vismem,
                           'CORES': st.prefs.nthread}
        imgranges = [[(min(st.get_search_ints(segment, dmind, dtind)),
                     max(st.get_search_ints(segment, dmind, dtind)))
                      for dtind in range(len(st.dtarr))]
                     for dmind in range(len(st.dmarr))]
        wisdom = cl.submit(search.set_wisdom, st.npixx, st.npixy,
                           resources={'CORES': 1})

        for dmind in range(len(st.dmarr)):
            delay = cl.submit(util.calc_delay, st.freq, st.freq.max(),
                              st.dmarr[dmind], st.inttime,
                              resources={'CORES': 1})
            for dtind in range(len(st.dtarr)):
                data_corr = cl.submit(search.dedisperseresample, data_prep,
                                      delay, st.dtarr[dtind],
                                      parallel=st.prefs.nthread > 1,
                                      resources={'MEMORY': 2*st.vismem,
                                                 'CORES': st.prefs.nthread})

                im0, im1 = imgranges[dmind][dtind]
                integrationlist = [list(range(im0, im1)[i:i+st.chunksize])
                                   for i in range(0, im1-im0, st.chunksize)]
                for integrations in integrationlist:
                    saved.append(cl.submit(search.search_thresh_fftw, st,
                                           segment, data_corr, dmind, dtind,
                                           integrations=integrations,
                                           wisdom=wisdom,
                                           resources=searchresources))

    elif st.fftmode == "cuda":
        for dmind in range(len(st.dmarr)):
            saved.append(cl.submit(search.dedisperse_image_cuda, st, segment,
                                   data_prep, dmind,
                                   resources={'GPU': 1,
                                              'CORES': st.prefs.nthread},
                                   priority=2))

    # TODO: put these on dedicated worker to ensure quick processing?
    canddatalist = cl.submit(mergelists, saved,
                             resources={'CORES': 1},
                             priority=3)
    candcollection = cl.submit(candidates.calc_features, canddatalist,
                               resources={'CORES': 1}, priority=4)
    futures['candcollection'] = candcollection

    return futures

def pipeline_scan_delayed(st, segments=None, cl=None, host=None, cfile=None,
                          vys_timeout=vys_timeout_default):
    """ Submit pipeline processing of a single segment to scheduler.
    Uses delayed function and client.compute to schedule.

    Returns a list of dicts with futures of data, collection jobs.
    """

    if cl is None:
        if host is not None:
            cl = distributed.Client('{0}:{1}'.format(host, '8786'))

    if not isinstance(segments, list):
        segments = range(st.nsegment)

    futures = []
    for segment in segments:
        future = {}
        resources = {}

        logger.info('Building dask for observation {0}, scan {1}, segment {2}.'
                    .format(st.metadata.datasetId, st.metadata.scan, segment))

        data = delayed(source.read_segment)(st, segment, cfile, vys_timeout)
        resources[tuple(data.__dask_keys__())] = {'READER': 1}
        if cl is not None:
            future['data'] = cl.compute(data, resources=resources)

        assert st.fftmode == "cuda", "only cuda fftmode supported"
#        data_prep = delayed(source.data_prep)(st, segment, data)
#        canddatalist = delayed(search.dedisperse_image_cuda)(st, segment,
#                                                             data_prep)
#        candcollection = delayed(candidates.calc_features)(canddatalist)
        candcollection = delayed(prep_and_search)(st, segment, data)

        resources[tuple(candcollection.__dask_keys__())] = {'GPU': 1}
        if cl is not None:
            future['candcollection'] = cl.compute(candcollection,
                                                  resources=resources)
            futures.append(future)
        else:
            futures.append(candcollection)

    return futures

def prep_and_search(st, segment, data):
    """ Bundles prep and search functions to improve performance in distributed.
    """

    data_prep = source.data_prep(st, segment, data)
    canddatalist = search.dedisperse_image_cuda(st, segment, data_prep)
    candcollection = candidates.calc_features(canddatalist)

    return candcollection

def mergelists(futlists):
    """ Take list of lists and return single list
    ** TODO: could put logic here to find islands, peaks, etc?
    """

    return [fut for futlist in futlists for fut in futlist]

def lazy_read_segment(st, segment, cfile=None,
                      timeout=vys_timeout_default):
    """ rfpipe read_segment as a dask array.
    equivalent to making delayed version of function and then:
    arr = dask.array.from_delayed(dd, st.datashape, np.complex64).
    """

    shape = st.datashape
    chunks = ((shape[0],), (shape[1],), (shape[2],), (shape[3],))

    name = 'read_segment-' + tokenize([st, segment])
    dask = {(name, 0, 0, 0, 0): (source.read_segment, st, segment,
                                 cfile, timeout)}

    return array.Array(dask=dask, name=name, chunks=chunks, dtype=np.complex64)
<|endoftext|>


<commit_before># vim:nowrap:textwidth=0
#
# Sheared -- non-blocking network programming library for Python
# Copyright (C) 2003  Sune Kirkeby <sune@mel.interspace.dk>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#

import unittest, os, random, signal, time

from sheared import reactor
from sheared import error
from sheared.protocol import http
from sheared.web import server, subserver, virtualhost, resource
from sheared.web.collections.collection import Collection
from sheared.web.server.request import HTTPRequest
from sheared.python import commands

from tests import transport

class FakeRequest(HTTPRequest):
    def __init__(self, requestline='GET / HTTP/1.0', headers='', body=''):
        requestline = http.HTTPRequestLine(requestline)
        headers = http.HTTPHeaders(headers)
        HTTPRequest.__init__(self, requestline, headers, body)

class FakeReply:
    def __init__(self, head_only=0):
        self.headers = http.HTTPHeaders()
        self.sent = ''
        self.status = 200
        self.head_only = head_only

    def setStatusCode(self, code):
        self.status = code

    def send(self, data):
        self.sent = self.sent + data
    def sendfile(self, file):
        self.send(file.read())
    def done(self):
        self.done = 1

class FakeResource(resource.NormalResource):
    def __init__(self, name='foo'):
        resource.NormalResource.__init__(self)
        self.name = name

    def handle(self, request, reply, subpath):
        content = """Welcome to %s!\r\n""" % self.name
        last_mod = http.HTTPDateTime(300229200)

        reply.headers.setHeader('Content-Type', 'text/plain')
        reply.headers.setHeader('Content-Length', len(content))
        reply.headers.setHeader('Last-Modified', last_mod)
        reply.headers.setHeader('ETag', 'abc')

        if not request.head_only:
            if request.headers.has_key('If-None-Match'):
                if request.headers['If-None-Match'] == 'abc':
                    raise error.web.NotModified

            if request.headers.has_key('If-Modified-Since'):
                when = http.HTTPDateTime(request.headers['If-Modified-Since'])
                if not last_mod > when:
                    raise error.web.NotModified

        reply.send(content)
        reply.done()

    def authenticate(self, request, reply):
        pass

class SimpleCollection(Collection):
    def __init__(self, name):
        resource.GettableResource.__init__(self)
        self.name = name
        self.resource = FakeResource(name)

    def getChild(self, request, reply, subpath):
        if subpath == '':
            return self.resource
        elif subpath == 'moved':
            reply.headers.setHeader('Location', '/')
            raise error.web.MovedPermanently
        elif subpath == 'abuse-me':
            reply.headers.setHeader('Foo', 'fubar')
            raise error.web.ForbiddenError, 'Sod off, cretin!'
        elif subpath == 'post':
            return self.resource
        else:
            raise error.web.NotFoundError

def parseReply(reply):
    try:
        headers, body = reply.split('\r\n\r\n', 1)
        status, headers = headers.split('\r\n', 1)
        status = http.HTTPStatusLine(status)
        headers = http.HTTPHeaders(headers)

    except ValueError:
        raise 'Bad reply: %r' % reply

    return status, headers, body
<commit_msg>Add hostname to FakeRequest.<commit_after># vim:nowrap:textwidth=0
#
# Sheared -- non-blocking network programming library for Python
# Copyright (C) 2003  Sune Kirkeby <sune@mel.interspace.dk>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#

import unittest, os, random, signal, time

from sheared import reactor
from sheared import error
from sheared.protocol import http
from sheared.web import server, subserver, virtualhost, resource
from sheared.web.collections.collection import Collection
from sheared.web.server.request import HTTPRequest
from sheared.python import commands

from tests import transport

class FakeRequest(HTTPRequest):
    def __init__(self, requestline='GET / HTTP/1.0', headers='', body=''):
        requestline = http.HTTPRequestLine(requestline)
        headers = http.HTTPHeaders(headers)
        HTTPRequest.__init__(self, requestline, headers, body)
        self.hostname = headers.get('Host', None) or 'foo.com'

class FakeReply:
    def __init__(self, head_only=0):
        self.headers = http.HTTPHeaders()
        self.sent = ''
        self.status = 200
        self.head_only = head_only

    def setStatusCode(self, code):
        self.status = code

    def send(self, data):
        self.sent = self.sent + data
    def sendfile(self, file):
        self.send(file.read())
    def done(self):
        self.done = 1

class FakeResource(resource.NormalResource):
    def __init__(self, name='foo'):
        resource.NormalResource.__init__(self)
        self.name = name

    def handle(self, request, reply, subpath):
        content = """Welcome to %s!\r\n""" % self.name
        last_mod = http.HTTPDateTime(300229200)

        reply.headers.setHeader('Content-Type', 'text/plain')
        reply.headers.setHeader('Content-Length', len(content))
        reply.headers.setHeader('Last-Modified', last_mod)
        reply.headers.setHeader('ETag', 'abc')

        if not request.head_only:
            if request.headers.has_key('If-None-Match'):
                if request.headers['If-None-Match'] == 'abc':
                    raise error.web.NotModified

            if request.headers.has_key('If-Modified-Since'):
                when = http.HTTPDateTime(request.headers['If-Modified-Since'])
                if not last_mod > when:
                    raise error.web.NotModified

        reply.send(content)
        reply.done()

    def authenticate(self, request, reply):
        pass

class SimpleCollection(Collection):
    def __init__(self, name):
        resource.GettableResource.__init__(self)
        self.name = name
        self.resource = FakeResource(name)

    def getChild(self, request, reply, subpath):
        if subpath == '':
            return self.resource
        elif subpath == 'moved':
            reply.headers.setHeader('Location', '/')
            raise error.web.MovedPermanently
        elif subpath == 'abuse-me':
            reply.headers.setHeader('Foo', 'fubar')
            raise error.web.ForbiddenError, 'Sod off, cretin!'
        elif subpath == 'post':
            return self.resource
        else:
            raise error.web.NotFoundError

def parseReply(reply):
    try:
        headers, body = reply.split('\r\n\r\n', 1)
        status, headers = headers.split('\r\n', 1)
        status = http.HTTPStatusLine(status)
        headers = http.HTTPHeaders(headers)

    except ValueError:
        raise 'Bad reply: %r' % reply

    return status, headers, body
<|endoftext|>


<commit_before>#!/usr/bin/python
#
# Copyright (c) 2010 rPath, Inc.
#
# This program is distributed under the terms of the MIT License as found
# in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/mit-license.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the MIT License for full details.
#
import os
import sys

import bootstrap  # pyflakes=ignore

from testrunner import pathManager, testhelp
from testrunner.testhelp import SkipTestException  # pyflakes=ignore

EXCLUDED_PATHS = ['scripts/.*', 'epdb.py', 'stackutil.py', 'test/.*']

def setup():
    pathManager.addExecPath('XOBJ_PATH')
    robjPath = pathManager.addExecPath('ROBJ_PATH')

    pathManager.addResourcePath('TEST_PATH', os.path.join(robjPath, 'test'))
    pathManager.addResourcePath('ROBJ_ARCHIVE_PATH',
        path=os.path.join(robjPath, 'test', 'archive'))

def main(argv=None, individual=True):
    if argv is None:
        argv = list(sys.argv)

    from conary.lib import util
    from conary.lib import coveragehook  # pyflakes=ignore
    sys.excepthook = util.genExcepthook(True, catchSIGUSR1=False)

    from robj.lib import log
    log.setupLogging()

    handlerClass = testhelp.getHandlerClass(testhelp.ConaryTestSuite,
            lambda handler, environ: os.getenv('ROBJ_PATH'),
            lambda handler, environ: EXCLUDED_PATHS)

    handler = handlerClass(individual=individual)
    results = handler.main(argv)
    return results.getExitCode()

class TestCase(testhelp.TestCase):
    def getArchiveContents(self, filename):
        path = os.path.join(self.archivePath, filename)
        return open(path).read()

    def getResponse(self, uri):
        cclass, pathvars = self.server.controllers.get(uri)
        controller = cclass(self.server.data, None, pathvars)
        response = controller.do_GET()
        return response

    def getModel(self, uri):
        response = self.getResponse(uri)
        return response._model

    def getXML(self, uri):
        response = self.getResponse(uri)
        return response.message

    def setUp(self):
        testhelp.TestCase.setUp(self)

        self.archivePath = pathManager.getPath('ROBJ_ARCHIVE_PATH')

        import testserver
        ports = testhelp.findPorts(num=1, closeSockets=True)
        self.server = testserver.ThreadServer(port=ports[0])

    def tearDown(self):
        self.server.shutdown()

if __name__ == '__main__':
    setup()
    sys.exit(main(sys.argv, individual=False))
<commit_msg>only set the debug flag if run with --debug<commit_after>#!/usr/bin/python
#
# Copyright (c) 2010 rPath, Inc.
#
# This program is distributed under the terms of the MIT License as found
# in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/mit-license.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the MIT License for full details.
#
import os
import sys

import bootstrap  # pyflakes=ignore

from testrunner import pathManager, testhelp
from testrunner.testhelp import SkipTestException  # pyflakes=ignore

EXCLUDED_PATHS = ['scripts/.*', 'epdb.py', 'stackutil.py', 'test/.*']

def setup():
    pathManager.addExecPath('XOBJ_PATH')
    robjPath = pathManager.addExecPath('ROBJ_PATH')

    pathManager.addResourcePath('TEST_PATH', os.path.join(robjPath, 'test'))
    pathManager.addResourcePath('ROBJ_ARCHIVE_PATH',
        path=os.path.join(robjPath, 'test', 'archive'))

def main(argv=None, individual=True):
    if argv is None:
        argv = list(sys.argv)

    from conary.lib import util
    from conary.lib import coveragehook  # pyflakes=ignore
    sys.excepthook = util.genExcepthook(True, catchSIGUSR1=False)

    from robj.lib import log
    log.setupLogging()

    handlerClass = testhelp.getHandlerClass(testhelp.ConaryTestSuite,
            lambda handler, environ: os.getenv('ROBJ_PATH'),
            lambda handler, environ: EXCLUDED_PATHS)

    handler = handlerClass(individual=individual)
    results = handler.main(argv)
    return results.getExitCode()

class TestCase(testhelp.TestCase):
    def getArchiveContents(self, filename):
        path = os.path.join(self.archivePath, filename)
        return open(path).read()

    def getResponse(self, uri):
        cclass, pathvars = self.server.controllers.get(uri)
        controller = cclass(self.server.data, None, pathvars)
        response = controller.do_GET()
        return response

    def getModel(self, uri):
        response = self.getResponse(uri)
        return response._model

    def getXML(self, uri):
        response = self.getResponse(uri)
        return response.message

    def run(self, results=None):
        if results is not None:
            self._debug_flag = results.debug
        return testhelp.TestCase.run(self, results)

    def setUp(self):
        testhelp.TestCase.setUp(self)

        self.archivePath = pathManager.getPath('ROBJ_ARCHIVE_PATH')

        import testserver
        ports = testhelp.findPorts(num=1, closeSockets=True)
        self.server = testserver.ThreadServer(port=ports[0],
            debug=self._debug_flag)

    def tearDown(self):
        self.server.shutdown()

if __name__ == '__main__':
    setup()
    sys.exit(main(sys.argv, individual=False))
<|endoftext|>


<commit_before># Copyright 2015 Spotify AB. All rights reserved.
#
# The contents of this file are licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with the
# License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.

class TestNetworkDriver:

    @classmethod
    def tearDownClass(cls):
        cls.device.close()

    def test_loading_config(self):
        self.device.load_replace_candidate(filename='%s/new_good.conf' % self.vendor)
        self.device.commit_config()
        diff = self.device.compare_config()

        # Reverting changes
        self.device.load_replace_candidate(filename='%s/initial.conf' % self.vendor)
        self.device.commit_config()

        self.assertEqual(len(diff), 0)

    def test_loading_config_with_typo(self):
        self.device.load_replace_candidate(filename='%s/new_typo.conf' % self.vendor)
        self.assertRaises(Exception, self.device.commit_config)

    def test_loading_modified_config_and_diff(self):
        self.device.load_replace_candidate(filename='%s/new_good.conf' % self.vendor)
        diff = self.device.compare_config()
        self.assertGreater(len(diff), 0)

    def test_loading_modified_config_replace_config_and_rollback(self):
        self.device.load_replace_candidate(filename='%s/new_good.conf' % self.vendor)
        orig_diff = self.device.compare_config()
        self.device.commit_config()
        replace_config_diff = self.device.compare_config()
        self.device.rollback()
        last_diff = self.device.compare_config()

        result = (orig_diff == last_diff) and ( len(replace_config_diff) == 0 )

        self.assertTrue(result)
<commit_msg>More unittests<commit_after># Copyright 2015 Spotify AB. All rights reserved.
#
# The contents of this file are licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with the
# License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.

from napalm import exceptions

class TestNetworkDriver:

    @classmethod
    def tearDownClass(cls):
        cls.device.close()

    @staticmethod
    def read_file(filename):
        with open(filename, 'r') as f:
            return f.read()

    def test_loading_and_committing_config(self):
        self.device.load_replace_candidate(filename='%s/new_good.conf' % self.vendor)
        self.device.commit_config()

        # The diff should be empty as the configuration has been committed already
        diff = self.device.compare_config()

        # Reverting changes
        self.device.load_replace_candidate(filename='%s/initial.conf' % self.vendor)
        self.device.commit_config()

        self.assertEqual(len(diff), 0)

    def test_loading_config_with_typo(self):
        self.device.load_replace_candidate(filename='%s/new_typo.conf' % self.vendor)
        self.assertRaises(Exception, self.device.commit_config)

    def test_loading_modified_config_and_diff_and_discard(self):
        intended_diff = self.read_file('%s/new_good.diff' % self.vendor)

        self.device.load_replace_candidate(filename='%s/new_good.conf' % self.vendor)
        commit_diff = self.device.compare_config()
        self.device.discard_config()
        discard_diff = self.device.compare_config()

        result = (commit_diff == intended_diff) and (discard_diff == '')
        self.assertTrue(result)

    def test_loading_modified_config_replace_config_and_rollback(self):
        self.device.load_replace_candidate(filename='%s/new_good.conf' % self.vendor)
        orig_diff = self.device.compare_config()
        self.device.commit_config()
        replace_config_diff = self.device.compare_config()
        self.device.rollback()
        last_diff = self.device.compare_config()

        result = (orig_diff == last_diff) and ( len(replace_config_diff) == 0 )

        self.assertTrue(result)

    def test_merge_configuration(self):
        intended_diff = self.read_file('%s/merge_good.diff' % self.vendor)

        self.device.load_merge_candidate(filename='%s/merge_good.conf' % self.vendor)
        self.device.commit_config()

        # Reverting changes
        self.device.load_replace_candidate(filename='%s/initial.conf' % self.vendor)
        diff = self.device.compare_config()
        self.device.commit_config()

        self.assertEqual(diff, intended_diff)

    def test_merge_configuration_typo_and_rollback(self):
        self.device.load_merge_candidate(filename='%s/merge_typo.conf' % self.vendor)
        diff = self.device.compare_config()

        try:
            self.device.commit_config()
            result = False
        except exceptions.MergeConfigException:
            # We load the original config as candidate. If the commit failed cleanly the compare_config should be empty
            self.device.load_replace_candidate(filename='%s/initial.conf' % self.vendor)
            result = (diff > 0) and ( self.device.compare_config() == '')

        self.assertTrue(result)
<|endoftext|>