<commit_before>"""
Copyright (c) 2017 Red Hat, Inc
All rights reserved.

This software may be modified and distributed under the terms
of the BSD license. See the LICENSE file for details.
"""

from __future__ import unicode_literals

import os
import subprocess
import pytest

from atomic_reactor.inner import DockerBuildWorkflow
from atomic_reactor.plugin import PreBuildPluginsRunner
from atomic_reactor.plugins.pre_add_help import AddHelpPlugin
from atomic_reactor.util import ImageName, df_parser
from tests.constants import MOCK_SOURCE
from tests.fixtures import docker_tasker

import atomic_reactor
from tests.test_inner import FakeLogger
from textwrap import dedent
from flexmock import flexmock

class Y(object):
    pass

class X(object):
    image_id = "xxx"
    source = Y()
    source.dockerfile_path = None
    source.path = None
    base_image = ImageName(repo="qwe", tag="asd")

def generate_a_file(destpath, contents):
    with open(destpath, 'w') as f:
        f.write(dedent(contents))

@pytest.mark.parametrize('filename', ['help.md', 'other_file.md'])
def test_add_help_plugin(tmpdir, docker_tasker, filename):
    df_content = dedent("""
        FROM fedora
        RUN yum install -y python-django
        CMD blabla""")
    df = df_parser(str(tmpdir))
    df.content = df_content

    workflow = DockerBuildWorkflow(MOCK_SOURCE, 'test-image')
    workflow.builder = X
    workflow.builder.df_path = df.dockerfile_path
    workflow.builder.df_dir = str(tmpdir)

    help_markdown_path = os.path.join(workflow.builder.df_dir, filename)
    generate_a_file(help_markdown_path, "foo")
    help_man_path = os.path.join(workflow.builder.df_dir, AddHelpPlugin.man_filename)
    generate_a_file(help_man_path, "bar")

    cmd = ['go-md2man', '-in={}'.format(help_markdown_path), '-out={}'.format(help_man_path)]

    def check_cmd(received_cmd, stderr):
        assert received_cmd == cmd
        assert stderr == subprocess.STDOUT

    (flexmock(subprocess)
         .should_receive("check_output")
         .replace_with(check_cmd))

    runner = PreBuildPluginsRunner(
        docker_tasker,
        workflow,
        [{
            'name': AddHelpPlugin.key,
            'args': {'help_file': filename}
        }]
    )
    runner.run()

    assert  df.content == dedent("""
        FROM fedora
        RUN yum install -y python-django
        ADD %s /%s
        CMD blabla""" % (AddHelpPlugin.man_filename, AddHelpPlugin.man_filename))

@pytest.mark.parametrize('filename', ['help.md', 'other_file.md'])
def test_add_help_no_help_file(request, tmpdir, docker_tasker, filename):
    df_content = "FROM fedora"
    df = df_parser(str(tmpdir))
    df.content = df_content

    workflow = DockerBuildWorkflow(MOCK_SOURCE, 'test-image')
    workflow.builder = X
    workflow.builder.df_path = df.dockerfile_path
    workflow.builder.df_dir = str(tmpdir)

    runner = PreBuildPluginsRunner(
        docker_tasker,
        workflow,
        [{
            'name': AddHelpPlugin.key,
            'args': {'help_file': filename}
        }]
    )
    # Runner should not crash if no help.md found
    runner.run()

@pytest.mark.parametrize('filename', ['help.md', 'other_file.md'])
@pytest.mark.parametrize('go_md2man_result', ['binary_missing', 'result_missing', 'fail', 'pass'])
def test_add_help_md2man_error(request, tmpdir, docker_tasker, filename, go_md2man_result):
    df_content = "FROM fedora"
    df = df_parser(str(tmpdir))
    df.content = df_content

    workflow = DockerBuildWorkflow(MOCK_SOURCE, 'test-image')
    workflow.builder = X
    workflow.builder.df_path = df.dockerfile_path
    workflow.builder.df_dir = str(tmpdir)

    help_markdown_path = os.path.join(workflow.builder.df_dir, filename)
    generate_a_file(help_markdown_path, "foo")
    help_man_path = os.path.join(workflow.builder.df_dir, AddHelpPlugin.man_filename)
    if go_md2man_result != 'result_missing':
        generate_a_file(help_man_path, "bar")

    cmd = [u'go-md2man', u'-in={}'.format(help_markdown_path), u'-out={}'.format(help_man_path)]

    def check_cmd_pass(received_cmd, stderr):
        assert received_cmd == cmd
        assert stderr == subprocess.STDOUT

    def check_cmd_binary_missing(received_cmd, stderr):
        check_cmd_pass(received_cmd, stderr)
        raise subprocess.CalledProcessError(returncode=127, cmd=received_cmd)

    def check_cmd_fail(received_cmd, stderr):
        check_cmd_pass(received_cmd, stderr)
        raise subprocess.CalledProcessError(returncode=1, cmd=received_cmd)

    if go_md2man_result == 'binary_missing':
        (flexmock(subprocess)
             .should_receive("check_output")
             .replace_with(check_cmd_binary_missing))
    elif go_md2man_result == 'fail':
        (flexmock(subprocess)
             .should_receive("check_output")
             .replace_with(check_cmd_fail))
    elif go_md2man_result in ['pass', 'result_missing']:
        (flexmock(subprocess)
             .should_receive("check_output")
             .replace_with(check_cmd_pass))

    fake_logger = FakeLogger()
    existing_logger = atomic_reactor.plugin.logger

    def restore_logger():
        atomic_reactor.plugin.logger = existing_logger

    request.addfinalizer(restore_logger)
    atomic_reactor.plugin.logger = fake_logger

    runner = PreBuildPluginsRunner(
        docker_tasker,
        workflow,
        [{
            'name': AddHelpPlugin.key,
            'args': {'help_file': filename}
        }]
    )

    runner.run()

    if go_md2man_result == 'binary_missing':
        error_message = "Help file is available, but go-md2man is not present in a buildroot"
    elif go_md2man_result == 'fail':
        error_message = "CalledProcessError()"
    elif go_md2man_result == 'result_missing':
        error_message = "go-md2man run complete, but man file is not found"

    if go_md2man_result != 'pass':
        # Python 2 prints the error message as "RuntimeError(u'...", but not py3
        # The test checks for correct plugin and error message text in two passes
        assert "plugin 'add_help' raised an exception: RuntimeError" in fake_logger.warnings[-1][0]
        assert error_message in fake_logger.warnings[-1][0]
<commit_msg>Improve unit testing for add_help plugin<commit_after>"""
Copyright (c) 2017 Red Hat, Inc
All rights reserved.

This software may be modified and distributed under the terms
of the BSD license. See the LICENSE file for details.
"""

import os
import subprocess
import pytest

from atomic_reactor.inner import DockerBuildWorkflow
from atomic_reactor.plugin import PreBuildPluginsRunner
from atomic_reactor.plugins.pre_add_help import AddHelpPlugin
from atomic_reactor.util import ImageName, df_parser
from tests.constants import MOCK_SOURCE
from tests.fixtures import docker_tasker

import atomic_reactor
from textwrap import dedent
from flexmock import flexmock

class Y(object):
    pass

class X(object):
    image_id = "xxx"
    source = Y()
    source.dockerfile_path = None
    source.path = None
    base_image = ImageName(repo="qwe", tag="asd")

class MockedPopen(object):
    def __init__(self, *args, **kwargs):
        self.args = []

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        pass

    def poll(self):
        pass

    def communicate(self, input=None, timeout=None):
        return ('', '')

def generate_a_file(destpath, contents):
    with open(destpath, 'w') as f:
        f.write(dedent(contents))

@pytest.mark.parametrize('filename', ['help.md', 'other_file.md'])
def test_add_help_plugin(tmpdir, docker_tasker, filename):
    df_content = dedent("""
        FROM fedora
        RUN yum install -y python-django
        CMD blabla""")
    df = df_parser(str(tmpdir))
    df.content = df_content

    workflow = DockerBuildWorkflow(MOCK_SOURCE, 'test-image')
    workflow.builder = X
    workflow.builder.df_path = df.dockerfile_path
    workflow.builder.df_dir = str(tmpdir)

    help_markdown_path = os.path.join(workflow.builder.df_dir, filename)
    generate_a_file(help_markdown_path, "foo")
    help_man_path = os.path.join(workflow.builder.df_dir, AddHelpPlugin.man_filename)
    generate_a_file(help_man_path, "bar")

    cmd = ['go-md2man', '-in={}'.format(help_markdown_path), '-out={}'.format(help_man_path)]

    def check_popen(*args, **kwargs):
        assert args[0] == cmd
        return MockedPopen()

    (flexmock(subprocess)
         .should_receive("Popen")
         .once()
         .replace_with(check_popen))

    runner = PreBuildPluginsRunner(
        docker_tasker,
        workflow,
        [{
            'name': AddHelpPlugin.key,
            'args': {'help_file': filename}
        }]
    )
    runner.run()

    assert  df.content == dedent("""
        FROM fedora
        RUN yum install -y python-django
        ADD %s /%s
        CMD blabla""" % (AddHelpPlugin.man_filename, AddHelpPlugin.man_filename))

@pytest.mark.parametrize('filename', ['help.md', 'other_file.md'])
def test_add_help_no_help_file(request, tmpdir, docker_tasker, filename):
    df_content = "FROM fedora"
    df = df_parser(str(tmpdir))
    df.content = df_content

    workflow = DockerBuildWorkflow(MOCK_SOURCE, 'test-image')
    workflow.builder = X
    workflow.builder.df_path = df.dockerfile_path
    workflow.builder.df_dir = str(tmpdir)

    runner = PreBuildPluginsRunner(
        docker_tasker,
        workflow,
        [{
            'name': AddHelpPlugin.key,
            'args': {'help_file': filename}
        }]
    )
    # Runner should not crash if no help.md found
    runner.run()

@pytest.mark.parametrize('filename', ['help.md', 'other_file.md'])
@pytest.mark.parametrize('go_md2man_result', ['binary_missing', 'result_missing', 'fail', 'pass'])
def test_add_help_md2man_error(request, tmpdir, docker_tasker, filename,
                               go_md2man_result, caplog):
    df_content = "FROM fedora"
    df = df_parser(str(tmpdir))
    df.content = df_content

    workflow = DockerBuildWorkflow(MOCK_SOURCE, 'test-image')
    workflow.builder = X
    workflow.builder.df_path = df.dockerfile_path
    workflow.builder.df_dir = str(tmpdir)

    help_markdown_path = os.path.join(workflow.builder.df_dir, filename)
    generate_a_file(help_markdown_path, "foo")
    help_man_path = os.path.join(workflow.builder.df_dir, AddHelpPlugin.man_filename)
    if go_md2man_result != 'result_missing':
        generate_a_file(help_man_path, "bar")

    cmd = ['go-md2man',
           '-in={}'.format(help_markdown_path),
           '-out={}'.format(help_man_path)]

    def check_popen_pass(*args, **kwargs):
        assert args[0] == cmd
        return MockedPopen()

    def check_popen_binary_missing(*args, **kwargs):
        check_popen_pass(*args, **kwargs)
        raise OSError(2, "No such file or directory")

    def check_popen_fail(*args, **kwargs):
        check_popen_pass(*args, **kwargs)
        raise subprocess.CalledProcessError(returncode=1, cmd=args[0])

    if go_md2man_result == 'binary_missing':
        (flexmock(subprocess)
             .should_receive("Popen")
             .once()
             .replace_with(check_popen_binary_missing))
    elif go_md2man_result == 'fail':
        (flexmock(subprocess)
             .should_receive("Popen")
             .once()
             .replace_with(check_popen_fail))
    elif go_md2man_result in ['pass', 'result_missing']:
        (flexmock(subprocess)
             .should_receive("Popen")
             .once()
             .replace_with(check_popen_pass))

    runner = PreBuildPluginsRunner(
        docker_tasker,
        workflow,
        [{
            'name': AddHelpPlugin.key,
            'args': {'help_file': filename}
        }]
    )

    runner.run()

    if go_md2man_result == 'binary_missing':
        error_message = "Help file is available, but go-md2man is not present in a buildroot"
    elif go_md2man_result == 'fail':
        error_message = "CalledProcessError()"
    elif go_md2man_result == 'result_missing':
        error_message = "go-md2man run complete, but man file is not found"

    if go_md2man_result != 'pass':
        # Python 2 prints the error message as "RuntimeError(u'...", but not py3
        # The test checks for correct plugin and error message text in two passes
        assert "plugin 'add_help' raised an exception: RuntimeError" in caplog.text()
        assert error_message in caplog.text()
<|endoftext|>


<commit_before>from dxr.testing import DxrInstanceTestCase

from nose.tools import eq_, ok_

class BasicTests(DxrInstanceTestCase):
    """Tests for functionality that isn't specific to particular filters"""

    def test_text(self):
        """Assert that a plain text search works."""
        self.found_files_eq('main', ['main.c', 'makefile'])

    def test_and(self):
        """Finding 2 words should find only the lines that contain both."""
        self.found_line_eq(
            'main int',
            'int main(int argc, char* argv[]){',
            4)

    def test_case_sensitive(self):
        """Make sure case-sensitive searching is case-sensitive.

        This tests trilite's substr-extents query type.

        """
        self.found_files_eq('really',
                            ['README.mkd'],
                            is_case_sensitive=True)
        self.found_nothing('REALLY',
                           is_case_sensitive=True)

    def test_case_insensitive(self):
        """Test case-insensitive free-text searching without extents.

        Also test negation of text queries.

        This tests trilite's isubstr query type.

        """
        results = self.search_results(
            'path:makefile -CODE', is_case_sensitive=False)
        eq_(results,
            [{"path": "makefile",
              "lines": [
                {"line_number": 3,
                  "line": "$(CXX) -o $@ $^"},
                {"line_number": 4,
                  "line": "clean:"}],
              "icon": "mimetypes/unknown"}])

    def test_case_insensitive_extents(self):
        """Test case-insensitive free-text searching with extents.

        This tests trilite's isubstr-extents query type.

        """
        self.found_files_eq('MAIN',
                            ['main.c', 'makefile'],
                            is_case_sensitive=False)

    def test_index(self):
        """Make sure the index controller redirects."""
        response = self.client().get('/')
        eq_(response.status_code, 302)
        ok_(response.headers['Location'].endswith('/code/source/'))

    def test_file_based_search(self):
        """Make sure searches that return files and not lines work.

        Specifically, test behavior when SearchFilter.has_lines is False.

        """
        eq_(self.search_results('path:makefile'),
            [{"path": "makefile",
              "lines": [],
              "icon": "mimetypes/unknown"}])
<commit_msg>Add a test for ANDing a structural with a text filter.<commit_after>from dxr.testing import DxrInstanceTestCase

from nose.tools import eq_, ok_

class BasicTests(DxrInstanceTestCase):
    """Tests for functionality that isn't specific to particular filters"""

    def test_text(self):
        """Assert that a plain text search works."""
        self.found_files_eq('main', ['main.c', 'makefile'])

    def test_and(self):
        """Finding 2 words should find only the lines that contain both."""
        self.found_line_eq(
            'main int',
            'int main(int argc, char* argv[]){',
            4)

    def test_structural_and(self):
        """Try ANDing a structural with a text filter."""
        self.found_line_eq(
            'function:main int',
            'int main(int argc, char* argv[]){',
            4)

    def test_case_sensitive(self):
        """Make sure case-sensitive searching is case-sensitive.

        This tests trilite's substr-extents query type.

        """
        self.found_files_eq('really',
                            ['README.mkd'],
                            is_case_sensitive=True)
        self.found_nothing('REALLY',
                           is_case_sensitive=True)

    def test_case_insensitive(self):
        """Test case-insensitive free-text searching without extents.

        Also test negation of text queries.

        This tests trilite's isubstr query type.

        """
        results = self.search_results(
            'path:makefile -CODE', is_case_sensitive=False)
        eq_(results,
            [{"path": "makefile",
              "lines": [
                {"line_number": 3,
                  "line": "$(CXX) -o $@ $^"},
                {"line_number": 4,
                  "line": "clean:"}],
              "icon": "mimetypes/unknown"}])

    def test_case_insensitive_extents(self):
        """Test case-insensitive free-text searching with extents.

        This tests trilite's isubstr-extents query type.

        """
        self.found_files_eq('MAIN',
                            ['main.c', 'makefile'],
                            is_case_sensitive=False)

    def test_index(self):
        """Make sure the index controller redirects."""
        response = self.client().get('/')
        eq_(response.status_code, 302)
        ok_(response.headers['Location'].endswith('/code/source/'))

    def test_file_based_search(self):
        """Make sure searches that return files and not lines work.

        Specifically, test behavior when SearchFilter.has_lines is False.

        """
        eq_(self.search_results('path:makefile'),
            [{"path": "makefile",
              "lines": [],
              "icon": "mimetypes/unknown"}])
<|endoftext|>


<commit_before>from os import path
from parcels import (
    FieldSet, ParticleSet, ScipyParticle, JITParticle, ErrorCode, KernelError,
    OutOfBoundsError, AdvectionRK4
)
import numpy as np
import pytest

ptype = {'scipy': ScipyParticle, 'jit': JITParticle}

def DoNothing(particle, fieldset, time):
    return ErrorCode.Success

def fieldset(xdim=20, ydim=20):
    """ Standard unit mesh fieldset """
    lon = np.linspace(0., 1., xdim, dtype=np.float32)
    lat = np.linspace(0., 1., ydim, dtype=np.float32)
    U, V = np.meshgrid(lat, lon)
    data = {'U': np.array(U, dtype=np.float32), 'V': np.array(V, dtype=np.float32)}
    dimensions = {'lat': lat, 'lon': lon}
    return FieldSet.from_data(data, dimensions, mesh='flat')

@pytest.fixture(name="fieldset")
def fieldset_fixture(xdim=20, ydim=20):
    return fieldset(xdim=xdim, ydim=ydim)

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
@pytest.mark.parametrize('start, end, substeps, dt', [
    (0., 10., 1, 1.),
    (0., 10., 4, 1.),
    (0., 10., 1, 3.),
    (2., 16., 5, 3.),
    (20., 10., 4, -1.),
    (20., -10., 7, -2.),
])
def test_execution_endtime(fieldset, mode, start, end, substeps, dt, npart=10):
    pset = ParticleSet(fieldset, pclass=ptype[mode], time=start,
                       lon=np.linspace(0, 1, npart),
                       lat=np.linspace(1, 0, npart))
    pset.execute(DoNothing, endtime=end, dt=dt)
    assert np.allclose(pset.time, end)

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
@pytest.mark.parametrize('start, end, substeps, dt', [
    (0., 10., 1, 1.),
    (0., 10., 4, 1.),
    (0., 10., 1, 3.),
    (2., 16., 5, 3.),
    (20., 10., 4, -1.),
    (20., -10., 7, -2.),
])
def test_execution_runtime(fieldset, mode, start, end, substeps, dt, npart=10):
    pset = ParticleSet(fieldset, pclass=ptype[mode], time=start,
                       lon=np.linspace(0, 1, npart),
                       lat=np.linspace(1, 0, npart))
    t_step = abs(end - start) / substeps
    for _ in range(substeps):
        pset.execute(DoNothing, runtime=t_step, dt=dt)
    assert np.allclose(pset.time, end)

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
@pytest.mark.parametrize('time', [0., 1])
@pytest.mark.parametrize('dt', [0., 1])
def test_pset_execute_dt_0(fieldset, mode, time, dt, npart=2):
    def SetLat(particle, fieldset, time):
        particle.lat = .6
    lon = np.linspace(0, 1, npart)
    lat = np.linspace(1, 0, npart)

    pset = ParticleSet(fieldset, pclass=ptype[mode], lon=lon, lat=lat)
    pset.execute(SetLat, endtime=time, dt=dt)
    assert np.allclose(pset.lon, lon)
    assert np.allclose(pset.lat, [.6])
    assert np.allclose(pset.time, min([time, dt]))

    pset = ParticleSet(fieldset, pclass=ptype[mode], lon=lon, lat=lat)
    pset.execute(SetLat, runtime=time, dt=dt)
    assert np.allclose(pset.lon, lon)
    assert np.allclose(pset.lat, [.6])
    assert np.allclose(pset.time, min([time, dt]))

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
def test_execution_fail_timed(fieldset, mode, npart=10):
    def TimedFail(particle, fieldset, time):
        if particle.time >= 10.:
            return ErrorCode.Error
        else:
            return ErrorCode.Success

    pset = ParticleSet(fieldset, pclass=ptype[mode],
                       lon=np.linspace(0, 1, npart),
                       lat=np.linspace(1, 0, npart))
    error_thrown = False
    try:
        pset.execute(TimedFail, endtime=20., dt=2.)
    except KernelError:
        error_thrown = True
    assert error_thrown
    assert len(pset) == npart
    assert np.allclose(pset.time, 10.)

@pytest.mark.parametrize('mode', ['scipy'])
def test_execution_fail_python_exception(fieldset, mode, npart=10):
    def PythonFail(particle, fieldset, time):
        if particle.time >= 10.:
            raise RuntimeError("Enough is enough!")
        else:
            return ErrorCode.Success

    pset = ParticleSet(fieldset, pclass=ptype[mode],
                       lon=np.linspace(0, 1, npart),
                       lat=np.linspace(1, 0, npart))
    error_thrown = False
    try:
        pset.execute(PythonFail, endtime=20., dt=2.)
    except KernelError:
        error_thrown = True
    assert error_thrown
    assert len(pset) == npart
    assert np.allclose(pset.time, 10.)

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
def test_execution_fail_out_of_bounds(fieldset, mode, npart=10):
    def MoveRight(particle, fieldset, time):
        fieldset.U[time, particle.depth, particle.lat, particle.lon + 0.1]
        particle.lon += 0.1

    pset = ParticleSet(fieldset, pclass=ptype[mode],
                       lon=np.linspace(0, 1, npart),
                       lat=np.linspace(1, 0, npart))
    error_thrown = False
    try:
        pset.execute(MoveRight, endtime=10., dt=1.)
    except OutOfBoundsError:
        error_thrown = True
    assert error_thrown
    assert len(pset) == npart
    assert (pset.lon - 1. > -1.e12).all()

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
def test_execution_recover_out_of_bounds(fieldset, mode, npart=2):
    def MoveRight(particle, fieldset, time):
        fieldset.U[time, particle.depth, particle.lat, particle.lon + 0.1]
        particle.lon += 0.1

    def MoveLeft(particle, fieldset, time):
        particle.lon -= 1.

    lon = np.linspace(0.05, 0.95, npart)
    lat = np.linspace(1, 0, npart)
    pset = ParticleSet(fieldset, pclass=ptype[mode], lon=lon, lat=lat)
    pset.execute(MoveRight, endtime=10., dt=1.,
                 recovery={ErrorCode.ErrorOutOfBounds: MoveLeft})
    assert len(pset) == npart
    assert np.allclose(pset.lon, lon, rtol=1e-5)
    assert np.allclose(pset.lat, lat, rtol=1e-5)

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
def test_execution_delete_out_of_bounds(fieldset, mode, npart=10):
    def MoveRight(particle, fieldset, time):
        fieldset.U[time, particle.depth, particle.lat, particle.lon + 0.1]
        particle.lon += 0.1

    def DeleteMe(particle, fieldset, time):
        particle.delete()

    lon = np.linspace(0.05, 0.95, npart)
    lat = np.linspace(1, 0, npart)
    pset = ParticleSet(fieldset, pclass=ptype[mode], lon=lon, lat=lat)
    pset.execute(MoveRight, endtime=10., dt=1.,
                 recovery={ErrorCode.ErrorOutOfBounds: DeleteMe})
    assert len(pset) == 0

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
def test_kernel_add_no_new_variables(fieldset, mode):
    def MoveEast(particle, fieldset, time):
        particle.lon += 0.1

    def MoveNorth(particle, fieldset, time):
        particle.lat += 0.1

    pset = ParticleSet(fieldset, pclass=ptype[mode], lon=[0.5], lat=[0.5])
    pset.execute(pset.Kernel(MoveEast) + pset.Kernel(MoveNorth),
                 endtime=1., dt=1.)
    assert np.allclose(pset.lon, 0.6, rtol=1e-5)
    assert np.allclose(pset.lat, 0.6, rtol=1e-5)

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
def test_multi_kernel_duplicate_varnames(fieldset, mode):
    # Testing for merging of two Kernels with the same variable declared
    # Should throw a warning, but go ahead regardless
    def MoveEast(particle, fieldset, time):
        add_lon = 0.1
        particle.lon += add_lon

    def MoveWest(particle, fieldset, time):
        add_lon = -0.3
        particle.lon += add_lon

    pset = ParticleSet(fieldset, pclass=ptype[mode], lon=[0.5], lat=[0.5])
    pset.execute(pset.Kernel(MoveEast) + pset.Kernel(MoveWest),
                 endtime=1., dt=1.)
    assert np.allclose(pset.lon, 0.3, rtol=1e-5)

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
def test_multi_kernel_reuse_varnames(fieldset, mode):
    # Testing for merging of two Kernels with the same variable declared
    # Should throw a warning, but go ahead regardless
    def MoveEast1(particle, fieldset, time):
        add_lon = 0.2
        particle.lon += add_lon

    def MoveEast2(particle, fieldset, time):
        particle.lon += add_lon  # NOQA - no flake8 testing of this line

    pset = ParticleSet(fieldset, pclass=ptype[mode], lon=[0.5], lat=[0.5])
    pset.execute(pset.Kernel(MoveEast1) + pset.Kernel(MoveEast2),
                 endtime=1., dt=1.)
    assert np.allclose(pset.lon, [0.9], rtol=1e-5)  # should be 0.5 + 0.2 + 0.2 = 0.9

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
def test_update_kernel_in_script(fieldset, mode):
    # Testing what happens when kernels are updated during runtime of a script
    # Should throw a warning, but go ahead regardless
    def MoveEast(particle, fieldset, time):
        add_lon = 0.1
        particle.lon += add_lon

    def MoveWest(particle, fieldset, time):
        add_lon = -0.3
        particle.lon += add_lon

    pset = ParticleSet(fieldset, pclass=ptype[mode], lon=[0.5], lat=[0.5])
    pset.execute(pset.Kernel(MoveEast), endtime=1., dt=1.)
    pset.execute(pset.Kernel(MoveWest), endtime=2., dt=1.)
    assert np.allclose(pset.lon, 0.3, rtol=1e-5)  # should be 0.5 + 0.1 - 0.3 = 0.3

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
def test_errorcode_repeat(fieldset, mode):
    def simpleKernel(particle, fieldset, time):
        if particle.lon > .1 and time < 1.:
            # if particle.lon is not re-setted before kernel repetition, it will break here
            return ErrorCode.Error
        particle.lon += 0.1
        if particle.dt > 1.49:
            # dt is used to leave the repetition loop (dt is the only variable not re-setted)
            return ErrorCode.Success
        particle.dt += .1
        return ErrorCode.Repeat

    pset = ParticleSet(fieldset, pclass=ptype[mode], lon=[0.], lat=[0.])
    pset.execute(pset.Kernel(simpleKernel), endtime=3., dt=1.)

@pytest.mark.parametrize('delete_cfiles', [True, False])
def test_execution_keep_cfiles_and_nocompilation_warnings(fieldset, delete_cfiles):
    pset = ParticleSet(fieldset, pclass=JITParticle, lon=[0.], lat=[0.])
    pset.execute(pset.Kernel(AdvectionRK4, delete_cfiles=delete_cfiles), endtime=1., dt=1.)
    cfile = pset.kernel.src_file
    logfile = pset.kernel.log_file
    del pset.kernel
    if delete_cfiles:
        assert not path.exists(cfile)
    else:
        assert path.exists(cfile)
        with open(logfile) as f:
            assert 'warning' not in f.read(), 'Compilation WARNING in log file'
<commit_msg>not testing for compiler warnings on windows<commit_after>from os import path
from parcels import (
    FieldSet, ParticleSet, ScipyParticle, JITParticle, ErrorCode, KernelError,
    OutOfBoundsError, AdvectionRK4
)
import numpy as np
import pytest
import sys

ptype = {'scipy': ScipyParticle, 'jit': JITParticle}

def DoNothing(particle, fieldset, time):
    return ErrorCode.Success

def fieldset(xdim=20, ydim=20):
    """ Standard unit mesh fieldset """
    lon = np.linspace(0., 1., xdim, dtype=np.float32)
    lat = np.linspace(0., 1., ydim, dtype=np.float32)
    U, V = np.meshgrid(lat, lon)
    data = {'U': np.array(U, dtype=np.float32), 'V': np.array(V, dtype=np.float32)}
    dimensions = {'lat': lat, 'lon': lon}
    return FieldSet.from_data(data, dimensions, mesh='flat')

@pytest.fixture(name="fieldset")
def fieldset_fixture(xdim=20, ydim=20):
    return fieldset(xdim=xdim, ydim=ydim)

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
@pytest.mark.parametrize('start, end, substeps, dt', [
    (0., 10., 1, 1.),
    (0., 10., 4, 1.),
    (0., 10., 1, 3.),
    (2., 16., 5, 3.),
    (20., 10., 4, -1.),
    (20., -10., 7, -2.),
])
def test_execution_endtime(fieldset, mode, start, end, substeps, dt, npart=10):
    pset = ParticleSet(fieldset, pclass=ptype[mode], time=start,
                       lon=np.linspace(0, 1, npart),
                       lat=np.linspace(1, 0, npart))
    pset.execute(DoNothing, endtime=end, dt=dt)
    assert np.allclose(pset.time, end)

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
@pytest.mark.parametrize('start, end, substeps, dt', [
    (0., 10., 1, 1.),
    (0., 10., 4, 1.),
    (0., 10., 1, 3.),
    (2., 16., 5, 3.),
    (20., 10., 4, -1.),
    (20., -10., 7, -2.),
])
def test_execution_runtime(fieldset, mode, start, end, substeps, dt, npart=10):
    pset = ParticleSet(fieldset, pclass=ptype[mode], time=start,
                       lon=np.linspace(0, 1, npart),
                       lat=np.linspace(1, 0, npart))
    t_step = abs(end - start) / substeps
    for _ in range(substeps):
        pset.execute(DoNothing, runtime=t_step, dt=dt)
    assert np.allclose(pset.time, end)

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
@pytest.mark.parametrize('time', [0., 1])
@pytest.mark.parametrize('dt', [0., 1])
def test_pset_execute_dt_0(fieldset, mode, time, dt, npart=2):
    def SetLat(particle, fieldset, time):
        particle.lat = .6
    lon = np.linspace(0, 1, npart)
    lat = np.linspace(1, 0, npart)

    pset = ParticleSet(fieldset, pclass=ptype[mode], lon=lon, lat=lat)
    pset.execute(SetLat, endtime=time, dt=dt)
    assert np.allclose(pset.lon, lon)
    assert np.allclose(pset.lat, [.6])
    assert np.allclose(pset.time, min([time, dt]))

    pset = ParticleSet(fieldset, pclass=ptype[mode], lon=lon, lat=lat)
    pset.execute(SetLat, runtime=time, dt=dt)
    assert np.allclose(pset.lon, lon)
    assert np.allclose(pset.lat, [.6])
    assert np.allclose(pset.time, min([time, dt]))

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
def test_execution_fail_timed(fieldset, mode, npart=10):
    def TimedFail(particle, fieldset, time):
        if particle.time >= 10.:
            return ErrorCode.Error
        else:
            return ErrorCode.Success

    pset = ParticleSet(fieldset, pclass=ptype[mode],
                       lon=np.linspace(0, 1, npart),
                       lat=np.linspace(1, 0, npart))
    error_thrown = False
    try:
        pset.execute(TimedFail, endtime=20., dt=2.)
    except KernelError:
        error_thrown = True
    assert error_thrown
    assert len(pset) == npart
    assert np.allclose(pset.time, 10.)

@pytest.mark.parametrize('mode', ['scipy'])
def test_execution_fail_python_exception(fieldset, mode, npart=10):
    def PythonFail(particle, fieldset, time):
        if particle.time >= 10.:
            raise RuntimeError("Enough is enough!")
        else:
            return ErrorCode.Success

    pset = ParticleSet(fieldset, pclass=ptype[mode],
                       lon=np.linspace(0, 1, npart),
                       lat=np.linspace(1, 0, npart))
    error_thrown = False
    try:
        pset.execute(PythonFail, endtime=20., dt=2.)
    except KernelError:
        error_thrown = True
    assert error_thrown
    assert len(pset) == npart
    assert np.allclose(pset.time, 10.)

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
def test_execution_fail_out_of_bounds(fieldset, mode, npart=10):
    def MoveRight(particle, fieldset, time):
        fieldset.U[time, particle.depth, particle.lat, particle.lon + 0.1]
        particle.lon += 0.1

    pset = ParticleSet(fieldset, pclass=ptype[mode],
                       lon=np.linspace(0, 1, npart),
                       lat=np.linspace(1, 0, npart))
    error_thrown = False
    try:
        pset.execute(MoveRight, endtime=10., dt=1.)
    except OutOfBoundsError:
        error_thrown = True
    assert error_thrown
    assert len(pset) == npart
    assert (pset.lon - 1. > -1.e12).all()

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
def test_execution_recover_out_of_bounds(fieldset, mode, npart=2):
    def MoveRight(particle, fieldset, time):
        fieldset.U[time, particle.depth, particle.lat, particle.lon + 0.1]
        particle.lon += 0.1

    def MoveLeft(particle, fieldset, time):
        particle.lon -= 1.

    lon = np.linspace(0.05, 0.95, npart)
    lat = np.linspace(1, 0, npart)
    pset = ParticleSet(fieldset, pclass=ptype[mode], lon=lon, lat=lat)
    pset.execute(MoveRight, endtime=10., dt=1.,
                 recovery={ErrorCode.ErrorOutOfBounds: MoveLeft})
    assert len(pset) == npart
    assert np.allclose(pset.lon, lon, rtol=1e-5)
    assert np.allclose(pset.lat, lat, rtol=1e-5)

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
def test_execution_delete_out_of_bounds(fieldset, mode, npart=10):
    def MoveRight(particle, fieldset, time):
        fieldset.U[time, particle.depth, particle.lat, particle.lon + 0.1]
        particle.lon += 0.1

    def DeleteMe(particle, fieldset, time):
        particle.delete()

    lon = np.linspace(0.05, 0.95, npart)
    lat = np.linspace(1, 0, npart)
    pset = ParticleSet(fieldset, pclass=ptype[mode], lon=lon, lat=lat)
    pset.execute(MoveRight, endtime=10., dt=1.,
                 recovery={ErrorCode.ErrorOutOfBounds: DeleteMe})
    assert len(pset) == 0

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
def test_kernel_add_no_new_variables(fieldset, mode):
    def MoveEast(particle, fieldset, time):
        particle.lon += 0.1

    def MoveNorth(particle, fieldset, time):
        particle.lat += 0.1

    pset = ParticleSet(fieldset, pclass=ptype[mode], lon=[0.5], lat=[0.5])
    pset.execute(pset.Kernel(MoveEast) + pset.Kernel(MoveNorth),
                 endtime=1., dt=1.)
    assert np.allclose(pset.lon, 0.6, rtol=1e-5)
    assert np.allclose(pset.lat, 0.6, rtol=1e-5)

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
def test_multi_kernel_duplicate_varnames(fieldset, mode):
    # Testing for merging of two Kernels with the same variable declared
    # Should throw a warning, but go ahead regardless
    def MoveEast(particle, fieldset, time):
        add_lon = 0.1
        particle.lon += add_lon

    def MoveWest(particle, fieldset, time):
        add_lon = -0.3
        particle.lon += add_lon

    pset = ParticleSet(fieldset, pclass=ptype[mode], lon=[0.5], lat=[0.5])
    pset.execute(pset.Kernel(MoveEast) + pset.Kernel(MoveWest),
                 endtime=1., dt=1.)
    assert np.allclose(pset.lon, 0.3, rtol=1e-5)

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
def test_multi_kernel_reuse_varnames(fieldset, mode):
    # Testing for merging of two Kernels with the same variable declared
    # Should throw a warning, but go ahead regardless
    def MoveEast1(particle, fieldset, time):
        add_lon = 0.2
        particle.lon += add_lon

    def MoveEast2(particle, fieldset, time):
        particle.lon += add_lon  # NOQA - no flake8 testing of this line

    pset = ParticleSet(fieldset, pclass=ptype[mode], lon=[0.5], lat=[0.5])
    pset.execute(pset.Kernel(MoveEast1) + pset.Kernel(MoveEast2),
                 endtime=1., dt=1.)
    assert np.allclose(pset.lon, [0.9], rtol=1e-5)  # should be 0.5 + 0.2 + 0.2 = 0.9

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
def test_update_kernel_in_script(fieldset, mode):
    # Testing what happens when kernels are updated during runtime of a script
    # Should throw a warning, but go ahead regardless
    def MoveEast(particle, fieldset, time):
        add_lon = 0.1
        particle.lon += add_lon

    def MoveWest(particle, fieldset, time):
        add_lon = -0.3
        particle.lon += add_lon

    pset = ParticleSet(fieldset, pclass=ptype[mode], lon=[0.5], lat=[0.5])
    pset.execute(pset.Kernel(MoveEast), endtime=1., dt=1.)
    pset.execute(pset.Kernel(MoveWest), endtime=2., dt=1.)
    assert np.allclose(pset.lon, 0.3, rtol=1e-5)  # should be 0.5 + 0.1 - 0.3 = 0.3

@pytest.mark.parametrize('mode', ['scipy', 'jit'])
def test_errorcode_repeat(fieldset, mode):
    def simpleKernel(particle, fieldset, time):
        if particle.lon > .1 and time < 1.:
            # if particle.lon is not re-setted before kernel repetition, it will break here
            return ErrorCode.Error
        particle.lon += 0.1
        if particle.dt > 1.49:
            # dt is used to leave the repetition loop (dt is the only variable not re-setted)
            return ErrorCode.Success
        particle.dt += .1
        return ErrorCode.Repeat

    pset = ParticleSet(fieldset, pclass=ptype[mode], lon=[0.], lat=[0.])
    pset.execute(pset.Kernel(simpleKernel), endtime=3., dt=1.)

@pytest.mark.parametrize('delete_cfiles', [True, False])
@pytest.mark.skipif(sys.platform.startswith("win"), reason="skipping windows test as windows compiler generates warning")
def test_execution_keep_cfiles_and_nocompilation_warnings(fieldset, delete_cfiles):
    pset = ParticleSet(fieldset, pclass=JITParticle, lon=[0.], lat=[0.])
    pset.execute(pset.Kernel(AdvectionRK4, delete_cfiles=delete_cfiles), endtime=1., dt=1.)
    cfile = pset.kernel.src_file
    logfile = pset.kernel.log_file
    del pset.kernel
    if delete_cfiles:
        assert not path.exists(cfile)
    else:
        assert path.exists(cfile)
        with open(logfile) as f:
            assert 'warning' not in f.read(), 'Compilation WARNING in log file'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2010-2011, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3
# only, as published by the Free Software Foundation.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License version 3 for more details
# (a copy is included in the LICENSE file that accompanied this code).
#
# You should have received a copy of the GNU Lesser General Public License
# version 3 along with OpenQuake.  If not, see
# <http://www.gnu.org/licenses/lgpl-3.0.txt> for a copy of the LGPLv3 License.

"""
Unit tests for the tools/oqbugs.py tool.
"""

import unittest
from tools.oqbugs import (CommitsOutput, filter_reviewers, filter_bugs,
        launchpad_lookup, fix_committed)
import mock
import os
from argparse import Namespace
import datetime

CACHE_DIR = os.path.expanduser("~/.launchpadlib-test/cache/")

def prepare_mock(bugs):
    """
        prepares mock objects for tests
    """

    mock_name = mock.Mock()
    mock_name.name = 'Funky Monk'
    attributes = {
     'date_fix_committed': str(datetime.datetime.now()),
     'assignee': mock_name,
     'web_link': 'http://openquake.org'}

    bugs_list = {}
    # prepares instances of mocked launchpad's bug objects
    for bug in bugs:
        magic_mock = mock.MagicMock()

        for prop, val in attributes.iteritems():
            magic_mock.bug_tasks[0].status = 'In Progress'
            magic_mock.title = 'a bug title'
            setattr(magic_mock.bug_tasks[0], prop, val)
            # prepares a dictionary for mocking launchpad_instance.bugs
            # lookup
            bugs_list.update({bug: magic_mock})
    return bugs_list

class OqBugsTestCase(unittest.TestCase):
    """Tests the mechanics of oqbugs ."""

    def setUp(self):
        self.bugs = {}

        def getitem(name):
            """ utility methods for returning dict data from a mock """
            return self.bugs[name]

        self.correct_commits = ['[r=favalex][f=827908]',
                '[r=mbarbon][f=*827366]',
                '[r=favalex][f=819689]', '[r=mbarbon] [f=827256]',
                '[r=larsbutler][f=807360]', '[r=al-maisan] [f=812698]',
                '[r=al-maisan][f=802396]', '[r=favalex][f=817081]',
                '[r=larsbutler][f=809217]', '[r=larsbutler] [f=82464]']

        self.skip_commits = ['[r=mbarbon][f=*827366]']
        self.multiple_bugs_commit = ['[r=mbarbon][f=827366,812698,809217]']
        self.wrong_lines = ['lorem', 'loren', 'sofia']

        self.launchpad = mock.MagicMock()
        self.launchpad.bugs.__getitem__ = mock.Mock(side_effect=getitem)

    def tearDown(self):
        pass

    def test_commits_output_is_empty(self):
        self.assertEquals(CommitsOutput.since('0 week'), [])

    # one month to be sure to have commit lines
    def test_commits_output_for_one_month(self):
        self.assertTrue(len(CommitsOutput.since('1 month')))

    def test_commits_output_reviewers(self):
        for commit in self.correct_commits:
            self.assertTrue(len(filter_reviewers(commit)))

    def test_filter_bugs_multiple(self):
        self.assertEquals(
                len(list(filter_bugs(self.multiple_bugs_commit[0]))), 3)

    def test_filter_bugs_skip(self):
        for commit in self.skip_commits:
            self.assertEquals(len(list(filter_bugs(commit))), 0)

    def test_filter_bugs_wrong_format(self):
        for bug in self.wrong_lines:
            self.assertEquals(filter_bugs(bug), [])

    def test_launchpad_instances(self):
        bugs_list = filter_bugs(self.multiple_bugs_commit[0])
        self.bugs = prepare_mock(bugs_list)
        self.assertEquals(len(launchpad_lookup(self.launchpad,
                self.bugs.keys())), 3)

    def test_launchpad_attributes(self):
        bugs_list = filter_bugs(self.correct_commits[0])
        self.bugs = prepare_mock(bugs_list)
        for bug_instance in launchpad_lookup(self.launchpad, self.bugs.keys()):
            self.assertTrue(isinstance(bug_instance.title, str))
            self.assertTrue(isinstance(
                bug_instance.bug_tasks[0].date_fix_committed, str))
            self.assertTrue(isinstance(
                bug_instance.bug_tasks[0].assignee.name, str))
            self.assertTrue(isinstance(
                bug_instance.bug_tasks[0].web_link, str))

    def test_commits_output_bugs(self):
        for commit in self.correct_commits:
            # preparing bugs for mocker
            self.bugs = prepare_mock(commit)
            for bug in launchpad_lookup(self.launchpad, self.bugs):
                self.assertTrue(isinstance(bug, mock.Mock))

    def test_fix_committed_mark(self):
        for commit in self.correct_commits:
            # preparing bugs for mocker
            self.bugs = prepare_mock(filter_bugs(commit))
            #tasks = launchpad_lookup(self.launchpad, self.bugs)

            # gets the FixCommitted reference by calling the fix_committed
            # method with args
            tasks = fix_committed(self.launchpad, commit)

            namespace = Namespace(time='1 week')

            # creates an argparse action instance and __call__[s] it passing
            # the namespace object
            bugs = tasks(None, None)(None, namespace,
                    None)

            for bug in bugs:
                self.assertEqual("Fix Committed", bug.bug_tasks[0].status)
<commit_msg>more meaninful test name<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2010-2011, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3
# only, as published by the Free Software Foundation.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License version 3 for more details
# (a copy is included in the LICENSE file that accompanied this code).
#
# You should have received a copy of the GNU Lesser General Public License
# version 3 along with OpenQuake.  If not, see
# <http://www.gnu.org/licenses/lgpl-3.0.txt> for a copy of the LGPLv3 License.

"""
Unit tests for the tools/oqbugs.py tool.
"""

import unittest
from tools.oqbugs import (CommitsOutput, filter_reviewers, filter_bugs,
        launchpad_lookup, fix_committed)
import mock
import os
from argparse import Namespace
import datetime

CACHE_DIR = os.path.expanduser("~/.launchpadlib-test/cache/")

def prepare_mock(bugs):
    """
        prepares mock objects for tests
    """

    mock_name = mock.Mock()
    mock_name.name = 'Funky Monk'
    attributes = {
     'date_fix_committed': str(datetime.datetime.now()),
     'assignee': mock_name,
     'web_link': 'http://openquake.org'}

    bugs_list = {}
    # prepares instances of mocked launchpad's bug objects
    for bug in bugs:
        magic_mock = mock.MagicMock()

        for prop, val in attributes.iteritems():
            magic_mock.bug_tasks[0].status = 'In Progress'
            magic_mock.title = 'a bug title'
            setattr(magic_mock.bug_tasks[0], prop, val)
            # prepares a dictionary for mocking launchpad_instance.bugs
            # lookup
            bugs_list.update({bug: magic_mock})
    return bugs_list

class OqBugsTestCase(unittest.TestCase):
    """Tests the mechanics of oqbugs ."""

    def setUp(self):
        self.bugs = {}

        def getitem(name):
            """ utility methods for returning dict data from a mock """
            return self.bugs[name]

        self.correct_commits = ['[r=favalex][f=827908]',
                '[r=mbarbon][f=*827366]',
                '[r=favalex][f=819689]', '[r=mbarbon] [f=827256]',
                '[r=larsbutler][f=807360]', '[r=al-maisan] [f=812698]',
                '[r=al-maisan][f=802396]', '[r=favalex][f=817081]',
                '[r=larsbutler][f=809217]', '[r=larsbutler] [f=82464]']

        self.skip_commits = ['[r=mbarbon][f=*827366]']
        self.multiple_bugs_commit = ['[r=mbarbon][f=827366,812698,809217]']
        self.wrong_lines = ['lorem', 'loren', 'sofia']

        self.launchpad = mock.MagicMock()
        self.launchpad.bugs.__getitem__ = mock.Mock(side_effect=getitem)

    def tearDown(self):
        pass

    def test_commits_output_is_empty(self):
        self.assertEquals(CommitsOutput.since('0 week'), [])

    # one month to be sure to have commit lines
    def test_commits_output_for_one_month(self):
        self.assertTrue(len(CommitsOutput.since('1 month')))

    def test_commits_output_reviewers(self):
        for commit in self.correct_commits:
            self.assertTrue(len(filter_reviewers(commit)))

    def test_filter_bugs_multiple(self):
        self.assertEquals(
                len(list(filter_bugs(self.multiple_bugs_commit[0]))), 3)

    def test_filter_bugs_skip(self):
        for commit in self.skip_commits:
            self.assertEquals(len(list(filter_bugs(commit))), 0)

    def test_filter_bugs_wrong_format(self):
        for bug in self.wrong_lines:
            self.assertEquals(filter_bugs(bug), [])

    def test_launchpad_instances(self):
        bugs_list = filter_bugs(self.multiple_bugs_commit[0])
        self.bugs = prepare_mock(bugs_list)
        self.assertEquals(len(launchpad_lookup(self.launchpad,
                self.bugs.keys())), 3)

    def test_launchpad_attributes(self):
        bugs_list = filter_bugs(self.correct_commits[0])
        self.bugs = prepare_mock(bugs_list)
        for bug_instance in launchpad_lookup(self.launchpad, self.bugs.keys()):
            self.assertTrue(isinstance(bug_instance.title, str))
            self.assertTrue(isinstance(
                bug_instance.bug_tasks[0].date_fix_committed, str))
            self.assertTrue(isinstance(
                bug_instance.bug_tasks[0].assignee.name, str))
            self.assertTrue(isinstance(
                bug_instance.bug_tasks[0].web_link, str))

    def test_commits_output_bugs(self):
        for commit in self.correct_commits:
            # preparing bugs for mocker
            self.bugs = prepare_mock(commit)
            for bug in launchpad_lookup(self.launchpad, self.bugs):
                self.assertTrue(isinstance(bug, mock.Mock))

    def test_fix_committed_status(self):
        for commit in self.correct_commits:
            # preparing bugs for mocker
            self.bugs = prepare_mock(filter_bugs(commit))
            #tasks = launchpad_lookup(self.launchpad, self.bugs)

            # gets the FixCommitted reference by calling the fix_committed
            # method with args
            tasks = fix_committed(self.launchpad, commit)

            namespace = Namespace(time='1 week')

            # creates an argparse action instance and __call__[s] it passing
            # the namespace object
            bugs = tasks(None, None)(None, namespace,
                    None)

            for bug in bugs:
                self.assertEqual("Fix Committed", bug.bug_tasks[0].status)
<|endoftext|>


<commit_before>import os
from time import sleep

import fabric
from entrypoint2 import entrypoint
from path import Path

import vagrant

# from fabric.api import env, execute, task, run, sudo, settings

# pip3 install fabric vncdotool python-vagrant entrypoint2

DIR = Path(__file__).parent.parent.parent

class Options:
    halt = True
    recreate = True
    destroy = False

def wrapcmd(cmd, guiproc):
    if guiproc:
        # copy env vars from graphical session
        cmd = f"sudo cat /proc/`pidof {guiproc}`/environ | tr '\\0' '\\n' > /tmp/x;export $(cat /tmp/x); {cmd}"
    print(f"command: {cmd}")
    return cmd

def run_box(options, vagrantfile, cmds, guiproc):
    env = os.environ
    env["VAGRANT_VAGRANTFILE"] = DIR / vagrantfile
    env["VAGRANT_DOTFILE_PATH"] = DIR / ".vagrant_" + vagrantfile
    v = vagrant.Vagrant(env=env, quiet_stdout=False, quiet_stderr=False)
    status = v.status()
    state = status[0].state
    print(status)

    if options.destroy:
        v.destroy()
        return

    if options.halt:
        v.halt()  # avoid screensaver

    if state == "not_created":
        # install programs in box
        v.up()
        # restart box
        v.halt()

    try:
        v.up()

        with fabric.Connection(
            v.user_hostname_port(), connect_kwargs={"key_filename": v.keyfile(),},
        ) as conn:
            with conn.cd("c:/vagrant" if options.win else "/vagrant"):
                if not options.win:
                    cmds = ["env | sort"] + cmds
                if guiproc:
                    pid = None
                    while not pid:
                        print(f"waiting for {guiproc}")
                        cmd = f"bash --login -c 'pidof {guiproc} || true'"
                        print(cmd)
                        pid = conn.run(cmd).stdout
                        sleep(1)
                    print(f"{guiproc} pid={pid}")
                sleep(1)
                for cmd in cmds:
                    if options.recreate:
                        if "tox" in cmd:
                            cmd += " -r"
                    conn.run(wrapcmd(cmd, guiproc))
    finally:
        if options.halt:
            v.halt()

config = {
    "server": ("Vagrantfile", ["tox"], "",),
    "lubuntu.18.04": (
        "Vagrantfile.lubuntu.18.04.rb",
        ["tox -e py27-desktop", "tox -e py3-desktop"],
        "lxsession",
    ),
    # TODO: add ubuntu 20.04 variants when it becomes stable
    # "lubuntu.20.04": (
    #     "Vagrantfile.lubuntu.20.04.rb",
    #     ["tox -e py27-desktop", "tox -e py3-desktop"],
    #     "lxsession",
    # ),
    "xubuntu.18.04": (
        "Vagrantfile.xubuntu.18.04.rb",
        ["tox -e py27-desktop", "tox -e py3-desktop"],
        "xfdesktop",
    ),
    # "xubuntu.20.04": (
    #     "Vagrantfile.xubuntu.20.04.rb",
    #     ["tox -e py27-desktop", "tox -e py3-desktop"],
    #     "xfdesktop",
    # ),
    "kubuntu.18.04": (
        "Vagrantfile.kubuntu.18.04.rb",
        ["tox -e py27-desktop", "tox -e py3-desktop"],
        "gnome-shell",
    ),
    # "kubuntu.20.04": (
    #     "Vagrantfile.kubuntu.20.04.rb",
    #     ["tox -e py27-desktop", "tox -e py3-desktop"],
    #     "gnome-shell",
    # ),
    "ubuntu.18.04": (
        "Vagrantfile.ubuntu.18.04.rb",
        ["tox -e py3-desktop"],
        "gnome-shell",
    ),
    "ubuntu.20.04": (
        "Vagrantfile.ubuntu.20.04.rb",
        ["tox -e py3-desktop"],
        "gnome-shell",
    ),
    "arch.kde.x11": (
        "Vagrantfile.arch.kde.x11.rb",
        ["tox -e py3-desktop"],
        "plasma_session",
    ),
    "arch.kde.wayland": (
        "Vagrantfile.arch.kde.wayland.rb",
        ["tox -e py3-desktop"],
        "plasma_session",
        # "Xwayland",
    ),
    "arch.gnome.wayland": (
        "Vagrantfile.arch.gnome.wayland.rb",
        ["tox -e py3-desktop"],
        "gnome-shell-calendar-server",
        # "Xwayland",
    ),
    # "win": ("Vagrantfile.win.rb", ["tox -e py3-win"], "",),
    "osx": (
        "Vagrantfile.osx.rb",
        ["bash --login -c 'python3 -m tox -e py3-osx'"],
        "Dock",
    ),
}

@entrypoint
def main(boxes="all", fast=False, destroy=False):
    options = Options()
    options.halt = not fast
    options.recreate = not fast
    options.destroy = destroy

    if boxes == "all":
        boxes = list(config.keys())
    else:
        boxes = boxes.split(",")
    for k, v in config.items():
        if k in boxes:
            options.win = k == "win"
            print("-----> %s %s %s" % (k, v[0], v[1]))
            run_box(options, v[0], v[1], v[2])
<commit_msg>test: vagrant: free -h<commit_after>import os
from time import sleep

import fabric
from entrypoint2 import entrypoint
from path import Path

import vagrant

# from fabric.api import env, execute, task, run, sudo, settings

# pip3 install fabric vncdotool python-vagrant entrypoint2

DIR = Path(__file__).parent.parent.parent

class Options:
    halt = True
    recreate = True
    destroy = False

def wrapcmd(cmd, guiproc):
    if guiproc:
        # copy env vars from graphical session
        cmd = f"sudo cat /proc/`pidof {guiproc}`/environ | tr '\\0' '\\n' > /tmp/x;export $(cat /tmp/x); {cmd}"
    print(f"command: {cmd}")
    return cmd

def run_box(options, vagrantfile, cmds, guiproc):
    env = os.environ
    env["VAGRANT_VAGRANTFILE"] = DIR / vagrantfile
    env["VAGRANT_DOTFILE_PATH"] = DIR / ".vagrant_" + vagrantfile
    v = vagrant.Vagrant(env=env, quiet_stdout=False, quiet_stderr=False)
    status = v.status()
    state = status[0].state
    print(status)

    if options.destroy:
        v.destroy()
        return

    if options.halt:
        v.halt()  # avoid screensaver

    if state == "not_created":
        # install programs in box
        v.up()
        # restart box
        v.halt()

    try:
        v.up()

        with fabric.Connection(
            v.user_hostname_port(), connect_kwargs={"key_filename": v.keyfile(),},
        ) as conn:
            with conn.cd("c:/vagrant" if options.win else "/vagrant"):
                if not options.win:
                    cmds = ["free -h", "env | sort"] + cmds + ["free -h"]
                if guiproc:
                    pid = None
                    while not pid:
                        print(f"waiting for {guiproc}")
                        cmd = f"bash --login -c 'pidof {guiproc} || true'"
                        print(cmd)
                        pid = conn.run(cmd).stdout
                        sleep(1)
                    print(f"{guiproc} pid={pid}")
                sleep(1)
                for cmd in cmds:
                    if options.recreate:
                        if "tox" in cmd:
                            cmd += " -r"
                    conn.run(wrapcmd(cmd, guiproc))
    finally:
        if options.halt:
            v.halt()

config = {
    "server": ("Vagrantfile", ["tox"], "",),
    "lubuntu.18.04": (
        "Vagrantfile.lubuntu.18.04.rb",
        ["tox -e py27-desktop", "tox -e py3-desktop"],
        "lxsession",
    ),
    # TODO: add ubuntu 20.04 variants when it becomes stable
    # "lubuntu.20.04": (
    #     "Vagrantfile.lubuntu.20.04.rb",
    #     ["tox -e py27-desktop", "tox -e py3-desktop"],
    #     "lxsession",
    # ),
    "xubuntu.18.04": (
        "Vagrantfile.xubuntu.18.04.rb",
        ["tox -e py27-desktop", "tox -e py3-desktop"],
        "xfdesktop",
    ),
    # "xubuntu.20.04": (
    #     "Vagrantfile.xubuntu.20.04.rb",
    #     ["tox -e py27-desktop", "tox -e py3-desktop"],
    #     "xfdesktop",
    # ),
    "kubuntu.18.04": (
        "Vagrantfile.kubuntu.18.04.rb",
        ["tox -e py27-desktop", "tox -e py3-desktop"],
        "gnome-shell",
    ),
    # "kubuntu.20.04": (
    #     "Vagrantfile.kubuntu.20.04.rb",
    #     ["tox -e py27-desktop", "tox -e py3-desktop"],
    #     "gnome-shell",
    # ),
    "ubuntu.18.04": (
        "Vagrantfile.ubuntu.18.04.rb",
        ["tox -e py3-desktop"],
        "gnome-shell",
    ),
    "ubuntu.20.04": (
        "Vagrantfile.ubuntu.20.04.rb",
        ["tox -e py3-desktop"],
        "gnome-shell",
    ),
    "arch.kde.x11": (
        "Vagrantfile.arch.kde.x11.rb",
        ["tox -e py3-desktop"],
        "plasma_session",
    ),
    "arch.kde.wayland": (
        "Vagrantfile.arch.kde.wayland.rb",
        ["tox -e py3-desktop"],
        "plasma_session",
        # "Xwayland",
    ),
    "arch.gnome.wayland": (
        "Vagrantfile.arch.gnome.wayland.rb",
        ["tox -e py3-desktop"],
        "gnome-shell-calendar-server",
        # "Xwayland",
    ),
    # "win": ("Vagrantfile.win.rb", ["tox -e py3-win"], "",),
    "osx": (
        "Vagrantfile.osx.rb",
        ["bash --login -c 'python3 -m tox -e py3-osx'"],
        "Dock",
    ),
}

@entrypoint
def main(boxes="all", fast=False, destroy=False):
    options = Options()
    options.halt = not fast
    options.recreate = not fast
    options.destroy = destroy

    if boxes == "all":
        boxes = list(config.keys())
    else:
        boxes = boxes.split(",")
    for k, v in config.items():
        if k in boxes:
            options.win = k == "win"
            print("-----> %s %s %s" % (k, v[0], v[1]))
            run_box(options, v[0], v[1], v[2])
<|endoftext|>


<commit_before>#!/usr/bin/env python

'''Test that the window caption can be set.

Expected behaviour:
    Two windows will be opened, one with the caption "Window caption 1"
    counting up every second; the other with a Unicode string including
    some non-ASCII characters.

    Press escape or close either window to finished the test.
'''

__docformat__ = 'restructuredtext'
__version__ = '$Id: $'

import time
import unittest
from pyglet import window

class WINDOW_CAPTION(unittest.TestCase):
    def test_caption(self):
        w1 = window.Window(200, 200)
        w2 = window.Window(200, 200)
        count = 1
        w1.set_caption('Window caption %d' % count)
        w2.set_caption(u'\u00bfHabla espa\u00f1ol?')
        last_time = time.time()
        while not (w1.has_exit or w2.has_exit):
            if time.time() - last_time > 1:
                count += 1
                w1.set_caption('Window caption %d' % count)
                last_time = time.time()
            w1.dispatch_events()
            w2.dispatch_events()
        w1.close()
        w2.close()

if __name__ == '__main__':
    unittest.main()
<commit_msg>Make windows bigger in this test so the captions can be read.<commit_after>#!/usr/bin/env python

'''Test that the window caption can be set.

Expected behaviour:
    Two windows will be opened, one with the caption "Window caption 1"
    counting up every second; the other with a Unicode string including
    some non-ASCII characters.

    Press escape or close either window to finished the test.
'''

__docformat__ = 'restructuredtext'
__version__ = '$Id: $'

import time
import unittest
from pyglet import window

class WINDOW_CAPTION(unittest.TestCase):
    def test_caption(self):
        w1 = window.Window(400, 200, resizable=True)
        w2 = window.Window(400, 200, resizable=True)
        count = 1
        w1.set_caption('Window caption %d' % count)
        w2.set_caption(u'\u00bfHabla espa\u00f1ol?')
        last_time = time.time()
        while not (w1.has_exit or w2.has_exit):
            if time.time() - last_time > 1:
                count += 1
                w1.set_caption('Window caption %d' % count)
                last_time = time.time()
            w1.dispatch_events()
            w2.dispatch_events()
        w1.close()
        w2.close()

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import plugintypes
import json
import subprocess
import os

from urllib.parse import urlparse

"""
Installable plugins are currently hardcoded here for early
development of the installation process. They will be centralized
online at a later date.

Plugin data will be downloaded in JSON format and inserted to a sqlite
database for local cache.
"""

PLUGIN_LINKS={ "Whiskey": "https://github.com/xlopo/tg-pybot-whiskey" }
PLUGIN_REPOS_DIR="plugins.repos"
PLUGINS_DIR="plugins"
GIT_BIN="/usr/bin/git"

class PluginsPlugin(plugintypes.TelegramPlugin):
    """
    Plugin to manage other plugins. Enable, disable or reload.
    """
    patterns = [
        "^!plugins$",
        "^!plugins? (enable) ([\w_.-]+)$",
        "^!plugins? (disable) ([\w_.-]+)$",
        "^!plugins? (install) ([\w_.-]+)$",
        # "^!plugins? (enable) ([\w_.-]+) (chat)",
        # "^!plugins? (disable) ([\w_.-]+) (chat)",
        "^!plugins? (reload)$"
    ]

    usage = [
        "!plugins: list all plugins.",
        "!plugins enable [plugin]: enable plugin.",
        "!plugins disable [plugin]: disable plugin.",
        # "!plugins disable [plugin] chat: disable plugin only this chat.",
        "!plugins reload: reloads all plugins."
    ]

    def activate_plugin(self):
        if not os.path.exists(PLUGIN_REPOS_DIR):
            os.makedirs(PLUGIN_REPOS_DIR)
        self.plugin_manager.updatePluginPlaces([PLUGIN_REPOS_DIR])

    def run(self, msg, matches):
        if matches.group(0) == "!plugins":
            return self.list_plugins()

        if matches.group(1) == "enable":
            return self.enable_plugin(matches)

        if matches.group(1) == "disable":
            return self.disable_plugin(matches)

        if matches.group(1) == "install":
            return self.install_plugin(matches)

        if matches.group(1) == "reload":
            return self.reload_plugins()

    def enable_plugin(self, matches):

        if self.plugin_manager.activatePluginByName(matches.group(2)):
            return "Plugin {} enabled".format(matches.group(2))
        else:
            return "Error loading plugin {}".format(matches.group(2))

    def disable_plugin(self, matches):
        if self.plugin_manager.activatePluginByName(matches.group(2)):
            return "Plugin {} disabled".format(matches.group(2))
        else:
            return "Error disabling plugin {}".format(matches.group(2))

    def __clone_repository(self, url):
            args = [GIT_BIN, "clone", url]
            p = subprocess.Popen(args, cwd=PLUGIN_REPOS_DIR)
            return p.wait()

    def install_plugin(self, matches):
        plugin = matches.group(2)
        urldata = urlparse(matches.group(2))

        url = None
        if urldata.scheme in [ "http", "https" ]:
            url = plugin
        elif plugin in PLUGIN_LINKS.keys():
            url = PLUGIN_LINKS[plugin]

        if not url:
            return "Invalid plugin or url: {}".format(plugin)

        if not self.__clone_repository(url):
            return "Error installing plugin: {}".format(plugin)

        self.reload_plugins()

        return "Successfully installed plugin: {}".format(plugin)

    def reload_plugins(self):
        self.plugin_manager.collectPlugins()
        return "Plugins reloaded"

    def list_plugins(self):
        text = ""
        for plugin in self.plugin_manager.getAllPlugins():
            text += "{0}: ({1})\n".format(plugin.name,
                                          "(Enabled)" if plugin.plugin_object.is_activated else "(Disabled)")

        return text
<commit_msg>Fixing success check on repo pull<commit_after>import plugintypes
import json
import subprocess
import os

from urllib.parse import urlparse

"""
Installable plugins are currently hardcoded here for early
development of the installation process. They will be centralized
online at a later date.

Plugin data will be downloaded in JSON format and inserted to a sqlite
database for local cache.
"""

PLUGIN_LINKS={ "Whiskey": "https://github.com/xlopo/tg-pybot-whiskey" }
PLUGIN_REPOS_DIR="plugins.repos"
PLUGINS_DIR="plugins"
GIT_BIN="/usr/bin/git"

class PluginsPlugin(plugintypes.TelegramPlugin):
    """
    Plugin to manage other plugins. Enable, disable or reload.
    """
    patterns = [
        "^!plugins$",
        "^!plugins? (enable) ([\w_.-]+)$",
        "^!plugins? (disable) ([\w_.-]+)$",
        "^!plugins? (install) (.*)$",
        # "^!plugins? (enable) ([\w_.-]+) (chat)",
        # "^!plugins? (disable) ([\w_.-]+) (chat)",
        "^!plugins? (reload)$"
    ]

    usage = [
        "!plugins: list all plugins.",
        "!plugins enable [plugin]: enable plugin.",
        "!plugins disable [plugin]: disable plugin.",
        # "!plugins disable [plugin] chat: disable plugin only this chat.",
        "!plugins reload: reloads all plugins."
    ]

    def activate_plugin(self):
        if not os.path.exists(PLUGIN_REPOS_DIR):
            os.makedirs(PLUGIN_REPOS_DIR)
        self.plugin_manager.updatePluginPlaces([PLUGIN_REPOS_DIR])

    def run(self, msg, matches):
        if matches.group(0) == "!plugins":
            return self.list_plugins()

        if matches.group(1) == "enable":
            return self.enable_plugin(matches)

        if matches.group(1) == "disable":
            return self.disable_plugin(matches)

        if matches.group(1) == "install":
            return self.install_plugin(matches)

        if matches.group(1) == "reload":
            return self.reload_plugins()

    def enable_plugin(self, matches):

        if self.plugin_manager.activatePluginByName(matches.group(2)):
            return "Plugin {} enabled".format(matches.group(2))
        else:
            return "Error loading plugin {}".format(matches.group(2))

    def disable_plugin(self, matches):
        if self.plugin_manager.activatePluginByName(matches.group(2)):
            return "Plugin {} disabled".format(matches.group(2))
        else:
            return "Error disabling plugin {}".format(matches.group(2))

    def __clone_repository(self, url):
            args = [GIT_BIN, "clone", url]
            p = subprocess.Popen(args, cwd=PLUGIN_REPOS_DIR)
            return p.wait()

    def install_plugin(self, matches):
        plugin = matches.group(2)
        urldata = urlparse(matches.group(2))

        url = None
        if urldata.scheme in [ "http", "https" ]:
            url = plugin
        elif plugin in PLUGIN_LINKS.keys():
            url = PLUGIN_LINKS[plugin]

        if not url:
            return "Invalid plugin or url: {}".format(plugin)

        if self.__clone_repository(url) != 0:
            return "Error installing plugin: {}".format(plugin)

        self.reload_plugins()

        return "Successfully installed plugin: {}".format(plugin)

    def reload_plugins(self):
        self.plugin_manager.collectPlugins()
        return "Plugins reloaded"

    def list_plugins(self):
        text = ""
        for plugin in self.plugin_manager.getAllPlugins():
            text += "{0}: ({1})\n".format(plugin.name,
                                          "(Enabled)" if plugin.plugin_object.is_activated else "(Disabled)")

        return text
<|endoftext|>


<commit_before>import plugintypes
import sys

sys.path.append('./plugins')

from py3owm import OpenWeatherMap

class WeatherPlugin(plugintypes.TelegramPlugin):
    """
    Weather
    """

    patterns = [
        "^!weather? ([0-9]{5})"
    ]

    usage = [
        "!weather <zip code>|<city>"
    ]

    def activate_plugin(self):
        if not self.has_option("api key"):
            self.write_option("api key", "")
        if not self.has_option("units"):
            self.write_option("units", "metric")

        api_key = self.read_option("api key")
        units = self.read_option("units")

        self.owm = OpenWeatherMap(api_key, units)

    def run(self, msg, matches):
        w = self.owm.weather_data(zipcode=matches.group(1))
        if w:
            report = "{} ({}) {}{}\n{}".format(w.name, w.country, w.temp, w.unit_symbol, w.description)
            return report
        return "Error getting weather data"

<commit_msg>adding weather emoji map<commit_after>import plugintypes
import sys

sys.path.append('./plugins')

from py3owm import OpenWeatherMap

OWM_icon_emoji_map = {
    "01d": u'\U0001F31E',
    "02d": u'\U000026C5',
    "03d": u'\U00002601',
    "03d": u'\U00002601',
    "09d": u'\U0001F302',
    "10d": u'\U00002614',
    "11d": u'\U000026A1',
    "13d": u'\U00002744',
    "50d": u'\U0001F301'
}

class WeatherPlugin(plugintypes.TelegramPlugin):
    """
    Weather
    """

    patterns = [
        "^!weather? ([0-9]{5})"
    ]

    usage = [
        "!weather <zip code>"
    ]

    def activate_plugin(self):
        if not self.has_option("api key"):
            self.write_option("api key", "")
        if not self.has_option("units"):
            self.write_option("units", "metric")

        api_key = self.read_option("api key")
        units = self.read_option("units")

        self.owm = OpenWeatherMap(api_key, units)

    def run(self, msg, matches):
        w = self.owm.weather_data(zipcode=matches.group(1))
        if w:
            report = "{} ({}) {}{}\n{}".format(w.name, w.country, w.temp, w.unit_symbol, w.description)
            return report
        return "Error getting weather data"

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Self-Generating Prototypes
"""

# Author: Dayvid Victor <victor.dvro@gmail.com>
#
# License: BSD 3 clause

import numpy as np
from scipy import sparse as sp

from sklearn.externals.six.moves import xrange
from sklearn.utils.validation import check_arrays, atleast2d_or_csr
from sklearn.neighbors.classification import KNeighborsClassifier
from sklearn.decomposition import PCA

from ..base import InstanceReductionMixin

class _Group(object):

    def __init__(self, X, label):
        self.X = X
        self.label = label
        if len(X) > 0:
            self.update_all()

    def __add__(self, other):
        X = np.vstack((self.X, other.X))
        return _Group(X, self.label, update=True)

    def __len__(self):
        length = self.X.shape[0] if self.X != None else 0
        return length

    def add_instances(self, X, update=False):
        self.X = np.vstack((self.X,X))
        if update:
            self.update_all()

    def remove_instances(self, indexes, update=False):
        _X = self.X[indexes]
        self.X = np.delete(self.X, indexes, axis=0)
        if update:
            self.update_all()
        return _X

    def update_all(self):
        self.rep_x = np.mean(self.X, axis=0)

class SGP(InstanceReductionMixin):
    """Self-Generating Prototypes

    The Self-Generating Prototypes generates instances is a centroid-based
    prototype generation algorithm that uses the space spliting mechanism
    to generate prototypes in the center of each cluster.

    Parameters
    ----------
    r_min: float, optional (default = 0.0)
        Determine the minimum size of a cluster [0.00, 0.20]

    r_mis: float, optional (default = 0.0)
        Determine the error tolerance before split a group

    Attributes
    ----------
    `X_` : array-like, shape = [indeterminated, n_features]
        Selected prototypes.

    `y_` : array-like, shape = [indeterminated]
        Labels of the selected prototypes.

    `reduction_` : float, percentual of reduction.

    Examples
    --------
    >>> from protopy.generation.sgp import SGP
    >>> import numpy as np
    >>> X = np.array([[i] for i in range(1,13)])
    >>> X = X + np.asarray([0.1,0,-0.1,0.1,0,-0.1,0.1,-0.1,0.1,-0.1,0.1,-0.1])
    >>> y = np.array([1, 1, 1, 2, 2, 2, 1, 1, 2, 2, 1, 1])
    >>> sgp = SGP()
    >>> sgp.fit(X, y)
    SGP(r_min=0.0, r_mis=0.0)
    >>> print sgp.predict(X)
    [1 1 1 2 2 2 1 1 2 2 1 1]
    >>> print sgp.reduction_
    0.5

    See also
    --------
    protopy.generation.sgp.SGP2: self-generating prototypes 2

    References
    ----------
    Hatem A. Fayed, Sherif R Hashem, and Amir F Atiya. Self-generating prototypes
    for pattern classification. Pattern Recognition, 40(5):1498–1509, 2007.
    """

    def __init__(self, r_min=0.0, r_mis=0.0):
        self.groups = None
        self.r_min = r_min
        self.r_mis = r_mis
        self.n_neighbors = 1
        self.classifier = None
        self.groups = None

    def reduce_data(self, X, y):
        X, y = check_arrays(X, y, sparse_format="csr")

        if self.classifier == None:
            self.classifier = KNeighborsClassifier(n_neighbors=self.n_neighbors)
        if self.classifier.n_neighbors != self.n_neighbors:
            self.classifier.n_neighbors = self.n_neighbors

        classes = np.unique(y)
        self.classes_ = classes

        # loading inicial groups
        self.groups = []
        for label in classes:
            mask = y == label
            self.groups = self.groups + [_Group(X[mask], label)]

        self.__main_loop()
        self.__generalization_step()

        self.X_ = np.asarray([g.rep_x for g in self.groups])
        self.y_ = np.asarray([g.label for g in self.groups])
        self.reduction_ = 1.0 - float(len(self.y_))/len(y)

        return self.X_, self.y_

    def __main_loop(self):
        exit_count = 0
        knn = KNeighborsClassifier(n_neighbors = 1, algorithm='brute')
        while exit_count < len(self.groups):
            index, exit_count = 0, 0
            while index < len(self.groups):

                group = self.groups[index]
                reps_x = np.asarray([g.rep_x for g in self.groups])
                reps_y = np.asarray([g.label for g in self.groups])
                knn.fit(reps_x, reps_y)

                nn_idx = knn.kneighbors(group.X, n_neighbors=1, return_distance=False)
                nn_idx = nn_idx.T[0]
                mask = nn_idx == index

                # if all are correctly classified
                if not (False in mask):
                    exit_count = exit_count + 1

                # if all are misclasified
                elif not (group.label in reps_y[nn_idx]):
                    pca = PCA(n_components=1)
                    pca.fit(group.X)
                    # maybe use a 'for' instead of creating array
                    d = pca.transform(reps_x[index])
                    dis = [pca.transform(inst)[0] for inst in group.X]
                    mask_split = (dis < d).flatten()

                    new_X = group.X[mask_split]
                    self.groups.append(_Group(new_X, group.label))
                    group.X = group.X[~mask_split]

                elif (reps_y[nn_idx] == group.label).all() and (nn_idx != index).any():
                    mask_mv = nn_idx != index
                    index_mv = np.asarray(range(len(group)))[mask_mv]
                    X_mv = group.remove_instances(index_mv)
                    G_mv = nn_idx[mask_mv]

                    for x, g in zip(X_mv, G_mv):
                        self.groups[g].add_instances([x])

                elif (reps_y[nn_idx] != group.label).sum()/float(len(group)) > self.r_mis:
                    mask_mv = reps_y[nn_idx] != group.label
                    new_X = group.X[mask_mv]
                    self.groups.append(_Group(new_X, group.label))
                    group.X = group.X[~mask_mv]
                else:
                   exit_count = exit_count + 1

                if len(group) == 0:
                    self.groups.remove(group)
                else:
                    index = index + 1

                for g in self.groups:
                    g.update_all()

        return self.groups

    def __generalization_step(self):
        larger = max([len(g) for g in self.groups])
        for group in self.groups:
            if len(group) < self.r_min * larger:
                self.groups.remove(group)
        return self.groups

<commit_msg>SGP2 created<commit_after># -*- coding: utf-8 -*-
"""
Self-Generating Prototypes
"""

# Author: Dayvid Victor <victor.dvro@gmail.com>
#
# License: BSD 3 clause

import numpy as np
from scipy import sparse as sp

from sklearn.externals.six.moves import xrange
from sklearn.utils.validation import check_arrays, atleast2d_or_csr
from sklearn.neighbors.classification import KNeighborsClassifier
from sklearn.decomposition import PCA

from ..base import InstanceReductionMixin

class _Group(object):

    def __init__(self, X, label):
        self.X = X
        self.label = label
        if len(X) > 0:
            self.update_all()

    def __add__(self, other):
        X = np.vstack((self.X, other.X))
        return _Group(X, self.label, update=True)

    def __len__(self):
        length = self.X.shape[0] if self.X != None else 0
        return length

    def add_instances(self, X, update=False):
        self.X = np.vstack((self.X,X))
        if update:
            self.update_all()

    def remove_instances(self, indexes, update=False):
        _X = self.X[indexes]
        self.X = np.delete(self.X, indexes, axis=0)
        if update:
            self.update_all()
        return _X

    def update_all(self):
        self.rep_x = np.mean(self.X, axis=0)

class SGP(InstanceReductionMixin):
    """Self-Generating Prototypes

    The Self-Generating Prototypes generates instances is a centroid-based
    prototype generation algorithm that uses the space spliting mechanism
    to generate prototypes in the center of each cluster.

    Parameters
    ----------
    r_min: float, optional (default = 0.0)
        Determine the minimum size of a cluster [0.00, 0.20]

    r_mis: float, optional (default = 0.0)
        Determine the error tolerance before split a group

    Attributes
    ----------
    `X_` : array-like, shape = [indeterminated, n_features]
        Selected prototypes.

    `y_` : array-like, shape = [indeterminated]
        Labels of the selected prototypes.

    `reduction_` : float, percentual of reduction.

    Examples
    --------
    >>> from protopy.generation.sgp import SGP
    >>> import numpy as np
    >>> X = np.array([[i] for i in range(1,13)])
    >>> X = X + np.asarray([0.1,0,-0.1,0.1,0,-0.1,0.1,-0.1,0.1,-0.1,0.1,-0.1])
    >>> y = np.array([1, 1, 1, 2, 2, 2, 1, 1, 2, 2, 1, 1])
    >>> sgp = SGP()
    >>> sgp.fit(X, y)
    SGP(r_min=0.0, r_mis=0.0)
    >>> print sgp.predict(X)
    [1 1 1 2 2 2 1 1 2 2 1 1]
    >>> print sgp.reduction_
    0.5

    Attributes
    ----------
    `X_` : array-like, shape = [indeterminated, n_features]
        Selected prototypes.

    `y_` : array-like, shape = [indeterminated]
        Labels of the selected prototypes.

    `reduction_` : float, percentual of reduction.

    See also
    --------
    protopy.generation.sgp.SGP2: self-generating prototypes 2

    References
    ----------
    Hatem A. Fayed, Sherif R Hashem, and Amir F Atiya. Self-generating prototypes
    for pattern classification. Pattern Recognition, 40(5):1498–1509, 2007.
    """

    def __init__(self, r_min=0.0, r_mis=0.0):
        self.groups = None
        self.r_min = r_min
        self.r_mis = r_mis
        self.n_neighbors = 1
        self.classifier = None
        self.groups = None

    def reduce_data(self, X, y):
        X, y = check_arrays(X, y, sparse_format="csr")

        if self.classifier == None:
            self.classifier = KNeighborsClassifier(n_neighbors=self.n_neighbors)
        if self.classifier.n_neighbors != self.n_neighbors:
            self.classifier.n_neighbors = self.n_neighbors

        classes = np.unique(y)
        self.classes_ = classes

        # loading inicial groups
        self.groups = []
        for label in classes:
            mask = y == label
            self.groups = self.groups + [_Group(X[mask], label)]

        self.__main_loop()
        self.__generalization_step()

        self.X_ = np.asarray([g.rep_x for g in self.groups])
        self.y_ = np.asarray([g.label for g in self.groups])
        self.reduction_ = 1.0 - float(len(self.y_))/len(y)

        return self.X_, self.y_

    def __main_loop(self):
        exit_count = 0
        knn = KNeighborsClassifier(n_neighbors = 1, algorithm='brute')
        while exit_count < len(self.groups):
            index, exit_count = 0, 0
            while index < len(self.groups):

                group = self.groups[index]
                reps_x = np.asarray([g.rep_x for g in self.groups])
                reps_y = np.asarray([g.label for g in self.groups])
                knn.fit(reps_x, reps_y)

                nn_idx = knn.kneighbors(group.X, n_neighbors=1, return_distance=False)
                nn_idx = nn_idx.T[0]
                mask = nn_idx == index

                # if all are correctly classified
                if not (False in mask):
                    exit_count = exit_count + 1

                # if all are misclasified
                elif not (group.label in reps_y[nn_idx]):
                    pca = PCA(n_components=1)
                    pca.fit(group.X)
                    # maybe use a 'for' instead of creating array
                    d = pca.transform(reps_x[index])
                    dis = [pca.transform(inst)[0] for inst in group.X]
                    mask_split = (dis < d).flatten()

                    new_X = group.X[mask_split]
                    self.groups.append(_Group(new_X, group.label))
                    group.X = group.X[~mask_split]

                elif (reps_y[nn_idx] == group.label).all() and (nn_idx != index).any():
                    mask_mv = nn_idx != index
                    index_mv = np.asarray(range(len(group)))[mask_mv]
                    X_mv = group.remove_instances(index_mv)
                    G_mv = nn_idx[mask_mv]

                    for x, g in zip(X_mv, G_mv):
                        self.groups[g].add_instances([x])

                elif (reps_y[nn_idx] != group.label).sum()/float(len(group)) > self.r_mis:
                    mask_mv = reps_y[nn_idx] != group.label
                    new_X = group.X[mask_mv]
                    self.groups.append(_Group(new_X, group.label))
                    group.X = group.X[~mask_mv]
                else:
                   exit_count = exit_count + 1

                if len(group) == 0:
                    self.groups.remove(group)
                else:
                    index = index + 1

                for g in self.groups:
                    g.update_all()

        return self.groups

    def __generalization_step(self):
        larger = max([len(g) for g in self.groups])
        for group in self.groups:
            if len(group) < self.r_min * larger:
                self.groups.remove(group)
        return self.groups

class SGP2(SGP):
    """Self-Generating Prototypes 2

    The Self-Generating Prototypes 2 is the second version of the
    Self-Generating Prototypes algorithm.
    It has a higher generalization power, including the procedures
    merge and pruning.

    Parameters
    ----------
    r_min: float, optional (default = 0.0)
        Determine the minimum size of a cluster [0.00, 0.20]

    r_mis: float, optional (default = 0.0)
        Determine the error tolerance before split a group

    Attributes
    ----------
    `X_` : array-like, shape = [indeterminated, n_features]
        Selected prototypes.

    `y_` : array-like, shape = [indeterminated]
        Labels of the selected prototypes.

    `reduction_` : float, percentual of reduction.

    `pos_label` : int, positive class label, the minority class.

    Examples
    --------
    >>> from sklearn.instance_reduction.sgp import SelfGeneratingPrototypes2
    >>> import numpy as np
    >>> X = np.array([[i] for i in range(1,13)])
    >>> X = X + np.asarray([0.1,0,-0.1,0.1,0,-0.1,0.1,-0.1,0.1,-0.1,0.1,-0.1])
    >>> y = np.array([1, 1, 1, 2, 2, 2, 1, 1, 2, 2, 1, 1])
    >>> sgp2 = SelfGeneratingPrototypes2()
    >>> sgp2.fit(X, y)
    SelfGeneratingPrototypes2(r_min=0.0, r_mis=0.0)
    >>> print sgp2.reduction_
    0.5

    See also
    --------
    sklearn.neighbors.KNeighborsClassifier: nearest neighbors classifier
    protopy.generation.SGP: self-generating prototypes

    References
    ----------
    Hatem A. Fayed, Sherif R Hashem, and Amir F Atiya. Self-generating prototypes
    for pattern classification. Pattern Recognition, 40(5):1498–1509, 2007.
    """
    def __init__(self, r_min=0.0, r_mis=0.0, pos_label=None):
        self.groups = None
        self.r_min = r_min
        self.r_mis = r_mis
        self.n_neighbors = 1
        self.classifier = None
        self.groups = None
        self.pos_label = pos_label

<|endoftext|>


<commit_before>class User(object):
    def __init__(self, *args):
        try:
            self.__dict__ = args[0]
        except:
            pass

class GroupChat(object):
    def __init__(self, *args):
        try:
            self.__dict__ = args[0]
        except:
            pass

class Message:
    def __init__(self, data):
        self.message_id = data["message_id"]
        self.from_user = User(data["from"])
        self.date = data["date"]

        # Finding if we have a GroupChat or an User
        if "title" in data["chat"]:
            self.chat = GroupChat(data["chat"])
        elif "first_name" in data["chat"]:
            self.chat = User(data["chat"])

        # Forward data
        # Idea : as date is there if there is from,
        # move date into from's condition?
        if "forward_from" in data:
            self.forward_from = User(data["forward_from"])

        if "forward_date" in data:
            self.forward_date = data["forward_date"]

        # Message types
        if "reply_to_message" in data:
            self.reply_to_message = Message(data["reply_to_message"])

        if "text" in data:
            self.text = data["text"]

        if "audio" in data:
            self.audio = Audio(data["audio"])

        if "document" in data:
            self.document = Document(data["document"])

        if "photo" in data:
            self.photo = []
            for photo in data["photo"]:
                self.photo.append(PhotoSize(photo))

        if "sticker" in data:
            self.sticker = Sticker(data["sticker"])

        if "video" in data:
            self.video = Video(data["video"])

        if "contact" in data:
            self.contact = Contact(data["contact"])

        if "location" in data:
            self.location = Location(data["location"])

        # What happened in the server
        if "new_chat_participant" in data:
            self.new_chat_participant = User(data["new_chat_participant"])

        if "left_chat_participant" in data:
            self.left_chat_participant = User(data["left_chat_participant"])

        if "new_chat_title" in data:
            self.new_chat_title = data["new_chat_title"]

        if "new_chat_photo" in data:
            self.new_chat_photo = []
            for photo in data["new_chat_photo"]:
                self.new_chat_photo.append(PhotoSize(photo))

        if "delete_chat_photo" in data:
            self.delete_chat_photo = data["delete_chat_photo"]

        if "group_chat_created" in data:
            self.group_chat_created = data["group_chat_created"]

# Todo? : Inheritance and create a File superclass
# for all file-related classes?
class PhotoSize:
    def __init__(self, data):
        if not data:
            return
        self.file_id = data["file_id"]
        self.width = data["width"]
        self.height = data["height"]
        self.file_size = data.get("file_size", -1)

class Audio:
    def __init__(self, data):
        self.file_id = data["file_id"]
        self.duration = data["duration"]
        self.mime_type = data["mime_type"]
        self.file_size = data.get("file_size", -1)

class Document:
    def __init__(self, data):
        self.file_id = data["file_id"]
        self.thumb = PhotoSize(data["thumb"])
        self.file_name = data.get("file_name", "")
        self.mime_type = data.get("mime_type", "")
        self.file_size = data.get("file_size", -1)

class Sticker:
    def __init__(self, data):
        self.file_id = data["file_id"]
        self.width = data["width"]
        self.height = data["height"]
        self.thumb = PhotoSize(data["thumb"])
        self.file_size = data.get("file_size", -1)

class Video:
    def __init__(self, data):
        self.file_id = data["file_id"]
        self.width = data["width"]
        self.height = data["height"]
        self.duration = data["duration"]
        self.thumb = PhotoSize(data["thumb"])
        self.mime_type = data.get("mime_type", "")
        self.file_size = data.get("file_size", -1)
        self.caption = data.get("caption", "")

class Contact:
    def __init__(self, data):
        self.phone_number = data["phone_number"]
        self.first_name = data["first_name"]
        self.last_name = data.get("last_name", "")
        self.user_id = data.get("user_id", "")

class Location:
    def __init__(self, data):
        self.longitude = data["longitude"]
        self.latitude = data["latitude"]

class UserProfilePhotos:
    def __init__(self, data):
        self.total_count = data["total_count"]
        self.photos = []
        for row in data["photos"]:
            self.photos.append(list(row))

# Todo? Keyboard superclass for inheritance
class ReplyKeybaordMarkup:
    def __init__(self, data):
        self.keyboard = []
        for row in data["keyboard"]:
            self.keyboard.append(list(row))
            self.reisze_keyboard = data.get("resize_keyboard", False)
            self.one_time_keyboard = data.get("one_time_keyboard", False)
            self.selective = data.get("selective", False)

class ReplyKeyboardHide:
    def __init__(self, data):
        self.force_reply = data["force_reply"]
        self.selective = data.get("selective", False)
<commit_msg>Message and Keyboards class merged and improved<commit_after>#Using __dict__ and *args for compulsory args and **kwargs for optional ones.

class User(object):
    def __init__(self, *args):
        try:
            self.__dict__ = args[0]
        except:
            pass

class GroupChat(object):
    def __init__(self, *args):
        try:
            self.__dict__ = args[0]
        except:
            pass

# Todo? : Inheritance and create a File superclass
# for all file-related classes?
class PhotoSize:
    def __init__(self, data):
        if not data:
            return
        self.file_id = data["file_id"]
        self.width = data["width"]
        self.height = data["height"]
        self.file_size = data.get("file_size", -1)

class Audio:
    def __init__(self, data):
        self.file_id = data["file_id"]
        self.duration = data["duration"]
        self.mime_type = data["mime_type"]
        self.file_size = data.get("file_size", -1)

class Document:
    def __init__(self, data):
        self.file_id = data["file_id"]
        self.thumb = PhotoSize(data["thumb"])
        self.file_name = data.get("file_name", "")
        self.mime_type = data.get("mime_type", "")
        self.file_size = data.get("file_size", -1)

class Sticker:
    def __init__(self, data):
        self.file_id = data["file_id"]
        self.width = data["width"]
        self.height = data["height"]
        self.thumb = PhotoSize(data["thumb"])
        self.file_size = data.get("file_size", -1)

class Video:
    def __init__(self, data):
        self.file_id = data["file_id"]
        self.width = data["width"]
        self.height = data["height"]
        self.duration = data["duration"]
        self.thumb = PhotoSize(data["thumb"])
        self.mime_type = data.get("mime_type", "")
        self.file_size = data.get("file_size", -1)
        self.caption = data.get("caption", "")

class Contact:
    def __init__(self, data):
        self.phone_number = data["phone_number"]
        self.first_name = data["first_name"]
        self.last_name = data.get("last_name", "")
        self.user_id = data.get("user_id", "")

class Location:
    def __init__(self, data):
        self.longitude = data["longitude"]
        self.latitude = data["latitude"]

class UserProfilePhotos:
    def __init__(self, data):
        self.total_count = data["total_count"]
        self.photos = []
        for row in data["photos"]:
            self.photos.append(list(row))

class ReplyKeyBoard(object):
    def __init__(self, **kwargs):
        self.selective = kwargs.get('selective', False)

class ReplyKeyboardMarkup(ReplyKeyBoard):

    def __init__(self, keyboard, **kwargs):
        ReplyKeyBoard.__init__(self, **kwargs)
        self.keyboard = keyboard
        self.reisze_keyboard = kwargs.get("resize_keyboard", False)
        self.one_time_keyboard = kwargs.get("one_time_keyboard", False)

class ReplyKeyboardHide(ReplyKeyBoard):

    def __init__(self, **kwargs):
        ReplyKeyBoard.__init__(self, **kwargs)
        self.hide_keyboard = True

class ForceReply(ReplyKeyBoard):

    def __init__(self, **kwargs):
        ReplyKeyBoard.__init__(self, **kwargs)
        self.force_reply = True

replace_dict = {'forward_from': User,
                   'audio': Audio,
                   'document': Document,
                   'sticker': Sticker,
                   'video': Video,
                   'contact': Contact,
                   'location': Location,
                   'new_chat_participant': User,
                   'left_chat_participant': User
                   }

class Message(object):
    def __init__(self, *args):
        message_dict = {}

        for attr, attr_value in args[0].iteritems():

            if attr == 'from':
                message_dict['from_user'] = User(attr_value)
            elif attr == 'chat':
                # Finding if we have a GroupChat or an User
                if 'first_name' in attr_value:
                    message_dict[attr] = User(attr_value)
                elif 'title' in attr_value:
                    message_dict[attr] = GroupChat(attr_value)
            elif attr in replace_dict:
                message_dict[attr] = replace_dict[attr](attr_value)
            elif attr == "reply_to_message":
                message_dict[attr] = Message(attr_value)
            elif attr in ("photo","new_chat_photo"):
                photos = []
                for photo in attr_value:
                    photos.append(PhotoSize(photo))
                message_dict[attr] = photos
            else:
                message_dict[attr] = attr_value

        self.__dict__ = message_dict
        <|endoftext|>


<commit_before>import json
import locale
import re
import urllib2

from util import hook

locale.setlocale(locale.LC_ALL, '')

youtube_re = re.compile(r'.*youtube.*v=([-_a-z0-9]+)', flags=re.I)
url = 'http://gdata.youtube.com/feeds/api/videos/%s?alt=json'

#@hook.command(hook=r'(.*)', prefix=False)
def youtube(inp):
    m = youtube_re.match(inp)
    if m:
        j = json.load(urllib2.urlopen(url % m.group(1)))['entry']

        out = '\x02%s\x02' % j['title']['$t']

        rating = j.get('gd$rating')
        if rating:
            out += ' - rated \x02%.2f/%.1f\x02 (%d)' % (rating['average'],
                    rating['max'], rating['numRaters'])

        stats = j.get('yt$statistics')
        if stats:
            view_count = int(stats['viewCount'])
            out += ' - views \x02%s\x02' % locale.format('%d', view_count, 1)

        out += ' - length \x02'
        length = int(j['media$group']['yt$duration']['seconds'])
        if length / 3600: # > 1 hour
            out += '%dh ' % (length / 3600)
        if length / 60:
            out += '%dm ' % (length / 60 % 60)
        out += "%ds" % (length % 60)

        return out
<commit_msg>rewrite youtube plugin again to use jsonc interface and report more information<commit_after>import json
import locale
import re
import time
import urllib2

from util import hook

locale.setlocale(locale.LC_ALL, '')

youtube_re = re.compile(r'youtube.*?v=([-_a-z0-9]+)', flags=re.I)
url = 'http://gdata.youtube.com/feeds/api/videos/%s?v=2&alt=jsonc'

#@hook.command(hook=r'(.*)', prefix=False)
def youtube(inp):
    m = youtube_re.search(inp)
    if m:
        j = json.load(urllib2.urlopen(url % m.group(1)))

        if j.get('error'):
            return

        j = j['data']

        out = '\x02%s\x02' % j['title']

        out += ' - length \x02'
        length = j['duration']
        if length / 3600: # > 1 hour
            out += '%dh ' % (length / 3600)
        if length / 60:
            out += '%dm ' % (length / 60 % 60)
        out += "%ds\x02" % (length % 60)

        if 'rating' in j:
            out += ' - rated \x02%.2f/5.0\x02 (%d)' % (j['rating'],
                    j['ratingCount'])

        if 'viewCount' in j:
            out += ' - \x02%s\x02 views' % locale.format('%d',
                                                         j['viewCount'], 1)

        upload_time = time.strptime(j['uploaded'], "%Y-%m-%dT%H:%M:%S.000Z")
        out += ' - \x02%s\x02 on \x02%s\x02' % (j['uploader'],
                    time.strftime("%Y.%m.%d", upload_time))

        return out
<|endoftext|>