<commit_before>from neomodel.exception import DoesNotExist

class FakeRelation(object):
    def __init__(self, origin):
        self.origin = origin
        self.connections = {}

    def __bool__(self):
        return len(self.connections) > 0

    def __len__(self):
        return len(self.connections)

    def __nonzero__(self):
        return len(self.connections) > 0

    def search(self, **kwargs):
        results = []
        for node in self.connections.values():
            match = True
            for (key, value) in kwargs.iteritems():
                if not hasattr(node, key) or getattr(node, key) != value:
                    match = False
                    break
            if match:
                results.append(node)
        return results

    def get(self, **kwargs):
        for n in self.connections.values():
            found = True
            for key in kwargs.keys():
                if not getattr(n, key, None) == kwargs[key]:
                    found = False
            if found:
                return n
        raise DoesNotExist

    def connect(self, node, properties=None):
        self.connections[node._id] = node

    def disconnect(self, node):
        del self.connections[node._id]

    def reconnect(self, node, new_node):
        del self.connections[node._id]
        self.connections[new_node._id] = new_node

    def all(self):
        return self.connections.values()

    def single(self):
        if self.connections.values():
            return self.connections.values()[0]
        else:
            return None

    def count(self):
        return len(self.connections)

    def is_connected(self, node):
        return node._id in self.connections

class FakeCategoryRelation(FakeRelation):
    def __init__(self, origin, nodes):
        super(FakeCategoryRelation, self).__init__(origin)
        self.connections = nodes

    def connect(self, *args, **kwargs):
        raise NotImplemented("Not available on category node rel")

    def reconnect(self, *args, **kwargs):
        raise NotImplemented("Not available on category node rel")

    def disconnect(self, *args, **kwargs):
        raise NotImplemented("Not available on category node rel")

    def single(self, *args, **kwargs):
        raise NotImplemented("Not available on category node rel")

    def is_connected(self, *args, **kwargs):
        raise NotImplemented("Not available on category node rel")
<commit_msg>Make use of those rel properties being passed in<commit_after>from neomodel.exception import DoesNotExist

class FakeRelation(object):
    def __init__(self, origin):
        self.origin = origin
        self.connections = {}
        self.relation_properties = {}

    def __bool__(self):
        return len(self.connections) > 0

    def __len__(self):
        return len(self.connections)

    def __nonzero__(self):
        return len(self.connections) > 0

    def search(self, **kwargs):
        results = []
        for node in self.connections.values():
            match = True
            for (key, value) in kwargs.iteritems():
                if not hasattr(node, key) or getattr(node, key) != value:
                    match = False
                    break
            if match:
                results.append(node)
        return results

    def get(self, **kwargs):
        for n in self.connections.values():
            found = True
            for key in kwargs.keys():
                if not getattr(n, key, None) == kwargs[key]:
                    found = False
            if found:
                return n
        raise DoesNotExist

    def connect(self, node, properties=None):
        self.connections[node._id] = node
        if properties is not None and isinstance(properties, dict):
            self.relation_properties[node._id] = properties

    def disconnect(self, node):
        del self.connections[node._id]

    def reconnect(self, node, new_node):
        del self.connections[node._id]
        self.connections[new_node._id] = new_node

    def all(self):
        return self.connections.values()

    def single(self):
        if self.connections.values():
            return self.connections.values()[0]
        else:
            return None

    def count(self):
        return len(self.connections)

    def is_connected(self, node):
        return node._id in self.connections

class FakeCategoryRelation(FakeRelation):
    def __init__(self, origin, nodes):
        super(FakeCategoryRelation, self).__init__(origin)
        self.connections = nodes

    def connect(self, *args, **kwargs):
        raise NotImplemented("Not available on category node rel")

    def reconnect(self, *args, **kwargs):
        raise NotImplemented("Not available on category node rel")

    def disconnect(self, *args, **kwargs):
        raise NotImplemented("Not available on category node rel")

    def single(self, *args, **kwargs):
        raise NotImplemented("Not available on category node rel")

    def is_connected(self, *args, **kwargs):
        raise NotImplemented("Not available on category node rel")
<|endoftext|>


<commit_before>import pytest
import uuid

from mlflow.utils.process import cache_return_value_per_process
from multiprocessing import Process, Queue

@cache_return_value_per_process
def _gen_random_str1(v):
    return str(v) + uuid.uuid4().hex

@cache_return_value_per_process
def _gen_random_str2(v):
    return str(v) + uuid.uuid4().hex

@cache_return_value_per_process
def _gen_random_no_arg():
    return uuid.uuid4().hex

def _test_cache_return_value_per_process_child_proc_target(path1, path3, queue):
    # in forked out child process
    child_path1 = _gen_random_str1(True)
    child_path2 = _gen_random_str1(False)
    result = len({path1, path3, child_path1, child_path2}) == 4
    queue.put(result)

def test_cache_return_value_per_process():

    path1 = _gen_random_str1(True)
    path2 = _gen_random_str1(True)

    assert path1 == path2

    path3 = _gen_random_str1(False)
    assert path3 != path2

    no_arg_path1 = _gen_random_no_arg()
    no_arg_path2 = _gen_random_no_arg()
    assert no_arg_path1 == no_arg_path2

    with pytest.raises(
        ValueError,
        match="The function decorated by `cache_return_value_per_process` is not allowed to be"
        "called with key-word style arguments.",
    ):
        _gen_random_str1(v=True)

    f2_path1 = _gen_random_str2(True)
    f2_path2 = _gen_random_str2(False)

    assert len({path1, path3, f2_path1, f2_path2}) == 4

    queue = Queue()
    child_proc = Process(
        target=_test_cache_return_value_per_process_child_proc_target, args=(path1, path3, queue)
    )
    child_proc.start()
    child_proc.join()
    assert queue.get(), "Testing inside child process failed."
<commit_msg>Use fork in test test_cache_return_value_per_process (#5636)<commit_after>import pytest
import uuid

from mlflow.utils.process import cache_return_value_per_process
import os

@cache_return_value_per_process
def _gen_random_str1(v):
    return str(v) + uuid.uuid4().hex

@cache_return_value_per_process
def _gen_random_str2(v):
    return str(v) + uuid.uuid4().hex

@cache_return_value_per_process
def _gen_random_no_arg():
    return uuid.uuid4().hex

def test_cache_return_value_per_process():

    path1 = _gen_random_str1(True)
    path2 = _gen_random_str1(True)

    assert path1 == path2

    path3 = _gen_random_str1(False)
    assert path3 != path2

    no_arg_path1 = _gen_random_no_arg()
    no_arg_path2 = _gen_random_no_arg()
    assert no_arg_path1 == no_arg_path2

    with pytest.raises(
        ValueError,
        match="The function decorated by `cache_return_value_per_process` is not allowed to be"
        "called with key-word style arguments.",
    ):
        _gen_random_str1(v=True)

    f2_path1 = _gen_random_str2(True)
    f2_path2 = _gen_random_str2(False)

    assert len({path1, path3, f2_path1, f2_path2}) == 4

    # Skip the following block on Windows which doesn't support `os.fork`
    if os.name != "nt":
        # Use `os.fork()` to make parent and child processes share the same global variable
        # `_per_process_value_cache_map`.
        pid = os.fork()
        if pid > 0:
            # in parent process
            child_pid = pid
            # check child process exit with return value 0.
            assert os.waitpid(child_pid, 0)[1] == 0
        else:
            # in forked out child process
            child_path1 = _gen_random_str1(True)
            child_path2 = _gen_random_str1(False)
            test_pass = len({path1, path3, child_path1, child_path2}) == 4
            # exit forked out child process with exit code representing testing pass or fail.
            os._exit(0 if test_pass else 1)
<|endoftext|>


<commit_before>import os
import stat
import sys
import tempfile
import unittest

from django.core.exceptions import SuspiciousOperation
from django.test import SimpleTestCase
from django.utils import archive

class TestArchive(unittest.TestCase):

    def setUp(self):
        self.testdir = os.path.join(os.path.dirname(__file__), 'archives')
        self.old_cwd = os.getcwd()
        os.chdir(self.testdir)

    def tearDown(self):
        os.chdir(self.old_cwd)

    def test_extract_function(self):
        for entry in os.scandir(self.testdir):
            with self.subTest(entry.name), tempfile.TemporaryDirectory() as tmpdir:
                archive.extract(entry.path, tmpdir)
                self.assertTrue(os.path.isfile(os.path.join(tmpdir, '1')))
                self.assertTrue(os.path.isfile(os.path.join(tmpdir, '2')))
                self.assertTrue(os.path.isfile(os.path.join(tmpdir, 'foo', '1')))
                self.assertTrue(os.path.isfile(os.path.join(tmpdir, 'foo', '2')))
                self.assertTrue(os.path.isfile(os.path.join(tmpdir, 'foo', 'bar', '1')))
                self.assertTrue(os.path.isfile(os.path.join(tmpdir, 'foo', 'bar', '2')))

    @unittest.skipIf(sys.platform == 'win32', 'Python on Windows has a limited os.chmod().')
    def test_extract_file_permissions(self):
        """archive.extract() preserves file permissions."""
        mask = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO
        umask = os.umask(0)
        os.umask(umask)  # Restore the original umask.
        for entry in os.scandir(self.testdir):
            if entry.name.startswith('leadpath_'):
                continue
            with self.subTest(entry.name), tempfile.TemporaryDirectory() as tmpdir:
                archive.extract(entry.path, tmpdir)
                # An executable file in the archive has executable permissions.
                filepath = os.path.join(tmpdir, 'executable')
                self.assertEqual(os.stat(filepath).st_mode & mask, 0o775)
                # A file is readable even if permission data is missing.
                filepath = os.path.join(tmpdir, 'no_permissions')
                self.assertEqual(os.stat(filepath).st_mode & mask, 0o666 & ~umask)

class TestArchiveInvalid(SimpleTestCase):
    def test_extract_function_traversal(self):
        archives_dir = os.path.join(os.path.dirname(__file__), 'traversal_archives')
        tests = [
            ('traversal.tar', '..'),
            ('traversal_absolute.tar', '/tmp/evil.py'),
        ]
        if sys.platform == 'win32':
            tests += [
                ('traversal_disk_win.tar', 'd:evil.py'),
                ('traversal_disk_win.zip', 'd:evil.py'),
            ]
        msg = "Archive contains invalid path: '%s'"
        for entry, invalid_path in tests:
            with self.subTest(entry), tempfile.TemporaryDirectory() as tmpdir:
                with self.assertRaisesMessage(SuspiciousOperation, msg % invalid_path):
                    archive.extract(os.path.join(archives_dir, entry), tmpdir)
<commit_msg>[3.2.x] Skipped test_archive tests when bz2/lzma module is not installed.<commit_after>import os
import stat
import sys
import tempfile
import unittest

from django.core.exceptions import SuspiciousOperation
from django.test import SimpleTestCase
from django.utils import archive

try:
    import bz2  # NOQA
    HAS_BZ2 = True
except ImportError:
    HAS_BZ2 = False

try:
    import lzma  # NOQA
    HAS_LZMA = True
except ImportError:
    HAS_LZMA = False

class TestArchive(unittest.TestCase):

    def setUp(self):
        self.testdir = os.path.join(os.path.dirname(__file__), 'archives')
        self.old_cwd = os.getcwd()
        os.chdir(self.testdir)

    def tearDown(self):
        os.chdir(self.old_cwd)

    def test_extract_function(self):
        for entry in os.scandir(self.testdir):
            with self.subTest(entry.name), tempfile.TemporaryDirectory() as tmpdir:
                if (
                    (entry.name.endswith('.bz2') and not HAS_BZ2) or
                    (entry.name.endswith(('.lzma', '.xz')) and not HAS_LZMA)
                ):
                    continue
                archive.extract(entry.path, tmpdir)
                self.assertTrue(os.path.isfile(os.path.join(tmpdir, '1')))
                self.assertTrue(os.path.isfile(os.path.join(tmpdir, '2')))
                self.assertTrue(os.path.isfile(os.path.join(tmpdir, 'foo', '1')))
                self.assertTrue(os.path.isfile(os.path.join(tmpdir, 'foo', '2')))
                self.assertTrue(os.path.isfile(os.path.join(tmpdir, 'foo', 'bar', '1')))
                self.assertTrue(os.path.isfile(os.path.join(tmpdir, 'foo', 'bar', '2')))

    @unittest.skipIf(sys.platform == 'win32', 'Python on Windows has a limited os.chmod().')
    def test_extract_file_permissions(self):
        """archive.extract() preserves file permissions."""
        mask = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO
        umask = os.umask(0)
        os.umask(umask)  # Restore the original umask.
        for entry in os.scandir(self.testdir):
            if (
                entry.name.startswith('leadpath_') or
                (entry.name.endswith('.bz2') and not HAS_BZ2) or
                (entry.name.endswith(('.lzma', '.xz')) and not HAS_LZMA)
            ):
                continue
            with self.subTest(entry.name), tempfile.TemporaryDirectory() as tmpdir:
                archive.extract(entry.path, tmpdir)
                # An executable file in the archive has executable permissions.
                filepath = os.path.join(tmpdir, 'executable')
                self.assertEqual(os.stat(filepath).st_mode & mask, 0o775)
                # A file is readable even if permission data is missing.
                filepath = os.path.join(tmpdir, 'no_permissions')
                self.assertEqual(os.stat(filepath).st_mode & mask, 0o666 & ~umask)

class TestArchiveInvalid(SimpleTestCase):
    def test_extract_function_traversal(self):
        archives_dir = os.path.join(os.path.dirname(__file__), 'traversal_archives')
        tests = [
            ('traversal.tar', '..'),
            ('traversal_absolute.tar', '/tmp/evil.py'),
        ]
        if sys.platform == 'win32':
            tests += [
                ('traversal_disk_win.tar', 'd:evil.py'),
                ('traversal_disk_win.zip', 'd:evil.py'),
            ]
        msg = "Archive contains invalid path: '%s'"
        for entry, invalid_path in tests:
            with self.subTest(entry), tempfile.TemporaryDirectory() as tmpdir:
                with self.assertRaisesMessage(SuspiciousOperation, msg % invalid_path):
                    archive.extract(os.path.join(archives_dir, entry), tmpdir)
<|endoftext|>


<commit_before>import math
import matplotlib.pyplot as plt
import numpy as np
import random
import time

from collections import defaultdict
from euclid import Circle, Point2, Vector2, LineSegment2

import tf_rl.utils.svg as svg

class GameObject(object):
    def __init__(self, position, speed, obj_type, settings):
        """Esentially represents circles of different kinds, which have
        position and speed."""
        self.settings = settings
        self.radius = self.settings["object_radius"]

        self.obj_type = obj_type
        self.position = position
        self.speed    = speed
        self.bounciness = 1.0

    def wall_collisions(self):
        """Update speed upon collision with the wall."""
        world_size = self.settings["world_size"]

        for dim in range(2):
            if self.position[dim] - self.radius       <= 0               and self.speed[dim] < 0:
                self.speed[dim] = - self.speed[dim] * self.bounciness
            elif self.position[dim] + self.radius + 1 >= world_size[dim] and self.speed[dim] > 0:
                self.speed[dim] = - self.speed[dim] * self.bounciness

    def move(self, dt):
        """Move as if dt seconds passed"""
        self.position += dt * self.speed
        self.position = Point2(*self.position)

    def step(self, dt):
        """Move and bounce of walls."""
        self.wall_collisions()
        self.move(dt)

    def as_circle(self):
        return Circle(self.position, float(self.radius))

    def draw(self):
        """Return svg object for this item."""
        color = self.settings["colors"][self.obj_type]
        return svg.Circle(self.position + Point2(10, 10), self.radius, color=color)

class KarpathyGame(object):
    def __init__(self, settings):
        """Initiallize game simulator with settings"""
        self.settings = settings
        self.size  = self.settings["world_size"]
        self.walls = [LineSegment2(Point2(0,0),                        Point2(0,self.size[1])),
                      LineSegment2(Point2(0,self.size[1]),             Point2(self.size[0], self.size[1])),
                      LineSegment2(Point2(self.size[0], self.size[1]), Point2(self.size[0], 0)),
                      LineSegment2(Point2(self.size[0], 0),            Point2(0,0))]

        self.hero = GameObject(Point2(*self.settings["hero_initial_position"]),
                               Vector2(*self.settings["hero_initial_speed"]),
                               "hero",
                               self.settings)
        if not self.settings["hero_bounces_off_walls"]:
            self.hero.bounciness = 0.0

        self.objects = []
        for obj_type, number in settings["num_objects"].items():
            for _ in range(number):
                self.spawn_object(obj_type)

        self.observation_lines = self.generate_observation_lines()

        self.object_reward = 0
        self.collected_rewards = []

        # every observation_line sees one of objects or wall and
        # two numbers representing speed of the object (if applicable)
        self.eye_observation_size = len(self.settings["objects"]) + 3
        # additionally there are two numbers representing agents own speed.
        self.observation_size = self.eye_observation_size * len(self.observation_lines) + 2

        self.directions = [Vector2(*d) for d in [[1,0], [0,1], [-1,0],[0,-1]]]
        self.num_actions      = len(self.directions)

        self.objects_eaten = defaultdict(lambda: 0)

    def perform_action(self, action_id):
        """Change speed to one of hero vectors"""
        assert 0 <= action_id < self.num_actions
        self.hero.speed *= 0.8
        self.hero.speed += self.directions[action_id] * self.settings["delta_v"]

    def spawn_object(self, obj_type):
        """Spawn object of a given type and add it to the objects array"""
        radius = self.settings["object_radius"]
        position = np.random.uniform([radius, radius], np.array(self.size) - radius)
        position = Point2(float(position[0]), float(position[1]))
        max_speed = np.array(self.settings["maximum_speed"])
        speed    = np.random.uniform(-max_speed, max_speed).astype(float)
        speed = Vector2(float(speed[0]), float(speed[1]))

        self.objects.append(GameObject(position, speed, obj_type, self.settings))

    def step(self, dt):
        """Simulate all the objects for a given ammount of time.

        Also resolve collisions with the hero"""
        for obj in self.objects + [self.hero] :
            obj.step(dt)
        self.resolve_collisions()

    def squared_distance(self, p1, p2):
        return (p1[0] - p2[0]) ** 2 + (p1[1] - p2[1]) ** 2

    def resolve_collisions(self):
        """If hero touches, hero eats. Also reward gets updated."""
        collision_distance = 2 * self.settings["object_radius"]
        collision_distance2 = collision_distance ** 2
        to_remove = []
        for obj in self.objects:
            if self.squared_distance(self.hero.position, obj.position) < collision_distance2:
                to_remove.append(obj)
        for obj in to_remove:
            self.objects.remove(obj)
            self.objects_eaten[obj.obj_type] += 1
            self.object_reward += self.settings["object_reward"][obj.obj_type]
            self.spawn_object(obj.obj_type)

    def inside_walls(self, point):
        """Check if the point is inside the walls"""
        EPS = 1e-4
        return (EPS <= point[0] < self.size[0] - EPS and
                EPS <= point[1] < self.size[1] - EPS)

    def observe(self):
        """Return observation vector. For all the observation directions it returns representation
        of the closest object to the hero - might be nothing, another object or a wall.
        Representation of observation for all the directions will be concatenated.
        """
        num_obj_types = len(self.settings["objects"]) + 1 # and wall
        max_speed_x, max_speed_y = self.settings["maximum_speed"]

        observable_distance = self.settings["observation_line_length"]

        relevant_objects = [obj for obj in self.objects
                            if obj.position.distance(self.hero.position) < observable_distance]
        # objects sorted from closest to furthest
        relevant_objects.sort(key=lambda x: x.position.distance(self.hero.position))

        observation        = np.zeros(self.observation_size)
        observation_offset = 0
        for i, observation_line in enumerate(self.observation_lines):
            # shift to hero position
            observation_line = LineSegment2(self.hero.position + Vector2(*observation_line.p1),
                                            self.hero.position + Vector2(*observation_line.p2))

            observed_object = None
            # if end of observation line is outside of walls, we see the wall.
            if not self.inside_walls(observation_line.p2):
                observed_object = "**wall**"
            for obj in relevant_objects:
                if observation_line.distance(obj.position) < self.settings["object_radius"]:
                    observed_object = obj
                    break
            object_type_id = None
            speed_x, speed_y = 0, 0
            proximity = 0
            if observed_object == "**wall**": # wall seen
                object_type_id = num_obj_types - 1
                # a wall has fairly low speed...
                speed_x, speed_y = 0, 0
                # best candidate is intersection between
                # observation_line and a wall, that's
                # closest to the hero
                best_candidate = None
                for wall in self.walls:
                    candidate = observation_line.intersect(wall)
                    if candidate is not None:
                        if (best_candidate is None or
                                best_candidate.distance(self.hero.position) >
                                candidate.distance(self.hero.position)):
                            best_candidate = candidate
                if best_candidate is None:
                    # assume it is due to rounding errors
                    # and wall is barely touching observation line
                    proximity = observable_distance
                else:
                    proximity = best_candidate.distance(self.hero.position)
            elif observed_object is not None: # agent seen
                object_type_id = self.settings["objects"].index(observed_object.obj_type)
                speed_x, speed_y = tuple(observed_object.speed)
                intersection_segment = obj.as_circle().intersect(observation_line)
                assert intersection_segment is not None
                try:
                    proximity = min(intersection_segment.p1.distance(self.hero.position),
                                    intersection_segment.p2.distance(self.hero.position))
                except AttributeError:
                    proximity = observable_distance
            for object_type_idx_loop in range(num_obj_types):
                observation[observation_offset + object_type_idx_loop] = 1.0
            if object_type_id is not None:
                observation[observation_offset + object_type_id] = proximity / observable_distance
            observation[observation_offset + num_obj_types] =     speed_x   / max_speed_x
            observation[observation_offset + num_obj_types + 1] = speed_y   / max_speed_y
            assert num_obj_types + 2 == self.eye_observation_size
            observation_offset += self.eye_observation_size

        observation[observation_offset]     = self.hero.speed[0] / max_speed_x
        observation[observation_offset + 1] = self.hero.speed[1] / max_speed_y
        assert observation_offset + 2 == self.observation_size

        return observation

    def distance_to_walls(self):
        """Returns distance of a hero to walls"""
        res = float('inf')
        for wall in self.walls:
            res = min(res, self.hero.position.distance(wall))
        return res - self.settings["object_radius"]

    def collect_reward(self):
        """Return accumulated object eating score + current distance to walls score"""
        wall_reward =  self.settings["wall_distance_penalty"] * \
                       np.exp(-self.distance_to_walls() / self.settings["tolerable_distance_to_wall"])
        assert wall_reward < 1e-3, "You are rewarding hero for being close to the wall!"
        total_reward = wall_reward + self.object_reward
        self.object_reward = 0
        self.collected_rewards.append(total_reward)
        return total_reward

    def plot_reward(self, smoothing = 30):
        """Plot evolution of reward over time."""
        plottable = self.collected_rewards[:]
        while len(plottable) > 1000:
            for i in range(0, len(plottable) - 1, 2):
                plottable[i//2] = (plottable[i] + plottable[i+1]) / 2
            plottable = plottable[:(len(plottable) // 2)]
        x = []
        for  i in range(smoothing, len(plottable)):
            chunk = plottable[i-smoothing:i]
            x.append(sum(chunk) / len(chunk))
        plt.plot(list(range(len(x))), x)

    def generate_observation_lines(self):
        """Generate observation segments in settings["num_observation_lines"] directions"""
        result = []
        start = Point2(0.0, 0.0)
        end   = Point2(self.settings["observation_line_length"],
                       self.settings["observation_line_length"])
        for angle in np.linspace(0, 2*np.pi, self.settings["num_observation_lines"], endpoint=False):
            rotation = Point2(math.cos(angle), math.sin(angle))
            current_start = Point2(start[0] * rotation[0], start[1] * rotation[1])
            current_end   = Point2(end[0]   * rotation[0], end[1]   * rotation[1])
            result.append( LineSegment2(current_start, current_end))
        return result

    def _repr_html_(self):
        return self.to_html()

    def to_html(self, stats=[]):
        """Return svg representation of the simulator"""

        stats = stats[:]
        recent_reward = self.collected_rewards[-100:] + [0]
        objects_eaten_str = ', '.join(["%s: %s" % (o,c) for o,c in self.objects_eaten.items()])
        stats.extend([
            "nearest wall = %.1f" % (self.distance_to_walls(),),
            "reward       = %.1f" % (sum(recent_reward)/len(recent_reward),),
            "objects eaten => %s" % (objects_eaten_str,),
        ])

        scene = svg.Scene((self.size[0] + 20, self.size[1] + 20 + 20 * len(stats)))
        scene.add(svg.Rectangle((10, 10), self.size))

        for line in self.observation_lines:
            scene.add(svg.Line(line.p1 + self.hero.position + Point2(10,10),
                               line.p2 + self.hero.position + Point2(10,10)))

        for obj in self.objects + [self.hero] :
            scene.add(obj.draw())

        offset = self.size[1] + 15
        for txt in stats:
            scene.add(svg.Text((10, offset + 20), txt, 15))
            offset += 20

        return scene

<commit_msg>Additional features / actions for karpathy game<commit_after>import math
import matplotlib.pyplot as plt
import numpy as np
import random
import time

from collections import defaultdict
from euclid import Circle, Point2, Vector2, LineSegment2

import tf_rl.utils.svg as svg

class GameObject(object):
    def __init__(self, position, speed, obj_type, settings):
        """Esentially represents circles of different kinds, which have
        position and speed."""
        self.settings = settings
        self.radius = self.settings["object_radius"]

        self.obj_type = obj_type
        self.position = position
        self.speed    = speed
        self.bounciness = 1.0

    def wall_collisions(self):
        """Update speed upon collision with the wall."""
        world_size = self.settings["world_size"]

        for dim in range(2):
            if self.position[dim] - self.radius       <= 0               and self.speed[dim] < 0:
                self.speed[dim] = - self.speed[dim] * self.bounciness
            elif self.position[dim] + self.radius + 1 >= world_size[dim] and self.speed[dim] > 0:
                self.speed[dim] = - self.speed[dim] * self.bounciness

    def move(self, dt):
        """Move as if dt seconds passed"""
        self.position += dt * self.speed
        self.position = Point2(*self.position)

    def step(self, dt):
        """Move and bounce of walls."""
        self.wall_collisions()
        self.move(dt)

    def as_circle(self):
        return Circle(self.position, float(self.radius))

    def draw(self):
        """Return svg object for this item."""
        color = self.settings["colors"][self.obj_type]
        return svg.Circle(self.position + Point2(10, 10), self.radius, color=color)

class KarpathyGame(object):
    def __init__(self, settings):
        """Initiallize game simulator with settings"""
        self.settings = settings
        self.size  = self.settings["world_size"]
        self.walls = [LineSegment2(Point2(0,0),                        Point2(0,self.size[1])),
                      LineSegment2(Point2(0,self.size[1]),             Point2(self.size[0], self.size[1])),
                      LineSegment2(Point2(self.size[0], self.size[1]), Point2(self.size[0], 0)),
                      LineSegment2(Point2(self.size[0], 0),            Point2(0,0))]

        self.hero = GameObject(Point2(*self.settings["hero_initial_position"]),
                               Vector2(*self.settings["hero_initial_speed"]),
                               "hero",
                               self.settings)
        if not self.settings["hero_bounces_off_walls"]:
            self.hero.bounciness = 0.0

        self.objects = []
        for obj_type, number in settings["num_objects"].items():
            for _ in range(number):
                self.spawn_object(obj_type)

        self.observation_lines = self.generate_observation_lines()

        self.object_reward = 0
        self.collected_rewards = []

        # every observation_line sees one of objects or wall and
        # two numbers representing speed of the object (if applicable)
        self.eye_observation_size = len(self.settings["objects"]) + 3
        # additionally there are two numbers representing agents own speed and position.
        self.observation_size = self.eye_observation_size * len(self.observation_lines) + 2 + 2

        self.directions = [Vector2(*d) for d in [[1,0], [0,1], [-1,0],[0,-1],[0.0,0.0]]]
        self.num_actions      = len(self.directions)

        self.objects_eaten = defaultdict(lambda: 0)

    def perform_action(self, action_id):
        """Change speed to one of hero vectors"""
        assert 0 <= action_id < self.num_actions
        self.hero.speed *= 0.5
        self.hero.speed += self.directions[action_id] * self.settings["delta_v"]

    def spawn_object(self, obj_type):
        """Spawn object of a given type and add it to the objects array"""
        radius = self.settings["object_radius"]
        position = np.random.uniform([radius, radius], np.array(self.size) - radius)
        position = Point2(float(position[0]), float(position[1]))
        max_speed = np.array(self.settings["maximum_speed"])
        speed    = np.random.uniform(-max_speed, max_speed).astype(float)
        speed = Vector2(float(speed[0]), float(speed[1]))

        self.objects.append(GameObject(position, speed, obj_type, self.settings))

    def step(self, dt):
        """Simulate all the objects for a given ammount of time.

        Also resolve collisions with the hero"""
        for obj in self.objects + [self.hero] :
            obj.step(dt)
        self.resolve_collisions()

    def squared_distance(self, p1, p2):
        return (p1[0] - p2[0]) ** 2 + (p1[1] - p2[1]) ** 2

    def resolve_collisions(self):
        """If hero touches, hero eats. Also reward gets updated."""
        collision_distance = 2 * self.settings["object_radius"]
        collision_distance2 = collision_distance ** 2
        to_remove = []
        for obj in self.objects:
            if self.squared_distance(self.hero.position, obj.position) < collision_distance2:
                to_remove.append(obj)
        for obj in to_remove:
            self.objects.remove(obj)
            self.objects_eaten[obj.obj_type] += 1
            self.object_reward += self.settings["object_reward"][obj.obj_type]
            self.spawn_object(obj.obj_type)

    def inside_walls(self, point):
        """Check if the point is inside the walls"""
        EPS = 1e-4
        return (EPS <= point[0] < self.size[0] - EPS and
                EPS <= point[1] < self.size[1] - EPS)

    def observe(self):
        """Return observation vector. For all the observation directions it returns representation
        of the closest object to the hero - might be nothing, another object or a wall.
        Representation of observation for all the directions will be concatenated.
        """
        num_obj_types = len(self.settings["objects"]) + 1 # and wall
        max_speed_x, max_speed_y = self.settings["maximum_speed"]

        observable_distance = self.settings["observation_line_length"]

        relevant_objects = [obj for obj in self.objects
                            if obj.position.distance(self.hero.position) < observable_distance]
        # objects sorted from closest to furthest
        relevant_objects.sort(key=lambda x: x.position.distance(self.hero.position))

        observation        = np.zeros(self.observation_size)
        observation_offset = 0
        for i, observation_line in enumerate(self.observation_lines):
            # shift to hero position
            observation_line = LineSegment2(self.hero.position + Vector2(*observation_line.p1),
                                            self.hero.position + Vector2(*observation_line.p2))

            observed_object = None
            # if end of observation line is outside of walls, we see the wall.
            if not self.inside_walls(observation_line.p2):
                observed_object = "**wall**"
            for obj in relevant_objects:
                if observation_line.distance(obj.position) < self.settings["object_radius"]:
                    observed_object = obj
                    break
            object_type_id = None
            speed_x, speed_y = 0, 0
            proximity = 0
            if observed_object == "**wall**": # wall seen
                object_type_id = num_obj_types - 1
                # a wall has fairly low speed...
                speed_x, speed_y = 0, 0
                # best candidate is intersection between
                # observation_line and a wall, that's
                # closest to the hero
                best_candidate = None
                for wall in self.walls:
                    candidate = observation_line.intersect(wall)
                    if candidate is not None:
                        if (best_candidate is None or
                                best_candidate.distance(self.hero.position) >
                                candidate.distance(self.hero.position)):
                            best_candidate = candidate
                if best_candidate is None:
                    # assume it is due to rounding errors
                    # and wall is barely touching observation line
                    proximity = observable_distance
                else:
                    proximity = best_candidate.distance(self.hero.position)
            elif observed_object is not None: # agent seen
                object_type_id = self.settings["objects"].index(observed_object.obj_type)
                speed_x, speed_y = tuple(observed_object.speed)
                intersection_segment = obj.as_circle().intersect(observation_line)
                assert intersection_segment is not None
                try:
                    proximity = min(intersection_segment.p1.distance(self.hero.position),
                                    intersection_segment.p2.distance(self.hero.position))
                except AttributeError:
                    proximity = observable_distance
            for object_type_idx_loop in range(num_obj_types):
                observation[observation_offset + object_type_idx_loop] = 1.0
            if object_type_id is not None:
                observation[observation_offset + object_type_id] = proximity / observable_distance
            observation[observation_offset + num_obj_types] =     speed_x   / max_speed_x
            observation[observation_offset + num_obj_types + 1] = speed_y   / max_speed_y
            assert num_obj_types + 2 == self.eye_observation_size
            observation_offset += self.eye_observation_size

        observation[observation_offset]     = self.hero.speed[0] / max_speed_x
        observation[observation_offset + 1] = self.hero.speed[1] / max_speed_y
        observation_offset += 2

        # add normalized locaiton of the hero in environment
        observation[observation_offset]     = self.hero.position[0] / 350.0 - 1.0
        observation[observation_offset + 1] = self.hero.position[1] / 250.0 - 1.0

        assert observation_offset + 2 == self.observation_size

        return observation

    def distance_to_walls(self):
        """Returns distance of a hero to walls"""
        res = float('inf')
        for wall in self.walls:
            res = min(res, self.hero.position.distance(wall))
        return res - self.settings["object_radius"]

    def collect_reward(self):
        """Return accumulated object eating score + current distance to walls score"""
        wall_reward =  self.settings["wall_distance_penalty"] * \
                       np.exp(-self.distance_to_walls() / self.settings["tolerable_distance_to_wall"])
        assert wall_reward < 1e-3, "You are rewarding hero for being close to the wall!"
        total_reward = wall_reward + self.object_reward
        self.object_reward = 0
        self.collected_rewards.append(total_reward)
        return total_reward

    def plot_reward(self, smoothing = 30):
        """Plot evolution of reward over time."""
        plottable = self.collected_rewards[:]
        while len(plottable) > 1000:
            for i in range(0, len(plottable) - 1, 2):
                plottable[i//2] = (plottable[i] + plottable[i+1]) / 2
            plottable = plottable[:(len(plottable) // 2)]
        x = []
        for  i in range(smoothing, len(plottable)):
            chunk = plottable[i-smoothing:i]
            x.append(sum(chunk) / len(chunk))
        plt.plot(list(range(len(x))), x)

    def generate_observation_lines(self):
        """Generate observation segments in settings["num_observation_lines"] directions"""
        result = []
        start = Point2(0.0, 0.0)
        end   = Point2(self.settings["observation_line_length"],
                       self.settings["observation_line_length"])
        for angle in np.linspace(0, 2*np.pi, self.settings["num_observation_lines"], endpoint=False):
            rotation = Point2(math.cos(angle), math.sin(angle))
            current_start = Point2(start[0] * rotation[0], start[1] * rotation[1])
            current_end   = Point2(end[0]   * rotation[0], end[1]   * rotation[1])
            result.append( LineSegment2(current_start, current_end))
        return result

    def _repr_html_(self):
        return self.to_html()

    def to_html(self, stats=[]):
        """Return svg representation of the simulator"""

        stats = stats[:]
        recent_reward = self.collected_rewards[-100:] + [0]
        objects_eaten_str = ', '.join(["%s: %s" % (o,c) for o,c in self.objects_eaten.items()])
        stats.extend([
            "nearest wall = %.1f" % (self.distance_to_walls(),),
            "reward       = %.1f" % (sum(recent_reward)/len(recent_reward),),
            "objects eaten => %s" % (objects_eaten_str,),
        ])

        scene = svg.Scene((self.size[0] + 20, self.size[1] + 20 + 20 * len(stats)))
        scene.add(svg.Rectangle((10, 10), self.size))

        for line in self.observation_lines:
            scene.add(svg.Line(line.p1 + self.hero.position + Point2(10,10),
                               line.p2 + self.hero.position + Point2(10,10)))

        for obj in self.objects + [self.hero] :
            scene.add(obj.draw())

        offset = self.size[1] + 15
        for txt in stats:
            scene.add(svg.Text((10, offset + 20), txt, 15))
            offset += 20

        return scene

<|endoftext|>


<commit_before>__author__ = 'Antony Cherepanov'

import argparse
import os
import multiprocessing
from PIL import Image

def parse_arguments():
    """ Parse arguments and start slice process
    :return list of arguments
    """

    parser = argparse.ArgumentParser(
        description="Multi-thread python app for slicing images")
    parser.add_argument("folder",
                        help="absolute path to the folder with images to slice")
    parser.add_argument("width", help="width of slice", type=int)
    parser.add_argument("height", help="height of slice", type=int)
    parser.add_argument("-add", action="store_true",
                        help="add extra space to the last slice")
    parser.add_argument("-s", "--save_to",
                        help="path to the folder where slices should be saved",
                        default="")

    args = parser.parse_args()
    return args.folder, args.width, args.height, args.add, args.save_to

def check_arguments(t_folder, t_width, t_height, t_save_folder):
    """ Check arguments
    :param t_folder: string - absolute path to the folder with images to slice
    :param t_width: int width of the slice
    :param t_height: int height of the slice
    :param t_save_folder: string absolute path to folder for slices
    :return boolean value. True if arguments are OK.
    """

    if check_existing_folder(t_folder) is False:
        print("Error: Invalid path to folder with images - " + t_folder)
        return False

    if t_width <= 0 or t_height <= 0:
        print("Error: Invalid slice size")
        return False

    if 0 < len(t_save_folder):
        if check_folder_path(t_save_folder) is True:
            if not os.path.exists(t_save_folder):
                os.makedirs(t_save_folder)
        else:
            print("Error: Invalid path to folder for slices - " + t_save_folder)
            return False

    return True

def check_existing_folder(t_path):
    """ Check if folder really exist
    :param t_path: string with absolute path to presumably existing folder
    :return: boolean value. True if folder exist, False if it's not
    """

    if not os.path.isabs(t_path) or\
            not os.path.exists(t_path) or\
            not os.path.isdir(t_path):
        return False
    return True

def check_folder_path(t_path):
    """ Check if path to folder is valid
    :param t_path: string with absolute path to some folder
    :return: boolean value. True if path could be path for folder.
    """

    if os.path.isabs(t_path) is True:
        return True
    return False

def start_slicing(t_folder, t_width, t_height, t_add_small_slice,
                  t_save_folder):
    """ Slice images
    :param t_folder: string with absolute path to the folder with images
     to slice
    :param t_width: int width of the slice
    :param t_height: int height of the slice
    :param t_add_small_slice: boolean value If True and last slice is too
     small, add it to the previous
    :param t_save_folder: string absolute path to folder for slices
    """

    images = get_images_paths(t_folder)
    cores_num = multiprocessing.cpu_count()
    img_chunks = list_split(images, cores_num)

    jobs = list()
    for i in range(cores_num):
        thread = multiprocessing.Process(target=slice_images,
                                         args=(next(img_chunks),
                                               t_width,
                                               t_height,
                                               t_add_small_slice,
                                               t_save_folder))
        jobs.append(thread)
        thread.start()

    for thread in jobs:
        thread.join()

def get_images_paths(t_folder):
    """ Check if folder contains images (on the first level) and return
     their paths
    :param t_folder: string with the absolute path to the folder
    :return: list with the absolute paths of the images in folder
    """

    if not os.path.isdir(t_folder):
        return list()

    image_extensions = ("jpg", "jpeg", "bmp", "png", "gif", "tiff")
    images = list()
    entries = os.listdir(t_folder)
    for entry in entries:
        file_path = os.path.join(t_folder, entry)
        extension = get_extension(file_path)
        if os.path.isfile(file_path) and extension in image_extensions:
            images.append(file_path)

    return images

def get_extension(t_path):
    """ Get extension of the file
    :param t_path: path or name of the file
    :return: string with extension of the file or empty string if we failed
     to get it
    """

    path_parts = str.split(t_path, '.')
    extension = path_parts[-1:][0]
    extension = extension.lower()
    return extension

def list_split(t_list, t_size):
    """ Generator that split list of elements into n chunks
    :param t_list - list of elements
    :param t_size - size of chunk
    :return generator of lists of chunks
    """

    new_length = int(len(t_list) / t_size)
    for i in range(0, t_size - 1):
        start = i * new_length
        yield t_list[start: start + new_length]
    yield t_list[t_size * new_length - new_length:]

def slice_images(t_images, t_width, t_height, t_add_small_slice, t_save_folder):
    """ Slicing images
    :param t_images: list of path to the images
    :param t_width: int width of the slice
    :param t_height: int height of the slice
    :param t_add_small_slice: boolean value If True and last slice is too
     small, add it to the previous
    :param t_save_folder: string absolute path to folder for slices
    :return list: created slices paths
    """

    slices_paths = list()
    for img_path in t_images:
        img = Image.open(img_path)
        img_width, img_height = img.size

        # Size of of image should bigger than a size of slice
        min_number_of_slices = 1
        # But if we can add extra space to the last slice, then size of image
        # should be bigger than two slices.
        if t_add_small_slice is True:
            min_number_of_slices = 2

        if (img_width // t_width < min_number_of_slices) or\
                (img_height // t_height < min_number_of_slices):
            print("Skip image " + img_path + " because it's too small")
            continue

        path, name, extension = parse_image_path(img_path)
        if 0 < len(t_save_folder):
            path = t_save_folder

        column = 0
        row = 0
        for hgt in range(0, img_height, t_height):
            hgt_end = hgt + t_height
            if img_height < hgt_end:
                break

            if t_add_small_slice is True and\
                    img_height < hgt_end + t_height:
                hgt_end = img_height

            for wdt in range(0, img_width, t_width):
                wdt_end = wdt + t_width
                if img_width < wdt_end:
                    break

                if t_add_small_slice is True and\
                        img_width < wdt_end + t_width:
                    wdt_end = img_width

                area = (wdt, hgt, wdt_end, hgt_end)
                img_slice = img.crop(area)

                filename = "{path}/{name}_{row:02d}_{col:02d}.{ext}".format(
                    path=path, name=name, col=column, row=row, ext=extension)

                img_slice.save(filename)
                slices_paths.append(filename)
                column += 1

            column = 0
            row += 1

    return slices_paths

def parse_image_path(t_img_path):
    """ Parse path to image and return it's parts: path, image name, extension
    :param t_img_path: string with path to image
    :return: tuple of strings that hold path to image file, image name and
     image extension
    """

    img_path_parts = str.split(t_img_path, os.sep)
    path_parts, image_name = img_path_parts[:-1], img_path_parts[-1]
    path = os.sep.join(path_parts)

    img_name_parts = str.split(image_name, '.')
    image_name_parts, extension = img_name_parts[:-1], img_name_parts[-1]
    name = ".".join(image_name_parts)

    return path, name, extension

if __name__ == '__main__':
    arguments = parse_arguments()
    isOk = check_arguments(arguments[0],
                           arguments[1],
                           arguments[2],
                           arguments[4])
    if isOk is True:
        start_slicing(arguments[0],
                      arguments[1],
                      arguments[2],
                      arguments[3],
                      arguments[4])
    else:
        print("Invalid arguments. Try again!")<commit_msg>Add some verbisity - output name of the processing image<commit_after>__author__ = 'Antony Cherepanov'

import argparse
import os
import multiprocessing
from PIL import Image

def parse_arguments():
    """ Parse arguments and start slice process
    :return list of arguments
    """

    parser = argparse.ArgumentParser(
        description="Multi-thread python app for slicing images")
    parser.add_argument("folder",
                        help="absolute path to the folder with images to slice")
    parser.add_argument("width", help="width of slice", type=int)
    parser.add_argument("height", help="height of slice", type=int)
    parser.add_argument("-add", action="store_true",
                        help="add extra space to the last slice")
    parser.add_argument("-s", "--save_to",
                        help="path to the folder where slices should be saved",
                        default="")

    args = parser.parse_args()
    return args.folder, args.width, args.height, args.add, args.save_to

def check_arguments(t_folder, t_width, t_height, t_save_folder):
    """ Check arguments
    :param t_folder: string - absolute path to the folder with images to slice
    :param t_width: int width of the slice
    :param t_height: int height of the slice
    :param t_save_folder: string absolute path to folder for slices
    :return boolean value. True if arguments are OK.
    """

    if check_existing_folder(t_folder) is False:
        print("Error: Invalid path to folder with images - " + t_folder)
        return False

    if t_width <= 0 or t_height <= 0:
        print("Error: Invalid slice size")
        return False

    if 0 < len(t_save_folder):
        if check_folder_path(t_save_folder) is True:
            if not os.path.exists(t_save_folder):
                os.makedirs(t_save_folder)
        else:
            print("Error: Invalid path to folder for slices - " + t_save_folder)
            return False

    return True

def check_existing_folder(t_path):
    """ Check if folder really exist
    :param t_path: string with absolute path to presumably existing folder
    :return: boolean value. True if folder exist, False if it's not
    """

    if not os.path.isabs(t_path) or\
            not os.path.exists(t_path) or\
            not os.path.isdir(t_path):
        return False
    return True

def check_folder_path(t_path):
    """ Check if path to folder is valid
    :param t_path: string with absolute path to some folder
    :return: boolean value. True if path could be path for folder.
    """

    if os.path.isabs(t_path) is True:
        return True
    return False

def start_slicing(t_folder, t_width, t_height, t_add_small_slice,
                  t_save_folder):
    """ Slice images
    :param t_folder: string with absolute path to the folder with images
     to slice
    :param t_width: int width of the slice
    :param t_height: int height of the slice
    :param t_add_small_slice: boolean value If True and last slice is too
     small, add it to the previous
    :param t_save_folder: string absolute path to folder for slices
    """

    images = get_images_paths(t_folder)
    cores_num = multiprocessing.cpu_count()
    img_chunks = list_split(images, cores_num)

    jobs = list()
    for i in range(cores_num):
        thread = multiprocessing.Process(target=slice_images,
                                         args=(next(img_chunks),
                                               t_width,
                                               t_height,
                                               t_add_small_slice,
                                               t_save_folder))
        jobs.append(thread)
        thread.start()

    for thread in jobs:
        thread.join()

def get_images_paths(t_folder):
    """ Check if folder contains images (on the first level) and return
     their paths
    :param t_folder: string with the absolute path to the folder
    :return: list with the absolute paths of the images in folder
    """

    if not os.path.isdir(t_folder):
        return list()

    image_extensions = ("jpg", "jpeg", "bmp", "png", "gif", "tiff")
    images = list()
    entries = os.listdir(t_folder)
    for entry in entries:
        file_path = os.path.join(t_folder, entry)
        extension = get_extension(file_path)
        if os.path.isfile(file_path) and extension in image_extensions:
            images.append(file_path)

    return images

def get_extension(t_path):
    """ Get extension of the file
    :param t_path: path or name of the file
    :return: string with extension of the file or empty string if we failed
     to get it
    """

    path_parts = str.split(t_path, '.')
    extension = path_parts[-1:][0]
    extension = extension.lower()
    return extension

def list_split(t_list, t_size):
    """ Generator that split list of elements into n chunks
    :param t_list - list of elements
    :param t_size - size of chunk
    :return generator of lists of chunks
    """

    new_length = int(len(t_list) / t_size)
    for i in range(0, t_size - 1):
        start = i * new_length
        yield t_list[start: start + new_length]
    yield t_list[t_size * new_length - new_length:]

def slice_images(t_images, t_width, t_height, t_add_small_slice, t_save_folder):
    """ Slicing images
    :param t_images: list of path to the images
    :param t_width: int width of the slice
    :param t_height: int height of the slice
    :param t_add_small_slice: boolean value If True and last slice is too
     small, add it to the previous
    :param t_save_folder: string absolute path to folder for slices
    :return list: created slices paths
    """

    slices_paths = list()
    for img_path in t_images:
        print("Slicing " + img_path)

        img = Image.open(img_path)
        img_width, img_height = img.size

        # Size of of image should bigger than a size of slice
        min_number_of_slices = 1
        # But if we can add extra space to the last slice, then size of image
        # should be bigger than two slices.
        if t_add_small_slice is True:
            min_number_of_slices = 2

        if (img_width // t_width < min_number_of_slices) or\
                (img_height // t_height < min_number_of_slices):
            print("Skip image " + img_path + " because it's too small")
            continue

        path, name, extension = parse_image_path(img_path)
        if 0 < len(t_save_folder):
            path = t_save_folder

        column = 0
        row = 0
        for hgt in range(0, img_height, t_height):
            hgt_end = hgt + t_height
            if img_height < hgt_end:
                break

            if t_add_small_slice is True and\
                    img_height < hgt_end + t_height:
                hgt_end = img_height

            for wdt in range(0, img_width, t_width):
                wdt_end = wdt + t_width
                if img_width < wdt_end:
                    break

                if t_add_small_slice is True and\
                        img_width < wdt_end + t_width:
                    wdt_end = img_width

                area = (wdt, hgt, wdt_end, hgt_end)
                img_slice = img.crop(area)

                filename = "{path}/{name}_{row:02d}_{col:02d}.{ext}".format(
                    path=path, name=name, col=column, row=row, ext=extension)

                img_slice.save(filename)
                slices_paths.append(filename)
                column += 1

            column = 0
            row += 1

    return slices_paths

def parse_image_path(t_img_path):
    """ Parse path to image and return it's parts: path, image name, extension
    :param t_img_path: string with path to image
    :return: tuple of strings that hold path to image file, image name and
     image extension
    """

    img_path_parts = str.split(t_img_path, os.sep)
    path_parts, image_name = img_path_parts[:-1], img_path_parts[-1]
    path = os.sep.join(path_parts)

    img_name_parts = str.split(image_name, '.')
    image_name_parts, extension = img_name_parts[:-1], img_name_parts[-1]
    name = ".".join(image_name_parts)

    return path, name, extension

if __name__ == '__main__':
    arguments = parse_arguments()
    isOk = check_arguments(arguments[0],
                           arguments[1],
                           arguments[2],
                           arguments[4])
    if isOk is True:
        start_slicing(arguments[0],
                      arguments[1],
                      arguments[2],
                      arguments[3],
                      arguments[4])
    else:
        print("Invalid arguments. Try again!")<|endoftext|>


<commit_before>import sqllite3
import os

conn = sqlite3.connect('database.db')#intalizing db
c = conn.cursor()

# If we are not given a path to list, use /tmp
root='/'
for dir_name, sub_dirs, files in os.walk(root): #dir_name is the current directory, sub_dirs are subs and files....
    #print '\n', dir_name
    # Make the subdirectory names stand out with /
    sub_dirs = [ '%s/' % n for n in sub_dirs ]
    # Mix the directory contents together
    contents = files  #originally sub_dirs + files
    contents.sort()
    for f in contents:
        #print f + " is in " + dir_name
        path = dir_name + '/' + f
        clustery = c.execute('SELECT * FROM clusters WHERE contents = '+ path);
        if (clustery === 'NULL'):
            cluster = "undefined" #if undefined alex will find it
        at=os.path.getatime(os.sep.join(dir_name, f]))#last access time of file
        c.execute('INSERT INTO scan (fpath, accessDate, cluster)]  VALUES (' + fpath + ',' + at + ',' + cluster')';')'#put something that is not retarted here

'''
OLD:

I'm tired and have to go to bed, here's what you need to know, first look at alex's dbsetup.py file
it should have everything you need, insert right shit into right problems, last access is a different story, ask this is irc:
 hey guys, does anyone know how I can access last modification of the files and not their paths, similiar to this: os.path.getatime(path

talk to alex about fpath., other than that you should be fine

'''
<commit_msg>fixed syntax/ import error<commit_after>import sqlite3
import os

conn = sqlite3.connect('database.db')#intalizing db
c = conn.cursor()

# If we are not given a path to list, use /tmp
root='/'
for dir_name, sub_dirs, files in os.walk(root): #dir_name is the current directory, sub_dirs are subs and files....
    #print '\n', dir_name
    # Make the subdirectory names stand out with /
    sub_dirs = [ '%s/' % n for n in sub_dirs ]
    # Mix the directory contents together
    contents = files  #originally sub_dirs + files
    contents.sort()
    for f in contents:
        #print f + " is in " + dir_name
        path = dir_name + '/' + f
        c.execute('SELECT * FROM clusters WHERE contents = '+ path);
        clustery = c.fetchone()
        if (len(clustery) == 0):
            cluster = "undefined"
        else:
            cluster = clustery[0]
        cluster = "undefined" #if undefined alex will find it
        at=os.path.getatime(os.sep.join(dir_name, f))#last access time of file
        c.execute('INSERT INTO scan (fpath, accessDate, cluster) VALUES (' + fpath + ',' + at + ',' + cluster + ')')#put something that is not retarted here'''
'''
OLD:

I'm tired and have to go to bed, here's what you need to know, first look at alex's dbsetup.py file
it should have everything you need, insert right shit into right problems, last access is a different story, ask this is irc:
 hey guys, does anyone know how I can access last modification of the files and not their paths, similiar to this: os.path.getatime(path

talk to alex about fpath., other than that you should be fine

'''
<|endoftext|>


<commit_before>from gflags import *
<commit_msg>Spoof gflags pseudo-package __name__ for backwards compatibility.<commit_after># gflags raises DuplicateFlagError when defining default flags from packages
# with different names, so this pseudo-package must mimic the core gflags
# package name.
__name__ += ".gflags"  # i.e. "third_party.py.gflags.gflags"

from gflags import *
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

""" decoder plugin based on aubio """

from timeside.core.decoder import Decoder, IDecoder, implements, interfacedoc
from timeside.plugins.decoder.utils import get_sha1, get_media_uri_info
import aubio
import mimetypes

class AubioDecoder(Decoder):
    """ File decoder based on aubio """
    implements(IDecoder)

    output_blocksize = 8 * 1024

    def __init__(self, uri, start=0, duration=None, sha1=None):
        super().__init__(start=start, duration=duration)
        self.uri = uri

        # create the source with default settings
        try:
            self.source = aubio.source(self.uri, hop_size=self.output_blocksize)
        except RuntimeError as e:
            raise IOError(e)
        self.input_samplerate = self.source.samplerate
        self.input_channels = self.source.channels

        # get the original file duration
        self.input_totalframes = self.source.duration
        self.input_duration = self.input_totalframes / self.input_samplerate
        self.uri_duration = self.input_duration

        self.start = start
        self.duration = duration

        # FIXME
        self.mimetype = mimetypes.guess_type(uri)[0]
        self.input_width = 8

        if sha1 is not None:
            self._sha1 = sha1
        else:
            self._sha1 = get_sha1(uri)

    def setup(self, channels=None, samplerate=None, blocksize=None):
        if self.start or self.duration:
            if self.start > self.uri_duration:
                raise ValueError ('Segment start time exceeds media duration')
            if self.duration is None:
                self.duration = self.uri_duration - self.start
            if self.start + self.duration > self.uri_duration:
                raise ValueError ('Segment duration exceeds media duration')

        kwargs = {}
        if channels is not None:
            kwargs.update ({'channels': channels})
        if samplerate is not None:
            kwargs.update ({'samplerate': samplerate})
        if blocksize is not None and blocksize != self.source.hop_size:
            kwargs.update ({'hop_size': blocksize})
        if len(kwargs):
            self.source = aubio.source(self.uri, **kwargs)

        self.output_blocksize = self.source.hop_size
        self.output_channels = self.source.channels
        self.output_samplerate  = self.source.samplerate

        self.frames_read = 0
        self.start_frame = int(self.start * self.output_samplerate)
        if self.duration:
            seconds_to_read = self.duration
            self.frames_to_read = int (seconds_to_read * self.output_samplerate)

        if self.duration:
            self.input_duration = self.duration
            self.input_totalframes = self.frames_to_read

        if self.start > 0:
            self.source.seek(self.start_frame)

    @staticmethod
    @interfacedoc
    def id():
        return "aubio_decoder"

    @staticmethod
    @interfacedoc
    def version():
        return "1.0"

    @interfacedoc
    def process(self):
        frames, read = self.source.do_multi()
        self.eod = (read < self.output_blocksize)
        if self.duration and self.frames_read + read >= self.frames_to_read:
            extra_read = self.frames_read + read - self.frames_to_read
            read = self.source.hop_size - extra_read
            self.eod = True
        self.frames_read += read
        frames = frames[:, :read].T
        return frames, self.eod

    @interfacedoc
    def mime_type(self):
        return self.mimetype

    @interfacedoc
    def resolution(self):
        return 0

    @interfacedoc
    def metadata(self):
        return {}

    @interfacedoc
    def totalframes(self):
        if self.input_samplerate == self.output_samplerate:
            return self.input_totalframes
        else:
            ratio = self.output_samplerate / self.input_samplerate
            return int(self.input_totalframes * ratio)
<commit_msg>[plugins] return a copy of the frames emitted by aubio decoder<commit_after># -*- coding: utf-8 -*-

""" decoder plugin based on aubio """

from timeside.core.decoder import Decoder, IDecoder, implements, interfacedoc
from timeside.plugins.decoder.utils import get_sha1, get_media_uri_info
import aubio
import mimetypes

class AubioDecoder(Decoder):
    """ File decoder based on aubio """
    implements(IDecoder)

    output_blocksize = 8 * 1024

    def __init__(self, uri, start=0, duration=None, sha1=None):
        super().__init__(start=start, duration=duration)
        self.uri = uri

        # create the source with default settings
        try:
            self.source = aubio.source(self.uri, hop_size=self.output_blocksize)
        except RuntimeError as e:
            raise IOError(e)
        self.input_samplerate = self.source.samplerate
        self.input_channels = self.source.channels

        # get the original file duration
        self.input_totalframes = self.source.duration
        self.input_duration = self.input_totalframes / self.input_samplerate
        self.uri_duration = self.input_duration

        self.start = start
        self.duration = duration

        # FIXME
        self.mimetype = mimetypes.guess_type(uri)[0]
        self.input_width = 8

        if sha1 is not None:
            self._sha1 = sha1
        else:
            self._sha1 = get_sha1(uri)

    def setup(self, channels=None, samplerate=None, blocksize=None):
        if self.start or self.duration:
            if self.start > self.uri_duration:
                raise ValueError ('Segment start time exceeds media duration')
            if self.duration is None:
                self.duration = self.uri_duration - self.start
            if self.start + self.duration > self.uri_duration:
                raise ValueError ('Segment duration exceeds media duration')

        kwargs = {}
        if channels is not None:
            kwargs.update ({'channels': channels})
        if samplerate is not None:
            kwargs.update ({'samplerate': samplerate})
        if blocksize is not None and blocksize != self.source.hop_size:
            kwargs.update ({'hop_size': blocksize})
        if len(kwargs):
            self.source = aubio.source(self.uri, **kwargs)

        self.output_blocksize = self.source.hop_size
        self.output_channels = self.source.channels
        self.output_samplerate  = self.source.samplerate

        self.frames_read = 0
        self.start_frame = int(self.start * self.output_samplerate)
        if self.duration:
            seconds_to_read = self.duration
            self.frames_to_read = int (seconds_to_read * self.output_samplerate)

        if self.duration:
            self.input_duration = self.duration
            self.input_totalframes = self.frames_to_read

        if self.start > 0:
            self.source.seek(self.start_frame)

    @staticmethod
    @interfacedoc
    def id():
        return "aubio_decoder"

    @staticmethod
    @interfacedoc
    def version():
        return "1.0"

    @interfacedoc
    def process(self):
        frames, read = self.source.do_multi()
        self.eod = (read < self.output_blocksize)
        if self.duration and self.frames_read + read >= self.frames_to_read:
            extra_read = self.frames_read + read - self.frames_to_read
            read = self.source.hop_size - extra_read
            self.eod = True
        self.frames_read += read
        frames = frames[:, :read].T
        return frames.copy(), self.eod

    @interfacedoc
    def mime_type(self):
        return self.mimetype

    @interfacedoc
    def resolution(self):
        return 0

    @interfacedoc
    def metadata(self):
        return {}

    @interfacedoc
    def totalframes(self):
        if self.input_samplerate == self.output_samplerate:
            return self.input_totalframes
        else:
            ratio = self.output_samplerate / self.input_samplerate
            return int(self.input_totalframes * ratio)
<|endoftext|>


<commit_before>import module_helper, os

def init(param):
   module_helper.checkParameter(param,key='cutadapt_exec',dType=str)
   module_helper.checkParameter(param,key='cutadapt_python_version',dType=str)
   module_helper.checkParameter(param,key='cutadapt_first_adapter',dType=str)
   if param['paired']:
       module_helper.checkParameter(param,key='cutadapt_second_adapter',dType=str)
   module_helper.checkParameter(param,key='cutadapt_m',dType=str)

def run_cutadapt(param,infile,outfile,adapter):
    #run the clipper

    call = param['cutadapt_python_version']+' '+param['cutadapt_exec']+' -m '+param['cutadapt_m']+' -a '+adapter+' -o "'+outfile+'" "'+param[infile]+'"'
    print '\n'
    print call
    print '\n'

    output,error = subprocess.Popen(call ,stdout = subprocess.PIPE, shell=True, stderr= subprocess.PIPE).communicate()
    print output
    print error

    param['file_handle'].write(error)
    param['file_handle'].write(output)

    with open(outfile+'.txt','w') as f:
        f.write(output)

    # Error handling
    if (not os.path.exists(outfile)) or os.stat(outfile).st_size<1000000:
        sys.exit(0)

if __name__ == "__main__":
    import subprocess, sys, os
    param=module_helper.initialize_module()

    #run fastqc
    outfile = param['module_dir']+param['stub'][param['file_index']]+'.clipped.fastq.gz'
    run_cutadapt(param,'working_file',outfile,param['cutadapt_first_adapter'])

    if not param['paired']:
        module_helper.wrapup_module(param,[outfile])
    #calling it on the second fastq file if it is paired
    else:
        outfile2 = param['module_dir']+param['stub'][param['file_index']]+'.clipped.2.fastq.gz'
        run_cutadapt(param,'working_file2',outfile2,param['cutadapt_second_adapter'])
        module_helper.wrapup_module(param,[outfile,outfile2])

<commit_msg>run cutadapt as the main exec, not python<commit_after>import module_helper, os

def init(param):
   module_helper.checkParameter(param,key='cutadapt_exec',dType=str)
   module_helper.checkParameter(param,key='cutadapt_python_version',dType=str)
   module_helper.checkParameter(param,key='cutadapt_first_adapter',dType=str)
   if param['paired']:
       module_helper.checkParameter(param,key='cutadapt_second_adapter',dType=str)
   module_helper.checkParameter(param,key='cutadapt_m',dType=str)

def run_cutadapt(param,infile,outfile,adapter):
    #run the clipper

    call = param['cutadapt_exec']+' -m '+param['cutadapt_m']+' -a '+adapter+' -o "'+outfile+'" "'+param[infile]+'"'
    print '\n'
    print call
    print '\n'

    output,error = subprocess.Popen(call ,stdout = subprocess.PIPE, shell=True, stderr= subprocess.PIPE).communicate()
    print output
    print error

    param['file_handle'].write(error)
    param['file_handle'].write(output)

    with open(outfile+'.txt','w') as f:
        f.write(output)

    # Error handling
    if (not os.path.exists(outfile)) or os.stat(outfile).st_size<1000000:
        sys.exit(0)

if __name__ == "__main__":
    import subprocess, sys, os
    param=module_helper.initialize_module()

    #run fastqc
    outfile = param['module_dir']+param['stub'][param['file_index']]+'.clipped.fastq.gz'
    run_cutadapt(param,'working_file',outfile,param['cutadapt_first_adapter'])

    if not param['paired']:
        module_helper.wrapup_module(param,[outfile])
    #calling it on the second fastq file if it is paired
    else:
        outfile2 = param['module_dir']+param['stub'][param['file_index']]+'.clipped.2.fastq.gz'
        run_cutadapt(param,'working_file2',outfile2,param['cutadapt_second_adapter'])
        module_helper.wrapup_module(param,[outfile,outfile2])

<|endoftext|>


<commit_before>#!/usr/bin/python
# Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# heapcheck_test.py

"""Wrapper for running the test under heapchecker and analyzing the output."""

import datetime
import logging
import os
import re

import common
import google.path_utils
import suppressions

class HeapcheckWrapper(object):
  TMP_FILE = 'heapcheck.log'
  SANITY_TEST_SUPPRESSION = "Heapcheck sanity test"

  def __init__(self, supp_files):
    self._mode = 'strict'
    self._timeout = 1200
    self._nocleanup_on_exit = False
    self._suppressions = []
    for fname in supp_files:
      self._suppressions.extend(suppressions.ReadSuppressionsFromFile(fname))
    if os.path.exists(self.TMP_FILE):
      os.remove(self.TMP_FILE)

  def PutEnvAndLog(self, env_name, env_value):
    """Sets the env var |env_name| to |env_value| and writes to logging.info.
    """
    os.putenv(env_name, env_value)
    logging.info('export %s=%s', env_name, env_value)

  def Execute(self):
    """Executes the app to be tested."""
    logging.info('starting execution...')
    proc = ['sh', google.path_utils.ScriptDir() + '/heapcheck_std.sh']
    proc += self._args
    self.PutEnvAndLog('G_SLICE', 'always-malloc')
    self.PutEnvAndLog('NSS_DISABLE_ARENA_FREE_LIST', '1')
    self.PutEnvAndLog('GTEST_DEATH_TEST_USE_FORK', '1')
    self.PutEnvAndLog('HEAPCHECK', self._mode)
    self.PutEnvAndLog('HEAP_CHECK_MAX_LEAKS', '-1')
    self.PutEnvAndLog('PPROF_PATH',
        google.path_utils.ScriptDir() +
        '/../../third_party/tcmalloc/chromium/src/pprof')
    self.PutEnvAndLog('LD_PRELOAD', '/usr/lib/debug/libstdc++.so')

    common.RunSubprocess(proc, self._timeout)

    # Always return true, even if running the subprocess failed. We depend on
    # Analyze to determine if the run was valid. (This behaviour copied from
    # the purify_test.py script.)
    return True

  def Analyze(self, log_lines, check_sanity=False):
    """Analyzes the app's output and applies suppressions to the reports.

    Analyze() searches the logs for leak reports and tries to apply
    suppressions to them. Unsuppressed reports and other log messages are
    dumped as is.

    If |check_sanity| is True, the list of suppressed reports is searched for a
    report starting with SANITY_TEST_SUPPRESSION. If there isn't one, Analyze
    returns 2 regardless of the unsuppressed reports.

    Args:
      log_lines:      An iterator over the app's log lines.
      check_sanity:   A flag that determines whether we should check the tool's
                      sanity.
    Returns:
      2, if the sanity check fails,
      1, if unsuppressed reports remain in the output and the sanity check
      passes,
      0, if all the errors are suppressed and the sanity check passes.
    """
    leak_report = re.compile(
        'Leak of ([0-9]*) bytes in ([0-9]*) objects allocated from:')
    stack_line = re.compile('\s*@\s*(?:0x)?[0-9a-fA-F]*\s*([^\n]*)')
    return_code = 0
    # leak signature: [number of bytes, number of objects]
    cur_leak_signature = None
    cur_stack = []
    cur_report = []
    # Statistics grouped by suppression description:
    # [hit count, bytes, objects].
    used_suppressions = {}
    for line in log_lines:
      line = line.rstrip()  # remove the trailing \n
      match = stack_line.match(line)
      if match:
        cur_stack.append(match.groups()[0])
        cur_report.append(line)
        continue
      else:
        if cur_stack:
          # Try to find the suppression that applies to the current leak stack.
          description = ''
          for supp in self._suppressions:
            if supp.Match(cur_stack):
              cur_stack = []
              description = supp.description
              break
          if cur_stack:
            # Print the report and set the return code to 1.
            print ('Leak of %d bytes in %d objects allocated from:'
                   % tuple(cur_leak_signature))
            print '\n'.join(cur_report)
            return_code = 1
          else:
            # Update the suppressions histogram.
            if description in used_suppressions:
              hits, bytes, objects = used_suppressions[description]
              hits += 1
              bytes += cur_leak_signature[0]
              objects += cur_leak_signature[1]
              used_suppressions[description] = [hits, bytes, objects]
            else:
              used_suppressions[description] = [1] + cur_leak_signature
        cur_stack = []
        cur_report = []
        cur_leak_signature = None
        match = leak_report.match(line)
        if match:
          cur_leak_signature = map(int, match.groups())
        else:
          print line
    # Print the list of suppressions used.
    is_sane = False
    if used_suppressions:
      print
      print '-----------------------------------------------------'
      print 'Suppressions used:'
      print '   count    bytes  objects name'
      histo = {}
      for description in used_suppressions:
        if description.startswith(HeapcheckWrapper.SANITY_TEST_SUPPRESSION):
          is_sane = True
        hits, bytes, objects = used_suppressions[description]
        line = '%8d %8d %8d %s' % (hits, bytes, objects, description)
        if hits in histo:
          histo[hits].append(line)
        else:
          histo[hits] = [line]
      keys = histo.keys()
      keys.sort()
      for count in keys:
        for line in histo[count]:
          print line
      print '-----------------------------------------------------'
    if check_sanity and not is_sane:
      logging.error("Sanity check failed")
      return 2
    else:
      return return_code

  def RunTestsAndAnalyze(self, check_sanity):
    self.Execute()
    log_file = file(self.TMP_FILE, 'r')
    ret = self.Analyze(log_file, check_sanity)
    log_file.close()
    return ret

  def Main(self, args, check_sanity=False):
    self._args = args
    start = datetime.datetime.now()
    retcode = -1
    retcode = self.RunTestsAndAnalyze(check_sanity)
    end = datetime.datetime.now()
    seconds = (end - start).seconds
    hours = seconds / 3600
    seconds %= 3600
    minutes = seconds / 60
    seconds %= 60
    logging.info('elapsed time: %02d:%02d:%02d', hours, minutes, seconds)
    logging.info('For more information on the Heapcheck bot see '
                 'http://dev.chromium.org/developers/how-tos/'
                 'using-the-heap-leak-checker')
    return retcode

def RunTool(args, supp_files, module):
  tool = HeapcheckWrapper(supp_files)
  MODULES_TO_SANITY_CHECK = ["base"]
  check_sanity = module in MODULES_TO_SANITY_CHECK
  return tool.Main(args[1:], check_sanity)
<commit_msg>Print suppressions for the leaks reported by heap leak checker.<commit_after>#!/usr/bin/python
# Copyright (c) 2006-2009 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# heapcheck_test.py

"""Wrapper for running the test under heapchecker and analyzing the output."""

import datetime
import logging
import os
import re

import common
import google.path_utils
import suppressions

class HeapcheckWrapper(object):
  TMP_FILE = 'heapcheck.log'
  SANITY_TEST_SUPPRESSION = "Heapcheck sanity test"

  def __init__(self, supp_files):
    self._mode = 'strict'
    self._timeout = 1200
    self._nocleanup_on_exit = False
    self._suppressions = []
    for fname in supp_files:
      self._suppressions.extend(suppressions.ReadSuppressionsFromFile(fname))
    if os.path.exists(self.TMP_FILE):
      os.remove(self.TMP_FILE)

  def PutEnvAndLog(self, env_name, env_value):
    """Sets the env var |env_name| to |env_value| and writes to logging.info.
    """
    os.putenv(env_name, env_value)
    logging.info('export %s=%s', env_name, env_value)

  def Execute(self):
    """Executes the app to be tested."""
    logging.info('starting execution...')
    proc = ['sh', google.path_utils.ScriptDir() + '/heapcheck_std.sh']
    proc += self._args
    self.PutEnvAndLog('G_SLICE', 'always-malloc')
    self.PutEnvAndLog('NSS_DISABLE_ARENA_FREE_LIST', '1')
    self.PutEnvAndLog('GTEST_DEATH_TEST_USE_FORK', '1')
    self.PutEnvAndLog('HEAPCHECK', self._mode)
    self.PutEnvAndLog('HEAP_CHECK_MAX_LEAKS', '-1')
    self.PutEnvAndLog('PPROF_PATH',
        google.path_utils.ScriptDir() +
        '/../../third_party/tcmalloc/chromium/src/pprof')
    self.PutEnvAndLog('LD_PRELOAD', '/usr/lib/debug/libstdc++.so')

    common.RunSubprocess(proc, self._timeout)

    # Always return true, even if running the subprocess failed. We depend on
    # Analyze to determine if the run was valid. (This behaviour copied from
    # the purify_test.py script.)
    return True

  def Analyze(self, log_lines, check_sanity=False):
    """Analyzes the app's output and applies suppressions to the reports.

    Analyze() searches the logs for leak reports and tries to apply
    suppressions to them. Unsuppressed reports and other log messages are
    dumped as is.

    If |check_sanity| is True, the list of suppressed reports is searched for a
    report starting with SANITY_TEST_SUPPRESSION. If there isn't one, Analyze
    returns 2 regardless of the unsuppressed reports.

    Args:
      log_lines:      An iterator over the app's log lines.
      check_sanity:   A flag that determines whether we should check the tool's
                      sanity.
    Returns:
      2, if the sanity check fails,
      1, if unsuppressed reports remain in the output and the sanity check
      passes,
      0, if all the errors are suppressed and the sanity check passes.
    """
    leak_report = re.compile(
        'Leak of ([0-9]*) bytes in ([0-9]*) objects allocated from:')
    stack_line = re.compile('\s*@\s*(?:0x)?[0-9a-fA-F]*\s*([^\n]*)')
    return_code = 0
    # leak signature: [number of bytes, number of objects]
    cur_leak_signature = None
    cur_stack = []
    cur_report = []
    # Statistics grouped by suppression description:
    # [hit count, bytes, objects].
    used_suppressions = {}
    for line in log_lines:
      line = line.rstrip()  # remove the trailing \n
      match = stack_line.match(line)
      if match:
        cur_stack.append(match.groups()[0])
        cur_report.append(line)
        continue
      else:
        if cur_stack:
          # Try to find the suppression that applies to the current leak stack.
          description = ''
          for supp in self._suppressions:
            if supp.Match(cur_stack):
              cur_stack = []
              description = supp.description
              break
          if cur_stack:
            # Print the report and set the return code to 1.
            print ('Leak of %d bytes in %d objects allocated from:'
                   % tuple(cur_leak_signature))
            print '\n'.join(cur_report)
            return_code = 1
            # Generate the suppression iff the stack contains more than one
            # frame (otherwise it's likely to be broken)
            if len(cur_stack) > 1:
              print 'Suppression:\n{'
              print '   <insert_a_suppression_name_here>'
              print '   Heapcheck:Leak'
              for frame in cur_stack:
                print '   fun:' + frame
              print '}'
            else:
              print ('This stack may be broken due to omitted frame pointers. '
                     'It''s not recommended to suppress it')
          else:
            # Update the suppressions histogram.
            if description in used_suppressions:
              hits, bytes, objects = used_suppressions[description]
              hits += 1
              bytes += cur_leak_signature[0]
              objects += cur_leak_signature[1]
              used_suppressions[description] = [hits, bytes, objects]
            else:
              used_suppressions[description] = [1] + cur_leak_signature
        cur_stack = []
        cur_report = []
        cur_leak_signature = None
        match = leak_report.match(line)
        if match:
          cur_leak_signature = map(int, match.groups())
        else:
          print line
    # Print the list of suppressions used.
    is_sane = False
    if used_suppressions:
      print
      print '-----------------------------------------------------'
      print 'Suppressions used:'
      print '   count    bytes  objects name'
      histo = {}
      for description in used_suppressions:
        if description.startswith(HeapcheckWrapper.SANITY_TEST_SUPPRESSION):
          is_sane = True
        hits, bytes, objects = used_suppressions[description]
        line = '%8d %8d %8d %s' % (hits, bytes, objects, description)
        if hits in histo:
          histo[hits].append(line)
        else:
          histo[hits] = [line]
      keys = histo.keys()
      keys.sort()
      for count in keys:
        for line in histo[count]:
          print line
      print '-----------------------------------------------------'
    if check_sanity and not is_sane:
      logging.error("Sanity check failed")
      return 2
    else:
      return return_code

  def RunTestsAndAnalyze(self, check_sanity):
    self.Execute()
    log_file = file(self.TMP_FILE, 'r')
    ret = self.Analyze(log_file, check_sanity)
    log_file.close()
    return ret

  def Main(self, args, check_sanity=False):
    self._args = args
    start = datetime.datetime.now()
    retcode = -1
    retcode = self.RunTestsAndAnalyze(check_sanity)
    end = datetime.datetime.now()
    seconds = (end - start).seconds
    hours = seconds / 3600
    seconds %= 3600
    minutes = seconds / 60
    seconds %= 60
    logging.info('elapsed time: %02d:%02d:%02d', hours, minutes, seconds)
    logging.info('For more information on the Heapcheck bot see '
                 'http://dev.chromium.org/developers/how-tos/'
                 'using-the-heap-leak-checker')
    return retcode

def RunTool(args, supp_files, module):
  tool = HeapcheckWrapper(supp_files)
  MODULES_TO_SANITY_CHECK = ["base"]
  check_sanity = module in MODULES_TO_SANITY_CHECK
  return tool.Main(args[1:], check_sanity)
<|endoftext|>