<commit_before>"""
An Event is a collection of Particles.
"""

import numpy as np
from .particle import Particle

def frominput(iterable):
    """
    Generate Events from standard particle info.

    Arguments
    ---------
    iterable -- containing standard particle lines [bytes or strings]

    Yields
    ------
    Event()

    """

    # init. empty list of particles
    particles = []

    # scan through lines
    for l in iterable:
        # if l is a blank line, then l.strip() will be an empty string

        if l.isspace():
            # this is a blank line => have reached the end of this event
            if particles:
                yield Event(particles)
                particles = []

        else:
            # this is a particle line
            ID,pT,phi,eta = l.split()

            # create a new Particle and add it to the list
            particles.append(Particle(
                    ID=int(ID),
                    pT=float(pT),
                    phi=float(phi),
                    eta=float(eta)
                )
            )

    # lines have been exhausted
    # typically there will be one last event to yield
    if particles:
        yield Event(particles)

class Event:
    """
    Stores a list of Particles and provides methods for calculating observables.

    Usage
    -----
    >>> Event([particle1, particle2, ...])

    """

    def __init__(self,particles=[]):
        self._particles = particles
        self._vn = []

    def _phi(self):
        # a list of phi for each Particle in the Event
        return [p.phi for p in self._particles]

    def _calc_flows(self,vnmin=2,vnmax=6):
        # event-plane method
        phi = np.asarray(self._phi(), dtype='float128')

        for n in range(vnmin,vnmax+1):
            vx = np.mean(np.cos(n*phi))
            vy = np.mean(np.sin(n*phi))
            self._vn.extend([vx,vy])

    def flows(self,vnmin=2,vnmax=6):
        """
        Retrieve the Event's flow coefficients v_n.  Only do the calculation if
        necessary.

        Arguments
        ---------
        vnmin, vnmax -- Range of v_n to calculate.

        Returns
        -------
        [vnmin_x, vnmin_y, ..., vnmax_x, vnmax_y]

        """

        if not self._vn:
            self._calc_flows(vnmin,vnmax)

        return self._vn
<commit_msg>Minor optimizations to Event.flows() method.<commit_after>"""
An Event is a collection of Particles.
"""

import numpy as np
from .particle import Particle

def frominput(iterable):
    """
    Generate Events from standard particle info.

    Arguments
    ---------
    iterable -- containing standard particle lines [bytes or strings]

    Yields
    ------
    Event()

    """

    # init. empty list of particles
    particles = []

    # scan through lines
    for l in iterable:
        # if l is a blank line, then l.strip() will be an empty string

        if l.isspace():
            # this is a blank line => have reached the end of this event
            if particles:
                yield Event(particles)
                particles = []

        else:
            # this is a particle line
            ID,pT,phi,eta = l.split()

            # create a new Particle and add it to the list
            particles.append(Particle(
                    ID=int(ID),
                    pT=float(pT),
                    phi=float(phi),
                    eta=float(eta)
                )
            )

    # lines have been exhausted
    # typically there will be one last event to yield
    if particles:
        yield Event(particles)

class Event:
    """
    Stores a list of Particles and provides methods for calculating observables.

    Usage
    -----
    >>> Event([particle1, particle2, ...])

    """

    def __init__(self,particles=[]):
        self._particles = particles

    def _phi(self):
        # phi of each Particle in the Event
        return np.array([p.phi for p in self._particles])

    def flows(self,vnmin,vnmax,mean=np.mean,sin=np.sin,cos=np.cos):
        """
        Calculate the Event's flow coefficients v_n.

        Arguments
        ---------
        vnmin, vnmax -- range of v_n to calculate

        Returns
        -------
        [vnmin_x, vnmin_y, ..., vnmax_x, vnmax_y]

        """

        self._vn = []

        phi = self._phi()

        for n in range(vnmin,vnmax+1):
            # event-plane method
            vx = mean(cos(n*phi))
            vy = mean(sin(n*phi))
            self._vn.extend([vx,vy])

        return self._vn
<|endoftext|>


<commit_before>#!/usr/bin/env python

import inspect
import logging
import os.path
import re
import subprocess

# Supported hooks - see
# https://www.kernel.org/pub/software/scm/git/docs/githooks.html
supported_hooks = [
    'applypatch-msg',
    'pre-applypatch',
    'post-applypatch',
    'pre-commit',
    'prepare-commit-msg',
    'commit-msg',
    'post-commit',
    'pre-rebase',
    'post-checkout',
    'post-merge',
    'pre-receive',
    'update',
    'post-receive',
    'post-update',
    'pre-auto-gc',
    'post-rewrite',
]

class CyclicalDependencyException(Exception):
    """Unresolvable group dependency encountered"""
    pass

class UnknownDependencyException(Exception):
    """Unknown group dependency encountered"""
    pass

class UnknownConfigElementException(Exception):
    """Unexpected configuration element encountered"""
    pass

class NoSuchRepoGroupException(Exception):
    """Repository group does not exist"""
    pass

class NoSuchHookGroupException(Exception):
    """Hook group does not exist"""
    pass

class CptHookConfig(object):
    """An object representing a cpthook configuration"""

    def __init__(self, config_file):
        if not os.path.isfile(config_file):
            raise IOError('No such file {0}'.format(config_file))

        global_config, repo_groups, hook_groups = self._parse_config(config_file)

        self.config_file = config_file
        self.global_config = global_config
        self.repo_groups = repo_groups
        self.hook_groups = hook_groups

        self._normalise_repo_groups('members')
        self._normalise_repo_groups('hooks')
        self._set_missing_globals()

    def _set_missing_globals(self):
        """Set global configuration for all repositories

        Performed in case global settings are not configured in the
        cpthook configuration block in the config file"""

        if 'script-path' not in self.global_config:
            # Default location of hooks.d to config directory
            self.global_config['script-path'] = os.path.join(
                os.path.dirname(self.config_file), 'hooks.d')

    def _normalise_repo_groups(self, option):
        """Resolve inherited memberships"""

        data = self.repo_groups
        tainted = data.keys()
        round_ = 0
        while tainted:
            round_ += 1
            logging.debug('Normalise {0}: round {1}'.format(option, round_))

            did_work = False

            for item in tainted:
                try:
                    members = data[item][option]
                except KeyError:
                    logging.debug('Removed empty item {0}'.format(item))
                    tainted.remove(item)
                    did_work = True
                    continue

                unresolved = [x for x in members if x.startswith('@')]
                if len(unresolved) == 0:
                    logging.debug('Nothing to resolve in {0}'.format(item))
                    tainted.remove(item)
                    did_work = True
                    continue

                resolved = []
                dirty = False
                for member in unresolved:
                    mem = member.lstrip('@')
                    try:
                        new_members = data[mem][option]
                    except KeyError:
                        raise UnknownDependencyException(member)
                    for new_mem in new_members:
                        if new_mem.startswith('@'):
                            # Unresolved membership in upstream group
                            dirty = True
                            break
                    resolved += new_members

                if not dirty:
                    # No dependencies remain - replace resolved groups
                    for member in unresolved:
                        members.remove(member)
                    members += resolved
                    data[item][option] = members
                    did_work = True

            if did_work is False:
                raise CyclicalDependencyException(','.join(tainted))
        self.repo_groups = data

    def _parse_config(self, filename):
        """Parse config file and return global, repo and hook config"""

        import ConfigParser
        parser = ConfigParser.SafeConfigParser()
        parser.read(filename)

        # Record the groups as defined in the config
        conf_repos = {}
        conf_hooks = {}
        conf = {}

        for section in parser.sections():
            logging.debug('Evaluating block {0}'.format(section))
            if section.startswith('repos '):
                repo_group = re.sub('^repos\s+', '', section)
                logging.debug('Found repo {0}'.format(repo_group))
                conf_repos[repo_group] = {}
                for option in ['members', 'hooks']:
                    try:
                        values = parser.get(section, option).split()
                        logging.debug('{0} -> {1} -> {2}'.format(
                            section, option, values))
                        conf_repos[repo_group][option] = values
                    except ConfigParser.NoOptionError:
                        # No members
                        logging.debug('No {0} in {1}'.format(
                            option, section))
            elif section.startswith('hooks '):
                hook_group = re.sub('^hooks\s+', '', section)
                logging.debug('Found hook {0}'.format(hook_group))
                conf_hooks[hook_group] = {}
                for type_ in supported_hooks:
                    try:
                        vals = parser.get(section, type_).split()
                        conf_hooks[hook_group][type_] = vals
                    except ConfigParser.NoOptionError:
                        # No hooks of that type
                        pass
            elif section == 'cpthook':
                try:
                    sp = parser.get(section, 'script-path').split()
                    conf['script-path'] = sp[0]
                except ConfigParser.NoOptionError:
                    # No defined repository search path
                    pass
            else:
                raise UnknownConfigElementException(
                    'Unknown config element {0}'.format(section))

        return conf, conf_repos, conf_hooks

    def _aggregate_hooks(self, hook_groups):
        if not hasattr(hook_groups, '__iter__'):
            # Check for __iter__ attribute rather than iter(),
            # which also captures strings.
            raise ValueError('hook_groups must be iterable')

        hooks = {}
        logging.debug('Aggregating hooks for hook groups {0}'.format(
            hook_groups))
        for hook_group in hook_groups:
            logging.debug('Evaluating hook group {0}'.format(hook_group))
            try:
                hg = self.hook_groups[hook_group]
                logging.debug('hg {0} -> {1}'.format(hook_group, hg))
            except KeyError:
                raise NoSuchHookGroupException(hook_group)
            for hook_type, hook_list in hg.items():
                if hook_type not in hooks:
                    hooks[hook_type] = hook_list
                else:
                    for hook in hook_list:
                        if hook not in hooks[hook_type]:
                            hooks[hook_type].append(hook)
        return hooks

    def repo_group_membership(self, repo):
        """Returns list of repo group membership for repo"""

        membership = []
        for repo_group, data in self.repo_groups.items():
            try:
                group_members = data['members']
            except KeyError:
                continue
            if repo in group_members:
                if repo not in membership:
                    membership.append(repo_group)
        logging.debug('{0} is a member of {1}'.format(repo, membership))
        return membership

    def repo_group_hook_groups(self, repo):
        """Returns list of repo group membership for repo"""

        # 1. Get repo group membership for repo
        repo_groups = self.repo_group_membership(repo)

        # 2. Combine lists of hook groups from repo groups
        membership = []
        for repo_group in repo_groups:
            try:
                hook_groups = self.repo_groups[repo_group]['hooks']
            except KeyError:
                logging.debug('No hook groups in {0}'.format(repo_group))
                continue
            for hook_group in hook_groups:
                if hook_group not in membership:
                    membership.append(hook_group)
        if not len(membership):
            logging.debug('No hook groups for {0}'.format(repo))
        return membership

    def hooks_for_repo(self, repo):
        """Returns dict of hooks to be applied to a repository"""

        # 1. Get hook group membership for repo
        hook_groups = self.repo_group_hook_groups(repo)

        # 2. Build dict of hooks from those hook groups
        hooks = self._aggregate_hooks(hook_groups)

        return hooks

    def repos(self):
        """Returns list of known repos"""

        rg = self.repo_groups
        members_by_group = map(lambda x: x['members'], rg.values())
        members = reduce(lambda x, y: set(x + y), members_by_group)
        return list(members)

class CptHook(object):

    def __init__(self, config_file):
        self.config_file = config_file
        self.config = CptHookConfig(config_file)
        self.dry_run = False

    def _script_name(self):
        file_ = inspect.getfile(inspect.currentframe())
        return os.path.abspath(os.path.realpath(file_))

    def add_hooks_to_repo(self, repo_path, hooks):
        template = (
            "#!/bin/sh\n"
            "#\n"
            "# MAGIC STRING: cpthook-wrapper (do not remove)\n"
            "{0} --config={1} --hook={2}\n"
        )

        config_file = os.path.realpath(self.config_file)
        hook_path = os.path.join(repo_path, 'hooks')
        if not os.path.isdir(hook_path):
            logging.warn('Hook path {0} is not a directory'.format(hook_path))
            return

        cpthook = self._script_name()

        for hook_type in hooks:
            target = os.path.join(repo_path, 'hooks', hook_type)
            if os.path.exists(target):
                if os.path.isfile(target):
                    try:
                        f = open(target, 'r')
                    except:
                        logging.info('Could not read {0}'.format(target))
                        continue
                    header = f.read(100)
                    if header.find('cpthook-wrapper') == -1:
                        logging.warn('{0} hook {1} is not managed by cpthook. '
                                     'Refusing to overwrite'.format(
                                         os.path.basename(repo_path), hook_type))
                        continue
                else:
                    logging.info('{0} exists but is not a file'.format(target))
                    continue

            if self.dry_run:
                logging.info('Dry run. Skipping write to {0}'.format(target))
                continue

            try:
                f = open(target, 'w')
            except:
                logging.warn('Could not write wrapper {0}'.format(target))
                continue

            try:
                wrapper = template.format(cpthook, config_file, hook_type)
                f.write(wrapper)
                f.close()
                os.chmod(target, 0755)
                logging.debug('Wrote {0} hook {1}'.format(
                    os.path.basename(repo_path), hook_type))
            except:
                logging.warn('Failed to create wrapper {0}'.format(target))

    def _locate_repo(self, repo):
        if os.path.exists(os.path.join(repo, 'hooks')):
            return repo
        if os.path.exists(os.path.join(repo, '.git', 'hooks')):
            return '{0}/.git'
        return None

    def install_hooks(self):
        for repo in self.config.repos():
            logging.debug('Examining repo {0}'.format(repo))
            repo_path = self._locate_repo(repo)
            if repo_path is None:
                logging.info('Could not locate repo {0}'.format(repo))
                continue
            hooks = self.config.hooks_for_repo(repo).keys()
            self.add_hooks_to_repo(repo_path, hooks)

    def run_hook(self, hook, args):
        ret = subprocess.call(['git', 'rev-parse'])
        if ret != 0:
            logging.warn('{0} is not a git repo?'.format(
                os.path.realpath(os.path.curdir)))
            return -1
        # Work out the repository name from the current directory
        repo = os.path.basename(os.path.realpath(os.path.curdir))
        # Determine script path to be in cpthook directory.
        # Work out where that is through some introspection
        script_path = os.path.join(os.path.dirname(self._script_name()),
            'hooks.d', hook)
        # TODO: make script path a config option
        # script_path = '/'.join(opts.hooksd, hook)
        hooks = self.config.hooks_for_repo(repo)
        if hook in hooks:
            logging.info('Found {0} hooks'.format(hook))
            for script in hooks[hook]:
                if self.dry_run:
                    logging.info('Dry-run: skipping script {0}'.format(
                        script))
                    continue
                logging.info('Running {0} hook {1}'.format(hook, script))
                ret = subprocess.call([os.path.join(script_path, script)] + args)
                if ret != 0:
                    logging.info(
                        'Received non-zero return code from {0}'.format(script))
                    return ret
        return 0
<commit_msg>Improve hook script path determination<commit_after>#!/usr/bin/env python

import inspect
import logging
import os.path
import re
import subprocess

# Supported hooks - see
# https://www.kernel.org/pub/software/scm/git/docs/githooks.html
supported_hooks = [
    'applypatch-msg',
    'pre-applypatch',
    'post-applypatch',
    'pre-commit',
    'prepare-commit-msg',
    'commit-msg',
    'post-commit',
    'pre-rebase',
    'post-checkout',
    'post-merge',
    'pre-receive',
    'update',
    'post-receive',
    'post-update',
    'pre-auto-gc',
    'post-rewrite',
]

class CyclicalDependencyException(Exception):
    """Unresolvable group dependency encountered"""
    pass

class UnknownDependencyException(Exception):
    """Unknown group dependency encountered"""
    pass

class UnknownConfigElementException(Exception):
    """Unexpected configuration element encountered"""
    pass

class NoSuchRepoGroupException(Exception):
    """Repository group does not exist"""
    pass

class NoSuchHookGroupException(Exception):
    """Hook group does not exist"""
    pass

class CptHookConfig(object):
    """An object representing a cpthook configuration"""

    def __init__(self, config_file):
        if not os.path.isfile(config_file):
            raise IOError('No such file {0}'.format(config_file))

        global_config, repo_groups, hook_groups = self._parse_config(config_file)

        self.config_file = config_file
        self.global_config = global_config
        self.repo_groups = repo_groups
        self.hook_groups = hook_groups

        self._normalise_repo_groups('members')
        self._normalise_repo_groups('hooks')
        self._set_missing_globals()

    def _set_missing_globals(self):
        """Set global configuration for all repositories

        Performed in case global settings are not configured in the
        cpthook configuration block in the config file"""

        if 'script-path' not in self.global_config:
            # Default location of hooks.d to config directory
            self.global_config['script-path'] = os.path.join(
                os.path.dirname(self.config_file), 'hooks.d')

    def _normalise_repo_groups(self, option):
        """Resolve inherited memberships"""

        data = self.repo_groups
        tainted = data.keys()
        round_ = 0
        while tainted:
            round_ += 1
            logging.debug('Normalise {0}: round {1}'.format(option, round_))

            did_work = False

            for item in tainted:
                try:
                    members = data[item][option]
                except KeyError:
                    logging.debug('Removed empty item {0}'.format(item))
                    tainted.remove(item)
                    did_work = True
                    continue

                unresolved = [x for x in members if x.startswith('@')]
                if len(unresolved) == 0:
                    logging.debug('Nothing to resolve in {0}'.format(item))
                    tainted.remove(item)
                    did_work = True
                    continue

                resolved = []
                dirty = False
                for member in unresolved:
                    mem = member.lstrip('@')
                    try:
                        new_members = data[mem][option]
                    except KeyError:
                        raise UnknownDependencyException(member)
                    for new_mem in new_members:
                        if new_mem.startswith('@'):
                            # Unresolved membership in upstream group
                            dirty = True
                            break
                    resolved += new_members

                if not dirty:
                    # No dependencies remain - replace resolved groups
                    for member in unresolved:
                        members.remove(member)
                    members += resolved
                    data[item][option] = members
                    did_work = True

            if did_work is False:
                raise CyclicalDependencyException(','.join(tainted))
        self.repo_groups = data

    def _parse_config(self, filename):
        """Parse config file and return global, repo and hook config"""

        import ConfigParser
        parser = ConfigParser.SafeConfigParser()
        parser.read(filename)

        # Record the groups as defined in the config
        conf_repos = {}
        conf_hooks = {}
        conf = {}

        for section in parser.sections():
            logging.debug('Evaluating block {0}'.format(section))
            if section.startswith('repos '):
                repo_group = re.sub('^repos\s+', '', section)
                logging.debug('Found repo {0}'.format(repo_group))
                conf_repos[repo_group] = {}
                for option in ['members', 'hooks']:
                    try:
                        values = parser.get(section, option).split()
                        logging.debug('{0} -> {1} -> {2}'.format(
                            section, option, values))
                        conf_repos[repo_group][option] = values
                    except ConfigParser.NoOptionError:
                        # No members
                        logging.debug('No {0} in {1}'.format(
                            option, section))
            elif section.startswith('hooks '):
                hook_group = re.sub('^hooks\s+', '', section)
                logging.debug('Found hook {0}'.format(hook_group))
                conf_hooks[hook_group] = {}
                for type_ in supported_hooks:
                    try:
                        vals = parser.get(section, type_).split()
                        conf_hooks[hook_group][type_] = vals
                    except ConfigParser.NoOptionError:
                        # No hooks of that type
                        pass
            elif section == 'cpthook':
                try:
                    sp = parser.get(section, 'script-path').split()
                    conf['script-path'] = sp[0]
                except ConfigParser.NoOptionError:
                    # No defined repository search path
                    pass
            else:
                raise UnknownConfigElementException(
                    'Unknown config element {0}'.format(section))

        return conf, conf_repos, conf_hooks

    def _aggregate_hooks(self, hook_groups):
        if not hasattr(hook_groups, '__iter__'):
            # Check for __iter__ attribute rather than iter(),
            # which also captures strings.
            raise ValueError('hook_groups must be iterable')

        hooks = {}
        logging.debug('Aggregating hooks for hook groups {0}'.format(
            hook_groups))
        for hook_group in hook_groups:
            logging.debug('Evaluating hook group {0}'.format(hook_group))
            try:
                hg = self.hook_groups[hook_group]
                logging.debug('hg {0} -> {1}'.format(hook_group, hg))
            except KeyError:
                raise NoSuchHookGroupException(hook_group)
            for hook_type, hook_list in hg.items():
                if hook_type not in hooks:
                    hooks[hook_type] = hook_list
                else:
                    for hook in hook_list:
                        if hook not in hooks[hook_type]:
                            hooks[hook_type].append(hook)
        return hooks

    def repo_group_membership(self, repo):
        """Returns list of repo group membership for repo"""

        membership = []
        for repo_group, data in self.repo_groups.items():
            try:
                group_members = data['members']
            except KeyError:
                continue
            if repo in group_members:
                if repo not in membership:
                    membership.append(repo_group)
        logging.debug('{0} is a member of {1}'.format(repo, membership))
        return membership

    def repo_group_hook_groups(self, repo):
        """Returns list of repo group membership for repo"""

        # 1. Get repo group membership for repo
        repo_groups = self.repo_group_membership(repo)

        # 2. Combine lists of hook groups from repo groups
        membership = []
        for repo_group in repo_groups:
            try:
                hook_groups = self.repo_groups[repo_group]['hooks']
            except KeyError:
                logging.debug('No hook groups in {0}'.format(repo_group))
                continue
            for hook_group in hook_groups:
                if hook_group not in membership:
                    membership.append(hook_group)
        if not len(membership):
            logging.debug('No hook groups for {0}'.format(repo))
        return membership

    def hooks_for_repo(self, repo):
        """Returns dict of hooks to be applied to a repository"""

        # 1. Get hook group membership for repo
        hook_groups = self.repo_group_hook_groups(repo)

        # 2. Build dict of hooks from those hook groups
        hooks = self._aggregate_hooks(hook_groups)

        return hooks

    def repos(self):
        """Returns list of known repos"""

        rg = self.repo_groups
        members_by_group = map(lambda x: x['members'], rg.values())
        members = reduce(lambda x, y: set(x + y), members_by_group)
        return list(members)

class CptHook(object):

    def __init__(self, config_file):
        self.config_file = config_file
        self.config = CptHookConfig(config_file)
        self.dry_run = False

    def _script_name(self):
        file_ = inspect.getfile(inspect.currentframe())
        return os.path.abspath(os.path.realpath(file_))

    def add_hooks_to_repo(self, repo_path, hooks):
        template = (
            "#!/bin/sh\n"
            "#\n"
            "# MAGIC STRING: cpthook-wrapper (do not remove)\n"
            "{0} --config={1} --hook={2}\n"
        )

        config_file = os.path.realpath(self.config_file)
        hook_path = os.path.join(repo_path, 'hooks')
        if not os.path.isdir(hook_path):
            logging.warn('Hook path {0} is not a directory'.format(hook_path))
            return

        cpthook = self._script_name()

        for hook_type in hooks:
            target = os.path.join(repo_path, 'hooks', hook_type)
            if os.path.exists(target):
                if os.path.isfile(target):
                    try:
                        f = open(target, 'r')
                    except:
                        logging.info('Could not read {0}'.format(target))
                        continue
                    header = f.read(100)
                    if header.find('cpthook-wrapper') == -1:
                        logging.warn('{0} hook {1} is not managed by cpthook. '
                                     'Refusing to overwrite'.format(
                                         os.path.basename(repo_path), hook_type))
                        continue
                else:
                    logging.info('{0} exists but is not a file'.format(target))
                    continue

            if self.dry_run:
                logging.info('Dry run. Skipping write to {0}'.format(target))
                continue

            try:
                f = open(target, 'w')
            except:
                logging.warn('Could not write wrapper {0}'.format(target))
                continue

            try:
                wrapper = template.format(cpthook, config_file, hook_type)
                f.write(wrapper)
                f.close()
                os.chmod(target, 0755)
                logging.debug('Wrote {0} hook {1}'.format(
                    os.path.basename(repo_path), hook_type))
            except:
                logging.warn('Failed to create wrapper {0}'.format(target))

    def _locate_repo(self, repo):
        if os.path.exists(os.path.join(repo, 'hooks')):
            return repo
        if os.path.exists(os.path.join(repo, '.git', 'hooks')):
            return '{0}/.git'
        return None

    def install_hooks(self):
        for repo in self.config.repos():
            logging.debug('Examining repo {0}'.format(repo))
            repo_path = self._locate_repo(repo)
            if repo_path is None:
                logging.info('Could not locate repo {0}'.format(repo))
                continue
            hooks = self.config.hooks_for_repo(repo).keys()
            self.add_hooks_to_repo(repo_path, hooks)

    def _abs_script_name(self, hook, script):
        hooksd_path = self.config.global_config['script-path']
        logging.debug('Script path {0}'.format(script_file))
        return os.path.join(hooksd_path, hook, script)

    def run_hook(self, hook, args):
        ret = subprocess.call(['git', 'rev-parse'])
        if ret != 0:
            logging.warn('{0} is not a git repo?'.format(
                os.path.realpath(os.path.curdir)))
            return -1
        # Work out the repository name from the current directory
        repo = os.path.basename(os.path.realpath(os.path.curdir))

        hooks = self.config.hooks_for_repo(repo)
        if hook in hooks:
            logging.info('Found {0} hooks'.format(hook))
            for script in hooks[hook]:
                script_file = self._abs_script_name(hook, script)
                if not os.path.exists(script_file):
                    logging.info('{0} hook {1} does not exist'.format(
                        hook, script))
                if self.dry_run:
                    logging.info('Dry-run: skipping {0} script {1}'.format(
                        repo, script))
                    continue
                logging.info('Running {0} hook {1}'.format(hook, script))
                ret = subprocess.call([script_file] + args)
                if ret != 0:
                    logging.info(
                        'Received non-zero return code from {0}'.format(script))
                    return ret
        return 0
<|endoftext|>


<commit_before>import config
import cv2
import cv2.cv as cv
import numpy as np
from pylab import array

class Frame:
    def __init__(self, original):
        self.original = original
        self.processed = None
        self.printed = False

    def process(self):
        # Frame can be processed only once...
        if self.processed is not None: return self.processed

        # convert to grayscale
        gray = cv2.cvtColor(self.original, cv2.COLOR_BGR2GRAY)

        # threshold the image
        # ret, thresh = cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY)

        thresh = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,\
            cv2.THRESH_BINARY, 11, 2)

        # th2 = cv2.adaptiveThreshold(gray,255,cv2.ADAPTIVE_THRESH_MEAN_C,\
        #     cv2.THRESH_BINARY,11,2)

        self.processed = thresh

    def find_glints(self):
        # convert to grayscale
        self.processed = self.original.copy()
        img = cv2.cvtColor(self.processed, cv2.COLOR_BGR2GRAY)

        # erode remaining white areas
        kernel = np.ones((5,5), np.uint8)
        img = cv2.morphologyEx(img, cv2.MORPH_OPEN, kernel)

        # cv2.imshow('4', img)
        cv2.imshow('frame', img)

        # find edges
        # values 50, 130 work well
        # values 40, 130 work well
        edges = cv2.Canny(img, 40, 130)
        # kernel = np.ones((2,2), np.uint8)
        # edges = cv2.dilate(edges, kernel, iterations = 1)
        # cv2.imshow('2', edges)

        cv2.imshow('4', edges)

        # find contours
        contours, hierarchy = cv2.findContours(edges, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

        largest_area = 0
        contour = None
        for i in contours:
            area = cv2.contourArea(i)
            # find only largest contour
            if area < largest_area:
                continue

            # discard contours that are not in the center of the image
            # discard contours which bounding circle is too large
            (x,y),radius = cv2.minEnclosingCircle(i)
            if y < 50 or y > 250 or radius > 30 or radius < 15:
                continue

            contour = i
            largest_area = area

        # if not self.printed:
        #     print e
        # self.printed = True

        # minLineLength = 100
        # maxLineGap = 10
        # lines = cv2.HoughLinesP(contours,1,np.pi/180,100,minLineLength,maxLineGap)
        # for x1,y1,x2,y2 in lines[0]:
        #     cv2.line(img,(x1,y1),(x2,y2),(0,255,0),2)

        if contour is not None and contour.any():
            (x,y),radius = cv2.minEnclosingCircle(contour)
            center = (int(x),int(y))
            radius = int(radius)
            img = cv2.circle(self.processed, center, radius,(0,255,0),2)
            img = cv2.circle(self.processed, center, 2,(0,0,255),2)
            #
            # cv2.drawContours(self.processed, contour, -1, (0,255,0), 3)
            font = cv2.FONT_HERSHEY_SIMPLEX
            cv2.putText(self.processed, 'Largest contour area: ' + str(round(largest_area, 2)), (10,30), font, 0.5, (255,255,255), 2)
            cv2.putText(self.processed, 'Center of the contour: (' + str(round(x, 2)) + ', ' + str(round(y, 2)) + ')', (10,60), font, 0.5, (255,255,255), 2)
            cv2.putText(self.processed, 'Radius of the contour: ' + str(round(radius, 2)), (10,90), font, 0.5, (255,255,255), 2)

        cv2.imshow('3', self.processed)

        # cv2.imshow(config.WINDOW_NAME, img)
        # edges = cv2.Canny(img, 0, 200)
        # #
        # cv2.imshow(config.WINDOW_NAME, edges)

        # ret, thresh = cv2.threshold(img, 200, 255, cv2.THRESH_BINARY)
        # #
        # contours, hierarchy = cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)
        # cv2.drawContours(self.processed, contours, -1, (0,255,0), 3)
        #
        # if not self.printed:
        #     print len(contours)
        # self.printed = True

        # img = img[c1:c1+25,r1:r1+25]

    def show(self):
        cv2.imshow(config.WINDOW_NAME, self.processed)

    def show_original(self):
        cv2.imshow(config.WINDOW_NAME, self.original)
<commit_msg>Update<commit_after>import config
import cv2
import cv2.cv as cv
import numpy as np
from pylab import array

class Frame:
    def __init__(self, original):
        self.original = original
        self.processed = None
        self.printed = False

    def process(self):
        # Frame can be processed only once...
        if self.processed is not None: return self.processed

        # convert to grayscale
        gray = cv2.cvtColor(self.original, cv2.COLOR_BGR2GRAY)

        # threshold the image
        # ret, thresh = cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY)

        thresh = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,\
            cv2.THRESH_BINARY, 11, 2)

        # th2 = cv2.adaptiveThreshold(gray,255,cv2.ADAPTIVE_THRESH_MEAN_C,\
        #     cv2.THRESH_BINARY,11,2)

        self.processed = thresh

    def find_glints(self):
        # convert to grayscale
        self.processed = self.original.copy()
        img = cv2.cvtColor(self.processed, cv2.COLOR_BGR2GRAY)

        # erode remaining white areas
        kernel = np.ones((10,10), np.uint8)
        img = cv2.morphologyEx(img, cv2.MORPH_OPEN, kernel)

        # cv2.imshow('4', img)
        cv2.imshow('frame', img)

        # find edges
        # values 50, 130 work well
        # values 40, 130 work well
        edges = cv2.Canny(img, 40, 130)
        # kernel = np.ones((2,2), np.uint8)
        # edges = cv2.dilate(edges, kernel, iterations = 1)
        # cv2.imshow('2', edges)

        cv2.imshow('4', edges)

        # find contours
        contours, hierarchy = cv2.findContours(edges, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

        largest_area = 0
        contour = None
        for i in contours:
            area = cv2.contourArea(i)
            # find only largest contour
            if area < largest_area:
                continue

            # discard contours that are not in the center of the image
            # discard contours which bounding circle is too large
            (x,y),radius = cv2.minEnclosingCircle(i)
            if y < 50 or y > 250 or radius > 30 or radius < 15:
                continue

            contour = i
            largest_area = area

        # if not self.printed:
        #     print e
        # self.printed = True

        # minLineLength = 100
        # maxLineGap = 10
        # lines = cv2.HoughLinesP(contours,1,np.pi/180,100,minLineLength,maxLineGap)
        # for x1,y1,x2,y2 in lines[0]:
        #     cv2.line(img,(x1,y1),(x2,y2),(0,255,0),2)

        if contour is not None and contour.any():
            (x,y),radius = cv2.minEnclosingCircle(contour)
            center = (int(x),int(y))
            radius = int(radius)
            img = cv2.circle(self.processed, center, radius,(0,255,0),2)
            img = cv2.circle(self.processed, center, 2,(0,0,255),2)
            #
            # cv2.drawContours(self.processed, contour, -1, (0,255,0), 3)
            font = cv2.FONT_HERSHEY_SIMPLEX
            cv2.putText(self.processed, 'Largest contour area: ' + str(round(largest_area, 2)), (10,30), font, 0.5, (255,255,255), 2)
            cv2.putText(self.processed, 'Center of the contour: (' + str(round(x, 2)) + ', ' + str(round(y, 2)) + ')', (10,60), font, 0.5, (255,255,255), 2)
            cv2.putText(self.processed, 'Radius of the contour: ' + str(round(radius, 2)), (10,90), font, 0.5, (255,255,255), 2)
            cv2.putText(self.processed, 'x: ' + str(round((x - radius), 2)), (10,90), font, 0.5, (255,255,255), 2)

        cv2.imshow('3', self.processed)

        # cv2.imshow(config.WINDOW_NAME, img)
        # edges = cv2.Canny(img, 0, 200)
        # #
        # cv2.imshow(config.WINDOW_NAME, edges)

        # ret, thresh = cv2.threshold(img, 200, 255, cv2.THRESH_BINARY)
        # #
        # contours, hierarchy = cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)
        # cv2.drawContours(self.processed, contours, -1, (0,255,0), 3)
        #
        # if not self.printed:
        #     print len(contours)
        # self.printed = True

        # img = img[c1:c1+25,r1:r1+25]

    def show(self):
        cv2.imshow(config.WINDOW_NAME, self.processed)

    def show_original(self):
        cv2.imshow(config.WINDOW_NAME, self.original)
<|endoftext|>


<commit_before>import config
import cv2
import cv2.cv as cv
import numpy as np

class Frame:
    def __init__(self, original):
        self.original = original
        self.processed = None
        self.printed = False

    def process(self):
        # Frame can be processed only once...
        if self.processed is not None: return self.processed

        # convert to grayscale
        gray = cv2.cvtColor(self.original, cv2.COLOR_BGR2GRAY)

        # threshold the image
        # ret, thresh = cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY)

        thresh = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,\
            cv2.THRESH_BINARY, 11, 2)

        # th2 = cv2.adaptiveThreshold(gray,255,cv2.ADAPTIVE_THRESH_MEAN_C,\
        #     cv2.THRESH_BINARY,11,2)

        self.processed = thresh

    def find_glints(self):
        # convert to grayscale
        self.processed = self.original.copy()
        img = cv2.cvtColor(self.processed, cv2.COLOR_BGR2GRAY)

        # erode remaining white areas
        kernel = np.ones((10,10), np.uint8)
        img = cv2.morphologyEx(img, cv2.MORPH_OPEN, kernel)

        img = (255.0/1)*(img/255.0/1))**2
        img = array(newImage1,dtype=uint8)

        # cv2.imshow('4', img)
        cv2.imshow('frame', img)

        # find edges
        # values 50, 130 work well
        edges = cv2.Canny(img, 40, 100)

        # kernel = np.ones((3,3), np.uint8)
        # edges = cv2.dilate(edges, kernel, iterations = 1)

        edges = cv2.Canny(img, 50, 120)
        cv2.imshow('2', edges)

        edges = cv2.Canny(img, 40, 130)
        cv2.imshow('3', edges)

        edges = cv2.Canny(img, 70, 100)
        cv2.imshow('4', edges)

        # find contours
        contours, hierarchy = cv2.findContours(edges, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

        largest_area = 0
        contour = None
        for i in contours:
            area = cv2.contourArea(i)
            # find only largest contour
            if area < largest_area:
                continue

            # discard contours that are not in the center of the image
            # discard contours which bounding circle is too large
            (x,y),radius = cv2.minEnclosingCircle(i)
            if y < 50 or y > 250 or radius > 50 or radius < 15:
                continue

            contour = i
            largest_area = area

        # if not self.printed:
        #     print e
        # self.printed = True

        if contour is not None and contour.any():
            (x,y),radius = cv2.minEnclosingCircle(contour)
            center = (int(x),int(y))
            radius = int(radius)
            img = cv2.circle(self.processed, center, radius,(0,255,0),2)
            img = cv2.circle(self.processed, center, 2,(0,0,255),2)
            #
            # cv2.drawContours(self.processed, contour, -1, (0,255,0), 3)
            font = cv2.FONT_HERSHEY_SIMPLEX
            cv2.putText(self.processed, 'Largest contour area: ' + str(round(largest_area, 2)), (10,10), font, 0.5, (255,255,255), 2)
            cv2.putText(self.processed, 'Center of the contour: (' + str(round(x, 2)) + ', ' + str(round(y, 2)) + ')', (10,30), font, 0.5, (255,255,255), 2)
            cv2.putText(self.processed, 'Radius of the contour: ' + str(round(radius, 2)), (10,60), font, 0.5, (255,255,255), 2)

        # cv2.imshow('3', self.processed)

        # cv2.imshow(config.WINDOW_NAME, img)
        # edges = cv2.Canny(img, 0, 200)
        # #
        # cv2.imshow(config.WINDOW_NAME, edges)

        # ret, thresh = cv2.threshold(img, 200, 255, cv2.THRESH_BINARY)
        # #
        # contours, hierarchy = cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)
        # cv2.drawContours(self.processed, contours, -1, (0,255,0), 3)
        #
        # if not self.printed:
        #     print len(contours)
        # self.printed = True

        # img = img[c1:c1+25,r1:r1+25]

    def show(self):
        cv2.imshow(config.WINDOW_NAME, self.processed)

    def show_original(self):
        cv2.imshow(config.WINDOW_NAME, self.original)
<commit_msg>Update<commit_after>import config
import cv2
import cv2.cv as cv
import numpy as np

class Frame:
    def __init__(self, original):
        self.original = original
        self.processed = None
        self.printed = False

    def process(self):
        # Frame can be processed only once...
        if self.processed is not None: return self.processed

        # convert to grayscale
        gray = cv2.cvtColor(self.original, cv2.COLOR_BGR2GRAY)

        # threshold the image
        # ret, thresh = cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY)

        thresh = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,\
            cv2.THRESH_BINARY, 11, 2)

        # th2 = cv2.adaptiveThreshold(gray,255,cv2.ADAPTIVE_THRESH_MEAN_C,\
        #     cv2.THRESH_BINARY,11,2)

        self.processed = thresh

    def find_glints(self):
        # convert to grayscale
        self.processed = self.original.copy()
        img = cv2.cvtColor(self.processed, cv2.COLOR_BGR2GRAY)

        # erode remaining white areas
        kernel = np.ones((10,10), np.uint8)
        img = cv2.morphologyEx(img, cv2.MORPH_OPEN, kernel)

        img = (255.0/1) * (img/(255.0/1))**2
        img = array(img,dtype=uint8)

        # cv2.imshow('4', img)
        cv2.imshow('frame', img)

        # find edges
        # values 50, 130 work well
        edges = cv2.Canny(img, 40, 100)

        # kernel = np.ones((3,3), np.uint8)
        # edges = cv2.dilate(edges, kernel, iterations = 1)

        edges = cv2.Canny(img, 50, 120)
        cv2.imshow('2', edges)

        edges = cv2.Canny(img, 40, 130)
        cv2.imshow('3', edges)

        edges = cv2.Canny(img, 70, 100)
        cv2.imshow('4', edges)

        # find contours
        contours, hierarchy = cv2.findContours(edges, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

        largest_area = 0
        contour = None
        for i in contours:
            area = cv2.contourArea(i)
            # find only largest contour
            if area < largest_area:
                continue

            # discard contours that are not in the center of the image
            # discard contours which bounding circle is too large
            (x,y),radius = cv2.minEnclosingCircle(i)
            if y < 50 or y > 250 or radius > 50 or radius < 15:
                continue

            contour = i
            largest_area = area

        # if not self.printed:
        #     print e
        # self.printed = True

        if contour is not None and contour.any():
            (x,y),radius = cv2.minEnclosingCircle(contour)
            center = (int(x),int(y))
            radius = int(radius)
            img = cv2.circle(self.processed, center, radius,(0,255,0),2)
            img = cv2.circle(self.processed, center, 2,(0,0,255),2)
            #
            # cv2.drawContours(self.processed, contour, -1, (0,255,0), 3)
            font = cv2.FONT_HERSHEY_SIMPLEX
            cv2.putText(self.processed, 'Largest contour area: ' + str(round(largest_area, 2)), (10,10), font, 0.5, (255,255,255), 2)
            cv2.putText(self.processed, 'Center of the contour: (' + str(round(x, 2)) + ', ' + str(round(y, 2)) + ')', (10,30), font, 0.5, (255,255,255), 2)
            cv2.putText(self.processed, 'Radius of the contour: ' + str(round(radius, 2)), (10,60), font, 0.5, (255,255,255), 2)

        # cv2.imshow('3', self.processed)

        # cv2.imshow(config.WINDOW_NAME, img)
        # edges = cv2.Canny(img, 0, 200)
        # #
        # cv2.imshow(config.WINDOW_NAME, edges)

        # ret, thresh = cv2.threshold(img, 200, 255, cv2.THRESH_BINARY)
        # #
        # contours, hierarchy = cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)
        # cv2.drawContours(self.processed, contours, -1, (0,255,0), 3)
        #
        # if not self.printed:
        #     print len(contours)
        # self.printed = True

        # img = img[c1:c1+25,r1:r1+25]

    def show(self):
        cv2.imshow(config.WINDOW_NAME, self.processed)

    def show_original(self):
        cv2.imshow(config.WINDOW_NAME, self.original)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Main function
"""
import os
import re
import sys
from datetime import datetime

import aiohttp
import asyncio
import requests

from local_settings import talk_id

class Crawler(object):
    """
    Crawler for download 755 photos.
    """
    url = 'http://7gogo.jp/api/talk/post/list'
    img_path = 'images'
    video_path = 'videos'

    def __init__(self):
        self.session = requests.session()
        pass

    def download_file(self, url, filename, file_type):
        """
        Download photo.

        Args:
            (String) url
            (String) file name (output)
        """
        req = self.session.get(url, stream=True)
        if req.status_code == 200:
            total_length = req.headers.get('content-length')
            dl_progress = 0

            if file_type == 'images':
                folder = self.img_path
            else:
                folder = self.video_path
            output_path = "{}{}{}".format(folder, os.sep, filename)
            if not os.path.exists(output_path):
                with open(output_path, 'wb') as o_file:
                    for chunk in req.iter_content(1024):
                        dl_progress += len(chunk)
                        o_file.write(chunk)

                        # Download progress report
                        percent = dl_progress / int(total_length)
                        sys.stdout.write("\r{}: {:.2%}".format(filename, percent))
                        sys.stdout.flush()

                print('')
            else:
                print('{}: File exist'.format(filename))
        else:
            print('Visit website fail')

    async def run(self, client, talk_id, stop_time=0):
        page_limit = 100

        payload = {
            'direction': 'PREV',
            'limit': page_limit,
            'postId': 6000,  # test 6000
            'talkId': talk_id,
        }

        r = self.session.get(self.url, params=payload)
        if r.status_code != 200:
            # handle connection fail
            sys.exit()
        else:
            raw_data = r.json()

            # handle no post
            if not raw_data['posts']:
                sys.exit()

            # Created directories for store files
            if not os.path.isdir(self.img_path):
                os.makedirs(self.img_path)
            if not os.path.isdir(self.video_path):
                os.makedirs(self.video_path)

            img_count = 1
            video_count = 1
            last_image_t = 0
            last_video_t = 0
            for i in range(page_limit):
                # if msg time too old, stop download
                post_time = int(raw_data['posts'][i]['time'])
                if int(post_time) < stop_time:
                    break

                url = raw_data['posts'][i]['body'][0].get('image', '')
                if url:
                    # file_date = url.split('/')[4]
                    file_date = datetime.utcfromtimestamp(post_time).strftime("%Y%m%d")
                    # handle duplication file
                    if file_date == last_image_t:
                        img_count += 1
                    else:
                        img_count = 1
                        last_image_t = file_date

                    self.download_file(url, "{}_{}.jpg".format(file_date, img_count), 'images')

                url = raw_data['posts'][i]['body'][0].get('movieUrlHq', '')
                if url:
                    file_date = datetime.utcfromtimestamp(post_time).strftime("%Y%m%d")
                    # handle duplication file
                    if file_date == last_video_t:
                        video_count += 1
                    else:
                        video_count = 1
                        last_video_t = file_date

                    self.download_file(url, "{}_{}.mp4".format(file_date, video_count), 'videos')

if __name__ == '__main__':
    my_cwawler = Crawler()
    stop_time = 1445687490

    loop = asyncio.get_event_loop()
    with aiohttp.ClientSession(loop=loop) as client:
        loop.run_until_complete(my_cwawler.run(client, talk_id, stop_time))
<commit_msg>Imp: Download all images and videos<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Main function
"""
import os
import re
import sys
from datetime import datetime

import aiohttp
import asyncio
import requests

from local_settings import talk_id

class Crawler(object):
    """
    Crawler for download 755 photos.
    """
    url = 'http://7gogo.jp/api/talk/post/list'
    img_path = 'images'
    video_path = 'videos'

    def __init__(self):
        self.session = requests.session()
        pass

    def download_file(self, url, filename, file_type):
        """
        Download photo.

        Args:
            (String) url
            (String) file name (output)
        """
        req = self.session.get(url, stream=True)
        if req.status_code == 200:
            total_length = req.headers.get('content-length')
            dl_progress = 0

            if file_type == 'images':
                folder = self.img_path
            else:
                folder = self.video_path
            output_path = "{}{}{}".format(folder, os.sep, filename)
            if not os.path.exists(output_path):
                with open(output_path, 'wb') as o_file:
                    for chunk in req.iter_content(1024):
                        dl_progress += len(chunk)
                        o_file.write(chunk)

                        # Download progress report
                        percent = dl_progress / int(total_length)
                        sys.stdout.write("\r{}: {:.2%}".format(filename, percent))
                        sys.stdout.flush()

                print('')
            else:
                print('{}: File exist'.format(filename))
        else:
            print('Visit website fail')

    async def run(self, client, talk_id, stop_time=0):
        page_limit = 100

        for post_rec in range(0, 99999999, 100):
            payload = {
                'direction': 'PREV',
                'limit': page_limit,
                'postId': post_rec,  # test 6000 (photos and videos)
                'talkId': talk_id,
            }

            r = self.session.get(self.url, params=payload)
            if r.status_code != 200:
                # handle connection fail
                sys.exit()
            else:
                raw_data = r.json()

                # handle no post
                if not raw_data['posts']:
                    print('Finished !')
                    sys.exit()

                # Created directories for store files
                if not os.path.isdir(self.img_path):
                    os.makedirs(self.img_path)
                if not os.path.isdir(self.video_path):
                    os.makedirs(self.video_path)

                img_count = 1
                video_count = 1
                last_image_t = 0
                last_video_t = 0
                for i in range(page_limit):
                    # if msg time too old, stop download
                    post_time = int(raw_data['posts'][i]['time'])
                    if int(post_time) < stop_time:
                        break

                    url = raw_data['posts'][i]['body'][0].get('image', '')
                    if url:
                        # file_date = url.split('/')[4]
                        file_date = datetime.utcfromtimestamp(post_time).strftime("%Y%m%d")
                        # handle duplication file
                        if file_date == last_image_t:
                            img_count += 1
                        else:
                            img_count = 1
                            last_image_t = file_date

                        self.download_file(url, "{}_{}.jpg".format(file_date, img_count), 'images')

                    url = raw_data['posts'][i]['body'][0].get('movieUrlHq', '')
                    if url:
                        file_date = datetime.utcfromtimestamp(post_time).strftime("%Y%m%d")
                        # handle duplication file
                        if file_date == last_video_t:
                            video_count += 1
                        else:
                            video_count = 1
                            last_video_t = file_date

                        self.download_file(url, "{}_{}.mp4".format(file_date, video_count), 'videos')

if __name__ == '__main__':
    my_cwawler = Crawler()
    stop_time = 1445687490

    loop = asyncio.get_event_loop()
    with aiohttp.ClientSession(loop=loop) as client:
        loop.run_until_complete(my_cwawler.run(client, talk_id, stop_time))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Crawler that sends notifications as soon as servers
on Kimsufi/OVH become available for purchase"""

import json
import sys
import logging
import importlib
import tornado.ioloop
import tornado.web
from tornado.httpclient import AsyncHTTPClient
from tornado.gen import coroutine

logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
_logger = logging.getLogger(__name__)

URL = "https://ws.ovh.com/dedicated/r2/ws.dispatcher/getAvailability2"

NOTIFIERS = {
    'email': 'notifiers.email_notifier.EmailNotifier',
    'osx': 'notifiers.osx_notifier.OSXNotifier',
    'popup': 'notifiers.popup_notifier.PopupNotifier',
    'smsapi': 'notifiers.smsapi_notifier.SmsApiNotifier',
}

SERVER_TYPES = {
    '142sk1': 'KS-1',
    '142sk7': 'KS-2',
    '142sk3': 'KS-3',
    '142sk4': 'KS-4',
    '142sk5': 'KS-5A',
    '142sk8': 'KS-5B',
    '142sk6': 'KS-6',
}

DATACENTERS = {
    'bhs': 'Beauharnois, Canada (Americas)',
    'gra': 'Gravelines, France',
    'rbx': 'Roubaix, France (Western Europe)',
    'sbg': 'Strasbourg, France (Central Europe)',
    'par': 'Paris, France',
}

STATES = {}

def update_state(state, value, message=False):
    """Update state of particular event"""
    # if new value is True and last saved was False
    if state not in STATES:
        STATES[state] = False
    if value is not STATES[state]:
        _logger.info("State change - %s: %s", state, value)
    if value and not STATES[state]:
        notifier.notify(**message)
    STATES[state] = value

@coroutine
def run_crawler():
    """Run a crawler iteration"""
    http_client = AsyncHTTPClient()
    # request OVH availablility API asynchronously
    response = yield http_client.fetch(URL)
    response_json = json.loads(response.body.decode('utf-8'))
    if not response_json or not response_json['answer']:
        return
    availability = response_json['answer']['availability']
    for item in availability:
        # look for servers of required types in OVH availability list
        if SERVER_TYPES.get(item['reference']) in config['servers']:
            # make a flat list of zones where servers are available
            available_zones = [e['zone'] for e in item['zones']
                               if e['availability'] != 'unavailable']
            # iterate over all tacked zones and set availability states
            for zone in config['zones']:
                server = SERVER_TYPES[item['reference']]
                state_id = '%s_available_in_%s' % (server, zone)
                # update state for each tracked zone
                text = "Server %s is available in %s" % (server, zone)
                message = {
                    'title': "Server %s available" % server,
                    'text': text,
                    'url': "http://www.kimsufi.com/fr/index.xml"
                }
                update_state(state_id, zone in available_zones, message)

if __name__ == "__main__":
    CONFIG_NAME = sys.argv[1] if len(sys.argv) == 2 else 'config.json'
    with open(CONFIG_NAME, 'r') as configfile:
        try:
            config = json.loads(configfile.read())
        except ValueError:
            _logger.error("Parsing JSON settings in config.json has failed. "
                          "Check syntax with a validator (i.e. jsonlint.com)")
            sys.exit(1)

    # Select notifier, 'email' by default
    if 'notifier' not in config:
        _logger.warning("No notifier selected in config, 'email' will be used")
        config['notifier'] = 'email'
    # Instantiate notifier class dynamically
    try:
        n_path = NOTIFIERS[config['notifier']]
        n_file, n_classname = n_path.rsplit('.', 1)
        n_module = importlib.import_module(n_file)
        notifier = getattr(n_module, n_classname)(config)
    except:
        _logger.exception("Notifier loading failed,"
                          "check configuration for errors")
        sys.exit(1)

    loop = tornado.ioloop.IOLoop.instance()
    tornado.ioloop.PeriodicCallback(run_crawler, 30000).start()
    _logger.info("Starting IO loop")

    try:
        loop.start()
    except KeyboardInterrupt:
        _logger.info("Terminated by user. Bye.")
        sys.exit(0)
<commit_msg>Update the product references with the new server types<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Crawler that sends notifications as soon as servers
on Kimsufi/OVH become available for purchase"""

import json
import sys
import logging
import importlib
import tornado.ioloop
import tornado.web
from tornado.httpclient import AsyncHTTPClient
from tornado.gen import coroutine

logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
_logger = logging.getLogger(__name__)

URL = "https://ws.ovh.com/dedicated/r2/ws.dispatcher/getAvailability2"

NOTIFIERS = {
    'email': 'notifiers.email_notifier.EmailNotifier',
    'osx': 'notifiers.osx_notifier.OSXNotifier',
    'popup': 'notifiers.popup_notifier.PopupNotifier',
    'smsapi': 'notifiers.smsapi_notifier.SmsApiNotifier',
}

SERVER_TYPES = {
    '150sk10': 'KS-1',
    '150sk20': 'KS-2a',
    '150sk21': 'KS-2b',
    '150sk22': 'KS-2c',
    '150sk30': 'KS-3',
    '150sk31': 'KS-3',
    '150sk40': 'KS-4',
    '150sk41': 'KS-4',
    '150sk42': 'KS-4',
    '150sk50': 'KS-5',
    '150sk60': 'KS-6',
}

DATACENTERS = {
    'bhs': 'Beauharnois, Canada (Americas)',
    'gra': 'Gravelines, France',
    'rbx': 'Roubaix, France (Western Europe)',
    'sbg': 'Strasbourg, France (Central Europe)',
    'par': 'Paris, France',
}

STATES = {}

def update_state(state, value, message=False):
    """Update state of particular event"""
    # if new value is True and last saved was False
    if state not in STATES:
        STATES[state] = False
    if value is not STATES[state]:
        _logger.info("State change - %s: %s", state, value)
    if value and not STATES[state]:
        notifier.notify(**message)
    STATES[state] = value

@coroutine
def run_crawler():
    """Run a crawler iteration"""
    http_client = AsyncHTTPClient()
    # request OVH availablility API asynchronously
    response = yield http_client.fetch(URL)
    response_json = json.loads(response.body.decode('utf-8'))
    if not response_json or not response_json['answer']:
        return
    availability = response_json['answer']['availability']
    for item in availability:
        # look for servers of required types in OVH availability list
        if SERVER_TYPES.get(item['reference']) in config['servers']:
            # make a flat list of zones where servers are available
            available_zones = [e['zone'] for e in item['zones']
                               if e['availability'] not in ['unavailable',
                                                            'unknown']]
            # iterate over all tacked zones and set availability states
            for zone in config['zones']:
                server = SERVER_TYPES[item['reference']]
                state_id = '%s_available_in_%s' % (server, zone)
                # update state for each tracked zone
                text = "Server %s is available in %s" % (server, zone)
                message = {
                    'title': "Server %s available" % server,
                    'text': text,
                    'url': "http://www.kimsufi.com/fr/index.xml"
                }
                update_state(state_id, zone in available_zones, message)

if __name__ == "__main__":
    CONFIG_NAME = sys.argv[1] if len(sys.argv) == 2 else 'config.json'
    with open(CONFIG_NAME, 'r') as configfile:
        try:
            config = json.loads(configfile.read())
        except ValueError:
            _logger.error("Parsing JSON settings in config.json has failed. "
                          "Check syntax with a validator (i.e. jsonlint.com)")
            sys.exit(1)

    # Select notifier, 'email' by default
    if 'notifier' not in config:
        _logger.warning("No notifier selected in config, 'email' will be used")
        config['notifier'] = 'email'
    # Instantiate notifier class dynamically
    try:
        n_path = NOTIFIERS[config['notifier']]
        n_file, n_classname = n_path.rsplit('.', 1)
        n_module = importlib.import_module(n_file)
        notifier = getattr(n_module, n_classname)(config)
    except:
        _logger.exception("Notifier loading failed,"
                          "check configuration for errors")
        sys.exit(1)

    loop = tornado.ioloop.IOLoop.instance()
    tornado.ioloop.PeriodicCallback(run_crawler, 30000).start()
    _logger.info("Starting IO loop")

    try:
        loop.start()
    except KeyboardInterrupt:
        _logger.info("Terminated by user. Bye.")
        sys.exit(0)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Main function
"""
import os
import sys
import time
import argparse
from datetime import datetime

import aiohttp
import asyncio
import requests

from talk_id import get_talk_id

class Crawler(object):
    """
    Crawler for download 755 photos.
    """
    url = 'http://7gogo.jp/api/talk/post/list'
    img_path = 'images'
    video_path = 'videos'

    def __init__(self):
        self.session = requests.session()
        pass

    def download_file(self, url, filename, dest_path):
        """
        Download photo.

        Args:
            (String) url
            (String) file name (output)
        """
        req = self.session.get(url, stream=True)
        if req.status_code == 200:
            total_length = req.headers.get('content-length')
            dl_progress = 0

            output_path = "{}{}{}".format(dest_path, os.sep, filename)
            if not os.path.exists(output_path):
                with open(output_path, 'wb') as o_file:
                    for chunk in req.iter_content(1024):
                        dl_progress += len(chunk)
                        o_file.write(chunk)

                        # Download progress report
                        percent = dl_progress / int(total_length)
                        sys.stdout.write("\r{}: {:.2%}".format(filename, percent))
                        sys.stdout.flush()

                print('')
            else:
                print('{}: File exist'.format(filename))
        else:
            print('Visit website fail')

    @asyncio.coroutine
    def run(self, client, talk_id, username, stop_time=0):
        page_limit = 100

        img_count = 1
        video_count = 1
        last_image_t = 0
        last_video_t = 0
        for post_rec in range(1, 99999999, 100):
            payload = {
                'direction': 'NEXT',
                'limit': page_limit,
                'postId': post_rec,  # test 6000 (photos and videos)
                'talkId': talk_id,
            }

            r = self.session.get(self.url, params=payload)
            if r.status_code != 200:
                # handle connection fail
                print('Error: Connection fail')
                return
            else:
                raw_data = r.json()

                # handle no post
                if not raw_data['posts']:
                    print('Finished !')
                    return

                # Created directories for store files
                dest_img_path = 'downloads{}{}{}'.format(
                    os.sep, username, os.sep, self.img_path)
                dest_video_path = 'downloads{}{}{}'.format(
                    os.sep, username, os.sep, self.video_path)
                if not os.path.isdir(dest_img_path):
                    os.makedirs(dest_img_path)
                if not os.path.isdir(dest_video_path):
                    os.makedirs(dest_video_path)

                for i in range(page_limit):
                    # handle one page doesn't have 100 posts
                    try:
                        post_time = int(raw_data['posts'][i]['time'])
                    except IndexError:
                        print('Finished !')
                        return

                    # if msg time too old, stop download
                    if int(post_time) < stop_time:
                        break

                    url = raw_data['posts'][i]['body'][0].get('image', '')
                    if url:
                        # file_date = url.split('/')[4]
                        file_date = datetime.utcfromtimestamp(post_time
                                                              ).strftime("%y%m%d%H%M%S")
                        # handle duplication file
                        if file_date == last_image_t:
                            img_count += 1
                        else:
                            img_count = 1
                            last_image_t = file_date

                        self.download_file(
                            url, "{}_{}.jpg".format(file_date, img_count), dest_img_path)

                    url = raw_data['posts'][i]['body'][0].get('movieUrlHq', '')
                    if url:
                        file_date = datetime.utcfromtimestamp(post_time
                                                              ).strftime("%y%m%d%H%M%S")
                        # handle duplication file
                        if file_date == last_video_t:
                            video_count += 1
                        else:
                            video_count = 1
                            last_video_t = file_date

                        self.download_file(
                            url, "{}_{}.mp4".format(file_date, video_count), dest_video_path)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Download 755 photos + videos.')
    parser.add_argument('url', type=str, nargs='?',
                        help='Target Url: ex. http://7gogo.jp/talks/examples')
    parser.add_argument('stop_time', type=str, nargs='?', default=None,
                        help='Date (yy/mm/dd): ex. 141105')
    args = parser.parse_args()

    if args.url and args.stop_time:
        try:
            args.stop_time = time.mktime(time.strptime(args.stop_time, "%y%m%d"))
        except ValueError:
            parser.print_help()
            print('Error: Stop time format')
            sys.exit()

        my_cwawler = Crawler()
        talk_id, username = get_talk_id(args.url)

        loop = asyncio.get_event_loop()
        with aiohttp.ClientSession(loop=loop) as client:
            loop.run_until_complete(my_cwawler.run(
                client, talk_id, username, args.stop_time))
    else:
        parser.print_help()
<commit_msg>Fix: photo and video path issue<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Main function
"""
import os
import sys
import time
import argparse
from datetime import datetime

import aiohttp
import asyncio
import requests

from talk_id import get_talk_id

class Crawler(object):
    """
    Crawler for download 755 photos.
    """
    url = 'http://7gogo.jp/api/talk/post/list'
    img_path = 'images'
    video_path = 'videos'

    def __init__(self):
        self.session = requests.session()
        pass

    def download_file(self, url, filename, dest_path):
        """
        Download photo.

        Args:
            (String) url
            (String) file name (output)
        """
        req = self.session.get(url, stream=True)
        if req.status_code == 200:
            total_length = req.headers.get('content-length')
            dl_progress = 0

            output_path = "{}{}{}".format(dest_path, os.sep, filename)
            if not os.path.exists(output_path):
                with open(output_path, 'wb') as o_file:
                    for chunk in req.iter_content(1024):
                        dl_progress += len(chunk)
                        o_file.write(chunk)

                        # Download progress report
                        percent = dl_progress / int(total_length)
                        sys.stdout.write("\r{}: {:.2%}".format(filename, percent))
                        sys.stdout.flush()

                print('')
            else:
                print('{}: File exist'.format(filename))
        else:
            print('Visit website fail')

    @asyncio.coroutine
    def run(self, client, talk_id, username, stop_time=0):
        page_limit = 100

        img_count = 1
        video_count = 1
        last_image_t = 0
        last_video_t = 0
        post_rec = 1
        while True:
            payload = {
                'direction': 'NEXT',
                'limit': page_limit,
                'postId': post_rec,
                'talkId': talk_id,
            }
            post_rec += page_limit

            r = self.session.get(self.url, params=payload)
            if r.status_code != 200:
                # handle connection fail
                print('Error: Connection fail')
                return
            else:
                raw_data = r.json()

                # handle no post
                if not raw_data['posts']:
                    print('Finished !')
                    return

                # Created directories for store files
                dest_img_path = 'downloads{}{}{}{}'.format(
                    os.sep, username, os.sep, self.img_path)
                dest_video_path = 'downloads{}{}{}{}'.format(
                    os.sep, username, os.sep, self.video_path)
                if not os.path.isdir(dest_img_path):
                    os.makedirs(dest_img_path)
                if not os.path.isdir(dest_video_path):
                    os.makedirs(dest_video_path)

                for i in range(page_limit):
                    # handle one page doesn't have 100 posts
                    try:
                        post_time = int(raw_data['posts'][i]['time'])
                    except IndexError:
                        print('Finished !')
                        return

                    # if msg time too old, stop download
                    if int(post_time) < stop_time:
                        break

                    url = raw_data['posts'][i]['body'][0].get('image', '')
                    if url:
                        # file_date = url.split('/')[4]
                        file_date = datetime.utcfromtimestamp(post_time
                                                              ).strftime("%y%m%d%H%M%S")
                        # handle duplication file
                        if file_date == last_image_t:
                            img_count += 1
                        else:
                            img_count = 1
                            last_image_t = file_date

                        self.download_file(
                            url, "{}_{}.jpg".format(file_date, img_count), dest_img_path)

                    url = raw_data['posts'][i]['body'][0].get('movieUrlHq', '')
                    if url:
                        file_date = datetime.utcfromtimestamp(post_time
                                                              ).strftime("%y%m%d%H%M%S")
                        # handle duplication file
                        if file_date == last_video_t:
                            video_count += 1
                        else:
                            video_count = 1
                            last_video_t = file_date

                        self.download_file(
                            url, "{}_{}.mp4".format(file_date, video_count), dest_video_path)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Download 755 photos + videos.')
    parser.add_argument('url', type=str, nargs='?',
                        help='Target Url: ex. http://7gogo.jp/talks/examples')
    parser.add_argument('stop_time', type=str, nargs='?', default=None,
                        help='Date (yy/mm/dd): ex. 141105')
    args = parser.parse_args()

    if args.url and args.stop_time:
        try:
            args.stop_time = time.mktime(time.strptime(args.stop_time, "%y%m%d"))
        except ValueError:
            parser.print_help()
            print('Error: Stop Date format is wrong')
            sys.exit()

        my_cwawler = Crawler()
        talk_id, username = get_talk_id(args.url)

        loop = asyncio.get_event_loop()
        with aiohttp.ClientSession(loop=loop) as client:
            loop.run_until_complete(my_cwawler.run(
                client, talk_id, username, args.stop_time))
    else:
        parser.print_help()
<|endoftext|>


<commit_before># Amara, universalsubtitles.org
#
# Copyright (C) 2012 Participatory Culture Foundation
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see
# http://www.gnu.org/licenses/agpl-3.0.html.

from django import template
from django.core.urlresolvers import reverse
from django.template.defaultfilters import linebreaks

from apps.subtitles.forms import SubtitlesUploadForm
from apps.videos.forms import CreateVideoUrlForm

register = template.Library()

# FIXME: remove when DRM is done
def format_time(milliseconds):
    t = int(round(milliseconds / 1000.0))
    s = t % 60
    s = s > 9 and s or '0%s' % s
    return '%s:%s' % (t / 60, s)

@register.inclusion_tag('videos/_upload_subtitles.html', takes_context=True)
def upload_subtitles(context, video):
    context['video'] = video
    initial = {}

    current_language = context.get('language')
    if current_language:
        initial['language_code'] = current_language.language_code

    primary_language = video.get_primary_audio_subtitle_language()
    if primary_language:
        initial['primary_audio_language_code'] = primary_language.language_code

    context['form'] = SubtitlesUploadForm(context['user'], video,
                                          initial=initial)

    return context

@register.simple_tag
def complete_color(language):
    if not language.subtitles_complete:
        return 'eighty'
    else:
        return 'full'

@register.inclusion_tag('videos/_video_url_panel.html', takes_context=True)
def video_url_panel(context):
    video = context['video']
    context['form'] = CreateVideoUrlForm(context['user'], initial={'video': video.pk})
    context['video_urls'] = video.videourl_set.all()
    return context

@register.simple_tag
def video_url_count(video):
    return video.videourl_set.count()

@register.simple_tag
def language_url(request, lang):
    """Return the absolute url for that subtitle language.

    Takens into consideration whether the video is private or public.  Also
    handles the language-without-language that should be going away soon.

    """
    lc = lang.language_code or 'unknown'
    return reverse('videos:translation_history',
                   args=[lang.video.video_id, lc, lang.pk])

@register.filter
def format_sub_time(t):
    return '' if t < 0 else format_time(t)

@register.filter
def display_subtitle(text):
    """
    Transforms our internal markup to html friendly diplay:
    use the default subtitle formatiing tags (i, b, u)
    and replace \n with

    """
    if not text:
        return ""
    # FIXME: implement from dfxp formatting to html
    txt = linebreaks(text)
    return txt

@register.filter
def is_synced_value(val):
    return bool(val) or val == 0

@register.inclusion_tag("videos/_diffing-subtitle.html")
def render_subtitle_diff(diff_item, first_version):
    """
    Use the diff item from babelsubs.storage.diff. -> subtitle_data
    first_version: if we're showing the older version on not
    """
    if first_version:
        subtitle = diff_item['subtitles'][0]
    else:
        subtitle = diff_item['subtitles'][1]
    return {
        'time_changed' : diff_item['time_changed'],
        'text_changed': diff_item['text_changed'],
        'subtitle': subtitle,
    }

<commit_msg>#1820 Fixed color for completedness indicators on video & subtitle views, for https://unisubs.sifterapp.com/issues/1820<commit_after># Amara, universalsubtitles.org
#
# Copyright (C) 2012 Participatory Culture Foundation
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see
# http://www.gnu.org/licenses/agpl-3.0.html.

from django import template
from django.core.urlresolvers import reverse
from django.template.defaultfilters import linebreaks

from apps.subtitles.forms import SubtitlesUploadForm
from apps.videos.forms import CreateVideoUrlForm

register = template.Library()

# FIXME: remove when DRM is done
def format_time(milliseconds):
    t = int(round(milliseconds / 1000.0))
    s = t % 60
    s = s > 9 and s or '0%s' % s
    return '%s:%s' % (t / 60, s)

@register.inclusion_tag('videos/_upload_subtitles.html', takes_context=True)
def upload_subtitles(context, video):
    context['video'] = video
    initial = {}

    current_language = context.get('language')
    if current_language:
        initial['language_code'] = current_language.language_code

    primary_language = video.get_primary_audio_subtitle_language()
    if primary_language:
        initial['primary_audio_language_code'] = primary_language.language_code

    context['form'] = SubtitlesUploadForm(context['user'], video,
                                          initial=initial)

    return context

@register.simple_tag
def complete_color(language):
    if language.subtitles_complete:
        return 'full language-is-complete'
    else:
        return 'twenty language-is-not-complete'

@register.inclusion_tag('videos/_video_url_panel.html', takes_context=True)
def video_url_panel(context):
    video = context['video']
    context['form'] = CreateVideoUrlForm(context['user'], initial={'video': video.pk})
    context['video_urls'] = video.videourl_set.all()
    return context

@register.simple_tag
def video_url_count(video):
    return video.videourl_set.count()

@register.simple_tag
def language_url(request, lang):
    """Return the absolute url for that subtitle language.

    Takens into consideration whether the video is private or public.  Also
    handles the language-without-language that should be going away soon.

    """
    lc = lang.language_code or 'unknown'
    return reverse('videos:translation_history',
                   args=[lang.video.video_id, lc, lang.pk])

@register.filter
def format_sub_time(t):
    return '' if t < 0 else format_time(t)

@register.filter
def display_subtitle(text):
    """
    Transforms our internal markup to html friendly diplay:
    use the default subtitle formatiing tags (i, b, u)
    and replace \n with

    """
    if not text:
        return ""
    # FIXME: implement from dfxp formatting to html
    txt = linebreaks(text)
    return txt

@register.filter
def is_synced_value(val):
    return bool(val) or val == 0

@register.inclusion_tag("videos/_diffing-subtitle.html")
def render_subtitle_diff(diff_item, first_version):
    """
    Use the diff item from babelsubs.storage.diff. -> subtitle_data
    first_version: if we're showing the older version on not
    """
    if first_version:
        subtitle = diff_item['subtitles'][0]
    else:
        subtitle = diff_item['subtitles'][1]
    return {
        'time_changed' : diff_item['time_changed'],
        'text_changed': diff_item['text_changed'],
        'subtitle': subtitle,
    }

<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding: utf-8

import urllib2
import re

class DamInfo(object):
    BASE_URL = 'http://www1.river.go.jp'
    REALTIME_URL = '/cgi-bin/DspDamData.exe?ID={damid}&KIND=3&PAGE=0'
    DEFAULT_ENCODE = 'euc-jp'
    USER_AGENT = 'Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko'
    REGXP_CSV_URL = r''

    def __init__(self, damid):
        self.damid = damid

    def get_latest_storage(self):
        info = self.get_realtime_daminfo()
        for line in reversed(info):
            attribute = line[11]
            if attribute != '-':
                date = line[0]
                time = line[1]
                percent = line[10]
                break
        else:
            raise "Could not find storage information"
        return date, time, percent

    def get_realtime_daminfo(self):
        contents = self._fetch_contents(self._get_realtime_url, 'euc-jp',
                                        self._get_csv_url, 'cp932')
        info = self._format_csv(contents)
        return info

    def _format_csv(self, data):
        lines = data.strip().split('\r\n')[9:]
        table = [l.split(',') for l in lines]
        return table

    def _fetch_contents(self, get_outer, outer_charset, get_inner, inner_charset):
        outer_url = get_outer()
        outer_contents = self._http_get(outer_url, outer_charset)
        inner_url = get_inner(outer_contents)
        try:
            contents = self._http_get(inner_url, inner_charset)
        except urllib2.HTTPError:
            # retry
            outer_contents = self._http_get(outer_url, outer_charset)
            inner_url = get_inner(outer_contents)
            contents = self._http_get(inner_url, inner_charset)
        return contents

    def _http_get(self, url, encode=DEFAULT_ENCODE):
        headers = {'User-Agent': self.USER_AGENT}
        req = urllib2.Request(url, None, headers)
        response = urllib2.urlopen(req)
        contents = response.read().decode(encode)
        return contents

    def _get_realtime_url(self):
        url = self.BASE_URL + self.REALTIME_URL.format(damid=self.damid)
        return url

    def _get_csv_url(self, contents):
        c_regxp = re.compile(self.REGXP_CSV_URL)
        mached = c_regxp.search(contents)
        if mached is not None:
            url = self.BASE_URL + mached.group(1)
            return url
        else:
            raise "Could not find csv url"
<commit_msg>calc_latest_storageを追加<commit_after>#!/usr/bin/env python
# coding: utf-8

import urllib2
import re

class DamInfo(object):
    BASE_URL = 'http://www1.river.go.jp'
    REALTIME_URL = '/cgi-bin/DspDamData.exe?ID={damid}&KIND=3&PAGE=0'
    DEFAULT_ENCODE = 'euc-jp'
    USER_AGENT = 'Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko'
    REGXP_CSV_URL = r''

    def __init__(self, damid, maximum_storage=0):
        self.damid = damid
        self.maximum_storage = maximum_storage

    def get_latest_storage(self):
        info = self.get_realtime_daminfo()
        for line in reversed(info):
            attribute = line[11]
            if attribute != '-':
                date = line[0]
                time = line[1]
                percent = line[10]
                break
        else:
            raise "Could not find storage information"
        return date, time, percent

    def calc_latest_storage(self):
        info = self.get_realtime_daminfo()
        date = info[-1][0]
        time = info[-1][1]
        percent = str(round(float(info[-1][4]) / self.maximum_storage * 100, 1))
        return date, time, percent

    def get_realtime_daminfo(self):
        contents = self._fetch_contents(self._get_realtime_url, 'euc-jp',
                                        self._get_csv_url, 'cp932')
        info = self._format_csv(contents)
        return info

    def _format_csv(self, data):
        lines = data.strip().split('\r\n')[9:]
        table = [l.split(',') for l in lines]
        return table

    def _fetch_contents(self, get_outer, outer_charset, get_inner, inner_charset):
        outer_url = get_outer()
        outer_contents = self._http_get(outer_url, outer_charset)
        inner_url = get_inner(outer_contents)
        try:
            contents = self._http_get(inner_url, inner_charset)
        except urllib2.HTTPError:
            # retry
            outer_contents = self._http_get(outer_url, outer_charset)
            inner_url = get_inner(outer_contents)
            contents = self._http_get(inner_url, inner_charset)
        return contents

    def _http_get(self, url, encode=DEFAULT_ENCODE):
        headers = {'User-Agent': self.USER_AGENT}
        req = urllib2.Request(url, None, headers)
        response = urllib2.urlopen(req)
        contents = response.read().decode(encode)
        return contents

    def _get_realtime_url(self):
        url = self.BASE_URL + self.REALTIME_URL.format(damid=self.damid)
        return url

    def _get_csv_url(self, contents):
        c_regxp = re.compile(self.REGXP_CSV_URL)
        mached = c_regxp.search(contents)
        if mached is not None:
            url = self.BASE_URL + mached.group(1)
            return url
        else:
            raise "Could not find csv url"
<|endoftext|>


<commit_before>
class Hunk:

    # src is assumed to be positioned at the right place to start
    # applying the patch, and the new lines are returned
    def apply(self, src):
	result = []
	fromLine = 0
	for line in self.lines:
	    if line[0] == " ":
		if src[fromLine] != line[1:]:
		    raise Conflict()
		result.append(src[fromLine])
		fromLine = fromLine + 1
	    elif line[0] == "+":
		result.append(line[1:])
	    elif line[0] == "-":
		fromLine = fromLine + 1

	assert(fromLine == self.fromLen)
	assert(len(result) == self.toLen)

	return result

    def __init__(self, fromStart, fromLen, toStart, toLen, lines):
	self.fromStart = fromStart
	self.toStart = toStart
	self.fromLen = fromLen
	self.toLen = toLen
	self.lines = lines

# this just applies hunks, not files... in other words, the --- and +++
# lines should be omitted, and only a single file can be patched
def patch(oldLines, unifiedDiff):
    i = 0
    last = len(unifiedDiff)
    hunks = []
    while i < last:
	(magic1, fromRange, toRange, magic2) = unifiedDiff[i].split()
	if magic1 != "@@" or magic2 != "@@" or fromRange[0] != "-" or \
	   toRange[0] != "+":
	    raise BadHunkHeader()

	(fromStart, fromLen) = fromRange.split(",")
	fromStart = int(fromStart[1:]) - 1
	fromLen = int(fromLen)

	(toStart, toLen) = toRange.split(",")
	toStart = int(toStart[1:]) - 1
	toLen = int(toLen)

	fromCount = 0
	toCount = 0
	lines = []
	i += 1
	while i < last and unifiedDiff[i][0] != '@':
	    lines.append(unifiedDiff[i])
	    ch = unifiedDiff[i][0]
	    if ch == " ":
		fromCount += 1
		toCount += 1
	    elif ch == "-":
		fromCount += 1
	    elif ch == "+":
		toCount += 1
	    else:
		raise BadHunk()

	    i += 1

	if toCount != toLen or fromCount != fromLen:
	    print toCount, toLen
	    raise BadHunk()

	hunks.append(Hunk(fromStart, fromLen, toStart, toLen, lines))

    i = 0
    last = len(unifiedDiff)
    result = []

    fromLine = 0

    for hunk in hunks:
	while (fromLine < hunk.fromStart):
	    result.append(oldLines[fromLine])
	    fromLine = fromLine + 1

	assert(hunk.toStart == len(result))

	result += hunk.apply(oldLines[fromLine:])
	fromLine += hunk.fromLen

    while (fromLine < len(oldLines)):
	result.append(oldLines[fromLine])
	fromLine = fromLine + 1

    return result

class BadHunkHeader(Exception):

    pass

class BadHunk(Exception):

    pass

class Conflict(Exception):

    pass
<commit_msg>look around for where a hunk really goes<commit_after>
class Hunk:

    # the new lines which result from this hunk are returned
    def apply(self, src, srcLine):
	result = []
	fromLine = srcLine
	for line in self.lines:
	    if line[0] == " ":
		if src[fromLine] != line[1:]:
		    raise Conflict()
		result.append(src[fromLine])
		fromLine = fromLine + 1
	    elif line[0] == "+":
		result.append(line[1:])
	    elif line[0] == "-":
		if src[fromLine] != line[1:]:
		    raise Conflict()

		fromLine = fromLine + 1

	assert(fromLine == self.fromLen + srcLine)
	assert(len(result) == self.toLen)

	return result

    def countConflicts(self, src, srcLine):
	conflicts = 0
	for line in self.lines:
	    if line[0] == " " or line[0] == "-":
		if src[srcLine] != line[1:]:
		    conflicts += 1
		srcLine += 1

	return conflicts

    def __init__(self, fromStart, fromLen, toStart, toLen, lines):
	self.fromStart = fromStart
	self.toStart = toStart
	self.fromLen = fromLen
	self.toLen = toLen
	self.lines = lines

# this just applies hunks, not files... in other words, the --- and +++
# lines should be omitted, and only a single file can be patched
def patch(oldLines, unifiedDiff):
    i = 0
    last = len(unifiedDiff)
    hunks = []
    while i < last:
	(magic1, fromRange, toRange, magic2) = unifiedDiff[i].split()
	if magic1 != "@@" or magic2 != "@@" or fromRange[0] != "-" or \
	   toRange[0] != "+":
	    raise BadHunkHeader()

	(fromStart, fromLen) = fromRange.split(",")
	fromStart = int(fromStart[1:]) - 1
	fromLen = int(fromLen)

	(toStart, toLen) = toRange.split(",")
	toStart = int(toStart[1:]) - 1
	toLen = int(toLen)

	fromCount = 0
	toCount = 0
	lines = []
	i += 1
	while i < last and unifiedDiff[i][0] != '@':
	    lines.append(unifiedDiff[i])
	    ch = unifiedDiff[i][0]
	    if ch == " ":
		fromCount += 1
		toCount += 1
	    elif ch == "-":
		fromCount += 1
	    elif ch == "+":
		toCount += 1
	    else:
		raise BadHunk()

	    i += 1

	if toCount != toLen or fromCount != fromLen:
	    print toCount, toLen
	    raise BadHunk()

	hunks.append(Hunk(fromStart, fromLen, toStart, toLen, lines))

    i = 0
    last = len(unifiedDiff)
    result = []

    fromLine = 0
    offset = 0

    for hunk in hunks:
	start = hunk.fromStart + offset
	conflicts = hunk.countConflicts(oldLines, start)

	i = 0
	before = 1000
	after = 1000
	while conflicts:
	    i = i + 1
	    if (start - abs(i) >= 0):
		before = hunk.countConflicts(oldLines, start - abs(i))
	    if (start + abs(i) < (len(oldLines) - hunk.fromLen)):
		after = hunk.countConflicts(oldLines, start + abs(i))

	    if before < after:
		conflicts = before
		i = -(abs(i))
	    else:
		conflicts = after
		i = abs(i)

	offset = i
	start += i

	while (fromLine < start):
	    result.append(oldLines[fromLine])
	    fromLine = fromLine + 1

	result += hunk.apply(oldLines, start)
	fromLine += hunk.fromLen

    while (fromLine < len(oldLines)):
	result.append(oldLines[fromLine])
	fromLine = fromLine + 1

    return result

class BadHunkHeader(Exception):

    pass

class BadHunk(Exception):

    pass

class Conflict(Exception):

    pass
<|endoftext|>