<commit_before>#!/usr/bin/env python
"""
A Python Statsd implementation with some datadog special sauce.
"""

# set up logging before importing any other components
from config import initialize_logging; initialize_logging('dogstatsd')

import os; os.umask(022)

# stdlib
import httplib as http_client
import logging
import optparse
import re
import select
import signal
import socket
import sys
from time import time
import threading
from urllib import urlencode

# project
from aggregator import MetricsAggregator
from checks.check_status import DogstatsdStatus
from config import get_config
from daemon import Daemon, AgentSupervisor
from util import json, PidFile, get_hostname

log = logging.getLogger('dogstatsd')

WATCHDOG_TIMEOUT = 120
UDP_SOCKET_TIMEOUT = 5
LOGGING_INTERVAL = 10

def serialize_metrics(metrics):
    return json.dumps({"series" : metrics})

def serialize_event(event):
    return json.dumps(event)

class Reporter(threading.Thread):
    """
    The reporter periodically sends the aggregated metrics to the
    server.
    """

    def __init__(self, interval, metrics_aggregator, api_host, api_key=None, use_watchdog=False):
        threading.Thread.__init__(self)
        self.interval = int(interval)
        self.finished = threading.Event()
        self.metrics_aggregator = metrics_aggregator
        self.flush_count = 0

        self.watchdog = None
        if use_watchdog:
            from util import Watchdog
            self.watchdog = Watchdog(WATCHDOG_TIMEOUT)

        self.api_key = api_key
        self.api_host = api_host

        self.http_conn_cls = http_client.HTTPSConnection

        match = re.match('^(https?)://(.*)', api_host)

        if match:
            self.api_host = match.group(2)
            if match.group(1) == 'http':
                self.http_conn_cls = http_client.HTTPConnection

    def stop(self):
        log.info("Stopping reporter")
        self.finished.set()

    def run(self):

        log.info("Reporting to %s every %ss" % (self.api_host, self.interval))
        log.debug("Watchdog enabled: %s" % bool(self.watchdog))

        # Persist a start-up message.
        DogstatsdStatus().persist()

        while not self.finished.isSet(): # Use camel case isSet for 2.4 support.
            self.finished.wait(self.interval)
            self.metrics_aggregator.send_packet_count('datadog.dogstatsd.packet.count')
            self.flush()
            if self.watchdog:
                self.watchdog.reset()

        # Clean up the status messages.
        log.debug("Stopped reporter")
        DogstatsdStatus.remove_latest_status()

    def flush(self):
        try:
            self.flush_count += 1
            packets_per_second = self.metrics_aggregator.packets_per_second(self.interval)
            packet_count = self.metrics_aggregator.total_count

            metrics = self.metrics_aggregator.flush()
            count = len(metrics)
            should_log = self.flush_count < LOGGING_INTERVAL or self.flush_count % LOGGING_INTERVAL == 0
            if not count:
                if should_log:
                    log.info("Flush #%s: No metrics to flush." % self.flush_count)
            else:
                if should_log:
                    log.info("Flush #%s: flushing %s metrics" % (self.flush_count, count))
                self.submit(metrics)

            events = self.metrics_aggregator.flush_events()
            event_count = len(events)
            if not event_count:
                    log.info("Flush #%s: No events to flush." % self.flush_count)
            else:
                log.info("Flush #%s: flushing %s events" % (self.flush_count, len(events)))
                self.submit_events(events)

            # Persist a status message.
            packet_count = self.metrics_aggregator.total_count
            DogstatsdStatus(
                flush_count=self.flush_count,
                packet_count=packet_count,
                packets_per_second=packets_per_second,
                metric_count=count,
                event_count=event_count
            ).persist()

        except Exception, e:
            log.exception("Error flushing metrics")

    def submit(self, metrics):
        # Copy and pasted from dogapi, because it's a bit of a pain to distribute python
        # dependencies with the agent.
        body = serialize_metrics(metrics)
        headers = {'Content-Type':'application/json'}
        method = 'POST'

        params = {}
        if self.api_key:
            params['api_key'] = self.api_key
        url = '/api/v1/series?%s' % urlencode(params)

        start_time = time()
        status = None
        conn = self.http_conn_cls(self.api_host)
        try:
            conn.request(method, url, body, headers)

            #FIXME: add timeout handling code here

            response = conn.getresponse()
            status = response.status
            response.close()
        finally:
            conn.close()
        duration = round((time() - start_time) * 1000.0, 4)
        log.debug("%s %s %s%s (%sms)" % (
                        status, method, self.api_host, url, duration))
        return duration

    def submit_events(self, events):
        headers = {'Content-Type':'application/json'}
        method = 'POST'

        params = {}
        if self.api_key:
            params['api_key'] = self.api_key
        url = '/api/v1/events?%s' % urlencode(params)

        status = None
        conn = self.http_conn_cls(self.api_host)
        try:
            for event in events:
                start_time = time()
                body = serialize_event(event)
                log.debug('Sending event: %s' % body)
                conn.request(method, url, body, headers)

                response = conn.getresponse()
                status = response.status
                response.close()
                duration = round((time() - start_time) * 1000.0, 4)
                log.debug("%s %s %s%s (%sms)" % (
                                status, method, self.api_host, url, duration))
        finally:
            conn.close()

class Server(object):
    """
    A statsd udp server.
    """

    def __init__(self, metrics_aggregator, host, port):
        self.host = host
        self.port = int(port)
        self.address = (self.host, self.port)
        self.metrics_aggregator = metrics_aggregator
        self.buffer_size = 1024 * 8

        self.running = False

    def start(self):
        """ Run the server. """
        # Bind to the UDP socket.
        # IPv4 only
        self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        self.socket.setblocking(0)
        self.socket.bind(self.address)

        log.info('Listening on host & port: %s' % str(self.address))

        # Inline variables for quick look-up.
        buffer_size = self.buffer_size
        aggregator_submit = self.metrics_aggregator.submit_packets
        sock = [self.socket]
        socket_recv = self.socket.recv
        select_select = select.select
        select_error = select.error
        timeout = UDP_SOCKET_TIMEOUT

        # Run our select loop.
        self.running = True
        while self.running:
            try:
                ready = select_select(sock, [], [], timeout)
                if ready[0]:
                    aggregator_submit(socket_recv(buffer_size))
            except select_error, se:
                # Ignore interrupted system calls from sigterm.
                errno = se[0]
                if errno != 4:
                    raise
            except (KeyboardInterrupt, SystemExit):
                break
            except Exception, e:
                log.exception('Error receiving datagram')

    def stop(self):
        self.running = False

class Dogstatsd(Daemon):
    """ This class is the dogstatsd daemon. """

    def __init__(self, pid_file, server, reporter, autorestart):
        Daemon.__init__(self, pid_file, autorestart=autorestart)
        self.server = server
        self.reporter = reporter

    def _handle_sigterm(self, signum, frame):
        log.debug("Caught sigterm. Stopping run loop.")
        self.server.stop()

    def stop(self):
        if self.autorestart:
            parent_pid = os.getpgid(self.pid())
            os.kill(parent_pid, signal.SIGTERM)
        super(Dogstatsd, self).stop()

    def run(self):
        # Gracefully exit on sigterm.
        signal.signal(signal.SIGTERM, self._handle_sigterm)

        # Handle Keyboard Interrupt
        signal.signal(signal.SIGINT, self._handle_sigterm)

        # Start the reporting thread before accepting data
        self.reporter.start()

        try:
            try:
                self.server.start()
            except Exception, e:
                log.exception('Error starting server')
                raise e
        finally:
            # The server will block until it's done. Once we're here, shutdown
            # the reporting thread.
            self.reporter.stop()
            self.reporter.join()
            log.info("Dogstatsd is stopped")
            # Restart if asked to restart
            if self.autorestart:
                sys.exit(AgentSupervisor.RESTART_EXIT_STATUS)

    def info(self):
        logging.getLogger().setLevel(logging.ERROR)
        return DogstatsdStatus.print_latest_status()

def init(config_path=None, use_watchdog=False, use_forwarder=False):
    """Configure the server and the reporting thread.
    """
    c = get_config(parse_args=False, cfg_path=config_path)
    log.debug("Configuration dogstatsd")

    port      = c['dogstatsd_port']
    interval  = int(c['dogstatsd_interval'])
    api_key   = c['api_key']
    non_local_traffic = c['non_local_traffic']

    target = c['dd_url']
    if use_forwarder:
        target = c['dogstatsd_target']

    hostname = get_hostname(c)

    # Create the aggregator (which is the point of communication between the
    # server and reporting threads.
    assert 0 < interval

    aggregator = MetricsAggregator(hostname, interval, recent_point_threshold=c.get('recent_point_threshold', None))

    # Start the reporting thread.
    reporter = Reporter(interval, aggregator, target, api_key, use_watchdog)

    # Start the server on an IPv4 stack
    # Default to loopback
    server_host = '127.0.0.1'
    # If specified, bind to all addressses
    if non_local_traffic:
        server_host = ''

    server = Server(aggregator, server_host, port)

    return reporter, server, c

def main(config_path=None):
    """ The main entry point for the unix version of dogstatsd. """
    parser = optparse.OptionParser("%prog [start|stop|restart|status]")
    parser.add_option('-u', '--use-local-forwarder', action='store_true',
                        dest="use_forwarder", default=False)
    opts, args = parser.parse_args()

    reporter, server, cnf = init(config_path, use_watchdog=True, use_forwarder=opts.use_forwarder)
    pid_file = PidFile('dogstatsd')
    daemon = Dogstatsd(pid_file.get_path(), server, reporter,
            cnf.get('autorestart', False))

    # If no args were passed in, run the server in the foreground.
    if not args:
        daemon.run()
        return 0

    # Otherwise, we're process the deamon command.
    else:
        command = args[0]

        if command == 'start':
            daemon.start()
        elif command == 'stop':
            daemon.stop()
        elif command == 'restart':
            daemon.restart()
        elif command == 'status':
            daemon.status()
        elif command == 'info':
            return daemon.info()
        else:
            sys.stderr.write("Unknown command: %s\n\n" % command)
            parser.print_help()
            return 1
        return 0

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Do not log statsd events at every flush<commit_after>#!/usr/bin/env python
"""
A Python Statsd implementation with some datadog special sauce.
"""

# set up logging before importing any other components
from config import initialize_logging; initialize_logging('dogstatsd')

import os; os.umask(022)

# stdlib
import httplib as http_client
import logging
import optparse
import re
import select
import signal
import socket
import sys
from time import time
import threading
from urllib import urlencode

# project
from aggregator import MetricsAggregator
from checks.check_status import DogstatsdStatus
from config import get_config
from daemon import Daemon, AgentSupervisor
from util import json, PidFile, get_hostname

log = logging.getLogger('dogstatsd')

WATCHDOG_TIMEOUT = 120
UDP_SOCKET_TIMEOUT = 5
LOGGING_INTERVAL = 10

def serialize_metrics(metrics):
    return json.dumps({"series" : metrics})

def serialize_event(event):
    return json.dumps(event)

class Reporter(threading.Thread):
    """
    The reporter periodically sends the aggregated metrics to the
    server.
    """

    def __init__(self, interval, metrics_aggregator, api_host, api_key=None, use_watchdog=False):
        threading.Thread.__init__(self)
        self.interval = int(interval)
        self.finished = threading.Event()
        self.metrics_aggregator = metrics_aggregator
        self.flush_count = 0

        self.watchdog = None
        if use_watchdog:
            from util import Watchdog
            self.watchdog = Watchdog(WATCHDOG_TIMEOUT)

        self.api_key = api_key
        self.api_host = api_host

        self.http_conn_cls = http_client.HTTPSConnection

        match = re.match('^(https?)://(.*)', api_host)

        if match:
            self.api_host = match.group(2)
            if match.group(1) == 'http':
                self.http_conn_cls = http_client.HTTPConnection

    def stop(self):
        log.info("Stopping reporter")
        self.finished.set()

    def run(self):

        log.info("Reporting to %s every %ss" % (self.api_host, self.interval))
        log.debug("Watchdog enabled: %s" % bool(self.watchdog))

        # Persist a start-up message.
        DogstatsdStatus().persist()

        while not self.finished.isSet(): # Use camel case isSet for 2.4 support.
            self.finished.wait(self.interval)
            self.metrics_aggregator.send_packet_count('datadog.dogstatsd.packet.count')
            self.flush()
            if self.watchdog:
                self.watchdog.reset()

        # Clean up the status messages.
        log.debug("Stopped reporter")
        DogstatsdStatus.remove_latest_status()

    def flush(self):
        try:
            self.flush_count += 1
            packets_per_second = self.metrics_aggregator.packets_per_second(self.interval)
            packet_count = self.metrics_aggregator.total_count

            metrics = self.metrics_aggregator.flush()
            count = len(metrics)
            should_log = self.flush_count < LOGGING_INTERVAL or self.flush_count % LOGGING_INTERVAL == 0
            if not count:
                if should_log:
                    log.info("Flush #%s: No metrics to flush." % self.flush_count)
            else:
                if should_log:
                    log.info("Flush #%s: flushing %s metrics" % (self.flush_count, count))
                self.submit(metrics)

            events = self.metrics_aggregator.flush_events()
            event_count = len(events)
            if not event_count:
                if should_log:
                    log.info("Flush #%s: No events to flush." % self.flush_count)
                else:
                    log.debug("Flush #%s: No events to flush." % self.flush_count)
            else:
                if should_log:
                    log.info("Flush #%s: flushing %s events" % (self.flush_count, len(events)))
                else:
                    log.debug("Flush #%s: flushing %s events" % (self.flush_count, len(events)))
                self.submit_events(events)

            # Persist a status message.
            packet_count = self.metrics_aggregator.total_count
            DogstatsdStatus(
                flush_count=self.flush_count,
                packet_count=packet_count,
                packets_per_second=packets_per_second,
                metric_count=count,
                event_count=event_count
            ).persist()

        except Exception, e:
            log.exception("Error flushing metrics")

    def submit(self, metrics):
        # Copy and pasted from dogapi, because it's a bit of a pain to distribute python
        # dependencies with the agent.
        body = serialize_metrics(metrics)
        headers = {'Content-Type':'application/json'}
        method = 'POST'

        params = {}
        if self.api_key:
            params['api_key'] = self.api_key
        url = '/api/v1/series?%s' % urlencode(params)

        start_time = time()
        status = None
        conn = self.http_conn_cls(self.api_host)
        try:
            conn.request(method, url, body, headers)

            #FIXME: add timeout handling code here

            response = conn.getresponse()
            status = response.status
            response.close()
        finally:
            conn.close()
        duration = round((time() - start_time) * 1000.0, 4)
        log.debug("%s %s %s%s (%sms)" % (
                        status, method, self.api_host, url, duration))
        return duration

    def submit_events(self, events):
        headers = {'Content-Type':'application/json'}
        method = 'POST'

        params = {}
        if self.api_key:
            params['api_key'] = self.api_key
        url = '/api/v1/events?%s' % urlencode(params)

        status = None
        conn = self.http_conn_cls(self.api_host)
        try:
            for event in events:
                start_time = time()
                body = serialize_event(event)
                log.debug('Sending event: %s' % body)
                conn.request(method, url, body, headers)

                response = conn.getresponse()
                status = response.status
                response.close()
                duration = round((time() - start_time) * 1000.0, 4)
                log.debug("%s %s %s%s (%sms)" % (
                                status, method, self.api_host, url, duration))
        finally:
            conn.close()

class Server(object):
    """
    A statsd udp server.
    """

    def __init__(self, metrics_aggregator, host, port):
        self.host = host
        self.port = int(port)
        self.address = (self.host, self.port)
        self.metrics_aggregator = metrics_aggregator
        self.buffer_size = 1024 * 8

        self.running = False

    def start(self):
        """ Run the server. """
        # Bind to the UDP socket.
        # IPv4 only
        self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        self.socket.setblocking(0)
        self.socket.bind(self.address)

        log.info('Listening on host & port: %s' % str(self.address))

        # Inline variables for quick look-up.
        buffer_size = self.buffer_size
        aggregator_submit = self.metrics_aggregator.submit_packets
        sock = [self.socket]
        socket_recv = self.socket.recv
        select_select = select.select
        select_error = select.error
        timeout = UDP_SOCKET_TIMEOUT

        # Run our select loop.
        self.running = True
        while self.running:
            try:
                ready = select_select(sock, [], [], timeout)
                if ready[0]:
                    aggregator_submit(socket_recv(buffer_size))
            except select_error, se:
                # Ignore interrupted system calls from sigterm.
                errno = se[0]
                if errno != 4:
                    raise
            except (KeyboardInterrupt, SystemExit):
                break
            except Exception, e:
                log.exception('Error receiving datagram')

    def stop(self):
        self.running = False

class Dogstatsd(Daemon):
    """ This class is the dogstatsd daemon. """

    def __init__(self, pid_file, server, reporter, autorestart):
        Daemon.__init__(self, pid_file, autorestart=autorestart)
        self.server = server
        self.reporter = reporter

    def _handle_sigterm(self, signum, frame):
        log.debug("Caught sigterm. Stopping run loop.")
        self.server.stop()

    def stop(self):
        if self.autorestart:
            parent_pid = os.getpgid(self.pid())
            os.kill(parent_pid, signal.SIGTERM)
        super(Dogstatsd, self).stop()

    def run(self):
        # Gracefully exit on sigterm.
        signal.signal(signal.SIGTERM, self._handle_sigterm)

        # Handle Keyboard Interrupt
        signal.signal(signal.SIGINT, self._handle_sigterm)

        # Start the reporting thread before accepting data
        self.reporter.start()

        try:
            try:
                self.server.start()
            except Exception, e:
                log.exception('Error starting server')
                raise e
        finally:
            # The server will block until it's done. Once we're here, shutdown
            # the reporting thread.
            self.reporter.stop()
            self.reporter.join()
            log.info("Dogstatsd is stopped")
            # Restart if asked to restart
            if self.autorestart:
                sys.exit(AgentSupervisor.RESTART_EXIT_STATUS)

    def info(self):
        logging.getLogger().setLevel(logging.ERROR)
        return DogstatsdStatus.print_latest_status()

def init(config_path=None, use_watchdog=False, use_forwarder=False):
    """Configure the server and the reporting thread.
    """
    c = get_config(parse_args=False, cfg_path=config_path)
    log.debug("Configuration dogstatsd")

    port      = c['dogstatsd_port']
    interval  = int(c['dogstatsd_interval'])
    api_key   = c['api_key']
    non_local_traffic = c['non_local_traffic']

    target = c['dd_url']
    if use_forwarder:
        target = c['dogstatsd_target']

    hostname = get_hostname(c)

    # Create the aggregator (which is the point of communication between the
    # server and reporting threads.
    assert 0 < interval

    aggregator = MetricsAggregator(hostname, interval, recent_point_threshold=c.get('recent_point_threshold', None))

    # Start the reporting thread.
    reporter = Reporter(interval, aggregator, target, api_key, use_watchdog)

    # Start the server on an IPv4 stack
    # Default to loopback
    server_host = '127.0.0.1'
    # If specified, bind to all addressses
    if non_local_traffic:
        server_host = ''

    server = Server(aggregator, server_host, port)

    return reporter, server, c

def main(config_path=None):
    """ The main entry point for the unix version of dogstatsd. """
    parser = optparse.OptionParser("%prog [start|stop|restart|status]")
    parser.add_option('-u', '--use-local-forwarder', action='store_true',
                        dest="use_forwarder", default=False)
    opts, args = parser.parse_args()

    reporter, server, cnf = init(config_path, use_watchdog=True, use_forwarder=opts.use_forwarder)
    pid_file = PidFile('dogstatsd')
    daemon = Dogstatsd(pid_file.get_path(), server, reporter,
            cnf.get('autorestart', False))

    # If no args were passed in, run the server in the foreground.
    if not args:
        daemon.run()
        return 0

    # Otherwise, we're process the deamon command.
    else:
        command = args[0]

        if command == 'start':
            daemon.start()
        elif command == 'stop':
            daemon.stop()
        elif command == 'restart':
            daemon.restart()
        elif command == 'status':
            daemon.status()
        elif command == 'info':
            return daemon.info()
        else:
            sys.stderr.write("Unknown command: %s\n\n" % command)
            parser.print_help()
            return 1
        return 0

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>import copy
import logging
from hashlib import sha1

import simplejson as json

from treeherder.log_parser.utils import validate_perf_data
from treeherder.model.models import OptionCollection
from treeherder.perf.models import (PerformanceDatum,
                                    PerformanceFramework,
                                    PerformanceSignature)
from treeherder.perf.tasks import generate_alerts

logger = logging.getLogger(__name__)

def _get_application_name(validated_perf_datum: dict):
    try:
        return validated_perf_datum['application']['name']
    except KeyError:
        return None

def _get_signature_hash(signature_properties):
    signature_prop_values = list(signature_properties.keys())
    str_values = []
    for value in signature_properties.values():
        if not isinstance(value, str):
            str_values.append(json.dumps(value, sort_keys=True))
        else:
            str_values.append(value)
    signature_prop_values.extend(str_values)

    sha = sha1()
    sha.update(''.join(map(str, sorted(signature_prop_values))).encode('utf-8'))

    return sha.hexdigest()

def _create_or_update_signature(repository, signature_hash, framework, defaults):
    signature, created = PerformanceSignature.objects.get_or_create(
        repository=repository,
        signature_hash=signature_hash,
        framework=framework,
        defaults=defaults)
    if not created:
        if signature.last_updated > defaults['last_updated']:
            defaults['last_updated'] = signature.last_updated
        signature, _ = PerformanceSignature.objects.update_or_create(
            repository=repository,
            signature_hash=signature_hash,
            framework=framework,
            defaults=defaults)
    return signature

def _load_perf_datum(job, perf_datum):
    validate_perf_data(perf_datum)

    extra_properties = {}
    extra_options = ''
    reference_data = {
        'option_collection_hash': job.signature.option_collection_hash,
        'machine_platform': job.signature.machine_platform
    }

    option_collection = OptionCollection.objects.get(
        option_collection_hash=job.signature.option_collection_hash)

    try:
        framework = PerformanceFramework.objects.get(
            name=perf_datum['framework']['name'])
    except PerformanceFramework.DoesNotExist:
        logger.warning("Performance framework %s does not exist, skipping "
                       "load of performance artifacts",
                       perf_datum['framework']['name'])
        return
    if not framework.enabled:
        logger.info("Performance framework %s is not enabled, skipping",
                    perf_datum['framework']['name'])
        return
    for suite in perf_datum['suites']:
        suite_extra_properties = copy.copy(extra_properties)
        suite_extra_options = copy.copy(extra_options)
        if suite.get('extraOptions'):
            suite_extra_properties = {
                'test_options': sorted(suite['extraOptions'])
            }
            # store extraOptions list as space separated string
            suite_extra_options = ' '.join(sorted(suite['extraOptions']))
        summary_signature_hash = None

        # if we have a summary value, create or get its signature by all its subtest
        # properties.
        if suite.get('value') is not None:
            # summary series
            summary_properties = {
                'suite': suite['name']
            }
            summary_properties.update(reference_data)
            summary_properties.update(suite_extra_properties)
            summary_signature_hash = _get_signature_hash(
                summary_properties)
            signature = _create_or_update_signature(
                job.repository, summary_signature_hash, framework, {
                    'test': '',
                    'suite': suite['name'],
                    'option_collection': option_collection,
                    'platform': job.machine_platform,
                    'extra_options': suite_extra_options,
                    'measurement_unit': suite.get('unit'),
                    'application': _get_application_name(perf_datum),
                    'lower_is_better': suite.get('lowerIsBetter', True),
                    'has_subtests': True,
                    # these properties below can be either True, False, or null
                    # (None). Null indicates no preference has been set.
                    'should_alert': suite.get('shouldAlert'),
                    'alert_change_type': PerformanceSignature._get_alert_change_type(
                        suite.get('alertChangeType')),
                    'alert_threshold': suite.get('alertThreshold'),
                    'min_back_window': suite.get('minBackWindow'),
                    'max_back_window': suite.get('maxBackWindow'),
                    'fore_window': suite.get('foreWindow'),
                    'last_updated': job.push.time
                })
            (_, datum_created) = PerformanceDatum.objects.get_or_create(
                repository=job.repository,
                job=job,
                push=job.push,
                signature=signature,
                push_timestamp=job.push.time,
                defaults={'value': suite['value']})
            if signature.should_alert is not False and datum_created and \
               job.repository.performance_alerts_enabled:
                generate_alerts.apply_async(args=[signature.id],
                                            queue='generate_perf_alerts')

        for subtest in suite['subtests']:
            subtest_properties = {
                'suite': suite['name'],
                'test': subtest['name']
            }
            subtest_properties.update(reference_data)
            subtest_properties.update(suite_extra_properties)

            summary_signature = None
            if summary_signature_hash is not None:
                subtest_properties.update({'parent_signature': summary_signature_hash})
                summary_signature = PerformanceSignature.objects.get(
                    repository=job.repository,
                    framework=framework,
                    signature_hash=summary_signature_hash)
            subtest_signature_hash = _get_signature_hash(subtest_properties)
            value = list(subtest['value'] for subtest in suite['subtests'] if
                         subtest['name'] == subtest_properties['test'])
            signature = _create_or_update_signature(
                job.repository, subtest_signature_hash, framework, {
                    'test': subtest_properties['test'],
                    'suite': suite['name'],
                    'option_collection': option_collection,
                    'platform': job.machine_platform,
                    'extra_options': suite_extra_options,
                    'measurement_unit': subtest.get('unit'),
                    'lower_is_better': subtest.get('lowerIsBetter', True),
                    'has_subtests': False,
                    # these properties below can be either True, False, or
                    # null (None). Null indicates no preference has been
                    # set.
                    'should_alert': subtest.get('shouldAlert'),
                    'alert_change_type': PerformanceSignature._get_alert_change_type(
                        subtest.get('alertChangeType')),
                    'alert_threshold': subtest.get('alertThreshold'),
                    'min_back_window': subtest.get('minBackWindow'),
                    'max_back_window': subtest.get('maxBackWindow'),
                    'fore_window': subtest.get('foreWindow'),
                    'parent_signature': summary_signature,
                    'last_updated': job.push.time
                })
            (_, datum_created) = PerformanceDatum.objects.get_or_create(
                repository=job.repository,
                job=job,
                push=job.push,
                signature=signature,
                push_timestamp=job.push.time,
                defaults={'value': value[0]})

            # by default if there is no summary, we should schedule a
            # generate alerts task for the subtest, since we have new data
            # (this can be over-ridden by the optional "should alert"
            # property)
            if ((signature.should_alert or (signature.should_alert is None and
                                            suite.get('value') is None)) and
                datum_created and job.repository.performance_alerts_enabled):
                generate_alerts.apply_async(args=[signature.id],
                                            queue='generate_perf_alerts')

def store_performance_artifact(job, artifact):
    blob = json.loads(artifact['blob'])
    performance_data = blob['performance_data']

    if isinstance(performance_data, list):
        for perfdatum in performance_data:
            _load_perf_datum(job, perfdatum)
    else:
        _load_perf_datum(job, performance_data)
<commit_msg>Bug 1593196 - Ingest application name for subtests<commit_after>import copy
import logging
from hashlib import sha1

import simplejson as json

from treeherder.log_parser.utils import validate_perf_data
from treeherder.model.models import OptionCollection
from treeherder.perf.models import (PerformanceDatum,
                                    PerformanceFramework,
                                    PerformanceSignature)
from treeherder.perf.tasks import generate_alerts

logger = logging.getLogger(__name__)

def _get_application_name(validated_perf_datum: dict):
    try:
        return validated_perf_datum['application']['name']
    except KeyError:
        return None

def _get_signature_hash(signature_properties):
    signature_prop_values = list(signature_properties.keys())
    str_values = []
    for value in signature_properties.values():
        if not isinstance(value, str):
            str_values.append(json.dumps(value, sort_keys=True))
        else:
            str_values.append(value)
    signature_prop_values.extend(str_values)

    sha = sha1()
    sha.update(''.join(map(str, sorted(signature_prop_values))).encode('utf-8'))

    return sha.hexdigest()

def _create_or_update_signature(repository, signature_hash, framework, defaults):
    signature, created = PerformanceSignature.objects.get_or_create(
        repository=repository,
        signature_hash=signature_hash,
        framework=framework,
        defaults=defaults)
    if not created:
        if signature.last_updated > defaults['last_updated']:
            defaults['last_updated'] = signature.last_updated
        signature, _ = PerformanceSignature.objects.update_or_create(
            repository=repository,
            signature_hash=signature_hash,
            framework=framework,
            defaults=defaults)
    return signature

def _load_perf_datum(job, perf_datum):
    validate_perf_data(perf_datum)

    extra_properties = {}
    extra_options = ''
    reference_data = {
        'option_collection_hash': job.signature.option_collection_hash,
        'machine_platform': job.signature.machine_platform
    }

    option_collection = OptionCollection.objects.get(
        option_collection_hash=job.signature.option_collection_hash)

    try:
        framework = PerformanceFramework.objects.get(
            name=perf_datum['framework']['name'])
    except PerformanceFramework.DoesNotExist:
        logger.warning("Performance framework %s does not exist, skipping "
                       "load of performance artifacts",
                       perf_datum['framework']['name'])
        return
    if not framework.enabled:
        logger.info("Performance framework %s is not enabled, skipping",
                    perf_datum['framework']['name'])
        return
    for suite in perf_datum['suites']:
        suite_extra_properties = copy.copy(extra_properties)
        suite_extra_options = copy.copy(extra_options)
        if suite.get('extraOptions'):
            suite_extra_properties = {
                'test_options': sorted(suite['extraOptions'])
            }
            # store extraOptions list as space separated string
            suite_extra_options = ' '.join(sorted(suite['extraOptions']))
        summary_signature_hash = None

        # if we have a summary value, create or get its signature by all its subtest
        # properties.
        if suite.get('value') is not None:
            # summary series
            summary_properties = {
                'suite': suite['name']
            }
            summary_properties.update(reference_data)
            summary_properties.update(suite_extra_properties)
            summary_signature_hash = _get_signature_hash(
                summary_properties)
            signature = _create_or_update_signature(
                job.repository, summary_signature_hash, framework, {
                    'test': '',
                    'suite': suite['name'],
                    'option_collection': option_collection,
                    'platform': job.machine_platform,
                    'extra_options': suite_extra_options,
                    'measurement_unit': suite.get('unit'),
                    'application': _get_application_name(perf_datum),
                    'lower_is_better': suite.get('lowerIsBetter', True),
                    'has_subtests': True,
                    # these properties below can be either True, False, or null
                    # (None). Null indicates no preference has been set.
                    'should_alert': suite.get('shouldAlert'),
                    'alert_change_type': PerformanceSignature._get_alert_change_type(
                        suite.get('alertChangeType')),
                    'alert_threshold': suite.get('alertThreshold'),
                    'min_back_window': suite.get('minBackWindow'),
                    'max_back_window': suite.get('maxBackWindow'),
                    'fore_window': suite.get('foreWindow'),
                    'last_updated': job.push.time
                })
            (_, datum_created) = PerformanceDatum.objects.get_or_create(
                repository=job.repository,
                job=job,
                push=job.push,
                signature=signature,
                push_timestamp=job.push.time,
                defaults={'value': suite['value']})
            if signature.should_alert is not False and datum_created and \
               job.repository.performance_alerts_enabled:
                generate_alerts.apply_async(args=[signature.id],
                                            queue='generate_perf_alerts')

        for subtest in suite['subtests']:
            subtest_properties = {
                'suite': suite['name'],
                'test': subtest['name']
            }
            subtest_properties.update(reference_data)
            subtest_properties.update(suite_extra_properties)

            summary_signature = None
            if summary_signature_hash is not None:
                subtest_properties.update({'parent_signature': summary_signature_hash})
                summary_signature = PerformanceSignature.objects.get(
                    repository=job.repository,
                    framework=framework,
                    signature_hash=summary_signature_hash)
            subtest_signature_hash = _get_signature_hash(subtest_properties)
            value = list(subtest['value'] for subtest in suite['subtests'] if
                         subtest['name'] == subtest_properties['test'])
            signature = _create_or_update_signature(
                job.repository, subtest_signature_hash, framework, {
                    'test': subtest_properties['test'],
                    'suite': suite['name'],
                    'option_collection': option_collection,
                    'platform': job.machine_platform,
                    'extra_options': suite_extra_options,
                    'measurement_unit': subtest.get('unit'),
                    'application': _get_application_name(perf_datum),
                    'lower_is_better': subtest.get('lowerIsBetter', True),
                    'has_subtests': False,
                    # these properties below can be either True, False, or
                    # null (None). Null indicates no preference has been
                    # set.
                    'should_alert': subtest.get('shouldAlert'),
                    'alert_change_type': PerformanceSignature._get_alert_change_type(
                        subtest.get('alertChangeType')),
                    'alert_threshold': subtest.get('alertThreshold'),
                    'min_back_window': subtest.get('minBackWindow'),
                    'max_back_window': subtest.get('maxBackWindow'),
                    'fore_window': subtest.get('foreWindow'),
                    'parent_signature': summary_signature,
                    'last_updated': job.push.time
                })
            (_, datum_created) = PerformanceDatum.objects.get_or_create(
                repository=job.repository,
                job=job,
                push=job.push,
                signature=signature,
                push_timestamp=job.push.time,
                defaults={'value': value[0]})

            # by default if there is no summary, we should schedule a
            # generate alerts task for the subtest, since we have new data
            # (this can be over-ridden by the optional "should alert"
            # property)
            if ((signature.should_alert or (signature.should_alert is None and
                                            suite.get('value') is None)) and
                datum_created and job.repository.performance_alerts_enabled):
                generate_alerts.apply_async(args=[signature.id],
                                            queue='generate_perf_alerts')

def store_performance_artifact(job, artifact):
    blob = json.loads(artifact['blob'])
    performance_data = blob['performance_data']

    if isinstance(performance_data, list):
        for perfdatum in performance_data:
            _load_perf_datum(job, perfdatum)
    else:
        _load_perf_datum(job, performance_data)
<|endoftext|>


<commit_before>from django.shortcuts import get_object_or_404
from annoying.decorators import render_to
from wiki.forms import WikiNewForm, WikiEditForm
from wiki.models import WikiPage

@render_to('wiki/index.html')
def wiki_index(request):
	""" Returns Index wiki page """
	page, created = WikiPage.objects.get_or_create(slug='Index',
		defaults={'title': u'Index'}
	)
	return {
		'page': page,
	}

@render_to('wiki/detail.html')
def wiki_detail(request, slug):
	page = get_object_or_404(WikiPage, slug=slug)
	return {
		'page': page,
	}

@render_to('wiki/new.html')
def wiki_new(request):
	form = WikiNewForm()
	if request.method == 'POST':
		form = WikiNewForm(request.POST)
		if form.is_valid():
			new_page = form.save()
			messages.success(request, "New page has been added to the wiki.")
			return redirect(new_page.get_absolute_url())
	return {
		'form': form,
	}

@render_to('wiki/edit.html')
def wiki_edit(request, slug):
	page = get_object_or_404(WikiPage, slug=slug)
	form = WikiEditForm(instance=page)
	if request.method == 'POST':
		form = WikiEditForm(request.POST, instance=page)
		if form.is_valid():
			form.save()
			messages.success(request,
				"Successfully updated \"{0}\" page.".format(page)
			)
			return redirect(page.get_absolute_url())
	return {
		'page': page,
		'form': form,
	}

@render_to('wiki/delete.html')
def wiki_delete(request, slug):
	page = get_object_or_404(WikiPage, slug=slug)
	if request.method == 'POST':
		page.delete()
		messages.success(request,
			"Successfully removed \"{0}\" page.".format(page)
		)
		return redirect(reverse('wiki_index'))
	return {
		'page': page,
	}

def wiki_history(request, slug):
	raise NotImplementedError

def wiki_history_detail(request, slug, rev):
	raise NotImplementedError

def wiki_compare(request, slug, rev_from, rev_to):
	raise NotImplementedError
<commit_msg>changed, so wiki_index will render to wiki/detail.html<commit_after>from django.shortcuts import get_object_or_404
from annoying.decorators import render_to
from wiki.forms import WikiNewForm, WikiEditForm
from wiki.models import WikiPage

@render_to('wiki/detail.html')
def wiki_index(request):
	""" Returns Index wiki page """
	page, created = WikiPage.objects.get_or_create(slug='Index',
		defaults={'title': u'Index'}
	)
	return {
		'page': page,
	}

@render_to('wiki/detail.html')
def wiki_detail(request, slug):
	page = get_object_or_404(WikiPage, slug=slug)
	return {
		'page': page,
	}

@render_to('wiki/new.html')
def wiki_new(request):
	form = WikiNewForm()
	if request.method == 'POST':
		form = WikiNewForm(request.POST)
		if form.is_valid():
			new_page = form.save()
			messages.success(request, "New page has been added to the wiki.")
			return redirect(new_page.get_absolute_url())
	return {
		'form': form,
	}

@render_to('wiki/edit.html')
def wiki_edit(request, slug):
	page = get_object_or_404(WikiPage, slug=slug)
	form = WikiEditForm(instance=page)
	if request.method == 'POST':
		form = WikiEditForm(request.POST, instance=page)
		if form.is_valid():
			form.save()
			messages.success(request,
				"Successfully updated \"{0}\" page.".format(page)
			)
			return redirect(page.get_absolute_url())
	return {
		'page': page,
		'form': form,
	}

@render_to('wiki/delete.html')
def wiki_delete(request, slug):
	page = get_object_or_404(WikiPage, slug=slug)
	if request.method == 'POST':
		page.delete()
		messages.success(request,
			"Successfully removed \"{0}\" page.".format(page)
		)
		return redirect(reverse('wiki_index'))
	return {
		'page': page,
	}

def wiki_history(request, slug):
	raise NotImplementedError

def wiki_history_detail(request, slug, rev):
	raise NotImplementedError

def wiki_compare(request, slug, rev_from, rev_to):
	raise NotImplementedError
<|endoftext|>


<commit_before>__all__ = ['coroutine',
           'iscoroutinefunction', 'iscoroutine']

import functools
import inspect
import opcode
import os
import sys
import traceback
import types

from . import compat
from . import events
from . import futures
from .log import logger

# Opcode of "yield from" instruction
_YIELD_FROM = opcode.opmap.get('YIELD_FROM', None)

# If you set _DEBUG to true, @coroutine will wrap the resulting
# generator objects in a CoroWrapper instance (defined below).  That
# instance will log a message when the generator is never iterated
# over, which may happen when you forget to use "yield" with a
# coroutine call.  Note that the value of the _DEBUG flag is taken
# when the decorator is used, so to be of any use it must be set
# before you define your coroutines.  A downside of using this feature
# is that tracebacks show entries for the CoroWrapper.__next__ method
# when _DEBUG is true.
_DEBUG = bool(os.environ.get('TROLLIUSDEBUG'))

if _YIELD_FROM is not None:
    # Check for CPython issue #21209
    exec('''if 1:
        def has_yield_from_bug():
            class MyGen:
                def __init__(self):
                    self.send_args = None
                def __iter__(self):
                    return self
                def __next__(self):
                    return 42
                def send(self, *what):
                    self.send_args = what
                    return None
            def yield_from_gen(gen):
                yield from gen
            value = (1, 2, 3)
            gen = MyGen()
            coro = yield_from_gen(gen)
            next(coro)
            coro.send(value)
            return gen.send_args != (value,)
''')
    _YIELD_FROM_BUG = has_yield_from_bug()
    del has_yield_from_bug
else:
    _YIELD_FROM_BUG = False

if compat.PY33:
    # Don't use the Return class on Python 3.3 and later to support asyncio
    # coroutines (to avoid the warning emited in Return destructor).
    #
    # The problem is that Return inherits from StopIteration.  "yield from
    # trollius_coroutine". Task._step() does not receive the Return exception,
    # because "yield from" handles it internally. So it's not possible to set
    # the raised attribute to True to avoid the warning in Return destructor.
    def Return(*args):
        if not args:
            value = None
        elif len(args) == 1:
            value = args[0]
        else:
            value = args
        return StopIteration(value)
else:
    class Return(StopIteration):
        def __init__(self, *args):
            StopIteration.__init__(self)
            if not args:
                self.value = None
            elif len(args) == 1:
                self.value = args[0]
            else:
                self.value = args
            self.raised = False
            if _DEBUG:
                frame = sys._getframe(1)
                self._source_traceback = traceback.extract_stack(frame)
                # explicitly clear the reference to avoid reference cycles
                frame = None
            else:
                self._source_traceback = None

        def __del__(self):
            if self.raised:
                return

            fmt = 'Return(%r) used without raise'
            if self._source_traceback:
                fmt += '\nReturn created at (most recent call last):\n'
                tb = ''.join(traceback.format_list(self._source_traceback))
                fmt += tb.rstrip()
            logger.error(fmt, self.value)

class CoroWrapper(object):
    # Wrapper for coroutine object in _DEBUG mode.

    def __init__(self, gen, func):
        assert inspect.isgenerator(gen), gen
        self.gen = gen
        self.func = func
        self._source_traceback = traceback.extract_stack(sys._getframe(1))
        # __name__, __qualname__, __doc__ attributes are set by the coroutine()
        # decorator

    def __repr__(self):
        coro_repr = _format_coroutine(self)
        if self._source_traceback:
            frame = self._source_traceback[-1]
            coro_repr += ', created at %s:%s' % (frame[0], frame[1])
        return '<%s %s>' % (self.__class__.__name__, coro_repr)

    def __iter__(self):
        return self

    def __next__(self):
        return next(self.gen)
    next = __next__

    if _YIELD_FROM_BUG:
        # For for CPython issue #21209: using "yield from" and a custom
        # generator, generator.send(tuple) unpacks the tuple instead of passing
        # the tuple unchanged. Check if the caller is a generator using "yield
        # from" to decide if the parameter should be unpacked or not.
        def send(self, *value):
            frame = sys._getframe()
            caller = frame.f_back
            assert caller.f_lasti >= 0
            if caller.f_code.co_code[caller.f_lasti] != _YIELD_FROM:
                value = value[0]
            return self.gen.send(value)
    else:
        def send(self, value):
            return self.gen.send(value)

    def throw(self, exc):
        return self.gen.throw(exc)

    def close(self):
        return self.gen.close()

    @property
    def gi_frame(self):
        return self.gen.gi_frame

    @property
    def gi_running(self):
        return self.gen.gi_running

    @property
    def gi_code(self):
        return self.gen.gi_code

    def __del__(self):
        # Be careful accessing self.gen.frame -- self.gen might not exist.
        gen = getattr(self, 'gen', None)
        frame = getattr(gen, 'gi_frame', None)
        if frame is not None and frame.f_lasti == -1:
            msg = '%r was never yielded from' % self
            tb = getattr(self, '_source_traceback', ())
            if tb:
                tb = ''.join(traceback.format_list(tb))
                msg += ('\nCoroutine object created at '
                        '(most recent call last):\n')
                msg += tb.rstrip()
            logger.error(msg)

if not compat.PY34:
    # Backport functools.update_wrapper() from Python 3.4:
    # - Python 2.7 fails if assigned attributes don't exist
    # - Python 2.7 and 3.1 don't set the __wrapped__ attribute
    # - Python 3.2 and 3.3 set __wrapped__ before updating __dict__
    def _update_wrapper(wrapper,
                       wrapped,
                       assigned = functools.WRAPPER_ASSIGNMENTS,
                       updated = functools.WRAPPER_UPDATES):
        """Update a wrapper function to look like the wrapped function

           wrapper is the function to be updated
           wrapped is the original function
           assigned is a tuple naming the attributes assigned directly
           from the wrapped function to the wrapper function (defaults to
           functools.WRAPPER_ASSIGNMENTS)
           updated is a tuple naming the attributes of the wrapper that
           are updated with the corresponding attribute from the wrapped
           function (defaults to functools.WRAPPER_UPDATES)
        """
        for attr in assigned:
            try:
                value = getattr(wrapped, attr)
            except AttributeError:
                pass
            else:
                setattr(wrapper, attr, value)
        for attr in updated:
            getattr(wrapper, attr).update(getattr(wrapped, attr, {}))
        # Issue #17482: set __wrapped__ last so we don't inadvertently copy it
        # from the wrapped function when updating __dict__
        wrapper.__wrapped__ = wrapped
        # Return the wrapper so this can be used as a decorator via partial()
        return wrapper

    def _wraps(wrapped,
              assigned = functools.WRAPPER_ASSIGNMENTS,
              updated = functools.WRAPPER_UPDATES):
        """Decorator factory to apply update_wrapper() to a wrapper function

           Returns a decorator that invokes update_wrapper() with the decorated
           function as the wrapper argument and the arguments to wraps() as the
           remaining arguments. Default arguments are as for update_wrapper().
           This is a convenience function to simplify applying partial() to
           update_wrapper().
        """
        return functools.partial(_update_wrapper, wrapped=wrapped,
                                 assigned=assigned, updated=updated)
else:
    _wraps = functools.wraps

def coroutine(func):
    """Decorator to mark coroutines.

    If the coroutine is not yielded from before it is destroyed,
    an error message is logged.
    """
    if inspect.isgeneratorfunction(func):
        coro = func
    else:
        @_wraps(func)
        def coro(*args, **kw):
            res = func(*args, **kw)
            if (isinstance(res, futures._FUTURE_CLASSES)
            or inspect.isgenerator(res)):
                res = yield From(res)
            raise Return(res)

    if not _DEBUG:
        wrapper = coro
    else:
        @_wraps(func)
        def wrapper(*args, **kwds):
            coro_wrapper = CoroWrapper(coro(*args, **kwds), func)
            if coro_wrapper._source_traceback:
                del coro_wrapper._source_traceback[-1]
            for attr in ('__name__', '__qualname__', '__doc__'):
                try:
                    value = getattr(func, attr)
                except AttributeError:
                    pass
                else:
                    setattr(coro_wrapper, attr, value)
            return coro_wrapper
        if not compat.PY3:
            wrapper.__wrapped__ = func

    wrapper._is_coroutine = True  # For iscoroutinefunction().
    return wrapper

def iscoroutinefunction(func):
    """Return True if func is a decorated coroutine function."""
    return getattr(func, '_is_coroutine', False)

_COROUTINE_TYPES = (types.GeneratorType, CoroWrapper)
if events.asyncio is not None:
    # Accept also asyncio CoroWrapper for interoperability
    if hasattr(events.asyncio, 'coroutines'):
        _COROUTINE_TYPES += (events.asyncio.coroutines.CoroWrapper,)
    else:
        # old Tulip/Python versions
        _COROUTINE_TYPES += (events.asyncio.tasks.CoroWrapper,)

def iscoroutine(obj):
    """Return True if obj is a coroutine object."""
    return isinstance(obj, _COROUTINE_TYPES)

def _format_coroutine(coro):
    assert iscoroutine(coro)
    coro_name = getattr(coro, '__qualname__', coro.__name__)

    filename = coro.gi_code.co_filename
    if (isinstance(coro, CoroWrapper)
    and not inspect.isgeneratorfunction(coro.func)):
        filename, lineno = events._get_function_source(coro.func)
        if coro.gi_frame is None:
            coro_repr = ('%s() done, defined at %s:%s'
                         % (coro_name, filename, lineno))
        else:
            coro_repr = ('%s() running, defined at %s:%s'
                         % (coro_name, filename, lineno))
    elif coro.gi_frame is not None:
        lineno = coro.gi_frame.f_lineno
        coro_repr = ('%s() running at %s:%s'
                     % (coro_name, filename, lineno))
    else:
        lineno = coro.gi_code.co_firstlineno
        coro_repr = ('%s() done, defined at %s:%s'
                     % (coro_name, filename, lineno))

    return coro_repr

class FromWrapper(object):
    __slots__ = ('obj',)

    def __init__(self, obj):
        if isinstance(obj, FromWrapper):
            obj = obj.obj
            assert not isinstance(obj, FromWrapper)
        self.obj = obj

def From(obj):
    if not _DEBUG:
        return obj
    else:
        return FromWrapper(obj)
<commit_msg>Backout f2ba35e96d46: "Remove unused private function"<commit_after>__all__ = ['coroutine',
           'iscoroutinefunction', 'iscoroutine']

import functools
import inspect
import opcode
import os
import sys
import traceback
import types

from . import compat
from . import events
from . import futures
from .log import logger

# Opcode of "yield from" instruction
_YIELD_FROM = opcode.opmap.get('YIELD_FROM', None)

# If you set _DEBUG to true, @coroutine will wrap the resulting
# generator objects in a CoroWrapper instance (defined below).  That
# instance will log a message when the generator is never iterated
# over, which may happen when you forget to use "yield" with a
# coroutine call.  Note that the value of the _DEBUG flag is taken
# when the decorator is used, so to be of any use it must be set
# before you define your coroutines.  A downside of using this feature
# is that tracebacks show entries for the CoroWrapper.__next__ method
# when _DEBUG is true.
_DEBUG = bool(os.environ.get('TROLLIUSDEBUG'))

if _YIELD_FROM is not None:
    # Check for CPython issue #21209
    exec('''if 1:
        def has_yield_from_bug():
            class MyGen:
                def __init__(self):
                    self.send_args = None
                def __iter__(self):
                    return self
                def __next__(self):
                    return 42
                def send(self, *what):
                    self.send_args = what
                    return None
            def yield_from_gen(gen):
                yield from gen
            value = (1, 2, 3)
            gen = MyGen()
            coro = yield_from_gen(gen)
            next(coro)
            coro.send(value)
            return gen.send_args != (value,)
''')
    _YIELD_FROM_BUG = has_yield_from_bug()
    del has_yield_from_bug
else:
    _YIELD_FROM_BUG = False

if compat.PY33:
    # Don't use the Return class on Python 3.3 and later to support asyncio
    # coroutines (to avoid the warning emited in Return destructor).
    #
    # The problem is that Return inherits from StopIteration.  "yield from
    # trollius_coroutine". Task._step() does not receive the Return exception,
    # because "yield from" handles it internally. So it's not possible to set
    # the raised attribute to True to avoid the warning in Return destructor.
    def Return(*args):
        if not args:
            value = None
        elif len(args) == 1:
            value = args[0]
        else:
            value = args
        return StopIteration(value)
else:
    class Return(StopIteration):
        def __init__(self, *args):
            StopIteration.__init__(self)
            if not args:
                self.value = None
            elif len(args) == 1:
                self.value = args[0]
            else:
                self.value = args
            self.raised = False
            if _DEBUG:
                frame = sys._getframe(1)
                self._source_traceback = traceback.extract_stack(frame)
                # explicitly clear the reference to avoid reference cycles
                frame = None
            else:
                self._source_traceback = None

        def __del__(self):
            if self.raised:
                return

            fmt = 'Return(%r) used without raise'
            if self._source_traceback:
                fmt += '\nReturn created at (most recent call last):\n'
                tb = ''.join(traceback.format_list(self._source_traceback))
                fmt += tb.rstrip()
            logger.error(fmt, self.value)

def _coroutine_at_yield_from(coro):
    """Test if the last instruction of a coroutine is "yield from".

    Return False if the coroutine completed.
    """
    frame = coro.gi_frame
    if frame is None:
        return False
    code = coro.gi_code
    assert frame.f_lasti >= 0
    offset = frame.f_lasti + 1
    instr = code.co_code[offset]
    return (instr == _YIELD_FROM)

class CoroWrapper(object):
    # Wrapper for coroutine object in _DEBUG mode.

    def __init__(self, gen, func):
        assert inspect.isgenerator(gen), gen
        self.gen = gen
        self.func = func
        self._source_traceback = traceback.extract_stack(sys._getframe(1))
        # __name__, __qualname__, __doc__ attributes are set by the coroutine()
        # decorator

    def __repr__(self):
        coro_repr = _format_coroutine(self)
        if self._source_traceback:
            frame = self._source_traceback[-1]
            coro_repr += ', created at %s:%s' % (frame[0], frame[1])
        return '<%s %s>' % (self.__class__.__name__, coro_repr)

    def __iter__(self):
        return self

    def __next__(self):
        return next(self.gen)
    next = __next__

    if _YIELD_FROM_BUG:
        # For for CPython issue #21209: using "yield from" and a custom
        # generator, generator.send(tuple) unpacks the tuple instead of passing
        # the tuple unchanged. Check if the caller is a generator using "yield
        # from" to decide if the parameter should be unpacked or not.
        def send(self, *value):
            frame = sys._getframe()
            caller = frame.f_back
            assert caller.f_lasti >= 0
            if caller.f_code.co_code[caller.f_lasti] != _YIELD_FROM:
                value = value[0]
            return self.gen.send(value)
    else:
        def send(self, value):
            return self.gen.send(value)

    def throw(self, exc):
        return self.gen.throw(exc)

    def close(self):
        return self.gen.close()

    @property
    def gi_frame(self):
        return self.gen.gi_frame

    @property
    def gi_running(self):
        return self.gen.gi_running

    @property
    def gi_code(self):
        return self.gen.gi_code

    def __del__(self):
        # Be careful accessing self.gen.frame -- self.gen might not exist.
        gen = getattr(self, 'gen', None)
        frame = getattr(gen, 'gi_frame', None)
        if frame is not None and frame.f_lasti == -1:
            msg = '%r was never yielded from' % self
            tb = getattr(self, '_source_traceback', ())
            if tb:
                tb = ''.join(traceback.format_list(tb))
                msg += ('\nCoroutine object created at '
                        '(most recent call last):\n')
                msg += tb.rstrip()
            logger.error(msg)

if not compat.PY34:
    # Backport functools.update_wrapper() from Python 3.4:
    # - Python 2.7 fails if assigned attributes don't exist
    # - Python 2.7 and 3.1 don't set the __wrapped__ attribute
    # - Python 3.2 and 3.3 set __wrapped__ before updating __dict__
    def _update_wrapper(wrapper,
                       wrapped,
                       assigned = functools.WRAPPER_ASSIGNMENTS,
                       updated = functools.WRAPPER_UPDATES):
        """Update a wrapper function to look like the wrapped function

           wrapper is the function to be updated
           wrapped is the original function
           assigned is a tuple naming the attributes assigned directly
           from the wrapped function to the wrapper function (defaults to
           functools.WRAPPER_ASSIGNMENTS)
           updated is a tuple naming the attributes of the wrapper that
           are updated with the corresponding attribute from the wrapped
           function (defaults to functools.WRAPPER_UPDATES)
        """
        for attr in assigned:
            try:
                value = getattr(wrapped, attr)
            except AttributeError:
                pass
            else:
                setattr(wrapper, attr, value)
        for attr in updated:
            getattr(wrapper, attr).update(getattr(wrapped, attr, {}))
        # Issue #17482: set __wrapped__ last so we don't inadvertently copy it
        # from the wrapped function when updating __dict__
        wrapper.__wrapped__ = wrapped
        # Return the wrapper so this can be used as a decorator via partial()
        return wrapper

    def _wraps(wrapped,
              assigned = functools.WRAPPER_ASSIGNMENTS,
              updated = functools.WRAPPER_UPDATES):
        """Decorator factory to apply update_wrapper() to a wrapper function

           Returns a decorator that invokes update_wrapper() with the decorated
           function as the wrapper argument and the arguments to wraps() as the
           remaining arguments. Default arguments are as for update_wrapper().
           This is a convenience function to simplify applying partial() to
           update_wrapper().
        """
        return functools.partial(_update_wrapper, wrapped=wrapped,
                                 assigned=assigned, updated=updated)
else:
    _wraps = functools.wraps

def coroutine(func):
    """Decorator to mark coroutines.

    If the coroutine is not yielded from before it is destroyed,
    an error message is logged.
    """
    if inspect.isgeneratorfunction(func):
        coro = func
    else:
        @_wraps(func)
        def coro(*args, **kw):
            res = func(*args, **kw)
            if (isinstance(res, futures._FUTURE_CLASSES)
            or inspect.isgenerator(res)):
                res = yield From(res)
            raise Return(res)

    if not _DEBUG:
        wrapper = coro
    else:
        @_wraps(func)
        def wrapper(*args, **kwds):
            coro_wrapper = CoroWrapper(coro(*args, **kwds), func)
            if coro_wrapper._source_traceback:
                del coro_wrapper._source_traceback[-1]
            for attr in ('__name__', '__qualname__', '__doc__'):
                try:
                    value = getattr(func, attr)
                except AttributeError:
                    pass
                else:
                    setattr(coro_wrapper, attr, value)
            return coro_wrapper
        if not compat.PY3:
            wrapper.__wrapped__ = func

    wrapper._is_coroutine = True  # For iscoroutinefunction().
    return wrapper

def iscoroutinefunction(func):
    """Return True if func is a decorated coroutine function."""
    return getattr(func, '_is_coroutine', False)

_COROUTINE_TYPES = (types.GeneratorType, CoroWrapper)
if events.asyncio is not None:
    # Accept also asyncio CoroWrapper for interoperability
    if hasattr(events.asyncio, 'coroutines'):
        _COROUTINE_TYPES += (events.asyncio.coroutines.CoroWrapper,)
    else:
        # old Tulip/Python versions
        _COROUTINE_TYPES += (events.asyncio.tasks.CoroWrapper,)

def iscoroutine(obj):
    """Return True if obj is a coroutine object."""
    return isinstance(obj, _COROUTINE_TYPES)

def _format_coroutine(coro):
    assert iscoroutine(coro)
    coro_name = getattr(coro, '__qualname__', coro.__name__)

    filename = coro.gi_code.co_filename
    if (isinstance(coro, CoroWrapper)
    and not inspect.isgeneratorfunction(coro.func)):
        filename, lineno = events._get_function_source(coro.func)
        if coro.gi_frame is None:
            coro_repr = ('%s() done, defined at %s:%s'
                         % (coro_name, filename, lineno))
        else:
            coro_repr = ('%s() running, defined at %s:%s'
                         % (coro_name, filename, lineno))
    elif coro.gi_frame is not None:
        lineno = coro.gi_frame.f_lineno
        coro_repr = ('%s() running at %s:%s'
                     % (coro_name, filename, lineno))
    else:
        lineno = coro.gi_code.co_firstlineno
        coro_repr = ('%s() done, defined at %s:%s'
                     % (coro_name, filename, lineno))

    return coro_repr

class FromWrapper(object):
    __slots__ = ('obj',)

    def __init__(self, obj):
        if isinstance(obj, FromWrapper):
            obj = obj.obj
            assert not isinstance(obj, FromWrapper)
        self.obj = obj

def From(obj):
    if not _DEBUG:
        return obj
    else:
        return FromWrapper(obj)
<|endoftext|>


<commit_before>from django.db import models

class VisibilityManagerMixin(object):
    """
    This manager should be used with a model that implements the Hideable
    mixin.
    """

    def __init__(self, *args, **kwargs):
        self.visible = kwargs.pop('visible', True)
        super().__init__(*args, **kwargs)

    def get_queryset(self):
        return super().get_queryset()
            .filter(hidden__isnull=self.visible)

class VisibilityManager(VisibilityManagerMixin, models.Manager):
    pass
<commit_msg>Added __all__ to managers.<commit_after>from django.db import models

__all__ = ['VisibilityManagerMixin', 'VisibilityManager']

class VisibilityManagerMixin(object):
    """
    This manager should be used with a model that implements the Hideable
    mixin.
    """

    def __init__(self, *args, **kwargs):
        self.visible = kwargs.pop('visible', True)
        super().__init__(*args, **kwargs)

    def get_queryset(self):
        return super().get_queryset()
            .filter(hidden__isnull=self.visible)

class VisibilityManager(VisibilityManagerMixin, models.Manager):
    pass
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
import pymongo
from Sneaker_Scraper.MongoDB.config import MONGODB_URI, MONGODB_DB

class NikePipeline(object):
    def process_item(self, item, spider):
        return item

#Pipeline for storing to MongoDB
class MongoPipeline(object):

    collection_name = 'Sneakers'

    def __init__(self, mongo_uri, mongo_db):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            mongo_uri=MONGODB_URI,
            mongo_db=MONGODB_DB
        )

    def open_spider(self, spider):
        self.client = pymongo.MongoClient(self.mongo_uri)
        self.db = self.client[self.mongo_db]

    def close_spider(self, spider):
        self.client.close()

    def process_item(self, item, spider):
        self.db[self.collection_name].insert(dict(item)) #store item to collection
        return item
<commit_msg>changed MongoDB collection storage to be organized by item's url source<commit_after># -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
import pymongo
from Sneaker_Scraper.MongoDB.config import MONGODB_URI, MONGODB_DB

class NikePipeline(object):
    def process_item(self, item, spider):
        return item

#Pipeline for storing to MongoDB
class MongoPipeline(object):

    def __init__(self, mongo_uri, mongo_db):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            mongo_uri=MONGODB_URI,
            mongo_db=MONGODB_DB
        )

    def open_spider(self, spider):
        self.client = pymongo.MongoClient(self.mongo_uri)
        self.db = self.client[self.mongo_db]

    def close_spider(self, spider):
        self.client.close()

    def process_item(self, item, spider):
        self.db[item['urlSource']].insert(dict(item)) #store item to collection by source
        return item
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import datetime
import unittest

from scrapy.http import HtmlResponse

from manoseimas.scrapy.items import Lobbyist
from manoseimas.scrapy.loaders import Loader
from manoseimas.scrapy.spiders.lobbyists import LobbyistsSpider
from manoseimas.scrapy.tests.utils import fixture

class TestLobbyistsSpider(unittest.TestCase):

    number_examples = [
        ('1.	', {}),
        ('35	', {}),
    ]

    name_examples = [
        ('BRONIUS ANTANAS RASIMAS	', {
            'name': 'Bronius Antanas Rasimas',
        }),
        ('MYKOLAS JUOZAPAVIČIUS	', {
            'name': u'Mykolas Juozapavičius',
            'url': 'http://www.lamco.lt/',
        }),
        ('NINA BARBORA EVANS 2012 02 21 sprendimu Nr. KS-16 (L) sustabdyta lobistinė veikla iki prašymo atnaujinti.	', {
            'name': 'Nina Barbora Evans',
            'status': u'2012 02 21 sprendimu Nr. KS-16 (L) sustabdyta lobistinė veikla iki prašymo atnaujinti.',
        }),
        ('MINDAUGAS VOLDEMARAS
 2015 03 04 sprendimu Nr. KS-20 (L) sustabdyta lobistinė veikla iki prašymo atnaujinti.	', {
            'name': 'Mindaugas Voldemaras',
            'status': u'2015 03 04 sprendimu Nr. KS-20 (L) sustabdyta lobistinė veikla iki prašymo atnaujinti.',
        }),
        ('
UAB "ERNST & YOUNG BALTIC"

Gintautas Bartkus, Jonas Platelis
	', {
            'name': 'UAB "ERNST & YOUNG BALTIC"',
            'representatives': 'Gintautas Bartkus, Jonas Platelis',
            'url': 'http://www.ey.com/GLOBAL/content.nsf/Lithuania_E/Lithuania_Home',
        }),
        ('
ADVOKATŲ PROFESINĖ BENDRIJA "BALTIC LEGAL SOLUTIONS LIETUVA"

Gytis Kaminskas, Gintautas Bartkus
	', {
            'name': u'ADVOKATŲ PROFESINĖ BENDRIJA "BALTIC LEGAL SOLUTIONS LIETUVA"',
            'representatives': 'Gytis Kaminskas, Gintautas Bartkus',
            'url': 'http://www.blslawfirm.com/bls-lithuania/home',
        }),
        ('
UAB "INLINEN"

Linas Pečiulaitis
	', {
            'name': 'UAB "INLINEN"',
            'representatives': u'Linas Pečiulaitis',
            'url': 'http://www.inlinen.eu/',
        }),
        ('UAB "VOX JURIS"
2015-04-15 sprendimu Nr. KS-31 (L) lobistinė veikla nutraukta.	', {
            'name': 'UAB "VOX JURIS"',
            'url': 'http://www.juris.lt/',
            'status': u'2015-04-15 sprendimu Nr. KS-31 (L) lobistinė veikla nutraukta.',
        }),
        ('
VŠĮ "MOKESČIŲ IR VERSLO PROCESŲ ADMINISTRAVIMO CENTRAS"

Sandra Šarkauskaitė
	', {
            'name': u'VŠĮ "MOKESČIŲ IR VERSLO PROCESŲ ADMINISTRAVIMO CENTRAS"',
            'representatives': u'Sandra Šarkauskaitė',
        }),
    ]

    company_code_examples = [
        ('123456789	', {'company_code': '123456789'}),
        (' 	', {'company_code': ''}),
    ]

    inclusion_examples = [
        ('2015 05 20 Nr. KS-36 (L)	', {
            'date_of_inclusion': datetime.date(2015, 05, 20),
            'decision': 'Nr. KS-36 (L)',
        }),
        ('2012 10 02 Nr. KS - 108 (L)	', {
            'date_of_inclusion': datetime.date(2012, 10, 02),
            'decision': 'Nr. KS - 108 (L)',
        }),
        ('2012 04 17 Nr. KS -22 (L)	', {
            'date_of_inclusion': datetime.date(2012, 04, 17),
            'decision': 'Nr. KS -22 (L)',
        }),
        ('2011 12 06 Nr. KS - 75 (L)	', {
            'date_of_inclusion': datetime.date(2011, 12, 06),
            'decision': 'Nr. KS - 75 (L)',
        }),
        ('2002 10 04 Nr. 23 (L)	', {
            'date_of_inclusion': datetime.date(2002, 10, 04),
            'decision': 'Nr. 23 (L)',
        }),
    ]

    lobbyist_examples = [
        ('''

            13.

UAB “GLAXOSMITHKLINE LIETUVA”

Kęstutis Čereška, Armindas Varkala, Arnas Beržanskis, Vilius Kirvaitis

            111785261
            2007 12 20 Nr. KS - 88 (L)

        ''', {
            'name': u'UAB “GLAXOSMITHKLINE LIETUVA”',
            'url': 'http://www.gsk.lt/',
            'representatives': u'Kęstutis Čereška, Armindas Varkala, Arnas Beržanskis, Vilius Kirvaitis',
            'company_code': '111785261',
            'date_of_inclusion': datetime.date(2007, 12, 20),
            'decision': 'Nr. KS - 88 (L)',
            'source_url': 'http://localhost/test.html',
            'raw_data': u'''

                13.

UAB “GLAXOSMITHKLINE LIETUVA”

Kęstutis Čereška, Armindas Varkala, Arnas Beržanskis, Vilius Kirvaitis

                111785261
                2007 12 20 Nr. KS - 88 (L)

            '''.strip().replace('\n    ', '\n'),
        }),
    ]

    maxDiff = None

    def setUp(self):
        self.spider = LobbyistsSpider()

    def run_examples(self, examples, wrapper, fn):
        for html, expected in examples:
            actual = wrapper(html, fn)
            if 'raw_data' in expected and 'raw_data' in actual:
                # let's see a useful diff of these
                self.assertEqual(expected['raw_data'], actual['raw_data'])
            self.assertEqual(expected, actual)

    def parse_field(self, html, fn):
        response = HtmlResponse('http://localhost/test.html',
                                body='

%s

' % html)
        row = response.css('tr')[0]
        node = response.css('td')[0]
        lobbyist = Loader(self.spider, response, Lobbyist(), row)
        lobbyist.add_value(None, fn(node))
        item = lobbyist.load_item()
        actual = dict(item)
        return actual

    def parse_row(self, html, fn):
        response = HtmlResponse('http://localhost/test.html',
                                body='
%s
' % html)
        row = response.css('tr')[0]
        item = fn(response, row)
        actual = dict(item)
        return actual

    def test_parse_number(self):
        self.run_examples(self.number_examples, self.parse_field,
                          self.spider._parse_number)

    def test_parse_name(self):
        self.run_examples(self.name_examples, self.parse_field,
                          self.spider._parse_name)

    def test_parse_company_code(self):
        self.run_examples(self.company_code_examples, self.parse_field,
                          self.spider._parse_company_code)

    def test_parse_inclusion(self):
        self.run_examples(self.inclusion_examples, self.parse_field,
                          self.spider._parse_inclusion)

    def test_parse_lobbyist(self):
        self.run_examples(self.lobbyist_examples, self.parse_row,
                          self.spider._parse_lobbyist)

    def test_maybe_titlecase_fixes_person_names(self):
        self.assertEqual(self.spider.maybe_titlecase(u'ALGIRDAS JUŠKYS'), u'Algirdas Juškys')
        self.assertEqual(self.spider.maybe_titlecase(u'BRONIUS ANTANAS RASIMAS'), u'Bronius Antanas Rasimas')

    def test_maybe_titlecase_leaves_company_names_alone(self):
        for name in ['UAB "VOX JURIS"',
                     u'VŠĮ "MOKESČIŲ IR VERSLO PROCESŲ ADMINISTRAVIMO CENTRAS"',
                     u'ADVOKATŲ PROFESINĖ BENDRIJA "BALTIC LEGAL SOLUTIONS LIETUVA"',
                     u'UAB “GLAXOSMITHKLINE LIETUVA”']:
            self.assertEqual(self.spider.maybe_titlecase(name), name)

    def test_parse(self):
        response = HtmlResponse('www.vtek.lt/index.php/deklaravimas', body=fixture('vtek_deklaravimas.html'))
        items = list(self.spider.parse(response))
        self.assertEqual(len(items), 35)
        self.assertEqual(items[0]['name'], 'Romas Stumbrys')
        self.assertEqual(items[-1]['name'], 'Arnas Marcinkus')
<commit_msg>Fix tests on Travis CI (older libxml2)<commit_after># -*- coding: utf-8 -*-
import datetime
import unittest

from scrapy.http import HtmlResponse

from manoseimas.scrapy.items import Lobbyist
from manoseimas.scrapy.loaders import Loader
from manoseimas.scrapy.spiders.lobbyists import LobbyistsSpider
from manoseimas.scrapy.tests.utils import fixture

class TestLobbyistsSpider(unittest.TestCase):

    number_examples = [
        ('1.	', {}),
        ('35	', {}),
    ]

    name_examples = [
        ('BRONIUS ANTANAS RASIMAS	', {
            'name': 'Bronius Antanas Rasimas',
        }),
        ('MYKOLAS JUOZAPAVIČIUS	', {
            'name': u'Mykolas Juozapavičius',
            'url': 'http://www.lamco.lt/',
        }),
        ('NINA BARBORA EVANS 2012 02 21 sprendimu Nr. KS-16 (L) sustabdyta lobistinė veikla iki prašymo atnaujinti.	', {
            'name': 'Nina Barbora Evans',
            'status': u'2012 02 21 sprendimu Nr. KS-16 (L) sustabdyta lobistinė veikla iki prašymo atnaujinti.',
        }),
        ('MINDAUGAS VOLDEMARAS
 2015 03 04 sprendimu Nr. KS-20 (L) sustabdyta lobistinė veikla iki prašymo atnaujinti.	', {
            'name': 'Mindaugas Voldemaras',
            'status': u'2015 03 04 sprendimu Nr. KS-20 (L) sustabdyta lobistinė veikla iki prašymo atnaujinti.',
        }),
        ('
UAB "ERNST & YOUNG BALTIC"

Gintautas Bartkus, Jonas Platelis
	', {
            'name': 'UAB "ERNST & YOUNG BALTIC"',
            'representatives': 'Gintautas Bartkus, Jonas Platelis',
            'url': 'http://www.ey.com/GLOBAL/content.nsf/Lithuania_E/Lithuania_Home',
        }),
        ('
ADVOKATŲ PROFESINĖ BENDRIJA "BALTIC LEGAL SOLUTIONS LIETUVA"

Gytis Kaminskas, Gintautas Bartkus
	', {
            'name': u'ADVOKATŲ PROFESINĖ BENDRIJA "BALTIC LEGAL SOLUTIONS LIETUVA"',
            'representatives': 'Gytis Kaminskas, Gintautas Bartkus',
            'url': 'http://www.blslawfirm.com/bls-lithuania/home',
        }),
        ('
UAB "INLINEN"

Linas Pečiulaitis
	', {
            'name': 'UAB "INLINEN"',
            'representatives': u'Linas Pečiulaitis',
            'url': 'http://www.inlinen.eu/',
        }),
        ('UAB "VOX JURIS"
2015-04-15 sprendimu Nr. KS-31 (L) lobistinė veikla nutraukta.	', {
            'name': 'UAB "VOX JURIS"',
            'url': 'http://www.juris.lt/',
            'status': u'2015-04-15 sprendimu Nr. KS-31 (L) lobistinė veikla nutraukta.',
        }),
        ('
VŠĮ "MOKESČIŲ IR VERSLO PROCESŲ ADMINISTRAVIMO CENTRAS"

Sandra Šarkauskaitė
	', {
            'name': u'VŠĮ "MOKESČIŲ IR VERSLO PROCESŲ ADMINISTRAVIMO CENTRAS"',
            'representatives': u'Sandra Šarkauskaitė',
        }),
    ]

    company_code_examples = [
        ('123456789	', {'company_code': '123456789'}),
        (' 	', {'company_code': ''}),
    ]

    inclusion_examples = [
        ('2015 05 20 Nr. KS-36 (L)	', {
            'date_of_inclusion': datetime.date(2015, 05, 20),
            'decision': 'Nr. KS-36 (L)',
        }),
        ('2012 10 02 Nr. KS - 108 (L)	', {
            'date_of_inclusion': datetime.date(2012, 10, 02),
            'decision': 'Nr. KS - 108 (L)',
        }),
        ('2012 04 17 Nr. KS -22 (L)	', {
            'date_of_inclusion': datetime.date(2012, 04, 17),
            'decision': 'Nr. KS -22 (L)',
        }),
        ('2011 12 06 Nr. KS - 75 (L)	', {
            'date_of_inclusion': datetime.date(2011, 12, 06),
            'decision': 'Nr. KS - 75 (L)',
        }),
        ('2002 10 04 Nr. 23 (L)	', {
            'date_of_inclusion': datetime.date(2002, 10, 04),
            'decision': 'Nr. 23 (L)',
        }),
    ]

    lobbyist_examples = [
        ('''

            13.

UAB “GLAXOSMITHKLINE LIETUVA”

Kęstutis Čereška, Armindas Varkala, Arnas Beržanskis, Vilius Kirvaitis

            111785261
            2007 12 20 Nr. KS - 88 (L)

        ''', {
            'name': u'UAB “GLAXOSMITHKLINE LIETUVA”',
            'url': 'http://www.gsk.lt/',
            'representatives': u'Kęstutis Čereška, Armindas Varkala, Arnas Beržanskis, Vilius Kirvaitis',
            'company_code': '111785261',
            'date_of_inclusion': datetime.date(2007, 12, 20),
            'decision': 'Nr. KS - 88 (L)',
            'source_url': 'http://localhost/test.html',
            'raw_data': u'''

                13.

UAB “GLAXOSMITHKLINE LIETUVA”

Kęstutis Čereška, Armindas Varkala, Arnas Beržanskis, Vilius Kirvaitis

                111785261
                2007 12 20 Nr. KS - 88 (L)

            '''.strip().replace('\n    ', '\n'),
        }),
    ]

    maxDiff = None

    def setUp(self):
        self.spider = LobbyistsSpider()

    def run_examples(self, examples, wrapper, fn):
        for html, expected in examples:
            actual = wrapper(html, fn)
            if 'raw_data' in expected and 'raw_data' in actual:
                # let's see a useful diff of these
                expected['raw_data'] = self.fixup_libxml2(expected['raw_data'])
                actual['raw_data'] = self.fixup_libxml2(actual['raw_data'])
                self.assertEqual(expected['raw_data'], actual['raw_data'])
            self.assertEqual(expected, actual)

    def fixup_libxml2(self, html):
        # Older libxml2 versions destroy some of the whitespace when you
        # parse and convert the html tree back to a string.  This causes
        # test failures on Travis CI.
        return html.replace('
', '
\n            ')

    def parse_field(self, html, fn):
        response = HtmlResponse('http://localhost/test.html',
                                body='

%s

' % html)
        row = response.css('tr')[0]
        node = response.css('td')[0]
        lobbyist = Loader(self.spider, response, Lobbyist(), row)
        lobbyist.add_value(None, fn(node))
        item = lobbyist.load_item()
        actual = dict(item)
        return actual

    def parse_row(self, html, fn):
        response = HtmlResponse('http://localhost/test.html',
                                body='
%s
' % html)
        row = response.css('tr')[0]
        item = fn(response, row)
        actual = dict(item)
        return actual

    def test_parse_number(self):
        self.run_examples(self.number_examples, self.parse_field,
                          self.spider._parse_number)

    def test_parse_name(self):
        self.run_examples(self.name_examples, self.parse_field,
                          self.spider._parse_name)

    def test_parse_company_code(self):
        self.run_examples(self.company_code_examples, self.parse_field,
                          self.spider._parse_company_code)

    def test_parse_inclusion(self):
        self.run_examples(self.inclusion_examples, self.parse_field,
                          self.spider._parse_inclusion)

    def test_parse_lobbyist(self):
        self.run_examples(self.lobbyist_examples, self.parse_row,
                          self.spider._parse_lobbyist)

    def test_maybe_titlecase_fixes_person_names(self):
        self.assertEqual(self.spider.maybe_titlecase(u'ALGIRDAS JUŠKYS'), u'Algirdas Juškys')
        self.assertEqual(self.spider.maybe_titlecase(u'BRONIUS ANTANAS RASIMAS'), u'Bronius Antanas Rasimas')

    def test_maybe_titlecase_leaves_company_names_alone(self):
        for name in ['UAB "VOX JURIS"',
                     u'VŠĮ "MOKESČIŲ IR VERSLO PROCESŲ ADMINISTRAVIMO CENTRAS"',
                     u'ADVOKATŲ PROFESINĖ BENDRIJA "BALTIC LEGAL SOLUTIONS LIETUVA"',
                     u'UAB “GLAXOSMITHKLINE LIETUVA”']:
            self.assertEqual(self.spider.maybe_titlecase(name), name)

    def test_parse(self):
        response = HtmlResponse('www.vtek.lt/index.php/deklaravimas', body=fixture('vtek_deklaravimas.html'))
        items = list(self.spider.parse(response))
        self.assertEqual(len(items), 35)
        self.assertEqual(items[0]['name'], 'Romas Stumbrys')
        self.assertEqual(items[-1]['name'], 'Arnas Marcinkus')
<|endoftext|>