<commit_before>from motobot import IRCBot, hook, request, Priority, Context, strip_control_codes
from motobot.modifiers import Modifier, CommandModifier, ParamsModifier, TrailingModifier, EatType
from itertools import tee

@request('HANDLE_MESSAGE')
def handle_message_request(bot, context, nick, channel, host, message):
    messages = list(split_messages(message, bot.command_prefix))

    break_priority = Priority.min
    for plugin in bot.request('GET_PLUGINS', channel):
        if break_priority > plugin.priority:
            break
        try:
            responses = handle_plugin(bot, plugin, nick, channel, host, messages)
            responses, eat = check_eat(responses)
            if eat:
                break_priority = plugin.priority
            yield responses
        except:
            bot.log_error()

def check_eat(responses):
    def extract_responses(responses):
        for x in responses:
            if isinstance(x, EatType):
                pass
            elif isinstance(x, str):
                yield x
            elif isinstance(x, Modifier):
                yield x
            elif hasattr(x, '__iter__'):
                yield extract_responses(x)

    def find_eat(responses):
        for x in responses:
            if isinstance(x, EatType):
                return True
            elif isinstance(x, str):
                pass
            elif isinstance(x, Modifier):
                pass
            elif hasattr(x, '__iter__'):
                if find_eat(x):
                    return True
        return False

    it1, it2 = tee(responses)
    return extract_responses(it1), find_eat(it2)

@hook('PRIVMSG')
def handle_privmsg(bot, context, message):
    """ Handle the privmsg commands.

    Will send messages to each plugin accounting for priority and level.
    Will then parse the responses from each plugin, if any, and send them
    via the bot.

    """
    nick = message.nick
    channel = message.params[0]
    host = message.host
    message = strip_control_codes(transform_action(nick, message.params[-1]))
    target = channel if channel != bot.nick else nick

    responses = bot.request('HANDLE_MESSAGE', nick, channel, host, message)
    handle_responses(bot, responses, [target])

def transform_action(nick, msg):
    """ Transform an action CTCP into a message. """
    if msg.startswith('\x01ACTION ') and msg.endswith('\x01'):
        return '*' + nick + msg[7:-1]
    else:
        return msg

def split_messages(message, command_prefix):
    messages = iter(message.split('|'))
    current_message = next(messages)

    for message in messages:
        test_message = message.lstrip(' ')
        if test_message.startswith(command_prefix):
            yield current_message.rstrip(' ')
            current_message = test_message
        else:
            current_message += '|' + message
    yield current_message.rstrip(' ')

def handle_plugin(bot, plugin, nick, channel, host, messages):
    responses = None

    for message in messages:
        if responses is None:
            responses = call_plugins([plugin], bot, nick, channel, host, message)
        else:
            responses = handle_pipe(bot, nick, channel, host, message, responses)

    return responses

def call_plugins(plugins, bot, nick, channel, host, message):
    for plugin in plugins:
        response = None
        module = plugin.module
        context = Context(nick, channel, host, bot.database.get_entry(module),
                          bot.sessions.get_entry(module))
        if plugin.type == IRCBot.command_plugin:
            response = handle_command(plugin, bot, context, message)
        elif plugin.type == IRCBot.match_plugin:
            response = handle_match(plugin, bot, context, message)
        elif plugin.type == IRCBot.sink_plugin:
            response = handle_sink(plugin, bot, context, message)
        if response is not None:
            yield response

def handle_pipe(bot, nick, channel, host, message, responses):
    plugins = list(filter(lambda x: x.type == IRCBot.command_plugin,
                          bot.request('GET_PLUGINS', channel)))
    for x in responses:
        if isinstance(x, str):
            yield call_plugins(plugins, bot, nick, channel, host, message.rstrip(' ') + ' ' + x)
        elif isinstance(x, Modifier):
            yield x
        elif hasattr(x, '__iter__'):
            yield handle_pipe(bot, nick, channel, host, message, x)

def handle_command(plugin, bot, context, message):
    trigger = bot.command_prefix + plugin.arg.trigger
    test = message.split(' ', 1)[0]

    if trigger == test:
        alt = bot.request('USERLEVEL', context.channel, context.nick) < plugin.level
        func = plugin.func if not alt else plugin.alt
        if func is not None:
            args = message[len(bot.command_prefix):].split(' ')
            return func(bot, context, message, args)

def handle_match(plugin, bot, context, message):
    match = plugin.arg.search(message)
    if match is not None:
        alt = bot.request('USERLEVEL', context.channel, context.nick) < plugin.level
        func = plugin.func if not alt else plugin.alt
        if func is not None:
            return func(bot, context, message, match)

def handle_sink(plugin, bot, context, message):
    alt = bot.request('USERLEVEL', context.channel, context.nick) < plugin.level
    func = plugin.func if not alt else plugin.alt
    if func is not None:
        return func(bot, context, message)

def handle_responses(bot, responses, params, command='PRIVMSG',
                     trailing_mods=None, require_trailing=True):
    trailings = []
    command_mods = []
    param_mods = []
    trailing_mods = [] if trailing_mods is None else trailing_mods
    iters = []
    extract_responses(responses, trailings, command_mods,
                      param_mods, trailing_mods, iters)

    for modifier in command_mods:
        require_trailing &= modifier.require_trailing
        command = modifier.modify_command(command)
    for modifier in param_mods:
        require_trailing &= modifier.require_trailing
        params = modifier.modify_params(params)

    if not require_trailing and not trailings:
        trailings = ['']

    for trailing in trailings:
        for modifier in trailing_mods:
            trailing = modifier.modify_trailing(trailing)
        message = form_message(command, params, trailing)
        bot.send(message)

    for iter in iters:
        handle_responses(bot, iter, params, command, trailing_mods, require_trailing)

def extract_responses(responses, trailings, command_mods,
                      param_mods, trailing_mods, iters):
    for x in responses:
        if isinstance(x, str):
            trailings.append(x)
        elif isinstance(x, Modifier):
            if isinstance(x, CommandModifier):
                command_mods.append(x)
            if isinstance(x, ParamsModifier):
                param_mods.append(x)
            if isinstance(x, TrailingModifier):
                trailing_mods.append(x)
        elif hasattr(x, '__iter__'):
            iters.append(x)

def form_message(command, params, trailing):
    message = command
    message += ' ' + ' '.join(params) if params else ''
    message += ' :' + trailing if trailing else ''
    return message
<commit_msg>Make check_eat not lazy<commit_after>from motobot import IRCBot, hook, request, Priority, Context, strip_control_codes
from motobot.modifiers import Modifier, CommandModifier, ParamsModifier, TrailingModifier, EatType
from itertools import tee

@request('HANDLE_MESSAGE')
def handle_message_request(bot, context, nick, channel, host, message):
    messages = list(split_messages(message, bot.command_prefix))

    break_priority = Priority.min
    for plugin in bot.request('GET_PLUGINS', channel):
        if break_priority > plugin.priority:
            break
        try:
            responses = handle_plugin(bot, plugin, nick, channel, host, messages)
            responses, eat = check_eat(responses)
            if eat:
                break_priority = plugin.priority
            yield responses
        except:
            bot.log_error()

def check_eat(responses):
    eat = False
    new_responses = []

    for x in responses:
            if isinstance(x, EatType):
                eat = True
            elif isinstance(x, str):
                new_responses.append(x)
            elif isinstance(x, Modifier):
                new_responses.append(x)
            elif hasattr(x, '__iter__'):
                x, eat = check_eat(x)
                new_responses.append(x)

    return new_responses, eat

@hook('PRIVMSG')
def handle_privmsg(bot, context, message):
    """ Handle the privmsg commands.

    Will send messages to each plugin accounting for priority and level.
    Will then parse the responses from each plugin, if any, and send them
    via the bot.

    """
    nick = message.nick
    channel = message.params[0]
    host = message.host
    message = strip_control_codes(transform_action(nick, message.params[-1]))
    target = channel if channel != bot.nick else nick

    responses = bot.request('HANDLE_MESSAGE', nick, channel, host, message)
    handle_responses(bot, responses, [target])

def transform_action(nick, msg):
    """ Transform an action CTCP into a message. """
    if msg.startswith('\x01ACTION ') and msg.endswith('\x01'):
        return '*' + nick + msg[7:-1]
    else:
        return msg

def split_messages(message, command_prefix):
    messages = iter(message.split('|'))
    current_message = next(messages)

    for message in messages:
        test_message = message.lstrip(' ')
        if test_message.startswith(command_prefix):
            yield current_message.rstrip(' ')
            current_message = test_message
        else:
            current_message += '|' + message
    yield current_message.rstrip(' ')

def handle_plugin(bot, plugin, nick, channel, host, messages):
    responses = None

    for message in messages:
        if responses is None:
            responses = call_plugins([plugin], bot, nick, channel, host, message)
        else:
            responses = handle_pipe(bot, nick, channel, host, message, responses)

    return responses

def call_plugins(plugins, bot, nick, channel, host, message):
    for plugin in plugins:
        response = None
        module = plugin.module
        context = Context(nick, channel, host, bot.database.get_entry(module),
                          bot.sessions.get_entry(module))
        if plugin.type == IRCBot.command_plugin:
            response = handle_command(plugin, bot, context, message)
        elif plugin.type == IRCBot.match_plugin:
            response = handle_match(plugin, bot, context, message)
        elif plugin.type == IRCBot.sink_plugin:
            response = handle_sink(plugin, bot, context, message)
        if response is not None:
            yield response

def handle_pipe(bot, nick, channel, host, message, responses):
    plugins = list(filter(lambda x: x.type == IRCBot.command_plugin,
                          bot.request('GET_PLUGINS', channel)))
    for x in responses:
        if isinstance(x, str):
            yield call_plugins(plugins, bot, nick, channel, host, message.rstrip(' ') + ' ' + x)
        elif isinstance(x, Modifier):
            yield x
        elif hasattr(x, '__iter__'):
            yield handle_pipe(bot, nick, channel, host, message, x)

def handle_command(plugin, bot, context, message):
    trigger = bot.command_prefix + plugin.arg.trigger
    test = message.split(' ', 1)[0]

    if trigger == test:
        alt = bot.request('USERLEVEL', context.channel, context.nick) < plugin.level
        func = plugin.func if not alt else plugin.alt
        if func is not None:
            args = message[len(bot.command_prefix):].split(' ')
            return func(bot, context, message, args)

def handle_match(plugin, bot, context, message):
    match = plugin.arg.search(message)
    if match is not None:
        alt = bot.request('USERLEVEL', context.channel, context.nick) < plugin.level
        func = plugin.func if not alt else plugin.alt
        if func is not None:
            return func(bot, context, message, match)

def handle_sink(plugin, bot, context, message):
    alt = bot.request('USERLEVEL', context.channel, context.nick) < plugin.level
    func = plugin.func if not alt else plugin.alt
    if func is not None:
        return func(bot, context, message)

def handle_responses(bot, responses, params, command='PRIVMSG',
                     trailing_mods=None, require_trailing=True):
    trailings = []
    command_mods = []
    param_mods = []
    trailing_mods = [] if trailing_mods is None else trailing_mods
    iters = []
    extract_responses(responses, trailings, command_mods,
                      param_mods, trailing_mods, iters)

    for modifier in command_mods:
        require_trailing &= modifier.require_trailing
        command = modifier.modify_command(command)
    for modifier in param_mods:
        require_trailing &= modifier.require_trailing
        params = modifier.modify_params(params)

    if not require_trailing and not trailings:
        trailings = ['']

    for trailing in trailings:
        for modifier in trailing_mods:
            trailing = modifier.modify_trailing(trailing)
        message = form_message(command, params, trailing)
        bot.send(message)

    for iter in iters:
        handle_responses(bot, iter, params, command, trailing_mods, require_trailing)

def extract_responses(responses, trailings, command_mods,
                      param_mods, trailing_mods, iters):
    for x in responses:
        if isinstance(x, str):
            trailings.append(x)
        elif isinstance(x, Modifier):
            if isinstance(x, CommandModifier):
                command_mods.append(x)
            if isinstance(x, ParamsModifier):
                param_mods.append(x)
            if isinstance(x, TrailingModifier):
                trailing_mods.append(x)
        elif hasattr(x, '__iter__'):
            iters.append(x)

def form_message(command, params, trailing):
    message = command
    message += ' ' + ' '.join(params) if params else ''
    message += ' :' + trailing if trailing else ''
    return message
<|endoftext|>


<commit_before>"""
This is a class for musical notes.
"""
from music21 import *

class MyNote:

    def __init__(self, p, d, v, c1, c2, c3, c4):
        self.pitch = p
        self.duration = d
        self.velocity = v
        self.mfcc = [c1, c2, c3, c4]

        n = note.Note()
        n.frequency = p
        self.m21 = n
<commit_msg>Fixed incorrect music21 usage<commit_after>"""
This is a class for musical notes.
"""
from music21 import *

class MyNote:

    def __init__(self, p, d, v, c1, c2, c3, c4):
        self.pitch = p
        self.duration = d
        self.velocity = v
        self.mfcc = [c1, c2, c3, c4]

        n = note.Note()
        #use n.frequency in future version that uses frequency as input instead of midi
        n.pitch.midi = p
        self.m21 = n

    def __repr__(self):
        """
        -Returns just the pitch and duration for now
        """
        return "{},{}".format(self.pitch, self.duration)
<|endoftext|>


<commit_before>import random
from Queue import Queue
import json
from threading import Thread, Lock, Semaphore
import time
import uuid
from boto.dynamodb2.exceptions import ItemNotFound, ConditionalCheckFailedException, \
    ProvisionedThroughputExceededException, LimitExceededException
from boto.dynamodb2.fields import HashKey
from boto.dynamodb2.items import Item
from boto.dynamodb2.table import Table
from boto.dynamodb2.types import STRING
from boto.exception import JSONResponseError
from motorway.messages import Message
from motorway.ramp import Ramp
import boto.kinesis
import boto.kinesis.exceptions
import logging

shard_election_logger = logging.getLogger("motorway.contrib.amazon_kinesis.shard_election")

class KinesisRamp(Ramp):
    stream_name = None
    heartbeat_timeout = 30  # Wait 10 seconds for a heartbeat update, or kill it

    def __init__(self, shard_threads_enabled=True):
        super(KinesisRamp, self).__init__()
        self.conn = boto.kinesis.connect_to_region(**self.connection_parameters())
        assert self.stream_name, "Please define attribute stream_name on your KinesisRamp"

        control_table_name = 'pipeline-control-%s' % self.stream_name

        self.worker_id = str(uuid.uuid4())
        self.semaphore = Semaphore()
        self.uncompleted_ids = {}

        if shard_threads_enabled:
            dynamodb_connection = boto.dynamodb2.connect_to_region(**self.connection_parameters())

            table_config = dict(
                connection=dynamodb_connection,
                schema=[
                    HashKey('shard_id', data_type=STRING),
                ],
                throughput={
                    'read': 10,
                    'write': 10,
                }
            )

            try:
                dynamodb_connection.describe_table(control_table_name)
            except JSONResponseError:
                Table.create(
                    control_table_name,
                    **table_config
                )

            self.control_table = Table(
                control_table_name,
                **table_config
            )

            shards = self.conn.describe_stream(self.stream_name)['StreamDescription']['Shards']
            threads = []
            self.insertion_queue = Queue()
            for i, shard in enumerate(shards):
                self.uncompleted_ids[shard['ShardId']] = []
                t = Thread(target=self.process_shard, name="%s-%s" % (self.__class__.__name__, i), args=(shard['ShardId'], ))
                threads.append(t)
                t.start()

    def claim_shard(self, shard_id):
        shard_election_logger.info("Claiming shard %s" % shard_id)
        control_record = self.control_table.get_item(shard_id=shard_id)
        control_record['worker_id'] = self.worker_id
        control_record['heartbeat'] = 0
        try:
            control_record.save()
        except ConditionalCheckFailedException:  # Someone else edited the record
            shard_election_logger.debug("Failed to claim %s to %s" % (shard_id, self.worker_id))
            return False
        return True

    def can_claim_shard(self, shard_id):
        control_record = self.control_table.get_item(shard_id=shard_id)
        heartbeat = control_record['heartbeat']
        time.sleep(self.heartbeat_timeout)
        if heartbeat == self.control_table.get_item(shard_id=shard_id)['heartbeat']:  # TODO: check worker_id as well
            shard_election_logger.debug("Heartbeat remained unchanged for defined time, taking over")
            return True
        else:
            shard_election_logger.debug("Heartbeat changed, continue sleeping")
        # Balance, if possible
        active_workers = {
            self.worker_id: True
        }
        heartbeats = {
            self.worker_id: 0
        }

        shards = list(self.control_table.scan())
        for shard in shards:  # Update active worker cache
            if shard['worker_id'] not in active_workers:
                heartbeats[shard['worker_id']] = shard['heartbeat']
            if heartbeats[shard['worker_id']] == shard['heartbeat']:
                active_workers[shard['worker_id']] = False
            else:
                active_workers[shard['worker_id']] = True
        number_of_active_workers = sum([1 for is_active in active_workers.values() if is_active])
        number_of_shards = len(shards)
        optimal_number_of_shards_per_worker = number_of_shards / number_of_active_workers
        workers = set([shard['worker_id'] for shard in shards])
        shards_per_worker = {worker: sum([1 for shard in shards if shard['worker_id'] == worker]) for worker in workers}
        for shard in shards:
            if shards_per_worker.get(shard['worker_id'], 0) > optimal_number_of_shards_per_worker:
                if shards_per_worker.get(self.worker_id, 0) < optimal_number_of_shards_per_worker:
                    if shard['shard_id'] == shard_id:
                        shard_election_logger.debug("Taking over %s from %s" % (shard_id, shard['worker_id']))
                        return True
        return False

    def process_shard(self, shard_id):
        while True:
            try:
                try:
                    while True:
                        with self.semaphore:
                            if self.can_claim_shard(shard_id):
                                if self.claim_shard(shard_id):
                                    break
                except ItemNotFound:
                    control_record = Item(self.control_table, data={
                        'shard_id': shard_id,
                        'checkpoint': 0,
                        'worker_id': self.worker_id,
                        'heartbeat': 0,
                    })
                    control_record.save()

                control_record = self.control_table.get_item(shard_id=shard_id)
                if control_record['checkpoint'] > 0:
                    iterator = self.conn.get_shard_iterator(
                        self.stream_name,
                        shard_id,
                        "AT_SEQUENCE_NUMBER",
                        starting_sequence_number=str(control_record['checkpoint'])
                    )['ShardIterator']
                else:
                    iterator = self.conn.get_shard_iterator(
                        self.stream_name,
                        shard_id,
                        "LATEST",
                    )['ShardIterator']
                while True:
                    control_record = self.control_table.get_item(shard_id=shard_id)
                    if not control_record['worker_id'] == self.worker_id:
                        shard_election_logger.info("Lost shard %s, going back to standby" % shard_id)
                        break
                    control_record['heartbeat'] += 1
                    if len(self.uncompleted_ids[shard_id]):  # Get the "youngest" uncompleted sequence number
                        control_record['checkpoint'] = min(self.uncompleted_ids[shard_id])
                    control_record.save()

                    result = self.conn.get_records(iterator)
                    for record in result['Records']:
                        self.uncompleted_ids[shard_id].append(record['SequenceNumber'])
                        self.insertion_queue.put(record)
                    iterator = result['NextShardIterator']
                    time.sleep(1)
            except ConditionalCheckFailedException:
                pass  # we're no longer worker for this shard
            except (ProvisionedThroughputExceededException, LimitExceededException, boto.kinesis.exceptions.ProvisionedThroughputExceededException, boto.kinesis.exceptions.LimitExceededException) as e:
                time.sleep(random.randrange(5, self.heartbeat_timeout))  # back off for a while

    def connection_parameters(self):
        return {
            'region_name': 'eu-west-1',
            # Add this or use ENV VARS
            # 'aws_access_key_id': '',
            # 'aws_secret_access_key': ''
        }

    def next(self):
        msg = self.insertion_queue.get()
        yield Message(msg['SequenceNumber'], json.loads(msg['Data']), grouping_value=msg['PartitionKey'])

    def success(self, _id):
        for shard_id, uncompleted_ids in self.uncompleted_ids.items():
            if _id in uncompleted_ids:
                uncompleted_ids.remove(_id)<commit_msg>motorway: Introduce some random delay to give different shards an even option<commit_after>import random
from Queue import Queue
import json
from threading import Thread, Lock, Semaphore
import time
import uuid
from boto.dynamodb2.exceptions import ItemNotFound, ConditionalCheckFailedException, \
    ProvisionedThroughputExceededException, LimitExceededException
from boto.dynamodb2.fields import HashKey
from boto.dynamodb2.items import Item
from boto.dynamodb2.table import Table
from boto.dynamodb2.types import STRING
from boto.exception import JSONResponseError
from motorway.messages import Message
from motorway.ramp import Ramp
import boto.kinesis
import boto.kinesis.exceptions
import logging

shard_election_logger = logging.getLogger("motorway.contrib.amazon_kinesis.shard_election")

class KinesisRamp(Ramp):
    stream_name = None
    heartbeat_timeout = 30  # Wait 10 seconds for a heartbeat update, or kill it

    def __init__(self, shard_threads_enabled=True):
        super(KinesisRamp, self).__init__()
        self.conn = boto.kinesis.connect_to_region(**self.connection_parameters())
        assert self.stream_name, "Please define attribute stream_name on your KinesisRamp"

        control_table_name = 'pipeline-control-%s' % self.stream_name

        self.worker_id = str(uuid.uuid4())
        self.semaphore = Semaphore()
        self.uncompleted_ids = {}

        if shard_threads_enabled:
            dynamodb_connection = boto.dynamodb2.connect_to_region(**self.connection_parameters())

            table_config = dict(
                connection=dynamodb_connection,
                schema=[
                    HashKey('shard_id', data_type=STRING),
                ],
                throughput={
                    'read': 10,
                    'write': 10,
                }
            )

            try:
                dynamodb_connection.describe_table(control_table_name)
            except JSONResponseError:
                Table.create(
                    control_table_name,
                    **table_config
                )

            self.control_table = Table(
                control_table_name,
                **table_config
            )

            shards = self.conn.describe_stream(self.stream_name)['StreamDescription']['Shards']
            threads = []
            self.insertion_queue = Queue()
            for i, shard in enumerate(shards):
                self.uncompleted_ids[shard['ShardId']] = []
                t = Thread(target=self.process_shard, name="%s-%s" % (self.__class__.__name__, i), args=(shard['ShardId'], ))
                threads.append(t)
                t.start()

    def claim_shard(self, shard_id):
        shard_election_logger.info("Claiming shard %s" % shard_id)
        control_record = self.control_table.get_item(shard_id=shard_id)
        control_record['worker_id'] = self.worker_id
        control_record['heartbeat'] = 0
        try:
            control_record.save()
        except ConditionalCheckFailedException:  # Someone else edited the record
            shard_election_logger.debug("Failed to claim %s to %s" % (shard_id, self.worker_id))
            return False
        return True

    def can_claim_shard(self, shard_id):
        control_record = self.control_table.get_item(shard_id=shard_id)
        heartbeat = control_record['heartbeat']
        time.sleep(self.heartbeat_timeout)
        if heartbeat == self.control_table.get_item(shard_id=shard_id)['heartbeat']:  # TODO: check worker_id as well
            shard_election_logger.debug("Heartbeat remained unchanged for defined time, taking over")
            return True
        else:
            shard_election_logger.debug("Heartbeat changed, continue sleeping")
        # Balance, if possible
        active_workers = {
            self.worker_id: True
        }
        heartbeats = {
            self.worker_id: 0
        }

        shards = list(self.control_table.scan())
        for shard in shards:  # Update active worker cache
            if shard['worker_id'] not in active_workers:
                heartbeats[shard['worker_id']] = shard['heartbeat']
            if heartbeats[shard['worker_id']] == shard['heartbeat']:
                active_workers[shard['worker_id']] = False
            else:
                active_workers[shard['worker_id']] = True
        number_of_active_workers = sum([1 for is_active in active_workers.values() if is_active])
        number_of_shards = len(shards)
        optimal_number_of_shards_per_worker = number_of_shards / number_of_active_workers
        workers = set([shard['worker_id'] for shard in shards])
        shards_per_worker = {worker: sum([1 for shard in shards if shard['worker_id'] == worker]) for worker in workers}
        for shard in shards:
            if shards_per_worker.get(shard['worker_id'], 0) > optimal_number_of_shards_per_worker:
                if shards_per_worker.get(self.worker_id, 0) < optimal_number_of_shards_per_worker:
                    if shard['shard_id'] == shard_id:
                        shard_election_logger.debug("Taking over %s from %s" % (shard_id, shard['worker_id']))
                        return True
        return False

    def process_shard(self, shard_id):
        while True:
            try:
                try:
                    while True:
                        with self.semaphore:
                            if self.can_claim_shard(shard_id):
                                if self.claim_shard(shard_id):
                                    break
                        time.sleep(random.randrange(2, 15))
                except ItemNotFound:
                    control_record = Item(self.control_table, data={
                        'shard_id': shard_id,
                        'checkpoint': 0,
                        'worker_id': self.worker_id,
                        'heartbeat': 0,
                    })
                    control_record.save()

                control_record = self.control_table.get_item(shard_id=shard_id)
                if control_record['checkpoint'] > 0:
                    iterator = self.conn.get_shard_iterator(
                        self.stream_name,
                        shard_id,
                        "AT_SEQUENCE_NUMBER",
                        starting_sequence_number=str(control_record['checkpoint'])
                    )['ShardIterator']
                else:
                    iterator = self.conn.get_shard_iterator(
                        self.stream_name,
                        shard_id,
                        "LATEST",
                    )['ShardIterator']
                while True:
                    control_record = self.control_table.get_item(shard_id=shard_id)
                    if not control_record['worker_id'] == self.worker_id:
                        shard_election_logger.info("Lost shard %s, going back to standby" % shard_id)
                        break
                    control_record['heartbeat'] += 1
                    if len(self.uncompleted_ids[shard_id]):  # Get the "youngest" uncompleted sequence number
                        control_record['checkpoint'] = min(self.uncompleted_ids[shard_id])
                    control_record.save()

                    result = self.conn.get_records(iterator)
                    for record in result['Records']:
                        self.uncompleted_ids[shard_id].append(record['SequenceNumber'])
                        self.insertion_queue.put(record)
                    iterator = result['NextShardIterator']
                    time.sleep(1)
            except ConditionalCheckFailedException:
                pass  # we're no longer worker for this shard
            except (ProvisionedThroughputExceededException, LimitExceededException, boto.kinesis.exceptions.ProvisionedThroughputExceededException, boto.kinesis.exceptions.LimitExceededException) as e:
                time.sleep(random.randrange(5, self.heartbeat_timeout))  # back off for a while

    def connection_parameters(self):
        return {
            'region_name': 'eu-west-1',
            # Add this or use ENV VARS
            # 'aws_access_key_id': '',
            # 'aws_secret_access_key': ''
        }

    def next(self):
        msg = self.insertion_queue.get()
        yield Message(msg['SequenceNumber'], json.loads(msg['Data']), grouping_value=msg['PartitionKey'])

    def success(self, _id):
        for shard_id, uncompleted_ids in self.uncompleted_ids.items():
            if _id in uncompleted_ids:
                uncompleted_ids.remove(_id)<|endoftext|>


<commit_before>''' Really simple HTTP server for development.

Do *NOT* attempt to use this as anything resembling a production server. It is
meant to be used as a development test server only.

You might ask, "Why do I need a development server for static pages?" One
hyphenated modifier: "root-relative." Since wok dumps all of the media files
in the root output directory, pages that reside inside subdirectories still
need to access these media files in a unified way.

E.g., if you include `base.css` in your `base.html` template, `base.css` should
be accessable to any page that uses `base.html`, even if it's a categorized
page, and thus, goes into a subdirectory. This way, your CSS include tag could
read `<link type='text/css' href='/base.css' />` (note the '/' in the `href`
property) and `base.css` can be accessed from anywhere.
'''

import sys
import os
from BaseHTTPServer import HTTPServer
from SimpleHTTPServer import SimpleHTTPRequestHandler

def run(address=None, port=None, serv_dir=None):
    ''' Run the development server on `address`:`port` and server the
    directory `serv_dir`. (If `serv_dir` is not provided, it will use
    the current working directory.)
    '''
    if not address:
        address = ''
    if not port:
        port = 8000
    else:
        port = int(port)
    if serv_dir:
        os.chdir(serv_dir)
    server_class = HTTPServer
    handler_class = SimpleHTTPRequestHandler
    httpd = server_class((address, port), handler_class)
    socketInfo = httpd.socket.getsockname()
    print "Development HTTP server running on http://%s:%s (Ctrl-c to stop)"\
            %(socketInfo[0], socketInfo[1])
    try:
        httpd.serve_forever()
    except KeyboardInterrupt:
        print "\nbye!"
        exit(0)
<commit_msg>Code cleanup (var naming consistency)<commit_after>''' Really simple HTTP *development* server

Do *NOT* attempt to use this as anything resembling a production server. It is
meant to be used as a development test server only.

You might ask, "Why do I need a development server for static pages?" One
hyphenated modifier: "root-relative." Since wok dumps all of the media files
in the root output directory, pages that reside inside subdirectories still
need to access these media files in a unified way.

E.g., if you include `base.css` in your `base.html` template, `base.css` should
be accessable to any page that uses `base.html`, even if it's a categorized
page, and thus, goes into a subdirectory. This way, your CSS include tag could
read `<link type='text/css' href='/base.css' />` (note the '/' in the `href`
property) and `base.css` can be accessed from anywhere.
'''

import sys
import os
from BaseHTTPServer import HTTPServer
from SimpleHTTPServer import SimpleHTTPRequestHandler

def run(address=None, port=None, serv_dir=None):
    ''' Run the development server on `address`:`port` and server the
    directory `serv_dir`. (If `serv_dir` is not provided, it will use
    the current working directory.)
    '''
    if not address:
        address = ''
    if not port:
        port = 8000
    else:
        port = int(port)
    if serv_dir:
        os.chdir(serv_dir)
    server = HTTPServer
    handler = SimpleHTTPRequestHandler
    httpd = server((address, port), handler)
    socket_info = httpd.socket.getsockname()
    print "Development HTTP server running on http://%s:%s (Ctrl-c to stop)"\
            %(socket_info[0], socket_info[1])
    try:
        httpd.serve_forever()
    except KeyboardInterrupt:
        print "\nbye!"
        exit(0)

<|endoftext|>


<commit_before>import random
from Queue import Queue
import json
from threading import Thread, Lock, Semaphore
import time
import uuid

import datetime
from boto.dynamodb2.exceptions import ItemNotFound, ConditionalCheckFailedException, \
    ProvisionedThroughputExceededException, LimitExceededException
from boto.dynamodb2.fields import HashKey
from boto.dynamodb2.items import Item
from boto.dynamodb2.table import Table
from boto.dynamodb2.types import STRING
from boto.exception import JSONResponseError
from motorway.messages import Message
from motorway.ramp import Ramp
import boto.kinesis
import boto.ec2.cloudwatch
import boto.kinesis.exceptions
import logging

shard_election_logger = logging.getLogger("motorway.contrib.amazon_kinesis.shard_election")

class KinesisRamp(Ramp):
    stream_name = None
    heartbeat_timeout = 30  # Wait 10 seconds for a heartbeat update, or kill it
    MAX_UNCOMPLETED_ITEMS = 3000
    GET_RECORDS_LIMIT = 1000

    def __init__(self, shard_threads_enabled=True, **kwargs):
        super(KinesisRamp, self).__init__(**kwargs)
        self.conn = boto.kinesis.connect_to_region(**self.connection_parameters())
        assert self.stream_name, "Please define attribute stream_name on your KinesisRamp"

        control_table_name = 'pipeline-control-%s' % self.stream_name

        self.worker_id = str(uuid.uuid4())
        self.semaphore = Semaphore()
        self.uncompleted_ids = {}

        if shard_threads_enabled:
            dynamodb_connection = boto.dynamodb2.connect_to_region(**self.connection_parameters())

            table_config = dict(
                connection=dynamodb_connection,
                schema=[
                    HashKey('shard_id', data_type=STRING),
                ],
                throughput={
                    'read': 10,
                    'write': 10,
                }
            )

            try:
                dynamodb_connection.describe_table(control_table_name)
            except JSONResponseError:
                Table.create(
                    control_table_name,
                    **table_config
                )

            self.control_table = Table(
                control_table_name,
                **table_config
            )

            shards = self.conn.describe_stream(self.stream_name)['StreamDescription']['Shards']
            threads = []
            self.insertion_queue = Queue()
            for i, shard in enumerate(shards):
                self.uncompleted_ids[shard['ShardId']] = set()
                t = Thread(target=self.process_shard, name="%s-%s" % (self.__class__.__name__, i), args=(shard['ShardId'], ))
                threads.append(t)
                t.start()

    def claim_shard(self, shard_id):
        shard_election_logger.info("Claiming shard %s" % shard_id)
        control_record = self.control_table.get_item(shard_id=shard_id)
        control_record['worker_id'] = self.worker_id
        control_record['heartbeat'] = 0
        try:
            control_record.save()
        except ConditionalCheckFailedException:  # Someone else edited the record
            shard_election_logger.debug("Failed to claim %s to %s" % (shard_id, self.worker_id))
            return False
        return True

    def can_claim_shard(self, shard_id):
        control_record = self.control_table.get_item(shard_id=shard_id)
        heartbeat = control_record['heartbeat']
        time.sleep(self.heartbeat_timeout)
        if heartbeat == self.control_table.get_item(shard_id=shard_id)['heartbeat']:  # TODO: check worker_id as well
            shard_election_logger.debug("Heartbeat remained unchanged for defined time, taking over")
            return True
        else:
            shard_election_logger.debug("Heartbeat changed, continue sleeping")
        # Balance, if possible
        active_workers = {
            self.worker_id: True
        }
        heartbeats = {
            self.worker_id: 0
        }

        shards = list(self.control_table.scan())
        for shard in shards:  # Update active worker cache
            if shard['worker_id'] not in active_workers:
                heartbeats[shard['worker_id']] = shard['heartbeat']
            if heartbeats[shard['worker_id']] == shard['heartbeat']:
                active_workers[shard['worker_id']] = False
            else:
                active_workers[shard['worker_id']] = True
        number_of_active_workers = sum([1 for is_active in active_workers.values() if is_active])
        number_of_shards = len(shards)
        optimal_number_of_shards_per_worker = number_of_shards / number_of_active_workers
        workers = set([shard['worker_id'] for shard in shards])
        shards_per_worker = {worker: sum([1 for shard in shards if shard['worker_id'] == worker]) for worker in workers}
        for shard in shards:
            if shards_per_worker.get(shard['worker_id'], 0) > optimal_number_of_shards_per_worker:
                if shards_per_worker.get(self.worker_id, 0) < optimal_number_of_shards_per_worker:
                    if shard['shard_id'] == shard_id:
                        shard_election_logger.debug("Taking over %s from %s" % (shard_id, shard['worker_id']))
                        return True
        return False

    def process_shard(self, shard_id):
        while True:
            try:
                try:
                    while True:
                        with self.semaphore:
                            if self.can_claim_shard(shard_id):
                                if self.claim_shard(shard_id):
                                    break
                        time.sleep(random.randrange(2, 15))
                except ItemNotFound:
                    control_record = Item(self.control_table, data={
                        'shard_id': shard_id,
                        'checkpoint': 0,
                        'worker_id': self.worker_id,
                        'heartbeat': 0,
                    })
                    control_record.save()

                control_record = self.control_table.get_item(shard_id=shard_id)
                if control_record['checkpoint'] > 0:
                    iterator = self.conn.get_shard_iterator(
                        self.stream_name,
                        shard_id,
                        "AT_SEQUENCE_NUMBER",
                        starting_sequence_number=str(control_record['checkpoint'])
                    )['ShardIterator']
                else:
                    iterator = self.conn.get_shard_iterator(
                        self.stream_name,
                        shard_id,
                        "LATEST",
                    )['ShardIterator']

                cloudwatch = boto.ec2.cloudwatch.connect_to_region(**self.connection_parameters())
                current_minute = lambda: datetime.datetime.now().minute
                minute = None
                latest_item = None
                while True:
                    control_record = self.control_table.get_item(shard_id=shard_id)
                    if not control_record['worker_id'] == self.worker_id:
                        shard_election_logger.info("Lost shard %s, going back to standby" % shard_id)
                        break
                    control_record['heartbeat'] += 1
                    if len(self.uncompleted_ids[shard_id]):  # Get the "youngest" uncompleted sequence number
                        control_record['checkpoint'] = min(self.uncompleted_ids[shard_id])
                    elif latest_item:
                        control_record['checkpoint'] = latest_item

                    control_record.save()  # Will fail if someone else modified it - ConditionalCheckFailedException

                    result = self.conn.get_records(iterator, limit=self.GET_RECORDS_LIMIT)
                    if len(self.uncompleted_ids) < self.MAX_UNCOMPLETED_ITEMS:
                        for record in result['Records']:
                            self.uncompleted_ids[shard_id].add(record['SequenceNumber'])
                            latest_item = record['SequenceNumber']
                            self.insertion_queue.put(record)
                        iterator = result['NextShardIterator']
                    else:
                        iterator = self.conn.get_shard_iterator(
                            self.stream_name,
                            shard_id,
                            "AT_SEQUENCE_NUMBER",
                            starting_sequence_number=str(control_record['checkpoint'])
                        )['ShardIterator']

                    # Push metrics to CloudWatch
                    delay = result['MillisBehindLatest']
                    if minute != current_minute():  # push once per minute to CloudWatch.
                        minute = current_minute()
                        cloudwatch.put_metric_data(
                            'Motorway/Kinesis',
                            'MillisecondsBehind',
                            value=delay,
                            unit='Milliseconds',
                            dimensions={
                                'Stream': self.stream_name,
                                'Shard': shard_id
                            }
                        )
                    time.sleep(1)  # recommended pause between fetches from AWS
            except ConditionalCheckFailedException:
                pass  # we're no longer worker for this shard
            except (ProvisionedThroughputExceededException, LimitExceededException, boto.kinesis.exceptions.ProvisionedThroughputExceededException, boto.kinesis.exceptions.LimitExceededException) as e:
                time.sleep(random.randrange(5, self.heartbeat_timeout/2))  # back off for a while

    def connection_parameters(self):
        return {
            'region_name': 'eu-west-1',
            # Add this or use ENV VARS
            # 'aws_access_key_id': '',
            # 'aws_secret_access_key': ''
        }

    def next(self):
        msg = self.insertion_queue.get()
        yield Message(msg['SequenceNumber'], json.loads(msg['Data']), grouping_value=msg['PartitionKey'])

    def success(self, _id):
        for uncompleted_ids in self.uncompleted_ids.values():
            if _id in uncompleted_ids:
                uncompleted_ids.remove(_id)
<commit_msg>stop sending duplicates in KinesisRamp<commit_after>import random
from Queue import Queue
import json
from threading import Thread, Lock, Semaphore
import time
import uuid

import datetime
from boto.dynamodb2.exceptions import ItemNotFound, ConditionalCheckFailedException, \
    ProvisionedThroughputExceededException, LimitExceededException
from boto.dynamodb2.fields import HashKey
from boto.dynamodb2.items import Item
from boto.dynamodb2.table import Table
from boto.dynamodb2.types import STRING
from boto.exception import JSONResponseError
from motorway.messages import Message
from motorway.ramp import Ramp
import boto.kinesis
import boto.ec2.cloudwatch
import boto.kinesis.exceptions
import logging

shard_election_logger = logging.getLogger("motorway.contrib.amazon_kinesis.shard_election")

class KinesisRamp(Ramp):
    stream_name = None
    heartbeat_timeout = 30  # Wait 10 seconds for a heartbeat update, or kill it
    MAX_UNCOMPLETED_ITEMS = 3000
    GET_RECORDS_LIMIT = 1000

    def __init__(self, shard_threads_enabled=True, **kwargs):
        super(KinesisRamp, self).__init__(**kwargs)
        self.conn = boto.kinesis.connect_to_region(**self.connection_parameters())
        assert self.stream_name, "Please define attribute stream_name on your KinesisRamp"

        control_table_name = 'pipeline-control-%s' % self.stream_name

        self.worker_id = str(uuid.uuid4())
        self.semaphore = Semaphore()
        self.uncompleted_ids = {}

        if shard_threads_enabled:
            dynamodb_connection = boto.dynamodb2.connect_to_region(**self.connection_parameters())

            table_config = dict(
                connection=dynamodb_connection,
                schema=[
                    HashKey('shard_id', data_type=STRING),
                ],
                throughput={
                    'read': 10,
                    'write': 10,
                }
            )

            try:
                dynamodb_connection.describe_table(control_table_name)
            except JSONResponseError:
                Table.create(
                    control_table_name,
                    **table_config
                )

            self.control_table = Table(
                control_table_name,
                **table_config
            )

            shards = self.conn.describe_stream(self.stream_name)['StreamDescription']['Shards']
            threads = []
            self.insertion_queue = Queue()
            for i, shard in enumerate(shards):
                self.uncompleted_ids[shard['ShardId']] = set()
                t = Thread(target=self.process_shard, name="%s-%s" % (self.__class__.__name__, i), args=(shard['ShardId'], ))
                threads.append(t)
                t.start()

    def claim_shard(self, shard_id):
        shard_election_logger.info("Claiming shard %s" % shard_id)
        control_record = self.control_table.get_item(shard_id=shard_id)
        control_record['worker_id'] = self.worker_id
        control_record['heartbeat'] = 0
        try:
            control_record.save()
        except ConditionalCheckFailedException:  # Someone else edited the record
            shard_election_logger.debug("Failed to claim %s to %s" % (shard_id, self.worker_id))
            return False
        return True

    def can_claim_shard(self, shard_id):
        control_record = self.control_table.get_item(shard_id=shard_id)
        heartbeat = control_record['heartbeat']
        time.sleep(self.heartbeat_timeout)
        if heartbeat == self.control_table.get_item(shard_id=shard_id)['heartbeat']:  # TODO: check worker_id as well
            shard_election_logger.debug("Heartbeat remained unchanged for defined time, taking over")
            return True
        else:
            shard_election_logger.debug("Heartbeat changed, continue sleeping")
        # Balance, if possible
        active_workers = {
            self.worker_id: True
        }
        heartbeats = {
            self.worker_id: 0
        }

        shards = list(self.control_table.scan())
        for shard in shards:  # Update active worker cache
            if shard['worker_id'] not in active_workers:
                heartbeats[shard['worker_id']] = shard['heartbeat']
            if heartbeats[shard['worker_id']] == shard['heartbeat']:
                active_workers[shard['worker_id']] = False
            else:
                active_workers[shard['worker_id']] = True
        number_of_active_workers = sum([1 for is_active in active_workers.values() if is_active])
        number_of_shards = len(shards)
        optimal_number_of_shards_per_worker = number_of_shards / number_of_active_workers
        workers = set([shard['worker_id'] for shard in shards])
        shards_per_worker = {worker: sum([1 for shard in shards if shard['worker_id'] == worker]) for worker in workers}
        for shard in shards:
            if shards_per_worker.get(shard['worker_id'], 0) > optimal_number_of_shards_per_worker:
                if shards_per_worker.get(self.worker_id, 0) < optimal_number_of_shards_per_worker:
                    if shard['shard_id'] == shard_id:
                        shard_election_logger.debug("Taking over %s from %s" % (shard_id, shard['worker_id']))
                        return True
        return False

    def process_shard(self, shard_id):
        while True:
            try:
                try:
                    while True:
                        with self.semaphore:
                            if self.can_claim_shard(shard_id):
                                if self.claim_shard(shard_id):
                                    break
                        time.sleep(random.randrange(2, 15))
                except ItemNotFound:
                    control_record = Item(self.control_table, data={
                        'shard_id': shard_id,
                        'checkpoint': 0,
                        'worker_id': self.worker_id,
                        'heartbeat': 0,
                    })
                    control_record.save()

                control_record = self.control_table.get_item(shard_id=shard_id)
                if control_record['checkpoint'] > 0:
                    iterator = self.conn.get_shard_iterator(
                        self.stream_name,
                        shard_id,
                        "AT_SEQUENCE_NUMBER",
                        starting_sequence_number=str(control_record['checkpoint'])
                    )['ShardIterator']
                else:
                    iterator = self.conn.get_shard_iterator(
                        self.stream_name,
                        shard_id,
                        "LATEST",
                    )['ShardIterator']

                cloudwatch = boto.ec2.cloudwatch.connect_to_region(**self.connection_parameters())
                current_minute = lambda: datetime.datetime.now().minute
                minute = None
                latest_item = None
                while True:
                    control_record = self.control_table.get_item(shard_id=shard_id)
                    if not control_record['worker_id'] == self.worker_id:
                        shard_election_logger.info("Lost shard %s, going back to standby" % shard_id)
                        break
                    control_record['heartbeat'] += 1
                    if len(self.uncompleted_ids[shard_id]):  # Get the "youngest" uncompleted sequence number
                        control_record['checkpoint'] = min(self.uncompleted_ids[shard_id])
                    elif latest_item:
                        control_record['checkpoint'] = latest_item

                    control_record.save()  # Will fail if someone else modified it - ConditionalCheckFailedException

                    result = self.conn.get_records(iterator, limit=self.GET_RECORDS_LIMIT)
                    if len(self.uncompleted_ids) < self.MAX_UNCOMPLETED_ITEMS:
                        for record in result['Records']:
                            self.uncompleted_ids[shard_id].add(record['SequenceNumber'])
                            latest_item = record['SequenceNumber']
                            self.insertion_queue.put(record)
                        iterator = result['NextShardIterator']
                    else:
                        next_iterator_number = latest_item if latest_item else str(control_record['checkpoint'])
                        iterator = self.conn.get_shard_iterator(
                            self.stream_name,
                            shard_id,
                            "AT_SEQUENCE_NUMBER",
                            starting_sequence_number=next_iterator_number
                        )['ShardIterator']

                    # Push metrics to CloudWatch
                    delay = result['MillisBehindLatest']
                    if minute != current_minute():  # push once per minute to CloudWatch.
                        minute = current_minute()
                        cloudwatch.put_metric_data(
                            'Motorway/Kinesis',
                            'MillisecondsBehind',
                            value=delay,
                            unit='Milliseconds',
                            dimensions={
                                'Stream': self.stream_name,
                                'Shard': shard_id
                            }
                        )
                    time.sleep(1)  # recommended pause between fetches from AWS
            except ConditionalCheckFailedException:
                pass  # we're no longer worker for this shard
            except (ProvisionedThroughputExceededException, LimitExceededException, boto.kinesis.exceptions.ProvisionedThroughputExceededException, boto.kinesis.exceptions.LimitExceededException) as e:
                time.sleep(random.randrange(5, self.heartbeat_timeout/2))  # back off for a while

    def connection_parameters(self):
        return {
            'region_name': 'eu-west-1',
            # Add this or use ENV VARS
            # 'aws_access_key_id': '',
            # 'aws_secret_access_key': ''
        }

    def next(self):
        msg = self.insertion_queue.get()
        yield Message(msg['SequenceNumber'], json.loads(msg['Data']), grouping_value=msg['PartitionKey'])

    def success(self, _id):
        for uncompleted_ids in self.uncompleted_ids.values():
            if _id in uncompleted_ids:
                uncompleted_ids.remove(_id)
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
##############################################################################
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see http://www.gnu.org/licenses/.
#
##############################################################################
from openerp import models, fields, api, exceptions, _

class MrpRepair(models.Model):
    _inherit = 'mrp.repair'

    analytic_account = fields.Many2one(
        'account.analytic.account', domain=[('type', '!=', 'view')],
        string='Analytic Account')

    @api.multi
    def create_repair_cost(self):
        analytic_line_obj = self.env['account.analytic.line']
        for record in self:
            if not record.analytic_account:
                continue
            lines = record.analytic_account.line_ids.filtered(
                lambda x: x.is_repair_cost and x.amount != 0)
            lines.unlink()
            for line in record.fees_lines.filtered('load_cost'):
                vals = record._catch_repair_line_information_for_analytic(line)
                if vals:
                    analytic_line_obj.create(vals)
            for line in record.operations.filtered('load_cost'):
                vals = record._catch_repair_line_information_for_analytic(line)
                if vals:
                    analytic_line_obj.create(vals)

    @api.one
    @api.model
    def action_repair_end(self):
        result = super(MrpRepair, self).action_repair_end()
        self.create_repair_cost()
        return result

    def _catch_repair_line_information_for_analytic(self, line):
        analytic_line_obj = self.env['account.analytic.line']
        journal = self.env.ref('mrp.analytic_journal_repair', False)
        if not journal:
            raise exceptions.Warning(_('Error!: Repair journal not found'))
        name = self.name
        if line.product_id.default_code:
            name += ' - ' + line.product_id.default_code
        categ_id = line.product_id.categ_id
        general_account = (line.product_id.property_account_income or
                           categ_id.property_account_income_categ or False)
        amount = line.product_id.standard_price * line.product_uom_qty * -1
        if not amount:
            return False
        vals = {'name': name,
                'user_id': line.user_id.id,
                'date': analytic_line_obj._get_default_date(),
                'product_id': line.product_id.id,
                'unit_amount': line.product_uom_qty,
                'product_uom_id': line.product_uom.id,
                'amount': amount,
                'journal_id': journal.id,
                'account_id': self.analytic_account.id,
                'is_repair_cost': True,
                'general_account_id': general_account.id
                }
        return vals

    @api.one
    @api.model
    def action_invoice_create(self, group=False):
        res = super(MrpRepair, self).action_invoice_create(group=group)
        for line in self.fees_lines:
            if line.invoice_line_id and self.analytic_account:
                line.invoice_line_id.write({'account_analytic_id':
                                            self.analytic_account.id})
        for line in self.operations:
            if line.invoice_line_id and self.analytic_account:
                line.invoice_line_id.write({'account_analytic_id':
                                            self.analytic_account.id})
        return res

class MrpRepairLine(models.Model):
    _inherit = 'mrp.repair.line'

    user_id = fields.Many2one('res.users', string='User', required=True,
                              default=lambda self: self.env.user)
    load_cost = fields.Boolean(string='Load Cost', default=True)

class MrpRepairFee(models.Model):
    _inherit = 'mrp.repair.fee'

    user_id = fields.Many2one('res.users', string='User', required=True,
                              default=lambda self: self.env.user)
    load_cost = fields.Boolean(string='Load Cost', default=True)
<commit_msg>[IMP] mrp_repair_analytic: don't load in costs operation lines to remove<commit_after># -*- encoding: utf-8 -*-
##############################################################################
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see http://www.gnu.org/licenses/.
#
##############################################################################
from openerp import models, fields, api, exceptions, _

class MrpRepair(models.Model):
    _inherit = 'mrp.repair'

    analytic_account = fields.Many2one(
        'account.analytic.account', domain=[('type', '!=', 'view')],
        string='Analytic Account')

    @api.multi
    def create_repair_cost(self):
        analytic_line_obj = self.env['account.analytic.line']
        for record in self:
            if not record.analytic_account:
                continue
            lines = record.analytic_account.line_ids.filtered(
                lambda x: x.is_repair_cost and x.amount != 0)
            lines.unlink()
            for line in record.fees_lines.filtered('load_cost'):
                vals = record._catch_repair_line_information_for_analytic(line)
                if vals:
                    analytic_line_obj.create(vals)
            for line in record.operations.filtered(
                    lambda x: x.load_cost and x.type == 'add'):
                vals = record._catch_repair_line_information_for_analytic(line)
                if vals:
                    analytic_line_obj.create(vals)

    @api.one
    @api.model
    def action_repair_end(self):
        result = super(MrpRepair, self).action_repair_end()
        self.create_repair_cost()
        return result

    def _catch_repair_line_information_for_analytic(self, line):
        analytic_line_obj = self.env['account.analytic.line']
        journal = self.env.ref('mrp.analytic_journal_repair', False)
        if not journal:
            raise exceptions.Warning(_('Error!: Repair journal not found'))
        name = self.name
        if line.product_id.default_code:
            name += ' - ' + line.product_id.default_code
        categ_id = line.product_id.categ_id
        general_account = (line.product_id.property_account_income or
                           categ_id.property_account_income_categ or False)
        amount = line.product_id.standard_price * line.product_uom_qty * -1
        if not amount:
            return False
        vals = {'name': name,
                'user_id': line.user_id.id,
                'date': analytic_line_obj._get_default_date(),
                'product_id': line.product_id.id,
                'unit_amount': line.product_uom_qty,
                'product_uom_id': line.product_uom.id,
                'amount': amount,
                'journal_id': journal.id,
                'account_id': self.analytic_account.id,
                'is_repair_cost': True,
                'general_account_id': general_account.id
                }
        return vals

    @api.one
    @api.model
    def action_invoice_create(self, group=False):
        res = super(MrpRepair, self).action_invoice_create(group=group)
        for line in self.fees_lines:
            if line.invoice_line_id and self.analytic_account:
                line.invoice_line_id.write({'account_analytic_id':
                                            self.analytic_account.id})
        for line in self.operations:
            if line.invoice_line_id and self.analytic_account:
                line.invoice_line_id.write({'account_analytic_id':
                                            self.analytic_account.id})
        return res

class MrpRepairLine(models.Model):
    _inherit = 'mrp.repair.line'

    user_id = fields.Many2one('res.users', string='User', required=True,
                              default=lambda self: self.env.user)
    load_cost = fields.Boolean(string='Load Cost', default=True)

class MrpRepairFee(models.Model):
    _inherit = 'mrp.repair.fee'

    user_id = fields.Many2one('res.users', string='User', required=True,
                              default=lambda self: self.env.user)
    load_cost = fields.Boolean(string='Load Cost', default=True)
<|endoftext|>


<commit_before>"""Attempt to generate templates for module reference with Sphinx

XXX - we exclude extension modules

To include extension modules, first identify them as valid in the
``_uri2path`` method, then handle them in the ``_parse_module`` script.

We get functions and classes by parsing the text of .py files.
Alternatively we could import the modules for discovery, and we'd have
to do that for extension modules.  This would involve changing the
``_parse_module`` method to work via import and introspection, and
might involve changing ``discover_modules`` (which determines which
files are modules, and therefore which module URIs will be passed to
``_parse_module``).

NOTE: this is a modified version of a script originally shipped with the
PyMVPA project, which we've adapted for NIPY use.  PyMVPA is an MIT-licensed
project."""

from __future__ import print_function

# Stdlib imports
import ast
import os
import re

class Obj(object):
    '''Namespace to hold arbitrary information.'''
    def __init__(self, **kwargs):
        for k, v in kwargs.items():
            setattr(self, k, v)

class FuncClsScanner(ast.NodeVisitor):
    """Scan a module for top-level functions and classes.

    Skips objects with an @undoc decorator, or a name starting with '_'.
    """
    def __init__(self):
        ast.NodeVisitor.__init__(self)
        self.classes = []
        self.classes_seen = set()
        self.functions = []

    @staticmethod
    def has_undoc_decorator(node):
        return any(isinstance(d, ast.Name) and d.id == 'undoc' \
                                for d in node.decorator_list)

    def visit_If(self, node):
        if isinstance(node.test, ast.Compare) \
                and isinstance(node.test.left, ast.Name) \
                and node.test.left.id == '__name__':
            return   # Ignore classes defined in "if __name__ == '__main__':"

        self.generic_visit(node)

    def visit_FunctionDef(self, node):
        if not (node.name.startswith('_') or self.has_undoc_decorator(node)) \
                and node.name not in self.functions:
            self.functions.append(node.name)

    def visit_ClassDef(self, node):
        if not (node.name.startswith('_') or self.has_undoc_decorator(node)) \
                and node.name not in self.classes_seen:
            cls = Obj(name=node.name)
            cls.has_init = any(isinstance(n, ast.FunctionDef) and \
                                n.name=='__init__' for n in node.body)
            self.classes.append(cls)
            self.classes_seen.add(node.name)

    def scan(self, mod):
        self.visit(mod)
        return self.functions, self.classes

# Functions and classes
class ApiDocWriter(object):
    ''' Class for automatic detection and parsing of API docs
    to Sphinx-parsable reST format'''

    # only separating first two levels
    rst_section_levels = ['*', '=', '-', '~', '^']

    def __init__(self,
                 package_name,
                 rst_extension='.rst',
                 package_skip_patterns=None,
                 module_skip_patterns=None,
                 ):
        ''' Initialize package for parsing

        Parameters
        ----------
        package_name : string
            Name of the top-level package.  *package_name* must be the
            name of an importable package
        rst_extension : string, optional
            Extension for reST files, default '.rst'
        package_skip_patterns : None or sequence of {strings, regexps}
            Sequence of strings giving URIs of packages to be excluded
            Operates on the package path, starting at (including) the
            first dot in the package path, after *package_name* - so,
            if *package_name* is ``sphinx``, then ``sphinx.util`` will
            result in ``.util`` being passed for earching by these
            regexps.  If is None, gives default. Default is:
            ['\.tests$']
        module_skip_patterns : None or sequence
            Sequence of strings giving URIs of modules to be excluded
            Operates on the module name including preceding URI path,
            back to the first dot after *package_name*.  For example
            ``sphinx.util.console`` results in the string to search of
            ``.util.console``
            If is None, gives default. Default is:
            ['\.setup$', '\._']
        '''
        if package_skip_patterns is None:
            package_skip_patterns = ['\\.tests$']
        if module_skip_patterns is None:
            module_skip_patterns = ['\\.setup$', '\\._']
        self.package_name = package_name
        self.rst_extension = rst_extension
        self.package_skip_patterns = package_skip_patterns
        self.module_skip_patterns = module_skip_patterns

    def get_package_name(self):
        return self._package_name

    def set_package_name(self, package_name):
        ''' Set package_name

        >>> docwriter = ApiDocWriter('sphinx')
        >>> import sphinx
        >>> docwriter.root_path == sphinx.__path__[0]
        True
        >>> docwriter.package_name = 'docutils'
        >>> import docutils
        >>> docwriter.root_path == docutils.__path__[0]
        True
        '''
        # It's also possible to imagine caching the module parsing here
        self._package_name = package_name
        self.root_module = __import__(package_name)
        self.root_path = self.root_module.__path__[0]
        self.written_modules = None

    package_name = property(get_package_name, set_package_name, None,
                            'get/set package_name')

    def _uri2path(self, uri):
        ''' Convert uri to absolute filepath

        Parameters
        ----------
        uri : string
            URI of python module to return path for

        Returns
        -------
        path : None or string
            Returns None if there is no valid path for this URI
            Otherwise returns absolute file system path for URI

        Examples
        --------
        >>> docwriter = ApiDocWriter('sphinx')
        >>> import sphinx
        >>> modpath = sphinx.__path__[0]
        >>> res = docwriter._uri2path('sphinx.builder')
        >>> res == os.path.join(modpath, 'builder.py')
        True
        >>> res = docwriter._uri2path('sphinx')
        >>> res == os.path.join(modpath, '__init__.py')
        True
        >>> docwriter._uri2path('sphinx.does_not_exist')

        '''
        if uri == self.package_name:
            return os.path.join(self.root_path, '__init__.py')
        path = uri.replace('.', os.path.sep)
        path = path.replace(self.package_name + os.path.sep, '')
        path = os.path.join(self.root_path, path)
        # XXX maybe check for extensions as well?
        if os.path.exists(path + '.py'): # file
            path += '.py'
        elif os.path.exists(os.path.join(path, '__init__.py')):
            path = os.path.join(path, '__init__.py')
        else:
            return None
        return path

    def _path2uri(self, dirpath):
        ''' Convert directory path to uri '''
        relpath = dirpath.replace(self.root_path, self.package_name)
        if relpath.startswith(os.path.sep):
            relpath = relpath[1:]
        return relpath.replace(os.path.sep, '.')

    def _parse_module(self, uri):
        ''' Parse module defined in *uri* '''
        filename = self._uri2path(uri)
        if filename is None:
            # nothing that we could handle here.
            return ([],[])
        with open(filename, 'rb') as f:
            mod = ast.parse(f.read())
        return FuncClsScanner().scan(mod)

    def generate_api_doc(self, uri):
        '''Make autodoc documentation template string for a module

        Parameters
        ----------
        uri : string
            python location of module - e.g 'sphinx.builder'

        Returns
        -------
        S : string
            Contents of API doc
        '''
        # get the names of all classes and functions
        functions, classes = self._parse_module(uri)
        if not len(functions) and not len(classes):
            print ('WARNING: Empty -', uri)  # dbg
            return ''

        # Make a shorter version of the uri that omits the package name for
        # titles
        uri_short = re.sub(r'^%s\.' % self.package_name,'',uri)

        ad = '.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n'

        # Set the chapter title to read 'Module:' for all modules except for the
        # main packages
        if '.' in uri:
            chap_title = 'Module: :mod:`' + uri_short + '`'
        else:
            chap_title = ':mod:`' + uri_short + '`'
        ad += chap_title + '\n' + self.rst_section_levels[1] * len(chap_title)

        ad += '\n.. automodule:: ' + uri + '\n'
        ad += '\n.. currentmodule:: ' + uri + '\n'

        if classes:
            subhead = str(len(classes)) + (' Classes' if len(classes) > 1 else ' Class')
            ad += '\n'+ subhead + '\n' + \
                  self.rst_section_levels[2] * len(subhead) + '\n'

        for c in classes:
            ad += '\n.. autoclass:: ' + c.name + '\n'
            # must NOT exclude from index to keep cross-refs working
            ad += '  :members:\n' \
                  '  :show-inheritance:\n'
            if c.has_init:
                  ad += '\n  .. automethod:: __init__\n'

        if functions:
            subhead = str(len(functions)) + (' Functions' if len(functions) > 1 else ' Function')
            ad += '\n'+ subhead + '\n' + \
                  self.rst_section_levels[2] * len(subhead) + '\n'
        for f in functions:
            # must NOT exclude from index to keep cross-refs working
            ad += '\n.. autofunction:: ' + uri + '.' + f + '\n\n'
        return ad

    def _survives_exclude(self, matchstr, match_type):
        ''' Returns True if *matchstr* does not match patterns

        ``self.package_name`` removed from front of string if present

        Examples
        --------
        >>> dw = ApiDocWriter('sphinx')
        >>> dw._survives_exclude('sphinx.okpkg', 'package')
        True
        >>> dw.package_skip_patterns.append('^\\.badpkg$')
        >>> dw._survives_exclude('sphinx.badpkg', 'package')
        False
        >>> dw._survives_exclude('sphinx.badpkg', 'module')
        True
        >>> dw._survives_exclude('sphinx.badmod', 'module')
        True
        >>> dw.module_skip_patterns.append('^\\.badmod$')
        >>> dw._survives_exclude('sphinx.badmod', 'module')
        False
        '''
        if match_type == 'module':
            patterns = self.module_skip_patterns
        elif match_type == 'package':
            patterns = self.package_skip_patterns
        else:
            raise ValueError('Cannot interpret match type "%s"'
                             % match_type)
        # Match to URI without package name
        L = len(self.package_name)
        if matchstr[:L] == self.package_name:
            matchstr = matchstr[L:]
        for pat in patterns:
            try:
                pat.search
            except AttributeError:
                pat = re.compile(pat)
            if pat.search(matchstr):
                return False
        return True

    def discover_modules(self):
        ''' Return module sequence discovered from ``self.package_name``

        Parameters
        ----------
        None

        Returns
        -------
        mods : sequence
            Sequence of module names within ``self.package_name``

        Examples
        --------
        >>> dw = ApiDocWriter('sphinx')
        >>> mods = dw.discover_modules()
        >>> 'sphinx.util' in mods
        True
        >>> dw.package_skip_patterns.append('\.util$')
        >>> 'sphinx.util' in dw.discover_modules()
        False
        >>>
        '''
        modules = [self.package_name]
        # raw directory parsing
        for dirpath, dirnames, filenames in os.walk(self.root_path):
            # Check directory names for packages
            root_uri = self._path2uri(os.path.join(self.root_path,
                                                   dirpath))
            for dirname in dirnames[:]: # copy list - we modify inplace
                package_uri = '.'.join((root_uri, dirname))
                if (self._uri2path(package_uri) and
                    self._survives_exclude(package_uri, 'package')):
                    modules.append(package_uri)
                else:
                    dirnames.remove(dirname)
            # Check filenames for modules
            for filename in filenames:
                module_name = filename[:-3]
                module_uri = '.'.join((root_uri, module_name))
                if (self._uri2path(module_uri) and
                    self._survives_exclude(module_uri, 'module')):
                    modules.append(module_uri)
        return sorted(modules)

    def write_modules_api(self, modules,outdir):
        # write the list
        written_modules = []
        for m in modules:
            api_str = self.generate_api_doc(m)
            if not api_str:
                continue
            # write out to file
            outfile = os.path.join(outdir,
                                   m + self.rst_extension)
            fileobj = open(outfile, 'wt')
            fileobj.write(api_str)
            fileobj.close()
            written_modules.append(m)
        self.written_modules = written_modules

    def write_api_docs(self, outdir):
        """Generate API reST files.

        Parameters
        ----------
        outdir : string
            Directory name in which to store files
            We create automatic filenames for each module

        Returns
        -------
        None

        Notes
        -----
        Sets self.written_modules to list of written modules
        """
        if not os.path.exists(outdir):
            os.mkdir(outdir)
        # compose list of modules
        modules = self.discover_modules()
        self.write_modules_api(modules,outdir)

    def write_index(self, outdir, path='gen.rst', relative_to=None):
        """Make a reST API index file from written files

        Parameters
        ----------
        outdir : string
            Directory to which to write generated index file
        path : string
            Filename to write index to
        relative_to : string
            path to which written filenames are relative.  This
            component of the written file path will be removed from
            outdir, in the generated index.  Default is None, meaning,
            leave path as it is.
        """
        if self.written_modules is None:
            raise ValueError('No modules written')
        # Get full filename path
        path = os.path.join(outdir, path)
        # Path written into index is relative to rootpath
        if relative_to is not None:
            relpath = outdir.replace(relative_to + os.path.sep, '')
        else:
            relpath = outdir
        idx = open(path,'wt')
        w = idx.write
        w('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n')
        w('.. toctree::\n\n')
        for f in self.written_modules:
            w('   %s\n' % os.path.join(relpath,f))
        idx.close()
<commit_msg>Don't warn on empty modules when scanning API<commit_after>"""Attempt to generate templates for module reference with Sphinx

XXX - we exclude extension modules

To include extension modules, first identify them as valid in the
``_uri2path`` method, then handle them in the ``_parse_module`` script.

We get functions and classes by parsing the text of .py files.
Alternatively we could import the modules for discovery, and we'd have
to do that for extension modules.  This would involve changing the
``_parse_module`` method to work via import and introspection, and
might involve changing ``discover_modules`` (which determines which
files are modules, and therefore which module URIs will be passed to
``_parse_module``).

NOTE: this is a modified version of a script originally shipped with the
PyMVPA project, which we've adapted for NIPY use.  PyMVPA is an MIT-licensed
project."""

from __future__ import print_function

# Stdlib imports
import ast
import os
import re

class Obj(object):
    '''Namespace to hold arbitrary information.'''
    def __init__(self, **kwargs):
        for k, v in kwargs.items():
            setattr(self, k, v)

class FuncClsScanner(ast.NodeVisitor):
    """Scan a module for top-level functions and classes.

    Skips objects with an @undoc decorator, or a name starting with '_'.
    """
    def __init__(self):
        ast.NodeVisitor.__init__(self)
        self.classes = []
        self.classes_seen = set()
        self.functions = []

    @staticmethod
    def has_undoc_decorator(node):
        return any(isinstance(d, ast.Name) and d.id == 'undoc' \
                                for d in node.decorator_list)

    def visit_If(self, node):
        if isinstance(node.test, ast.Compare) \
                and isinstance(node.test.left, ast.Name) \
                and node.test.left.id == '__name__':
            return   # Ignore classes defined in "if __name__ == '__main__':"

        self.generic_visit(node)

    def visit_FunctionDef(self, node):
        if not (node.name.startswith('_') or self.has_undoc_decorator(node)) \
                and node.name not in self.functions:
            self.functions.append(node.name)

    def visit_ClassDef(self, node):
        if not (node.name.startswith('_') or self.has_undoc_decorator(node)) \
                and node.name not in self.classes_seen:
            cls = Obj(name=node.name)
            cls.has_init = any(isinstance(n, ast.FunctionDef) and \
                                n.name=='__init__' for n in node.body)
            self.classes.append(cls)
            self.classes_seen.add(node.name)

    def scan(self, mod):
        self.visit(mod)
        return self.functions, self.classes

# Functions and classes
class ApiDocWriter(object):
    ''' Class for automatic detection and parsing of API docs
    to Sphinx-parsable reST format'''

    # only separating first two levels
    rst_section_levels = ['*', '=', '-', '~', '^']

    def __init__(self,
                 package_name,
                 rst_extension='.rst',
                 package_skip_patterns=None,
                 module_skip_patterns=None,
                 ):
        ''' Initialize package for parsing

        Parameters
        ----------
        package_name : string
            Name of the top-level package.  *package_name* must be the
            name of an importable package
        rst_extension : string, optional
            Extension for reST files, default '.rst'
        package_skip_patterns : None or sequence of {strings, regexps}
            Sequence of strings giving URIs of packages to be excluded
            Operates on the package path, starting at (including) the
            first dot in the package path, after *package_name* - so,
            if *package_name* is ``sphinx``, then ``sphinx.util`` will
            result in ``.util`` being passed for earching by these
            regexps.  If is None, gives default. Default is:
            ['\.tests$']
        module_skip_patterns : None or sequence
            Sequence of strings giving URIs of modules to be excluded
            Operates on the module name including preceding URI path,
            back to the first dot after *package_name*.  For example
            ``sphinx.util.console`` results in the string to search of
            ``.util.console``
            If is None, gives default. Default is:
            ['\.setup$', '\._']
        '''
        if package_skip_patterns is None:
            package_skip_patterns = ['\\.tests$']
        if module_skip_patterns is None:
            module_skip_patterns = ['\\.setup$', '\\._']
        self.package_name = package_name
        self.rst_extension = rst_extension
        self.package_skip_patterns = package_skip_patterns
        self.module_skip_patterns = module_skip_patterns

    def get_package_name(self):
        return self._package_name

    def set_package_name(self, package_name):
        ''' Set package_name

        >>> docwriter = ApiDocWriter('sphinx')
        >>> import sphinx
        >>> docwriter.root_path == sphinx.__path__[0]
        True
        >>> docwriter.package_name = 'docutils'
        >>> import docutils
        >>> docwriter.root_path == docutils.__path__[0]
        True
        '''
        # It's also possible to imagine caching the module parsing here
        self._package_name = package_name
        self.root_module = __import__(package_name)
        self.root_path = self.root_module.__path__[0]
        self.written_modules = None

    package_name = property(get_package_name, set_package_name, None,
                            'get/set package_name')

    def _uri2path(self, uri):
        ''' Convert uri to absolute filepath

        Parameters
        ----------
        uri : string
            URI of python module to return path for

        Returns
        -------
        path : None or string
            Returns None if there is no valid path for this URI
            Otherwise returns absolute file system path for URI

        Examples
        --------
        >>> docwriter = ApiDocWriter('sphinx')
        >>> import sphinx
        >>> modpath = sphinx.__path__[0]
        >>> res = docwriter._uri2path('sphinx.builder')
        >>> res == os.path.join(modpath, 'builder.py')
        True
        >>> res = docwriter._uri2path('sphinx')
        >>> res == os.path.join(modpath, '__init__.py')
        True
        >>> docwriter._uri2path('sphinx.does_not_exist')

        '''
        if uri == self.package_name:
            return os.path.join(self.root_path, '__init__.py')
        path = uri.replace('.', os.path.sep)
        path = path.replace(self.package_name + os.path.sep, '')
        path = os.path.join(self.root_path, path)
        # XXX maybe check for extensions as well?
        if os.path.exists(path + '.py'): # file
            path += '.py'
        elif os.path.exists(os.path.join(path, '__init__.py')):
            path = os.path.join(path, '__init__.py')
        else:
            return None
        return path

    def _path2uri(self, dirpath):
        ''' Convert directory path to uri '''
        relpath = dirpath.replace(self.root_path, self.package_name)
        if relpath.startswith(os.path.sep):
            relpath = relpath[1:]
        return relpath.replace(os.path.sep, '.')

    def _parse_module(self, uri):
        ''' Parse module defined in *uri* '''
        filename = self._uri2path(uri)
        if filename is None:
            # nothing that we could handle here.
            return ([],[])
        with open(filename, 'rb') as f:
            mod = ast.parse(f.read())
        return FuncClsScanner().scan(mod)

    def generate_api_doc(self, uri):
        '''Make autodoc documentation template string for a module

        Parameters
        ----------
        uri : string
            python location of module - e.g 'sphinx.builder'

        Returns
        -------
        S : string
            Contents of API doc
        '''
        # get the names of all classes and functions
        functions, classes = self._parse_module(uri)
        if not len(functions) and not len(classes):
            #print ('WARNING: Empty -', uri)  # dbg
            return ''

        # Make a shorter version of the uri that omits the package name for
        # titles
        uri_short = re.sub(r'^%s\.' % self.package_name,'',uri)

        ad = '.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n'

        # Set the chapter title to read 'Module:' for all modules except for the
        # main packages
        if '.' in uri:
            chap_title = 'Module: :mod:`' + uri_short + '`'
        else:
            chap_title = ':mod:`' + uri_short + '`'
        ad += chap_title + '\n' + self.rst_section_levels[1] * len(chap_title)

        ad += '\n.. automodule:: ' + uri + '\n'
        ad += '\n.. currentmodule:: ' + uri + '\n'

        if classes:
            subhead = str(len(classes)) + (' Classes' if len(classes) > 1 else ' Class')
            ad += '\n'+ subhead + '\n' + \
                  self.rst_section_levels[2] * len(subhead) + '\n'

        for c in classes:
            ad += '\n.. autoclass:: ' + c.name + '\n'
            # must NOT exclude from index to keep cross-refs working
            ad += '  :members:\n' \
                  '  :show-inheritance:\n'
            if c.has_init:
                  ad += '\n  .. automethod:: __init__\n'

        if functions:
            subhead = str(len(functions)) + (' Functions' if len(functions) > 1 else ' Function')
            ad += '\n'+ subhead + '\n' + \
                  self.rst_section_levels[2] * len(subhead) + '\n'
        for f in functions:
            # must NOT exclude from index to keep cross-refs working
            ad += '\n.. autofunction:: ' + uri + '.' + f + '\n\n'
        return ad

    def _survives_exclude(self, matchstr, match_type):
        ''' Returns True if *matchstr* does not match patterns

        ``self.package_name`` removed from front of string if present

        Examples
        --------
        >>> dw = ApiDocWriter('sphinx')
        >>> dw._survives_exclude('sphinx.okpkg', 'package')
        True
        >>> dw.package_skip_patterns.append('^\\.badpkg$')
        >>> dw._survives_exclude('sphinx.badpkg', 'package')
        False
        >>> dw._survives_exclude('sphinx.badpkg', 'module')
        True
        >>> dw._survives_exclude('sphinx.badmod', 'module')
        True
        >>> dw.module_skip_patterns.append('^\\.badmod$')
        >>> dw._survives_exclude('sphinx.badmod', 'module')
        False
        '''
        if match_type == 'module':
            patterns = self.module_skip_patterns
        elif match_type == 'package':
            patterns = self.package_skip_patterns
        else:
            raise ValueError('Cannot interpret match type "%s"'
                             % match_type)
        # Match to URI without package name
        L = len(self.package_name)
        if matchstr[:L] == self.package_name:
            matchstr = matchstr[L:]
        for pat in patterns:
            try:
                pat.search
            except AttributeError:
                pat = re.compile(pat)
            if pat.search(matchstr):
                return False
        return True

    def discover_modules(self):
        ''' Return module sequence discovered from ``self.package_name``

        Parameters
        ----------
        None

        Returns
        -------
        mods : sequence
            Sequence of module names within ``self.package_name``

        Examples
        --------
        >>> dw = ApiDocWriter('sphinx')
        >>> mods = dw.discover_modules()
        >>> 'sphinx.util' in mods
        True
        >>> dw.package_skip_patterns.append('\.util$')
        >>> 'sphinx.util' in dw.discover_modules()
        False
        >>>
        '''
        modules = [self.package_name]
        # raw directory parsing
        for dirpath, dirnames, filenames in os.walk(self.root_path):
            # Check directory names for packages
            root_uri = self._path2uri(os.path.join(self.root_path,
                                                   dirpath))
            for dirname in dirnames[:]: # copy list - we modify inplace
                package_uri = '.'.join((root_uri, dirname))
                if (self._uri2path(package_uri) and
                    self._survives_exclude(package_uri, 'package')):
                    modules.append(package_uri)
                else:
                    dirnames.remove(dirname)
            # Check filenames for modules
            for filename in filenames:
                module_name = filename[:-3]
                module_uri = '.'.join((root_uri, module_name))
                if (self._uri2path(module_uri) and
                    self._survives_exclude(module_uri, 'module')):
                    modules.append(module_uri)
        return sorted(modules)

    def write_modules_api(self, modules,outdir):
        # write the list
        written_modules = []
        for m in modules:
            api_str = self.generate_api_doc(m)
            if not api_str:
                continue
            # write out to file
            outfile = os.path.join(outdir,
                                   m + self.rst_extension)
            fileobj = open(outfile, 'wt')
            fileobj.write(api_str)
            fileobj.close()
            written_modules.append(m)
        self.written_modules = written_modules

    def write_api_docs(self, outdir):
        """Generate API reST files.

        Parameters
        ----------
        outdir : string
            Directory name in which to store files
            We create automatic filenames for each module

        Returns
        -------
        None

        Notes
        -----
        Sets self.written_modules to list of written modules
        """
        if not os.path.exists(outdir):
            os.mkdir(outdir)
        # compose list of modules
        modules = self.discover_modules()
        self.write_modules_api(modules,outdir)

    def write_index(self, outdir, path='gen.rst', relative_to=None):
        """Make a reST API index file from written files

        Parameters
        ----------
        outdir : string
            Directory to which to write generated index file
        path : string
            Filename to write index to
        relative_to : string
            path to which written filenames are relative.  This
            component of the written file path will be removed from
            outdir, in the generated index.  Default is None, meaning,
            leave path as it is.
        """
        if self.written_modules is None:
            raise ValueError('No modules written')
        # Get full filename path
        path = os.path.join(outdir, path)
        # Path written into index is relative to rootpath
        if relative_to is not None:
            relpath = outdir.replace(relative_to + os.path.sep, '')
        else:
            relpath = outdir
        idx = open(path,'wt')
        w = idx.write
        w('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n')
        w('.. toctree::\n\n')
        for f in self.written_modules:
            w('   %s\n' % os.path.join(relpath,f))
        idx.close()
<|endoftext|>