<commit_before>import re
import json
from random import randint

from plugins.battling.battle import Battle, Pokemon
from plugins.battling.battleLogic import getMove, getLead

class BattleHandler:
    def __init__(self, ws, name):
        self.ws = ws
        self.botName = name
        self.activeBattles = {}

    def respond(self, battle, msg):
        self.ws.send('{room}|{msg}'.format(room = battle, msg = msg))

    def getSpecies(self, details):
        return details.split(',')[0].replace('-*', '')

    def parse(self, battle, msg):
        if not msg: return
        if battle in self.activeBattles and 'init' in msg: return
        print(msg)
        msg = msg.split('|')
        if 'init' == msg[1] and 'battle' == msg[2]:
            self.activeBattles[battle] = Battle(battle)
            self.respond(battle, '/timer')
        elif 'request' == msg[1]:
            # This is where all the battle picking happen
            request = json.loads(msg[2])
            if 'rqid' in request:
                self.activeBattles[battle].rqid = request['rqid']
            sidedata = request['side']
            teamSlot = 1
            for poke in sidedata['pokemon']:
                self.activeBattles[battle].me.updateTeam(
                    Pokemon(self.getSpecies(poke['details']),poke['details'],poke['condition'],poke['active'],
                            poke['stats'],poke['moves'],poke['baseAbility'],poke['item'],poke['canMegaEvo'], teamSlot))
                teamSlot += 1
            if 'active' in request:
                self.activeBattles[battle].myActiveData = request['active']

        elif 'poke' == msg[1]:
            if not self.activeBattles[battle].me.id == msg[2]:
                self.activeBattles[battle].other.updateTeam(
                    Pokemon(self.getSpecies(msg[3]), msg[3], '100/100', False,
                            {'atk':1,'def':1,'spa':1,'spd':1,'spe':1}, ['','','',''], '', '', False, len(self.activeBattles[battle].other.team)+1))
        elif 'player' == msg[1]:
            if len(msg) < 4: return
            if msg[3] == self.botName:
                self.activeBattles[battle].setMe(msg[3], msg[2])
            else:
                self.activeBattles[battle].setOther(msg[3], msg[2])
        elif 'teampreview' == msg[1]:
            poke = getLead(self.activeBattles[battle].me.team, self.activeBattles[battle].other.team)
            self.respond(battle, '/team {nr}|{rqid}'.format(nr = poke, rqid = self.activeBattles[battle].rqid))
        elif 'turn' == msg[1]:
            active = self.activeBattles[battle].me.active
            moves = [m['move'].replace(' ','').lower() for m in self.activeBattles[battle].myActiveData[0]['moves'] if not m['disabled']]
            move = getMove(moves, active, self.activeBattles[battle].other.active)
            self.respond(battle, '/choose move {name}|{rqid}'.format(name = move, rqid = self.activeBattles[battle].rqid))
        elif 'switch' == msg[1]:
            btl = self.activeBattles[battle]
            if msg[2].startswith(btl.me.id):
                btl.me.setActive(btl.me.getPokemon(self.getSpecies(msg[3])))
            else:
                btl.other.setActive(btl.other.getPokemon(self.getSpecies(msg[3])))
        elif msg[1] in ['win', 'tie']:
            if msg[2] == self.botName:
                self.respond(battle, 'O-oh, I won? Sorry :(')
            else:
                self.respond(battle, "It's okay, you're better than me :>")
            self.respond(battle, '/leave')
<commit_msg>If locked into a single move, use that move<commit_after>import re
import json
from random import randint

from plugins.battling.battle import Battle, Pokemon
from plugins.battling.battleLogic import getMove, getLead

class BattleHandler:
    def __init__(self, ws, name):
        self.ws = ws
        self.botName = name
        self.activeBattles = {}

    def respond(self, battle, msg):
        self.ws.send('{room}|{msg}'.format(room = battle, msg = msg))

    def getSpecies(self, details):
        return details.split(',')[0].replace('-*', '')

    def parse(self, battle, msg):
        if not msg: return
        if battle in self.activeBattles and 'init' in msg: return
        print(msg)
        msg = msg.split('|')
        if 'init' == msg[1] and 'battle' == msg[2]:
            self.activeBattles[battle] = Battle(battle)
            self.respond(battle, '/timer')
        elif 'request' == msg[1]:
            # This is where all the battle picking happen
            request = json.loads(msg[2])
            if 'rqid' in request:
                self.activeBattles[battle].rqid = request['rqid']
            sidedata = request['side']
            teamSlot = 1
            for poke in sidedata['pokemon']:
                self.activeBattles[battle].me.updateTeam(
                    Pokemon(self.getSpecies(poke['details']),poke['details'],poke['condition'],poke['active'],
                            poke['stats'],poke['moves'],poke['baseAbility'],poke['item'],poke['canMegaEvo'], teamSlot))
                teamSlot += 1
            if 'active' in request:
                self.activeBattles[battle].myActiveData = request['active']

        elif 'poke' == msg[1]:
            if not self.activeBattles[battle].me.id == msg[2]:
                self.activeBattles[battle].other.updateTeam(
                    Pokemon(self.getSpecies(msg[3]), msg[3], '100/100', False,
                            {'atk':1,'def':1,'spa':1,'spd':1,'spe':1}, ['','','',''], '', '', False, len(self.activeBattles[battle].other.team)+1))
        elif 'player' == msg[1]:
            if len(msg) < 4: return
            if msg[3] == self.botName:
                self.activeBattles[battle].setMe(msg[3], msg[2])
            else:
                self.activeBattles[battle].setOther(msg[3], msg[2])
        elif 'teampreview' == msg[1]:
            poke = getLead(self.activeBattles[battle].me.team, self.activeBattles[battle].other.team)
            self.respond(battle, '/team {nr}|{rqid}'.format(nr = poke, rqid = self.activeBattles[battle].rqid))
        elif 'turn' == msg[1]:
            active = self.activeBattles[battle].me.active
            moveData = self.activeBattles[battle].myActiveData[0]['moves']
            moves = []
            if len(moveData) == 1:
                moves = [moveData[0]['move'].replace(' ','').lower()]
            else:
                moves = [m['move'].replace(' ','').lower() for m in moveData if not m['disabled']]
            move = getMove(moves, active, self.activeBattles[battle].other.active)
            self.respond(battle, '/choose move {name}|{rqid}'.format(name = move, rqid = self.activeBattles[battle].rqid))
        elif 'switch' == msg[1]:
            btl = self.activeBattles[battle]
            if msg[2].startswith(btl.me.id):
                btl.me.setActive(btl.me.getPokemon(self.getSpecies(msg[3])))
            else:
                btl.other.setActive(btl.other.getPokemon(self.getSpecies(msg[3])))
        elif msg[1] in ['win', 'tie']:
            if msg[2] == self.botName:
                self.respond(battle, 'O-oh, I won? Sorry :(')
            else:
                self.respond(battle, "It's okay, you're better than me :>")
            self.respond(battle, '/leave')
<|endoftext|>


<commit_before>import functools
import re

import sublime
import sublime_plugin

from ..lib.scope_data import COMPILED_HEADS
from ..lib import syntax_paths
from ..lib import inhibit_word_completions

__all__ = (
    'SyntaxDefCompletionsListener',
    'PackagedevCommitScopeCompletionCommand'
)

# a list of kinds used to denote the different kinds of completions
KIND_HEADER_BASE = (sublime.KIND_ID_NAMESPACE, 'K', 'Header Key')
KIND_HEADER_DICT = (sublime.KIND_ID_NAMESPACE, 'D', 'Header Dict')
KIND_HEADER_LIST = (sublime.KIND_ID_NAMESPACE, 'L', 'Header List')
KIND_BRANCH = (sublime.KIND_ID_NAVIGATION, 'b', 'Branch Point')
KIND_CONTEXT = (sublime.KIND_ID_KEYWORD, 'c', 'Context')
KIND_FUNCTION = (sublime.KIND_ID_FUNCTION, 'f', 'Function')
KIND_FUNCTION_TRUE = (sublime.KIND_ID_FUNCTION, 'f', 'Function')
KIND_FUNCTION_FALSE = (sublime.KIND_ID_FUNCTION, 'f', 'Function')
KIND_CAPTURUE = (sublime.KIND_ID_FUNCTION, 'c', 'Captures')
KIND_SCOPE = (sublime.KIND_ID_NAMESPACE, 's', 'Scope')
KIND_VARIABLE = (sublime.KIND_ID_VARIABLE, 'v', 'Variable')

PACKAGE_NAME = __package__.split('.')[0]

def status(msg, console=False):
    msg = "[%s] %s" % (PACKAGE_NAME, msg)
    sublime.status_message(msg)
    if console:
        print(msg)

def format_static_completions(templates):

    def format_item(trigger, kind, details):
        if kind in (KIND_HEADER_DICT, KIND_CAPTURUE, KIND_CONTEXT):
            completion_format = sublime.COMPLETION_FORMAT_SNIPPET
            suffix = ":\n  "
        elif kind is KIND_HEADER_LIST:
            completion_format = sublime.COMPLETION_FORMAT_SNIPPET
            suffix = ":\n  - "
        elif kind is KIND_FUNCTION_TRUE:
            completion_format = sublime.COMPLETION_FORMAT_SNIPPET
            suffix = ": ${1:true}"
        elif kind is KIND_FUNCTION_FALSE:
            completion_format = sublime.COMPLETION_FORMAT_SNIPPET
            suffix = ": ${1:false}"
        else:
            completion_format = sublime.COMPLETION_FORMAT_TEXT
            suffix = ": "

        return sublime.CompletionItem(
            trigger=trigger,
            kind=kind,
            details=details,
            completion=trigger + suffix,
            completion_format=completion_format,
        )

    return [format_item(*template) for template in templates]

def format_completions(items, annotation="", kind=sublime.KIND_AMBIGUOUS):
    format_string = "Defined at line {0}"
    return [
        sublime.CompletionItem(
            trigger=trigger,
            annotation=annotation,
            kind=kind,
            details=format_string.format(row) if row is not None else "",
        )
        for trigger, row in items
    ]

class SyntaxDefCompletionsListener(sublime_plugin.ViewEventListener):

    base_completions_root = format_static_completions(templates=(
        # base keys
        ('name', KIND_HEADER_BASE, "The display name of the syntax."),
        ('scope', KIND_HEADER_BASE, "The main scope of the syntax."),
        ('version', KIND_HEADER_BASE, "The sublime-syntax version."),
        ('extends', KIND_HEADER_BASE, "The syntax which is to be extended."),
        ('name', KIND_HEADER_BASE, "The display name of the syntax."),
        ('first_line_match', KIND_HEADER_BASE, "The pattern to identify a file by content."),
        # dict keys
        ('variables', KIND_HEADER_DICT, 'The variables definitions.'),
        ('contexts', KIND_HEADER_DICT, 'The syntax contexts.'),
        # list keys
        ('file_extensions', KIND_HEADER_LIST, "The list of file extensions."),
        ('hidden_extensions', KIND_HEADER_LIST, "The list of hidden file extensions.")
    ))

    base_completions_contexts = format_static_completions(templates=(
        # meta functions
        ('meta_append', KIND_FUNCTION_TRUE, "Add rules to the end of the inherit context."),
        ('meta_content_scope', KIND_FUNCTION, "A scope to apply to the content of a context."),
        ('meta_include_prototype', KIND_FUNCTION_FALSE, "Flag to in-/exclude `prototype`"),
        ('meta_prepend', KIND_FUNCTION_TRUE, "Add rules to the beginning of the inherit context."),
        ('meta_scope', KIND_FUNCTION, "A scope to apply to the full context."),
        ('clear_scopes', KIND_FUNCTION, "Clear meta scopes."),
        # matching tokens
        ('match', KIND_FUNCTION, "Pattern to match tokens."),
        # scoping
        ('scope', KIND_FUNCTION, "The scope to apply if a token matches"),
        ('captures', KIND_CAPTURUE, "Assigns scopes to the capture groups."),
        # contexts
        ('push', KIND_FUNCTION, "Push a context onto the stack."),
        ('set', KIND_FUNCTION, "Set a context onto the stack."),
        ('pop', KIND_FUNCTION_TRUE, 'Pop context(s) from the stack.'),
        ('with_prototype', KIND_FUNCTION, "Rules to prepend to each context."),
        # branching
        ('branch_point', KIND_FUNCTION, "Name of the point to rewind to if a branch fails."),
        ('branch', KIND_FUNCTION, "Push branches onto the stack."),
        ('fail', KIND_FUNCTION, "Fail the current branch."),
        # embedding
        ('embed', KIND_FUNCTION, "A context or syntax to embed."),
        ('embed_scope', KIND_FUNCTION, "A scope to apply to the embedded syntax."),
        ('escape', KIND_FUNCTION, "A pattern to denote the end of the embedded syntax."),
        ('escape_captures', KIND_CAPTURUE, "Assigns scopes to the capture groups."),
        # including
        ('include', KIND_FUNCTION, "Includes a context."),
        ('apply_prototype', KIND_FUNCTION_TRUE, "Apply prototype of included syntax."),
    ))

    # These instance variables are for communicating
    # with our PostCompletionsListener instance.
    base_suffix = None

    @classmethod
    def applies_to_primary_view_only(cls):
        return False

    @classmethod
    def is_applicable(cls, settings):
        return settings.get('syntax') == syntax_paths.SYNTAX_DEF

    @inhibit_word_completions
    def on_query_completions(self, prefix, locations):

        def match_selector(selector, offset=0):
            """Verify scope for each location."""
            return all(self.view.match_selector(point + offset, selector)
                       for point in locations)

        # None of our business
        if not match_selector("- comment - (source.regexp - keyword.other.variable)"):
            return None

        # Scope name completions based on our scope_data database
        if match_selector("meta.expect-scope, meta.scope", -1):
            return self._complete_scope(prefix, locations)

        # Auto-completion for include values using the 'contexts' keys and for
        if match_selector("meta.expect-context-list-or-content"
                          " | meta.context-list-or-content", -1):
            return (self._complete_keyword(prefix, locations)
                    + self._complete_context(prefix, locations))

        # Auto-completion for include values using the 'contexts' keys
        if match_selector("meta.expect-context-list | meta.expect-context"
                          " | meta.include | meta.context-list", -1):
            return self._complete_context(prefix, locations) or None

        # Auto-completion for branch points with 'fail' key
        if match_selector("meta.expect-branch-point-reference"
                          " | meta.branch-point-reference", -1):
            return self._complete_branch_point()

        # Auto-completion for variables in match patterns using 'variables' keys
        if match_selector("keyword.other.variable"):
            return self._complete_variable()

        # Standard completions for unmatched regions
        return self._complete_keyword(prefix, locations)

    def _line_prefix(self, point):
        _, col = self.view.rowcol(point)
        line = self.view.substr(self.view.line(point))
        return line[:col]

    def _complete_context(self, prefix, locations):
        # Verify that we're not looking for an external include
        for point in locations:
            line_prefix = self._line_prefix(point)
            real_prefix = re.search(r"[^,\[ ]*$", line_prefix).group(0)
            if real_prefix.startswith("scope:") or "/" in real_prefix:
                return []  # Don't show any completions here
            elif real_prefix != prefix:
                # print("Unexpected prefix mismatch: {} vs {}".format(real_prefix, prefix))
                return []

        return format_completions(
            [(self.view.substr(r), self.view.rowcol(r.begin())[0] + 1)
             for r in self.view.find_by_selector("entity.name.function.context")],
            annotation="",
            kind=KIND_CONTEXT,
        )

    def _complete_keyword(self, prefix, locations):

        def match_selector(selector, offset=0):
            """Verify scope for each location."""
            return all(self.view.match_selector(point + offset, selector)
                       for point in locations)

        prefixes = set()
        for point in locations:
            # Ensure that we are completing a key name everywhere
            line_prefix = self._line_prefix(point)
            real_prefix = re.sub(r"^ +(- +)*", " ", line_prefix)  # collapse leading whitespace
            prefixes.add(real_prefix)

        if len(prefixes) != 1:
            return None
        else:
            real_prefix = next(iter(prefixes))

        # (Supposedly) all keys start their own line
        match = re.match(r"^(\s*)[\w-]*$", real_prefix)
        if not match:
            return None
        elif not match.group(1):
            return self.base_completions_root
        elif match_selector("meta.block.contexts"):
            return self.base_completions_contexts
        else:
            return None

    def _complete_scope(self, prefix, locations):
        # Determine entire prefix
        prefixes = set()
        for point in locations:
            *_, real_prefix = self._line_prefix(point).rpartition(" ")
            prefixes.add(real_prefix)

        if len(prefixes) > 1:
            return None
        else:
            real_prefix = next(iter(prefixes))

        # Tokenize the current selector
        tokens = real_prefix.split(".")
        if len(tokens) <= 1:
            # No work to be done here, just return the heads
            return COMPILED_HEADS.to_completion()

        base_scope_completion = self._complete_base_scope(tokens[-1])
        # Browse the nodes and their children
        nodes = COMPILED_HEADS
        for i, token in enumerate(tokens[:-1]):
            node = nodes.find(token)
            if not node:
                status("`%s` not found in scope naming conventions" % '.'.join(tokens[:i + 1]))
                break
            nodes = node.children
            if not nodes:
                status("No nodes available in scope naming conventions after `%s`"
                       % '.'.join(tokens[:-1]))
                break
        else:
            # Offer to complete from conventions or base scope
            return nodes.to_completion() + base_scope_completion

        # Since we don't have anything to offer,
        # just complete the base scope appendix/suffix.
        return base_scope_completion

    def _complete_base_scope(self, last_token):
        regions = self.view.find_by_selector("meta.scope string - meta.block")
        if len(regions) != 1:
            status("Warning: Could not determine base scope uniquely", console=True)
            self.base_suffix = None
            return []

        base_scope = self.view.substr(regions[0])
        *_, base_suffix = base_scope.rpartition(".")
        # Only useful when the base scope suffix is not already the last one
        # In this case it is even useful to inhibit other completions completely
        if last_token == base_suffix:
            self.base_suffix = None
            return []

        self.base_suffix = base_suffix
        return format_completions([(base_suffix, None)], "base suffix", KIND_SCOPE)

    def _complete_variable(self):
        return format_completions(
            [(self.view.substr(r), self.view.rowcol(r.begin())[0] + 1)
             for r in self.view.find_by_selector("entity.name.constant")],
            annotation="",
            kind=KIND_VARIABLE,
        )

    def _complete_branch_point(self):
        return format_completions(
            [(self.view.substr(r), self.view.rowcol(r.begin())[0] + 1)
             for r in self.view.find_by_selector("entity.name.label.branch-point")],
            annotation="",
            kind=KIND_BRANCH,
        )

class PackagedevCommitScopeCompletionCommand(sublime_plugin.TextCommand):

    def run(self, edit):
        self.view.run_command("commit_completion")

        # Don't add duplicated dot, if scope is edited in the middle.
        if self.view.substr(self.view.sel()[0].a) == ".":
            return

        # Check if the completed value was the base suffix
        # and don't re-open auto complete in that case.
        listener = sublime_plugin.find_view_event_listener(self.view, SyntaxDefCompletionsListener)
        if listener and listener.base_suffix:
            point = self.view.sel()[0].a
            region = sublime.Region(point - len(listener.base_suffix) - 1, point)
            if self.view.substr(region) == "." + listener.base_suffix:
                return

        # Insert a . and trigger next completion
        self.view.run_command('insert', {'characters': "."})
        self.view.run_command('auto_complete', {'disable_auto_insert': True})
<commit_msg>Syntax: Remove unused import<commit_after>import re

import sublime
import sublime_plugin

from ..lib.scope_data import COMPILED_HEADS
from ..lib import syntax_paths
from ..lib import inhibit_word_completions

__all__ = (
    'SyntaxDefCompletionsListener',
    'PackagedevCommitScopeCompletionCommand'
)

# a list of kinds used to denote the different kinds of completions
KIND_HEADER_BASE = (sublime.KIND_ID_NAMESPACE, 'K', 'Header Key')
KIND_HEADER_DICT = (sublime.KIND_ID_NAMESPACE, 'D', 'Header Dict')
KIND_HEADER_LIST = (sublime.KIND_ID_NAMESPACE, 'L', 'Header List')
KIND_BRANCH = (sublime.KIND_ID_NAVIGATION, 'b', 'Branch Point')
KIND_CONTEXT = (sublime.KIND_ID_KEYWORD, 'c', 'Context')
KIND_FUNCTION = (sublime.KIND_ID_FUNCTION, 'f', 'Function')
KIND_FUNCTION_TRUE = (sublime.KIND_ID_FUNCTION, 'f', 'Function')
KIND_FUNCTION_FALSE = (sublime.KIND_ID_FUNCTION, 'f', 'Function')
KIND_CAPTURUE = (sublime.KIND_ID_FUNCTION, 'c', 'Captures')
KIND_SCOPE = (sublime.KIND_ID_NAMESPACE, 's', 'Scope')
KIND_VARIABLE = (sublime.KIND_ID_VARIABLE, 'v', 'Variable')

PACKAGE_NAME = __package__.split('.')[0]

def status(msg, console=False):
    msg = "[%s] %s" % (PACKAGE_NAME, msg)
    sublime.status_message(msg)
    if console:
        print(msg)

def format_static_completions(templates):

    def format_item(trigger, kind, details):
        if kind in (KIND_HEADER_DICT, KIND_CAPTURUE, KIND_CONTEXT):
            completion_format = sublime.COMPLETION_FORMAT_SNIPPET
            suffix = ":\n  "
        elif kind is KIND_HEADER_LIST:
            completion_format = sublime.COMPLETION_FORMAT_SNIPPET
            suffix = ":\n  - "
        elif kind is KIND_FUNCTION_TRUE:
            completion_format = sublime.COMPLETION_FORMAT_SNIPPET
            suffix = ": ${1:true}"
        elif kind is KIND_FUNCTION_FALSE:
            completion_format = sublime.COMPLETION_FORMAT_SNIPPET
            suffix = ": ${1:false}"
        else:
            completion_format = sublime.COMPLETION_FORMAT_TEXT
            suffix = ": "

        return sublime.CompletionItem(
            trigger=trigger,
            kind=kind,
            details=details,
            completion=trigger + suffix,
            completion_format=completion_format,
        )

    return [format_item(*template) for template in templates]

def format_completions(items, annotation="", kind=sublime.KIND_AMBIGUOUS):
    format_string = "Defined at line {0}"
    return [
        sublime.CompletionItem(
            trigger=trigger,
            annotation=annotation,
            kind=kind,
            details=format_string.format(row) if row is not None else "",
        )
        for trigger, row in items
    ]

class SyntaxDefCompletionsListener(sublime_plugin.ViewEventListener):

    base_completions_root = format_static_completions(templates=(
        # base keys
        ('name', KIND_HEADER_BASE, "The display name of the syntax."),
        ('scope', KIND_HEADER_BASE, "The main scope of the syntax."),
        ('version', KIND_HEADER_BASE, "The sublime-syntax version."),
        ('extends', KIND_HEADER_BASE, "The syntax which is to be extended."),
        ('name', KIND_HEADER_BASE, "The display name of the syntax."),
        ('first_line_match', KIND_HEADER_BASE, "The pattern to identify a file by content."),
        # dict keys
        ('variables', KIND_HEADER_DICT, 'The variables definitions.'),
        ('contexts', KIND_HEADER_DICT, 'The syntax contexts.'),
        # list keys
        ('file_extensions', KIND_HEADER_LIST, "The list of file extensions."),
        ('hidden_extensions', KIND_HEADER_LIST, "The list of hidden file extensions.")
    ))

    base_completions_contexts = format_static_completions(templates=(
        # meta functions
        ('meta_append', KIND_FUNCTION_TRUE, "Add rules to the end of the inherit context."),
        ('meta_content_scope', KIND_FUNCTION, "A scope to apply to the content of a context."),
        ('meta_include_prototype', KIND_FUNCTION_FALSE, "Flag to in-/exclude `prototype`"),
        ('meta_prepend', KIND_FUNCTION_TRUE, "Add rules to the beginning of the inherit context."),
        ('meta_scope', KIND_FUNCTION, "A scope to apply to the full context."),
        ('clear_scopes', KIND_FUNCTION, "Clear meta scopes."),
        # matching tokens
        ('match', KIND_FUNCTION, "Pattern to match tokens."),
        # scoping
        ('scope', KIND_FUNCTION, "The scope to apply if a token matches"),
        ('captures', KIND_CAPTURUE, "Assigns scopes to the capture groups."),
        # contexts
        ('push', KIND_FUNCTION, "Push a context onto the stack."),
        ('set', KIND_FUNCTION, "Set a context onto the stack."),
        ('pop', KIND_FUNCTION_TRUE, 'Pop context(s) from the stack.'),
        ('with_prototype', KIND_FUNCTION, "Rules to prepend to each context."),
        # branching
        ('branch_point', KIND_FUNCTION, "Name of the point to rewind to if a branch fails."),
        ('branch', KIND_FUNCTION, "Push branches onto the stack."),
        ('fail', KIND_FUNCTION, "Fail the current branch."),
        # embedding
        ('embed', KIND_FUNCTION, "A context or syntax to embed."),
        ('embed_scope', KIND_FUNCTION, "A scope to apply to the embedded syntax."),
        ('escape', KIND_FUNCTION, "A pattern to denote the end of the embedded syntax."),
        ('escape_captures', KIND_CAPTURUE, "Assigns scopes to the capture groups."),
        # including
        ('include', KIND_FUNCTION, "Includes a context."),
        ('apply_prototype', KIND_FUNCTION_TRUE, "Apply prototype of included syntax."),
    ))

    # These instance variables are for communicating
    # with our PostCompletionsListener instance.
    base_suffix = None

    @classmethod
    def applies_to_primary_view_only(cls):
        return False

    @classmethod
    def is_applicable(cls, settings):
        return settings.get('syntax') == syntax_paths.SYNTAX_DEF

    @inhibit_word_completions
    def on_query_completions(self, prefix, locations):

        def match_selector(selector, offset=0):
            """Verify scope for each location."""
            return all(self.view.match_selector(point + offset, selector)
                       for point in locations)

        # None of our business
        if not match_selector("- comment - (source.regexp - keyword.other.variable)"):
            return None

        # Scope name completions based on our scope_data database
        if match_selector("meta.expect-scope, meta.scope", -1):
            return self._complete_scope(prefix, locations)

        # Auto-completion for include values using the 'contexts' keys and for
        if match_selector("meta.expect-context-list-or-content"
                          " | meta.context-list-or-content", -1):
            return (self._complete_keyword(prefix, locations)
                    + self._complete_context(prefix, locations))

        # Auto-completion for include values using the 'contexts' keys
        if match_selector("meta.expect-context-list | meta.expect-context"
                          " | meta.include | meta.context-list", -1):
            return self._complete_context(prefix, locations) or None

        # Auto-completion for branch points with 'fail' key
        if match_selector("meta.expect-branch-point-reference"
                          " | meta.branch-point-reference", -1):
            return self._complete_branch_point()

        # Auto-completion for variables in match patterns using 'variables' keys
        if match_selector("keyword.other.variable"):
            return self._complete_variable()

        # Standard completions for unmatched regions
        return self._complete_keyword(prefix, locations)

    def _line_prefix(self, point):
        _, col = self.view.rowcol(point)
        line = self.view.substr(self.view.line(point))
        return line[:col]

    def _complete_context(self, prefix, locations):
        # Verify that we're not looking for an external include
        for point in locations:
            line_prefix = self._line_prefix(point)
            real_prefix = re.search(r"[^,\[ ]*$", line_prefix).group(0)
            if real_prefix.startswith("scope:") or "/" in real_prefix:
                return []  # Don't show any completions here
            elif real_prefix != prefix:
                # print("Unexpected prefix mismatch: {} vs {}".format(real_prefix, prefix))
                return []

        return format_completions(
            [(self.view.substr(r), self.view.rowcol(r.begin())[0] + 1)
             for r in self.view.find_by_selector("entity.name.function.context")],
            annotation="",
            kind=KIND_CONTEXT,
        )

    def _complete_keyword(self, prefix, locations):

        def match_selector(selector, offset=0):
            """Verify scope for each location."""
            return all(self.view.match_selector(point + offset, selector)
                       for point in locations)

        prefixes = set()
        for point in locations:
            # Ensure that we are completing a key name everywhere
            line_prefix = self._line_prefix(point)
            real_prefix = re.sub(r"^ +(- +)*", " ", line_prefix)  # collapse leading whitespace
            prefixes.add(real_prefix)

        if len(prefixes) != 1:
            return None
        else:
            real_prefix = next(iter(prefixes))

        # (Supposedly) all keys start their own line
        match = re.match(r"^(\s*)[\w-]*$", real_prefix)
        if not match:
            return None
        elif not match.group(1):
            return self.base_completions_root
        elif match_selector("meta.block.contexts"):
            return self.base_completions_contexts
        else:
            return None

    def _complete_scope(self, prefix, locations):
        # Determine entire prefix
        prefixes = set()
        for point in locations:
            *_, real_prefix = self._line_prefix(point).rpartition(" ")
            prefixes.add(real_prefix)

        if len(prefixes) > 1:
            return None
        else:
            real_prefix = next(iter(prefixes))

        # Tokenize the current selector
        tokens = real_prefix.split(".")
        if len(tokens) <= 1:
            # No work to be done here, just return the heads
            return COMPILED_HEADS.to_completion()

        base_scope_completion = self._complete_base_scope(tokens[-1])
        # Browse the nodes and their children
        nodes = COMPILED_HEADS
        for i, token in enumerate(tokens[:-1]):
            node = nodes.find(token)
            if not node:
                status("`%s` not found in scope naming conventions" % '.'.join(tokens[:i + 1]))
                break
            nodes = node.children
            if not nodes:
                status("No nodes available in scope naming conventions after `%s`"
                       % '.'.join(tokens[:-1]))
                break
        else:
            # Offer to complete from conventions or base scope
            return nodes.to_completion() + base_scope_completion

        # Since we don't have anything to offer,
        # just complete the base scope appendix/suffix.
        return base_scope_completion

    def _complete_base_scope(self, last_token):
        regions = self.view.find_by_selector("meta.scope string - meta.block")
        if len(regions) != 1:
            status("Warning: Could not determine base scope uniquely", console=True)
            self.base_suffix = None
            return []

        base_scope = self.view.substr(regions[0])
        *_, base_suffix = base_scope.rpartition(".")
        # Only useful when the base scope suffix is not already the last one
        # In this case it is even useful to inhibit other completions completely
        if last_token == base_suffix:
            self.base_suffix = None
            return []

        self.base_suffix = base_suffix
        return format_completions([(base_suffix, None)], "base suffix", KIND_SCOPE)

    def _complete_variable(self):
        return format_completions(
            [(self.view.substr(r), self.view.rowcol(r.begin())[0] + 1)
             for r in self.view.find_by_selector("entity.name.constant")],
            annotation="",
            kind=KIND_VARIABLE,
        )

    def _complete_branch_point(self):
        return format_completions(
            [(self.view.substr(r), self.view.rowcol(r.begin())[0] + 1)
             for r in self.view.find_by_selector("entity.name.label.branch-point")],
            annotation="",
            kind=KIND_BRANCH,
        )

class PackagedevCommitScopeCompletionCommand(sublime_plugin.TextCommand):

    def run(self, edit):
        self.view.run_command("commit_completion")

        # Don't add duplicated dot, if scope is edited in the middle.
        if self.view.substr(self.view.sel()[0].a) == ".":
            return

        # Check if the completed value was the base suffix
        # and don't re-open auto complete in that case.
        listener = sublime_plugin.find_view_event_listener(self.view, SyntaxDefCompletionsListener)
        if listener and listener.base_suffix:
            point = self.view.sel()[0].a
            region = sublime.Region(point - len(listener.base_suffix) - 1, point)
            if self.view.substr(region) == "." + listener.base_suffix:
                return

        # Insert a . and trigger next completion
        self.view.run_command('insert', {'characters': "."})
        self.view.run_command('auto_complete', {'disable_auto_insert': True})
<|endoftext|>


<commit_before>"""
Scrapes committee reports from PMG e.g.
http://www.pmg.org.za/committees
"""
from __future__ import print_function
from BeautifulSoup import BeautifulSoup
from dateutil import parser as date_parser
import scrapertools
import time
from pmg_scrapers import logger
from pmg_backend.models import *
from pmg_backend import db
import json
from random import shuffle

class ReportScraper(object):

    def __init__(self, session):
        self.session = session
        self.current_committee = None
        self.current_report = None
        self.current_url = None
        self.current_page = None
        self.stats = {
            "total_committee_reports": 0,
            "new_committee_reports": 0,
            "errors": []
        }

    @property
    def next_page(self):
        """
        Extract the 'next' link, if there is one.
        """
        soup = BeautifulSoup(self.current_page)
        reports_tab = soup.find(id="quicktabs_tabpage_committees_tabs_1")
        next_link = reports_tab.find("li", {"class": "pager-next"})
        if next_link:
            href = "http://www.pmg.org.za" + next_link.find('a').attrs[0][1]
            self.current_url = href
            self.current_page = scrapertools.URLFetcher(self.current_url, self.session).html
            return True
        return False

    @property
    def next_report(self):
        """
        Iterate over the reports listed on a particular page.
        """
        while True:
            soup = BeautifulSoup(self.current_page, convertEntities=BeautifulSoup.HTML_ENTITIES)
            reports_tab = soup.find(id="quicktabs_tabpage_committees_tabs_1")
            if reports_tab is None:
                logger.error("No reports tab for this committee: " + self.current_url)
                break
            table_body = reports_tab.find("tbody")

            if table_body:
                rows = table_body.findAll("tr")
                for row in rows:
                    try:
                        cells = row.findAll('td')
                        date = date_parser.parse(cells[1].find('span').contents[0]).date()
                        title = cells[2].find('a').contents[0]
                        href = "http://www.pmg.org.za" + cells[2].find('a').attrs[0][1]
                        yield date, title, href
                    except Exception:
                        msg = "Error reading committee report details from table row: "
                        msg += self.current_url
                        self.stats["errors"].append(msg)
                        pass
            else:
                logger.error("No table body")
            if not self.next_page:
                break
        return

    def scrape_committee(self):
        """
        Scrape all meeting reports for a particular committee.
        """

        for (j, (date, title, href_report)) in enumerate(self.next_report):
            logger.debug("\t\t" + str(date) + " - " + (title[0:45]) if len(title) > 45 else title)
            tmp_url = href_report
            html = scrapertools.URLFetcher(tmp_url, self.session).html
            soup = BeautifulSoup(html)
            content = soup.find(id="content")
            bills = scrapertools.find_bills(str(content))
            # only save report entries that can be tagged to bills
            if bills:
                self.current_report = {
                    "entry_type": "committee-meeting",
                    "bills": bills,
                    "url": tmp_url,
                    "date": date,
                    "title": title,
                    "agent": self.current_committee,
                    }

                # report URL may have changed after editing on pmg.org.za, check for this
                possible_duplicates = Entry.query.filter(Entry.agent == self.current_committee)\
                    .filter(Entry.url != tmp_url)\
                    .filter(Entry.type == "committee-meeting")\
                    .filter(Entry.is_deleted == False)\
                    .filter(Entry.date == date)\
                    .order_by(Entry.entry_id).all()
                deletion_flag = False
                if possible_duplicates:
                    logger.debug(str(len(possible_duplicates)) + " possible duplicates found")
                    for possible_duplicate in possible_duplicates:
                        redirect_url = scrapertools.URLFetcher(possible_duplicate.url, self.session).follow_redirect()
                        if possible_duplicate.url != redirect_url:
                            logger.debug('redirect encountered')
                            if redirect_url == tmp_url:
                                logger.info("Updating entry URL")
                                # update the existing record's URL
                                possible_duplicate.url = tmp_url
                                # # delete all but one entry, if there are multiple duplicates
                                # if deletion_flag:
                                #     logger.info('duplicate entry deleted')
                                #     possible_duplicate.is_deleted = True
                                db.session.add(possible_duplicate)
                                deletion_flag = True

                if self.current_committee.location:
                    self.current_report["location"] = self.current_committee.location
                try:
                    self.add_or_update()
                except Exception, e:
                    msg = "Could not add committee report to database: "
                    if self.current_report.get("title"):
                        msg += self.current_report["title"]
                    self.stats["errors"].append(msg)
                    logger.error(msg)
                    logger.exception(str(e))
                self.current_report = {}
            else:
                logger.debug('no bills found in committee meeting report')
        return

    def run_scraper(self):

        committees = Agent.query.filter(Agent.type == "committee").all()
        shuffle(committees)  # randomize the order, just to keep things interesting
        for i, committee in enumerate(committees):
            self.current_committee = committee
            self.current_url = committee.url
            try:
                self.current_page = scrapertools.URLFetcher(self.current_url, self.session).html
                logger.debug("Committee: " + str(committee.name))

                self.scrape_committee()
                # give some progress feedback
                logger.info(str(i + 1) + " out of " + str(len(committees)) + " committees' reports have been scraped.")
                logger.info(json.dumps(self.stats, indent=4))

                # commit entries to database, once per committee
                logger.debug("SAVING TO DATABASE")
                db.session.commit()
            except Exception as e:
                msg = "Error scraping committee's reports."
                self.stats["errors"].append(msg)
                logger.error(msg)
                logger.exception(str(e))
        return

    def add_or_update(self):
        """
        Add current_report to database, or update the record if it already exists.
        """

        report = Entry.query.filter(Entry.agent_id == self.current_committee.agent_id) \
            .filter(Entry.url == self.current_report['url'])\
            .filter(Entry.is_deleted == False).first()
        if report is None:
            report = Entry()
            self.stats["new_committee_reports"] += 1

        tmp_bills = None
        if self.current_report.get('bills'):
            tmp_bills = self.current_report['bills']
            logger.info(str(tmp_bills))
        report = scrapertools.populate_entry(report, self.current_report, tmp_bills)
        db.session.add(report)
        self.stats["total_committee_reports"] += 1
        self.current_report = {}
        return

if __name__ == "__main__":

    report_scraper = ReportScraper()
    report_scraper.run_scraper()
    logger.info(json.dumps(report_scraper.stats, indent=4))
<commit_msg>Don't try and scrape a committee if it has no URL.<commit_after>"""
Scrapes committee reports from PMG e.g.
http://www.pmg.org.za/committees
"""
from __future__ import print_function
from BeautifulSoup import BeautifulSoup
from dateutil import parser as date_parser
import scrapertools
import time
from pmg_scrapers import logger
from pmg_backend.models import *
from pmg_backend import db
import json
from random import shuffle

class ReportScraper(object):

    def __init__(self, session):
        self.session = session
        self.current_committee = None
        self.current_report = None
        self.current_url = None
        self.current_page = None
        self.stats = {
            "total_committee_reports": 0,
            "new_committee_reports": 0,
            "errors": []
        }

    @property
    def next_page(self):
        """
        Extract the 'next' link, if there is one.
        """
        soup = BeautifulSoup(self.current_page)
        reports_tab = soup.find(id="quicktabs_tabpage_committees_tabs_1")
        next_link = reports_tab.find("li", {"class": "pager-next"})
        if next_link:
            href = "http://www.pmg.org.za" + next_link.find('a').attrs[0][1]
            self.current_url = href
            self.current_page = scrapertools.URLFetcher(self.current_url, self.session).html
            return True
        return False

    @property
    def next_report(self):
        """
        Iterate over the reports listed on a particular page.
        """
        while True:
            soup = BeautifulSoup(self.current_page, convertEntities=BeautifulSoup.HTML_ENTITIES)
            reports_tab = soup.find(id="quicktabs_tabpage_committees_tabs_1")
            if reports_tab is None:
                logger.error("No reports tab for this committee: " + self.current_url)
                break
            table_body = reports_tab.find("tbody")

            if table_body:
                rows = table_body.findAll("tr")
                for row in rows:
                    try:
                        cells = row.findAll('td')
                        date = date_parser.parse(cells[1].find('span').contents[0]).date()
                        title = cells[2].find('a').contents[0]
                        href = "http://www.pmg.org.za" + cells[2].find('a').attrs[0][1]
                        yield date, title, href
                    except Exception:
                        msg = "Error reading committee report details from table row: "
                        msg += self.current_url
                        self.stats["errors"].append(msg)
                        pass
            else:
                logger.error("No table body")
            if not self.next_page:
                break
        return

    def scrape_committee(self):
        """
        Scrape all meeting reports for a particular committee.
        """

        for (j, (date, title, href_report)) in enumerate(self.next_report):
            logger.debug("\t\t" + str(date) + " - " + (title[0:45]) if len(title) > 45 else title)
            tmp_url = href_report
            html = scrapertools.URLFetcher(tmp_url, self.session).html
            soup = BeautifulSoup(html)
            content = soup.find(id="content")
            bills = scrapertools.find_bills(str(content))
            # only save report entries that can be tagged to bills
            if bills:
                self.current_report = {
                    "entry_type": "committee-meeting",
                    "bills": bills,
                    "url": tmp_url,
                    "date": date,
                    "title": title,
                    "agent": self.current_committee,
                    }

                # report URL may have changed after editing on pmg.org.za, check for this
                possible_duplicates = Entry.query.filter_by(Entry.agent == self.current_committee)\
                    .filter(Entry.url != tmp_url)\
                    .filter(Entry.type == "committee-meeting")\
                    .filter(Entry.is_deleted == False)\
                    .filter(Entry.date == date)\
                    .order_by(Entry.entry_id).all()
                deletion_flag = False
                if possible_duplicates:
                    logger.debug(str(len(possible_duplicates)) + " possible duplicates found")
                    for possible_duplicate in possible_duplicates:
                        redirect_url = scrapertools.URLFetcher(possible_duplicate.url, self.session).follow_redirect()
                        if possible_duplicate.url != redirect_url:
                            logger.debug('redirect encountered')
                            if redirect_url == tmp_url:
                                logger.info("Updating entry URL")
                                # update the existing record's URL
                                possible_duplicate.url = tmp_url
                                # # delete all but one entry, if there are multiple duplicates
                                # if deletion_flag:
                                #     logger.info('duplicate entry deleted')
                                #     possible_duplicate.is_deleted = True
                                db.session.add(possible_duplicate)
                                deletion_flag = True

                if self.current_committee.location:
                    self.current_report["location"] = self.current_committee.location
                try:
                    self.add_or_update()
                except Exception, e:
                    msg = "Could not add committee report to database: "
                    if self.current_report.get("title"):
                        msg += self.current_report["title"]
                    self.stats["errors"].append(msg)
                    logger.error(msg)
                    logger.exception(str(e))
                self.current_report = {}
            else:
                logger.debug('no bills found in committee meeting report')
        return

    def run_scraper(self):

        committees = Agent.query.filter(Agent.type == "committee").filter(Agent.url != None).all()
        shuffle(committees)  # randomize the order, just to keep things interesting
        for i, committee in enumerate(committees):
            self.current_committee = committee
            self.current_url = committee.url
            try:
                self.current_page = scrapertools.URLFetcher(self.current_url, self.session).html
                logger.debug("Committee: " + str(committee.name))

                self.scrape_committee()
                # give some progress feedback
                logger.info(str(i + 1) + " out of " + str(len(committees)) + " committees' reports have been scraped.")
                logger.info(json.dumps(self.stats, indent=4))

                # commit entries to database, once per committee
                logger.debug("SAVING TO DATABASE")
                db.session.commit()
            except Exception as e:
                msg = "Error scraping committee's reports."
                self.stats["errors"].append(msg)
                logger.error(msg)
                logger.exception(str(e))
        return

    def add_or_update(self):
        """
        Add current_report to database, or update the record if it already exists.
        """

        report = Entry.query.filter_by(agent_id=self.current_committee.agent_id) \
            .filter_by(url=self.current_report['url'])\
            .filter_by(is_deleted=False).first()
        if report is None:
            report = Entry()
            self.stats["new_committee_reports"] += 1

        tmp_bills = None
        if self.current_report.get('bills'):
            tmp_bills = self.current_report['bills']
            logger.info(str(tmp_bills))
        report = scrapertools.populate_entry(report, self.current_report, tmp_bills)
        db.session.add(report)
        self.stats["total_committee_reports"] += 1
        self.current_report = {}
        return

if __name__ == "__main__":

    report_scraper = ReportScraper()
    report_scraper.run_scraper()
    logger.info(json.dumps(report_scraper.stats, indent=4))
<|endoftext|>


<commit_before>'''
Created on 16/04/2013

@author: synchro
'''
<commit_msg>proyecto Controlador<commit_after>'''
Created on 14/04/2013

@author: synchro
'''
from entidad import Proyecto
from initdb import db_session, init_db

session = db_session()

def getProyectos():
    """Obtener usuarios
    """
    init_db()
    proyectos = session.query(Proyecto).all()
    return proyectos<|endoftext|>


<commit_before>from collections import namedtuple
from typing import Dict
import distutils.ccompiler
import casadi as ca
import numpy as np
import copy
import sys
import os
import fnmatch
import logging
import pickle
import contextlib

from pymoca import parser, tree, ast, __version__
from . import generator
from .model import CASADI_ATTRIBUTES, Model, Variable

logger = logging.getLogger("pymoca")

class CachedModel(Model):

    def __init__(self):
        self.states = []
        self.der_states = []
        self.alg_states = []
        self.inputs = []
        self.outputs = []
        self.constants = []
        self.parameters = []
        self.time = ca.MX.sym('time')
        self.delay_states = []

        self._dae_residual_function = None
        self._initial_residual_function = None
        self._variable_metadata_function = None
        self._delay_arguments_function = None

    def __str__(self):
        r = ""
        r += "Model\n"
        r += "time: " + str(self.time) + "\n"
        r += "states: " + str(self.states) + "\n"
        r += "der_states: " + str(self.der_states) + "\n"
        r += "alg_states: " + str(self.alg_states) + "\n"
        r += "inputs: " + str(self.inputs) + "\n"
        r += "outputs: " + str(self.outputs) + "\n"
        r += "constants: " + str(self.constants) + "\n"
        r += "parameters: " + str(self.parameters) + "\n"
        return r

    @property
    def dae_residual_function(self):
        return self._dae_residual_function

    @property
    def initial_residual_function(self):
        return self._initial_residual_function

    @property
    def variable_metadata_function(self):
        return self._variable_metadata_function

    @property
    def delay_arguments_function(self):
        return self._delay_arguments_function

    @property
    def equations(self):
        raise NotImplementedError("Cannot access individual equations on cached model.  Use residual function instead.")

    @property
    def initial_equations(self):
        raise NotImplementedError("Cannot access individual equations on cached model.  Use residual function instead.")

    @property
    def delay_arguments(self):
        raise NotImplementedError("Cannot access delay arguments on cached model.  Use delay arguments function instead.")

    def simplify(self, options):
        raise NotImplementedError("Cannot simplify cached model")

class InvalidCacheError(Exception):
    pass

def _compile_model(model_folder: str, model_name: str, compiler_options: Dict[str, str]):
    # Load folders
    tree = None
    for folder in [model_folder] + compiler_options.get('library_folders', []):
        for root, dir, files in os.walk(folder, followlinks=True):
            for item in fnmatch.filter(files, "*.mo"):
                logger.info("Parsing {}".format(item))

                with open(os.path.join(root, item), 'r') as f:
                    if tree is None:
                        tree = parser.parse(f.read())
                    else:
                        tree.extend(parser.parse(f.read()))

    # Compile
    logger.info("Generating CasADi model")

    model = generator.generate(tree, model_name, compiler_options)
    if compiler_options.get('check_balanced', True):
        model.check_balanced()

    model.simplify(compiler_options)

    if compiler_options.get('verbose', False):
        model.check_balanced()

    return model

def _codegen_model(model_folder: str, f: ca.Function, library_name: str):
    # Compile shared libraries
    if os.name == 'posix':
        compiler_flags = ['-O2', '-fPIC']
        linker_flags = ['-fPIC']
    else:
        compiler_flags = ['/O2', '/wd4101']  # Shut up unused local variable warnings.
        linker_flags = ['/DLL']

    # Generate C code
    logger.debug("Generating {}".format(library_name))

    cg = ca.CodeGenerator(library_name)
    cg.add(f, True) # Nondifferentiated function
    cg.add(f.forward(1), True) # Jacobian-times-vector product
    cg.add(f.reverse(1), True) # vector-times-Jacobian product
    cg.add(f.reverse(1).forward(1), True) # Hessian-times-vector product
    cg.generate(model_folder + '/')

    compiler = distutils.ccompiler.new_compiler()

    file_name = os.path.relpath(os.path.join(model_folder, library_name + '.c'))
    object_name = compiler.object_filenames([file_name])[0]
    library = os.path.join(model_folder, library_name + compiler.shared_lib_extension)
    try:
        # NOTE: For some reason running in debug mode in PyCharm (2017.1)
        # on Windows causes cl.exe to fail on its own binary name (?!) and
        # the include paths. This does not happen when running directly
        # from cmd.exe / PowerShell or e.g. with debug mode in VS Code.
        compiler.compile([file_name], extra_postargs=compiler_flags)

        # We do not want the "lib" prefix on POSIX systems, so we call
        # link() directly with our desired filename instead of
        # link_shared_lib().
        compiler.link(compiler.SHARED_LIBRARY, [object_name], library, extra_preargs=linker_flags)
    except:
        raise
    finally:
        with contextlib.suppress(FileNotFoundError):
            os.remove(file_name)
            os.remove(object_name)
    return library

def save_model(model_folder: str, model_name: str, model: Model,
               compiler_options: Dict[str, str]) -> None:
    """
    Saves a CasADi model to disk.

    :param model_folder: Folder where the precompiled CasADi model will be stored.
    :param model_name: Name of the model.
    :param model: Model instance.
    :param compiler_options: Dictionary of compiler options.
    """

    objects = {'dae_residual': None, 'initial_residual': None, 'variable_metadata': None, 'delay_arguments': None}
    for o in objects.keys():
        f = getattr(model, o + '_function')

        if compiler_options.get('codegen', False):
            objects[o] = _codegen_model(model_folder, f, '{}_{}'.format(model_name, o))
        else:
            objects[o] = f

    # Output metadata
    db_file = os.path.join(model_folder, model_name)
    with open(db_file, 'wb') as f:
        db = {}

        # Store version
        db['version'] = __version__

        # Include references to the shared libraries (codegen) or pickled functions (cache)
        db.update(objects)

        db['library_os'] = os.name

        db['options'] = compiler_options

        # Describe variables per category
        for key in ['states', 'der_states', 'alg_states', 'inputs', 'parameters', 'constants']:
            db[key] = [e.to_dict() for e in getattr(model, key)]

        db['outputs'] = model.outputs

        db['delay_states'] = model.delay_states

        pickle.dump(db, f)

def load_model(model_folder: str, model_name: str, compiler_options: Dict[str, str]) -> CachedModel:
    """
    Loads a precompiled CasADi model into a CachedModel instance.

    :param model_folder: Folder where the precompiled CasADi model is located.
    :param model_name: Name of the model.
    :param compiler_options: Dictionary of compiler options.

    :returns: CachedModel instance.
    """

    db_file = os.path.join(model_folder, model_name)

    if compiler_options.get('mtime_check', True):
        # Mtime check
        cache_mtime = os.path.getmtime(db_file)
        for folder in [model_folder] + compiler_options.get('library_folders', []):
            for root, dir, files in os.walk(folder, followlinks=True):
                for item in fnmatch.filter(files, "*.mo"):
                    filename = os.path.join(root, item)
                    if os.path.getmtime(filename) > cache_mtime:
                        raise InvalidCacheError("Cache out of date")

    # Create empty model object
    model = CachedModel()

    # Load metadata
    with open(db_file, 'rb') as f:
        db = pickle.load(f)

        if db['version'] != __version__:
            raise InvalidCacheError('Cache generated for a different version of pymoca')

        if db['options'] != compiler_options:
            raise InvalidCacheError('Cache generated for different compiler options')

        # Pickles are platform independent, but dynamic libraries are not
        if compiler_options['codegen']:
            if db['library_os'] != os.name:
                raise InvalidCacheError('Cache generated for incompatible OS')

        # Include references to the shared libraries
        for o in ['dae_residual', 'initial_residual', 'variable_metadata', 'delay_arguments']:
            if isinstance(db[o], str):
                # Path to codegen'd library
                f = ca.external(o, db[o])
            else:
                # Pickled CasADi Function; use as is
                assert isinstance(db[o], ca.Function)
                f = db[o]

            setattr(model, '_' + o + '_function', f)

        # Load variables per category
        variables_with_metadata = ['states', 'alg_states', 'inputs', 'parameters', 'constants']
        variable_dict = {}
        for key in variables_with_metadata:
            variables = getattr(model, key)
            for i, d in enumerate(db[key]):
                variable = Variable.from_dict(d)
                variables.append(variable)
                variable_dict[variable.symbol.name()] = variable

        model.der_states = [Variable.from_dict(d) for d in db['der_states']]
        model.outputs = db['outputs']
        model.delay_states = db['delay_states']

        # Evaluate variable metadata:
        parameter_vector = ca.veccat(*[v.symbol for v in model.parameters])
        metadata = dict(zip(variables_with_metadata, model.variable_metadata_function(parameter_vector)))
        independent_metadata = dict(zip(variables_with_metadata, model.variable_metadata_function(ca.veccat(*[np.nan for v in model.parameters]))))

        for k, key in enumerate(variables_with_metadata):
            for i, d in enumerate(db[key]):
                variable = variable_dict[d['name']]
                for j, tmp in enumerate(CASADI_ATTRIBUTES):
                    if ca.depends_on(ca.MX(metadata[key][i, j]), parameter_vector):
                        setattr(variable, tmp, metadata[key][i, j])
                    else:
                        setattr(variable, tmp, independent_metadata[key][i, j])

    # Done
    return model

def transfer_model(model_folder: str, model_name: str, compiler_options=None):
    if compiler_options is None:
        compiler_options = {}
    else:
        compiler_options = copy.copy(compiler_options)

    cache = compiler_options.setdefault('cache', False)
    codegen = compiler_options.setdefault('codegen', False)

    # Compilation takes precedence over caching, and disables it
    if cache and codegen:
        logger.warning("Both 'cache' and 'codegen' specified. Code generation will take precedence.")
        cache = compiler_options['cache'] = False

    if cache or codegen:
        # Until CasADi supports pickling MXFunctions, caching implies
        # expanding to SX. We only raise a warning when we have to (re)compile
        # the model though.
        raise_expand_warning = False
        if cache and not compiler_options.get('expand_mx', False):
            compiler_options['expand_mx'] = True
            raise_expand_warning = True

        try:
            return load_model(model_folder, model_name, compiler_options)
        except (FileNotFoundError, InvalidCacheError):
            if raise_expand_warning:
                logger.warning("Caching implies expanding to SX. Setting 'expand_mx' to True.")
            model = _compile_model(model_folder, model_name, compiler_options)
            save_model(model_folder, model_name, model, compiler_options)
            return model
    else:
        return _compile_model(model_folder, model_name, compiler_options)
<commit_msg>Exclude library_folders from cache options check<commit_after>from collections import namedtuple
from typing import Dict
import distutils.ccompiler
import casadi as ca
import numpy as np
import copy
import sys
import os
import fnmatch
import logging
import pickle
import contextlib

from pymoca import parser, tree, ast, __version__
from . import generator
from .model import CASADI_ATTRIBUTES, Model, Variable

logger = logging.getLogger("pymoca")

class CachedModel(Model):

    def __init__(self):
        self.states = []
        self.der_states = []
        self.alg_states = []
        self.inputs = []
        self.outputs = []
        self.constants = []
        self.parameters = []
        self.time = ca.MX.sym('time')
        self.delay_states = []

        self._dae_residual_function = None
        self._initial_residual_function = None
        self._variable_metadata_function = None
        self._delay_arguments_function = None

    def __str__(self):
        r = ""
        r += "Model\n"
        r += "time: " + str(self.time) + "\n"
        r += "states: " + str(self.states) + "\n"
        r += "der_states: " + str(self.der_states) + "\n"
        r += "alg_states: " + str(self.alg_states) + "\n"
        r += "inputs: " + str(self.inputs) + "\n"
        r += "outputs: " + str(self.outputs) + "\n"
        r += "constants: " + str(self.constants) + "\n"
        r += "parameters: " + str(self.parameters) + "\n"
        return r

    @property
    def dae_residual_function(self):
        return self._dae_residual_function

    @property
    def initial_residual_function(self):
        return self._initial_residual_function

    @property
    def variable_metadata_function(self):
        return self._variable_metadata_function

    @property
    def delay_arguments_function(self):
        return self._delay_arguments_function

    @property
    def equations(self):
        raise NotImplementedError("Cannot access individual equations on cached model.  Use residual function instead.")

    @property
    def initial_equations(self):
        raise NotImplementedError("Cannot access individual equations on cached model.  Use residual function instead.")

    @property
    def delay_arguments(self):
        raise NotImplementedError("Cannot access delay arguments on cached model.  Use delay arguments function instead.")

    def simplify(self, options):
        raise NotImplementedError("Cannot simplify cached model")

class InvalidCacheError(Exception):
    pass

def _compile_model(model_folder: str, model_name: str, compiler_options: Dict[str, str]):
    # Load folders
    tree = None
    for folder in [model_folder] + compiler_options.get('library_folders', []):
        for root, dir, files in os.walk(folder, followlinks=True):
            for item in fnmatch.filter(files, "*.mo"):
                logger.info("Parsing {}".format(item))

                with open(os.path.join(root, item), 'r') as f:
                    if tree is None:
                        tree = parser.parse(f.read())
                    else:
                        tree.extend(parser.parse(f.read()))

    # Compile
    logger.info("Generating CasADi model")

    model = generator.generate(tree, model_name, compiler_options)
    if compiler_options.get('check_balanced', True):
        model.check_balanced()

    model.simplify(compiler_options)

    if compiler_options.get('verbose', False):
        model.check_balanced()

    return model

def _codegen_model(model_folder: str, f: ca.Function, library_name: str):
    # Compile shared libraries
    if os.name == 'posix':
        compiler_flags = ['-O2', '-fPIC']
        linker_flags = ['-fPIC']
    else:
        compiler_flags = ['/O2', '/wd4101']  # Shut up unused local variable warnings.
        linker_flags = ['/DLL']

    # Generate C code
    logger.debug("Generating {}".format(library_name))

    cg = ca.CodeGenerator(library_name)
    cg.add(f, True) # Nondifferentiated function
    cg.add(f.forward(1), True) # Jacobian-times-vector product
    cg.add(f.reverse(1), True) # vector-times-Jacobian product
    cg.add(f.reverse(1).forward(1), True) # Hessian-times-vector product
    cg.generate(model_folder + '/')

    compiler = distutils.ccompiler.new_compiler()

    file_name = os.path.relpath(os.path.join(model_folder, library_name + '.c'))
    object_name = compiler.object_filenames([file_name])[0]
    library = os.path.join(model_folder, library_name + compiler.shared_lib_extension)
    try:
        # NOTE: For some reason running in debug mode in PyCharm (2017.1)
        # on Windows causes cl.exe to fail on its own binary name (?!) and
        # the include paths. This does not happen when running directly
        # from cmd.exe / PowerShell or e.g. with debug mode in VS Code.
        compiler.compile([file_name], extra_postargs=compiler_flags)

        # We do not want the "lib" prefix on POSIX systems, so we call
        # link() directly with our desired filename instead of
        # link_shared_lib().
        compiler.link(compiler.SHARED_LIBRARY, [object_name], library, extra_preargs=linker_flags)
    except:
        raise
    finally:
        with contextlib.suppress(FileNotFoundError):
            os.remove(file_name)
            os.remove(object_name)
    return library

def save_model(model_folder: str, model_name: str, model: Model,
               compiler_options: Dict[str, str]) -> None:
    """
    Saves a CasADi model to disk.

    :param model_folder: Folder where the precompiled CasADi model will be stored.
    :param model_name: Name of the model.
    :param model: Model instance.
    :param compiler_options: Dictionary of compiler options.
    """

    objects = {'dae_residual': None, 'initial_residual': None, 'variable_metadata': None, 'delay_arguments': None}
    for o in objects.keys():
        f = getattr(model, o + '_function')

        if compiler_options.get('codegen', False):
            objects[o] = _codegen_model(model_folder, f, '{}_{}'.format(model_name, o))
        else:
            objects[o] = f

    # Output metadata
    db_file = os.path.join(model_folder, model_name)
    with open(db_file, 'wb') as f:
        db = {}

        # Store version
        db['version'] = __version__

        # Include references to the shared libraries (codegen) or pickled functions (cache)
        db.update(objects)

        db['library_os'] = os.name

        db['options'] = compiler_options

        # Describe variables per category
        for key in ['states', 'der_states', 'alg_states', 'inputs', 'parameters', 'constants']:
            db[key] = [e.to_dict() for e in getattr(model, key)]

        db['outputs'] = model.outputs

        db['delay_states'] = model.delay_states

        pickle.dump(db, f)

def load_model(model_folder: str, model_name: str, compiler_options: Dict[str, str]) -> CachedModel:
    """
    Loads a precompiled CasADi model into a CachedModel instance.

    :param model_folder: Folder where the precompiled CasADi model is located.
    :param model_name: Name of the model.
    :param compiler_options: Dictionary of compiler options.

    :returns: CachedModel instance.
    """

    db_file = os.path.join(model_folder, model_name)

    if compiler_options.get('mtime_check', True):
        # Mtime check
        cache_mtime = os.path.getmtime(db_file)
        for folder in [model_folder] + compiler_options.get('library_folders', []):
            for root, dir, files in os.walk(folder, followlinks=True):
                for item in fnmatch.filter(files, "*.mo"):
                    filename = os.path.join(root, item)
                    if os.path.getmtime(filename) > cache_mtime:
                        raise InvalidCacheError("Cache out of date")

    # Create empty model object
    model = CachedModel()

    # Load metadata
    with open(db_file, 'rb') as f:
        db = pickle.load(f)

        if db['version'] != __version__:
            raise InvalidCacheError('Cache generated for a different version of pymoca')

        # Check compiler options. We ignore the library folders, as they have
        # already been checked, and checking them will impede platform
        # portability of the cache.
        exclude_options = ['library_folders']
        old_opts = {k: v for k, v in db['options'].items() if k not in exclude_options}
        new_opts = {k: v for k, v in compiler_options.items() if k not in exclude_options}

        if old_opts != new_opts:
            raise InvalidCacheError('Cache generated for different compiler options')

        # Pickles are platform independent, but dynamic libraries are not
        if compiler_options['codegen']:
            if db['library_os'] != os.name:
                raise InvalidCacheError('Cache generated for incompatible OS')

        # Include references to the shared libraries
        for o in ['dae_residual', 'initial_residual', 'variable_metadata', 'delay_arguments']:
            if isinstance(db[o], str):
                # Path to codegen'd library
                f = ca.external(o, db[o])
            else:
                # Pickled CasADi Function; use as is
                assert isinstance(db[o], ca.Function)
                f = db[o]

            setattr(model, '_' + o + '_function', f)

        # Load variables per category
        variables_with_metadata = ['states', 'alg_states', 'inputs', 'parameters', 'constants']
        variable_dict = {}
        for key in variables_with_metadata:
            variables = getattr(model, key)
            for i, d in enumerate(db[key]):
                variable = Variable.from_dict(d)
                variables.append(variable)
                variable_dict[variable.symbol.name()] = variable

        model.der_states = [Variable.from_dict(d) for d in db['der_states']]
        model.outputs = db['outputs']
        model.delay_states = db['delay_states']

        # Evaluate variable metadata:
        parameter_vector = ca.veccat(*[v.symbol for v in model.parameters])
        metadata = dict(zip(variables_with_metadata, model.variable_metadata_function(parameter_vector)))
        independent_metadata = dict(zip(variables_with_metadata, model.variable_metadata_function(ca.veccat(*[np.nan for v in model.parameters]))))

        for k, key in enumerate(variables_with_metadata):
            for i, d in enumerate(db[key]):
                variable = variable_dict[d['name']]
                for j, tmp in enumerate(CASADI_ATTRIBUTES):
                    if ca.depends_on(ca.MX(metadata[key][i, j]), parameter_vector):
                        setattr(variable, tmp, metadata[key][i, j])
                    else:
                        setattr(variable, tmp, independent_metadata[key][i, j])

    # Done
    return model

def transfer_model(model_folder: str, model_name: str, compiler_options=None):
    if compiler_options is None:
        compiler_options = {}
    else:
        compiler_options = copy.copy(compiler_options)

    cache = compiler_options.setdefault('cache', False)
    codegen = compiler_options.setdefault('codegen', False)

    # Compilation takes precedence over caching, and disables it
    if cache and codegen:
        logger.warning("Both 'cache' and 'codegen' specified. Code generation will take precedence.")
        cache = compiler_options['cache'] = False

    if cache or codegen:
        # Until CasADi supports pickling MXFunctions, caching implies
        # expanding to SX. We only raise a warning when we have to (re)compile
        # the model though.
        raise_expand_warning = False
        if cache and not compiler_options.get('expand_mx', False):
            compiler_options['expand_mx'] = True
            raise_expand_warning = True

        try:
            return load_model(model_folder, model_name, compiler_options)
        except (FileNotFoundError, InvalidCacheError):
            if raise_expand_warning:
                logger.warning("Caching implies expanding to SX. Setting 'expand_mx' to True.")
            model = _compile_model(model_folder, model_name, compiler_options)
            save_model(model_folder, model_name, model, compiler_options)
            return model
    else:
        return _compile_model(model_folder, model_name, compiler_options)
<|endoftext|>


<commit_before>import vtk
import time
import numpy as np
from ddapp.timercallback import TimerCallback

class OrbitController(TimerCallback):

    def __init__(self, view):
        TimerCallback.__init__(self)
        self.view = view
        self.orbitTime = 20.0

    def tick(self):
        speed = 360.0 / self.orbitTime
        degrees = self.elapsed * speed
        self.view.camera().Azimuth(degrees)
        self.view.render()

class Flyer(TimerCallback):

    def __init__(self, view):
        TimerCallback.__init__(self)
        self.view = view
        self.flyTime = 0.5
        self.startTime = 0.0
        self.maintainViewDirection = False
        self.positionZoom = 0.2

    def getCameraCopy(self):
        camera = vtk.vtkCamera()
        camera.DeepCopy(self.view.camera())
        return camera

    def zoomTo(self, newFocalPoint, newPosition=None):

        self.interp = vtk.vtkCameraInterpolator()
        self.interp.AddCamera(0.0, self.getCameraCopy())

        c = self.getCameraCopy()
        newFocalPoint = np.array(newFocalPoint)
        oldFocalPoint = np.array(c.GetFocalPoint())
        oldPosition = np.array(c.GetPosition())

        if newPosition is None:
            if self.maintainViewDirection:
                newPosition = oldPosition + (newFocalPoint - oldFocalPoint)
            else:
                newPosition = oldPosition
            newPosition += self.positionZoom*(newFocalPoint - newPosition)

        c.SetFocalPoint(newFocalPoint)
        c.SetPosition(newPosition)
        c.SetViewUp([0.0, 0.0, 1.0])

        self.interp.AddCamera(1.0, c)
        self.startTime = time.time()
        self.start()

    def tick(self):

        elapsed = time.time() - self.startTime
        t = elapsed / float(self.flyTime)

        self.interp.InterpolateCamera(t, self.view.camera())
        self.view.render()

        if t > 1.0:
            return False
<commit_msg>increase position zoom of the camera flyer<commit_after>import vtk
import time
import numpy as np
from ddapp.timercallback import TimerCallback

class OrbitController(TimerCallback):

    def __init__(self, view):
        TimerCallback.__init__(self)
        self.view = view
        self.orbitTime = 20.0

    def tick(self):
        speed = 360.0 / self.orbitTime
        degrees = self.elapsed * speed
        self.view.camera().Azimuth(degrees)
        self.view.render()

class Flyer(TimerCallback):

    def __init__(self, view):
        TimerCallback.__init__(self)
        self.view = view
        self.flyTime = 0.5
        self.startTime = 0.0
        self.maintainViewDirection = False
        self.positionZoom = 0.7

    def getCameraCopy(self):
        camera = vtk.vtkCamera()
        camera.DeepCopy(self.view.camera())
        return camera

    def zoomTo(self, newFocalPoint, newPosition=None):

        self.interp = vtk.vtkCameraInterpolator()
        self.interp.AddCamera(0.0, self.getCameraCopy())

        c = self.getCameraCopy()
        newFocalPoint = np.array(newFocalPoint)
        oldFocalPoint = np.array(c.GetFocalPoint())
        oldPosition = np.array(c.GetPosition())

        if newPosition is None:
            if self.maintainViewDirection:
                newPosition = oldPosition + (newFocalPoint - oldFocalPoint)
            else:
                newPosition = oldPosition
            newPosition += self.positionZoom*(newFocalPoint - newPosition)
            #newPosition = newFocalPoint - self.positionZoom*(newFocalPoint - newPosition)

        c.SetFocalPoint(newFocalPoint)
        c.SetPosition(newPosition)
        c.SetViewUp([0.0, 0.0, 1.0])

        self.interp.AddCamera(1.0, c)
        self.startTime = time.time()
        self.start()

    def tick(self):

        elapsed = time.time() - self.startTime
        t = elapsed / float(self.flyTime)

        self.interp.InterpolateCamera(t, self.view.camera())
        self.view.render()

        if t > 1.0:
            return False
<|endoftext|>


<commit_before>from __future__ import absolute_import

from datetime import timedelta
from django.db import IntegrityError, models, transaction
from django.utils import timezone

from sentry.utils.cache import cache
from sentry.utils.hashlib import md5_text
from sentry.db.models import (
    BoundedPositiveIntegerField, Model, sane_repr
)

class GroupRelease(Model):
    __core__ = False

    project_id = BoundedPositiveIntegerField(db_index=True)
    group_id = BoundedPositiveIntegerField()
    release_id = BoundedPositiveIntegerField(db_index=True)
    environment = models.CharField(max_length=64, default='')
    first_seen = models.DateTimeField(default=timezone.now)
    last_seen = models.DateTimeField(default=timezone.now, db_index=True)

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_grouprelease'
        unique_together = (('group_id', 'release_id', 'environment'),)

    __repr__ = sane_repr('group_id', 'release_id')

    @classmethod
    def get_cache_key(cls, group_id, release_id, environment):
        return 'grouprelease:1:{}:{}'.format(
            group_id,
            md5_text('{}:{}'.format(release_id, environment)).hexdigest(),
        )

    @classmethod
    def get_or_create(cls, group, release, environment, datetime, **kwargs):
        cache_key = cls.get_cache_key(group.id, release.id, environment.name)

        instance = cache.get(cache_key)
        if instance is None:
            try:
                with transaction.atomic():
                    instance, created = cls.objects.create(
                        release_id=release.id,
                        group_id=group.id,
                        environment=environment.name,
                        project_id=group.project_id,
                        first_seen=datetime,
                        last_seen=datetime,
                    ), True
            except IntegrityError:
                instance, created = cls.objects.get(
                    release_id=release.id,
                    group_id=group.id,
                    environment=environment.name,
                ), False
            cache.set(cache_key, instance, 3600)
        else:
            created = False

        # TODO(dcramer): this would be good to buffer, but until then we minimize
        # updates to once a minute, and allow Postgres to optimistically skip
        # it even if we can't
        if not created and instance.last_seen < datetime - timedelta(seconds=60):
            cls.objects.filter(
                id=instance.id,
                last_seen__lt=datetime - timedelta(seconds=60),
            ).update(
                last_seen=datetime,
            )
            instance.last_seen = datetime
            cache.set(cache_key, instance, 3600)
        return instance
<commit_msg>Environment names may have unicodes, so don't blow up (#4075)<commit_after>from __future__ import absolute_import

from datetime import timedelta
from django.db import IntegrityError, models, transaction
from django.utils import timezone

from sentry.utils.cache import cache
from sentry.utils.hashlib import md5_text
from sentry.db.models import (
    BoundedPositiveIntegerField, Model, sane_repr
)

class GroupRelease(Model):
    __core__ = False

    project_id = BoundedPositiveIntegerField(db_index=True)
    group_id = BoundedPositiveIntegerField()
    release_id = BoundedPositiveIntegerField(db_index=True)
    environment = models.CharField(max_length=64, default='')
    first_seen = models.DateTimeField(default=timezone.now)
    last_seen = models.DateTimeField(default=timezone.now, db_index=True)

    class Meta:
        app_label = 'sentry'
        db_table = 'sentry_grouprelease'
        unique_together = (('group_id', 'release_id', 'environment'),)

    __repr__ = sane_repr('group_id', 'release_id')

    @classmethod
    def get_cache_key(cls, group_id, release_id, environment):
        return 'grouprelease:1:{}:{}'.format(
            group_id,
            md5_text(u'{}:{}'.format(release_id, environment)).hexdigest(),
        )

    @classmethod
    def get_or_create(cls, group, release, environment, datetime, **kwargs):
        cache_key = cls.get_cache_key(group.id, release.id, environment.name)

        instance = cache.get(cache_key)
        if instance is None:
            try:
                with transaction.atomic():
                    instance, created = cls.objects.create(
                        release_id=release.id,
                        group_id=group.id,
                        environment=environment.name,
                        project_id=group.project_id,
                        first_seen=datetime,
                        last_seen=datetime,
                    ), True
            except IntegrityError:
                instance, created = cls.objects.get(
                    release_id=release.id,
                    group_id=group.id,
                    environment=environment.name,
                ), False
            cache.set(cache_key, instance, 3600)
        else:
            created = False

        # TODO(dcramer): this would be good to buffer, but until then we minimize
        # updates to once a minute, and allow Postgres to optimistically skip
        # it even if we can't
        if not created and instance.last_seen < datetime - timedelta(seconds=60):
            cls.objects.filter(
                id=instance.id,
                last_seen__lt=datetime - timedelta(seconds=60),
            ).update(
                last_seen=datetime,
            )
            instance.last_seen = datetime
            cache.set(cache_key, instance, 3600)
        return instance
<|endoftext|>


<commit_before>import unittest
import nose.tools
import numpy as np

import skimage.io as io
from scipy.ndimage.filters import gaussian_filter

from mammogram.orientated_bins import orientated_bins

class OrientatedBinsTest(unittest.TestCase):

    def test_with_pure_structure(self):
        size = 20
        linear_structure = np.zeros(shape=(size,size))
        linear_structure[:,size/2] = np.ones(size)

        line_strength = orientated_bins(linear_structure, 7)
        # line_strength[line_strength<0.15] = 0 #threshold
        io.imshow(line_strength)
        io.show()
        nose.tools.assert_equal(np.count_nonzero(line_strength), size)

    def test_with_burred_structure(self):
        linear_structure = np.identity(20)
        img = gaussian_filter(linear_structure, 0.8)

        line_strength = orientated_bins(img, 10)
        line_strength[line_strength<0.15] = 0 #threshold

        nose.tools.assert_equal(np.count_nonzero(line_strength), 20)

    def test_with_multiple_windows(self):
        size = 100
        linear_structure = np.zeros(shape=(size,size))
        linear_structure[:,15] = np.ones(size)

        linear_structure = np.identity(size)
        noise = np.random.rand(size, size) * 0.5
        linear_structure += noise
        img = gaussian_filter(linear_structure, 1.5)

        line_strength = orientated_bins(img, 10)

    def test_with_more_bins(self):
        size = 100
        linear_structure = np.zeros(shape=(size,size))
        linear_structure[:,15] = np.ones(size)

        linear_structure = np.identity(size)
        noise = np.random.rand(size, size) * 0.5
        linear_structure += noise
        img = gaussian_filter(linear_structure, 1.5)

        line_strength = orientated_bins(img, 5, nbins=8)

    # def test_real(self):
    #     path="../../data/p214-010-60001-cl.png"
    #     mask="../../data/f214-010-60001-cl_mask.png"
    #
    #     img = io.imread(path)
    #     msk = io.imread(mask, as_grey=True)
    #
    #     pyramid = pyramid_gaussian(img, downscale=4)
    #     pyramid.next()
    #     img = pyramid.next()
    #
    #     img = img * msk
    #
    #     line_strength = orientated_bins(img, 5, nbins=12)
    #
    #     # io.imshow(line_strength)
    #     # io.show()
<commit_msg>Update with slightly better testing<commit_after>import unittest
import nose.tools
import numpy as np
from skimage import filters, io, transform

from mammogram.orientated_bins import orientated_bins

class OrientatedBinsTest(unittest.TestCase):

    def test_with_pure_structure(self):
        linear_structure = generate_linear_structure(20)
        line_strength, line_orinetation = orientated_bins(linear_structure, 7)
        line_strength[line_strength<0.15] = 0

        nose.tools.assert_equal(np.count_nonzero(line_strength), 20)
        nose.tools.assert_equal(np.count_nonzero(line_orinetation), 60)

    def test_with_noisy_structure(self):
        linear_structure = generate_linear_structure(100, with_noise=True)
        line_strength, line_orinetation = orientated_bins(linear_structure, 10)
        line_strength[line_strength<0.05] = 0
        nose.tools.assert_almost_equal(np.count_nonzero(line_strength), 7, delta=2)

    def test_with_multiple_many_bins(self):
        linear_structure = generate_linear_structure(100, with_noise=True)
        line_strength, line_orinetation = orientated_bins(linear_structure, 10)
        line_strength[line_strength<0.05] = 0
        nose.tools.assert_almost_equal(np.count_nonzero(line_strength), 7, delta=2)

def generate_linear_structure(size, with_noise=False):
    """Generate a basic linear structure, possibly with noise"""
    linear_structure = np.zeros(shape=(size,size))
    linear_structure[:,size/2] = np.ones(size)

    if with_noise:
        linear_structure = np.identity(size)
        noise = np.random.rand(size, size) * 0.1
        linear_structure += noise
        linear_structure = filters.gaussian_filter(linear_structure, 1.5)

    return linear_structure
<|endoftext|>


<commit_before># Python rough implementation of a C# TCP client
import socket
import time
from datetime import datetime, timedelta
from io import BytesIO, BufferedWriter
from threading import Event, Lock
import errno

from ..errors import ReadCancelledError

class TcpClient:
    def __init__(self, proxy=None, timeout=timedelta(seconds=5)):
        self._proxy = proxy
        self._socket = None
        if isinstance(timeout, timedelta):
            self._timeout = timeout.seconds
        elif isinstance(timeout, int) or isinstance(timeout, float):
            self._timeout = float(timeout)
        else:
            raise ValueError('Invalid timeout type', type(timeout))

    def _recreate_socket(self, mode):
        if self._proxy is None:
            self._socket = socket.socket(mode, socket.SOCK_STREAM)
        else:
            import socks
            self._socket = socks.socksocket(mode, socket.SOCK_STREAM)
            if type(self._proxy) is dict:
                self._socket.set_proxy(**self._proxy)
            else:  # tuple, list, etc.
                self._socket.set_proxy(*self._proxy)

    def connect(self, ip, port):
        """Connects to the specified IP and port number.
           'timeout' must be given in seconds
        """
        if not self.connected:
            if ':' in ip:  # IPv6
                mode, address = socket.AF_INET6, (ip, port, 0, 0)
            else:
                mode, address = socket.AF_INET, (ip, port)

            self._recreate_socket(mode)
            self._socket.settimeout(self._timeout)
            self._socket.connect(address)

    def _get_connected(self):
        return self._socket is not None

    connected = property(fget=_get_connected)

    def close(self):
        """Closes the connection"""
        try:
            if self.connected:
                self._socket.shutdown(socket.SHUT_RDWR)
                self._socket.close()
        except OSError as e:
            if e.errno != errno.ENOTCONN:
                raise
        finally:
            self._socket = None

    def write(self, data):
        """Writes (sends) the specified bytes to the connected peer"""

        # TODO Timeout may be an issue when sending the data, Changed in v3.5:
        # The socket timeout is now the maximum total duration to send all data.
        try:
            self._socket.sendall(data)
        except socket.timeout as e:
            raise TimeoutError() from e
        except BrokenPipeError:
            self.close()
            raise

    def read(self, size):
        """Reads (receives) a whole block of 'size bytes
           from the connected peer.

           A timeout can be specified, which will cancel the operation if
           no data has been read in the specified time. If data was read
           and it's waiting for more, the timeout will NOT cancel the
           operation. Set to None for no timeout
        """
        # TODO Remove the timeout from this method, always use previous one
        with BufferedWriter(BytesIO(), buffer_size=size) as buffer:
            bytes_left = size
            while bytes_left != 0:
                try:
                    partial = self._socket.recv(bytes_left)
                except socket.timeout as e:
                    raise TimeoutError() from e

                if len(partial) == 0:
                    self.close()
                    raise ConnectionResetError(
                        'The server has closed the connection.')

                buffer.write(partial)
                bytes_left -= len(partial)

            # If everything went fine, return the read bytes
            buffer.flush()
            return buffer.raw.getvalue()
<commit_msg>Treat OSError.errno == EBADF as a ConnectionResetError<commit_after># Python rough implementation of a C# TCP client
import socket
import time
from datetime import datetime, timedelta
from io import BytesIO, BufferedWriter
from threading import Event, Lock
import errno

from ..errors import ReadCancelledError

class TcpClient:
    def __init__(self, proxy=None, timeout=timedelta(seconds=5)):
        self._proxy = proxy
        self._socket = None
        if isinstance(timeout, timedelta):
            self._timeout = timeout.seconds
        elif isinstance(timeout, int) or isinstance(timeout, float):
            self._timeout = float(timeout)
        else:
            raise ValueError('Invalid timeout type', type(timeout))

    def _recreate_socket(self, mode):
        if self._proxy is None:
            self._socket = socket.socket(mode, socket.SOCK_STREAM)
        else:
            import socks
            self._socket = socks.socksocket(mode, socket.SOCK_STREAM)
            if type(self._proxy) is dict:
                self._socket.set_proxy(**self._proxy)
            else:  # tuple, list, etc.
                self._socket.set_proxy(*self._proxy)

    def connect(self, ip, port):
        """Connects to the specified IP and port number.
           'timeout' must be given in seconds
        """
        if not self.connected:
            if ':' in ip:  # IPv6
                mode, address = socket.AF_INET6, (ip, port, 0, 0)
            else:
                mode, address = socket.AF_INET, (ip, port)

            self._recreate_socket(mode)
            self._socket.settimeout(self._timeout)
            self._socket.connect(address)

    def _get_connected(self):
        return self._socket is not None

    connected = property(fget=_get_connected)

    def close(self):
        """Closes the connection"""
        try:
            if self.connected:
                self._socket.shutdown(socket.SHUT_RDWR)
                self._socket.close()
        except OSError as e:
            if e.errno != errno.ENOTCONN:
                raise
        finally:
            self._socket = None

    def write(self, data):
        """Writes (sends) the specified bytes to the connected peer"""

        # TODO Timeout may be an issue when sending the data, Changed in v3.5:
        # The socket timeout is now the maximum total duration to send all data.
        try:
            self._socket.sendall(data)
        except socket.timeout as e:
            raise TimeoutError() from e
        except OSError as e:
            if e.errno == errno.EBADF:
                self._raise_connection_reset()
            else:
                raise
        except BrokenPipeError:
            self._raise_connection_reset()

    def read(self, size):
        """Reads (receives) a whole block of 'size bytes
           from the connected peer.

           A timeout can be specified, which will cancel the operation if
           no data has been read in the specified time. If data was read
           and it's waiting for more, the timeout will NOT cancel the
           operation. Set to None for no timeout
        """
        # TODO Remove the timeout from this method, always use previous one
        with BufferedWriter(BytesIO(), buffer_size=size) as buffer:
            bytes_left = size
            while bytes_left != 0:
                try:
                    partial = self._socket.recv(bytes_left)
                except socket.timeout as e:
                    raise TimeoutError() from e
                except OSError as e:
                    if e.errno == errno.EBADF:
                        self._raise_connection_reset()
                    else:
                        raise

                if len(partial) == 0:
                    self._raise_connection_reset()

                buffer.write(partial)
                bytes_left -= len(partial)

            # If everything went fine, return the read bytes
            buffer.flush()
            return buffer.raw.getvalue()

    def _raise_connection_reset(self):
        self.close()  # Connection reset -> flag as socket closed
        raise ConnectionResetError('The server has closed the connection.')
<|endoftext|>


<commit_before><commit_msg>-Change: silence flake8.<commit_after><|endoftext|>