<commit_before>"""command doit graph - display a graph with task dependencies"""

from __future__ import print_function

from doit import cmd_base
from doit.cmd_base import DoitCmdBase
from doit.exceptions import InvalidCommand
import pprint
from textwrap import dedent

import six

def _draw_matplotlib_graph(graph, template, show_status):
    import networkx as nx
    from matplotlib import pyplot as plt

    def find_node_attr(g, attr, value):
        return [n for n,d in g.nodes_iter(data=True) if d[attr] == value]
    def find_edge_attr(g, attr, value):
        return [(n1,n2) for n1,n2,d in g.edges_iter(data=True) if d[attr] == value]

    node_type_styles = {
        'task':     { 'node_color': 'g', 'node_shape': 's'},
        'file':     { 'node_color': 'b', 'node_shape': 'o'},
        'wildcard': { 'node_color': 'c', 'node_shape': '8'},
    }
    dep_type_styles = {
        ## TASK-dependencies
        'task_dep': { 'edge_color': 'k', 'style': 'dotted'},
        'setup_dep':{ 'edge_color': 'm', },
        'calc_dep': { 'edge_color': 'g', },
        ## DATA-dependencies
        'file_dep': { 'edge_color': 'b', },
        'wild_dep': { 'edge_color': 'b', 'style': 'dashed'},
        'target':   { 'edge_color': 'c', },
    }

    pos = nx.spring_layout(graph, dim=2)
    for item_type, style in node_type_styles.items():
        nodes       = find_node_attr(graph, 'type', item_type)
        nx.draw_networkx_nodes(graph, pos, nodes,
                               label=item_type, alpha=0.8,
                               **style)
    for item_type, style in dep_type_styles.items():
        edges       = find_edge_attr(graph, 'type', item_type)
        edge_col    = nx.draw_networkx_edges(graph, pos, edges,
                               label=item_type, alpha=0.5,
                               **style)
        if edge_col:
            edge_col.set_label(None)    ## Remove duplicate label on DiGraph.

    if template is None:
        template = '{name}'
        if show_status:
            template = '({status})' + template
    labels = {n: (template.format(name=n, **d) if d['type'] == 'task' else n)
                   for n,d in graph.nodes_iter(data=True)}
    nx.draw_networkx_labels(graph, pos, labels)

    ax = plt.gca()
    ax.legend(scatterpoints=1, framealpha=0.5)
    #ax.set_frame_on(False)
    ax.get_xaxis().set_visible(False)
    ax.get_yaxis().set_visible(False)
    plt.subplots_adjust(0,0,1,1)

    plt.show()

SUPPORTED_GRAPH_TYPES = {'matplotlib': _draw_matplotlib_graph}

opt_subtasks = {
    'name': 'subtasks',
    'short':'b',
    'long': 'subtasks',
    'type': bool,
    'default': False,
    'help': "include also sub-tasks"
            "(applies when task-list given)"
    }

opt_private = {
    'name': 'private',
    'short': 'p',
    'long': 'private',
    'type': bool,
    'default': False,
    'help': "include also private tasks starting with '_'"
            " (applies when task-list given)"
    }

opt_no_children = {
    'name': 'no_children',
    'short':'c',
    'long': 'no-children',
    'type': bool,
    'default': False,
    'help': "TODO: include only selected tasks"
            " (applies when task-list given)"
    }

opt_deps = {
    'name': 'deps',
    'short': 'd',
    'long': 'deps',
    'type': str,
    'default': '',
    'help': ("TODO: type of dependencies to include from selected tasks"
             "(list of: file|wildcard|task|calc|target|setup|none|all"
             "(default: all)")
    }

opt_show_status = {
    'name': 'show_status',
    'short': 's',
    'long': 'status',
    'type': bool,
    'default': False,
    'help': 'read task-status (R)un, (U)p-to-date, (I)gnored (see `--template`)'
    }

opt_template = {
    'name': 'template',
    'short': '',
    'long': 'template',
    'type': str,
    'default': None,
    'help': "template for task-labels (use %s to get all keywords)"
    }

opt_graph_type = {
    'name': 'graph_type',
    'short': 'g',
    'long': 'graph',
    'type': str,
    'default': 'matplotlib',
    'help': "selection of graph library"
            " (one of: %s)." % list(SUPPORTED_GRAPH_TYPES)
    }

opt_out_file = {
    'name': 'out_file',
    'short': 'O',
    'long': 'out-file',
    'type': str,
    'default': False,
    'help': "TODO: where to store graph, if textual"
    }

def my_safe_repr(obj, context, maxlevels, level):
    """pretty print supressing unicode prefix

    http://stackoverflow.com/questions/16888409/
           suppress-unicode-prefix-on-strings-when-using-pprint
    """
    typ = type(obj)
    if six.PY2 and typ is six.text_type:
        obj = str(obj)
    return pprint._safe_repr(obj, context, maxlevels, level)

class Graphx(DoitCmdBase):
    """command doit graph"""

    doc_purpose = "display a dependency-graph for all (or selected ) tasks"
    doc_usage = "[TASK ...]"
    doc_description = dedent("""\
        Without any options, includes all known taks.
        TODO: Task-selection works also with wildcards.

        Examples:
          doit graph
          doit graph --deps file,calc,target --private
          doit graph --out-file some.png
          doit graph --graph-type json --out-file some.png
        """)

    cmd_options = (opt_subtasks, opt_private, opt_no_children, opt_deps,
                   opt_show_status, opt_template, opt_graph_type,
                   opt_out_file)

    STATUS_MAP = {'ignore': 'I', 'up-to-date': 'U', 'run': 'R'}

    @staticmethod
    def _check_task_names(all_task_names, task_names):
        """repost if task 'task_names' """
        ## Note: simpler and user-friendlier than cmd_base.check_tasks_exist()
        if not set(task_names).issubset(all_task_names):
            bad_tasks = set(task_names) - all_task_names
            msg = "Task(s) not found: %s" % str(bad_tasks)
            raise InvalidCommand(msg)

    @staticmethod
    def _include_subtasks(tasks, task_names, include_subtasks):
        """append any subtasks of 'task_names' """
        # get task by name
        subtasks = []
        for name in task_names:
            subtasks.extend(cmd_base.subtasks_iter(tasks, tasks[name]))
        return subtasks

    @staticmethod
    def _get_task_status(dep_manager, task):
        """print a single task"""
        # FIXME group task status is never up-to-date
        if dep_manager.status_is_ignore(task):
            task_status = 'ignore'
        else:
            # FIXME:'ignore' handling is ugly
            task_status = dep_manager.get_status(task, None)
        return Graphx.STATUS_MAP[task_status]

    def _prepare_graph(self, all_tasks_map, filter_task_names, show_status):
        """
        Construct a *networkx* graph of nodes (Tasks/Files/Wildcards) and their dependencies (file/wildcard, task/setup,calc).

        :param filter_task_names: If None, graph includes all tasks
        """
        import networkx as nx

        task_attributes = {
            'task_dep':     {'node_type':'task'},
            'setup_tasks':  {'node_type':'task', 'edge_type':'setup_dep'},
            'calc_dep':     {'node_type':'task'},
            'file_dep':     {'node_type':'file'},
            'wild_dep':     {'node_type':'wildcard'},
        }

        graph = nx.DiGraph()
        def add_graph_node(node, node_type, add_deps=False):
            if node in graph:
                return
            if node_type != 'task':
                graph.add_node(node, type=node_type)
            else:
                task = all_tasks_map[node]
                status = ''
                if show_status:
                    status = Graphx._get_task_status(self.dep_manager, task)
                graph.add_node(node, type=node_type,
                               is_subtask=task.is_subtask, status=status)
                if add_deps:
                    for attr, attr_kws in task_attributes.items():
                        for dname in getattr(task, attr):
                            dig_deps = filter_task_names is None or dname in filter_task_names
                            add_graph_node(dname, attr_kws['node_type'], add_deps=dig_deps)
                            graph.add_edge(node, dname, type=attr_kws.get('edge_type', attr))
                    ## Above loop cannot add targets
                    #    because they are reversed.
                    #
                    for dname in task.targets:
                        add_graph_node(dname, 'file')
                        graph.add_edge(dname, node, type='target')

        ## Add all named-tasks
        #    and their dependencies.
        #
        for tname in (filter_task_names or all_tasks_map.keys()):
            add_graph_node(tname, 'task', add_deps=True)

        return graph

    def _display_graph(self, graph, graph_type, template, show_status, out_file):
        for gtype in SUPPORTED_GRAPH_TYPES:
            if gtype.startswith(graph_type):
                func = SUPPORTED_GRAPH_TYPES[gtype]
                func(graph, template, show_status)
                break
        else:
            msg = "Unsupported graph-type '%s'; should be one : %s"
            raise InvalidCommand(msg % (graph_type, list(SUPPORTED_GRAPH_TYPES)))

    def _execute(self, subtasks=False, no_children=True, show_status=False,
                 private=False, deps=False, template=None,
                 graph_type='matplotlib', out_file=None, pos_args=None):
        task_names=pos_args
        tasks_map = dict([(t.name, t) for t in self.task_list])

        if task_names:
            Graphx._check_task_names(tasks_map.keys(), task_names)
            if not private:
                task_names = [t for t in task_names if not t.startswith('_')]
            if subtasks:
                task_names = Graphx._include_subtasks(tasks_map, task_names, subtasks)

        graph = self._prepare_graph(tasks_map, task_names, show_status)

        self._display_graph(graph, graph_type, template, show_status, out_file)

<commit_msg>impl `--deps` to filter-out dep-types.<commit_after>"""command doit graph - display a graph with task dependencies"""

from __future__ import print_function

from doit import cmd_base
from doit.cmd_base import DoitCmdBase
from doit.exceptions import InvalidCommand
import pprint
import re
from textwrap import dedent

import six

def _draw_matplotlib_graph(graph, template, show_status):
    import networkx as nx
    from matplotlib import pyplot as plt

    def find_node_attr(g, attr, value):
        return [n for n,d in g.nodes_iter(data=True) if d[attr] == value]
    def find_edge_attr(g, attr, value):
        return [(n1,n2) for n1,n2,d in g.edges_iter(data=True) if d[attr] == value]

    node_type_styles = {
        'task':     { 'node_color': 'g', 'node_shape': 's'},
        'file':     { 'node_color': 'b', 'node_shape': 'o'},
        'wildcard': { 'node_color': 'c', 'node_shape': '8'},
    }
    dep_type_styles = {
        ## TASK-dependencies
        'task_dep': { 'edge_color': 'k', 'style': 'dotted'},
        'setup_dep':{ 'edge_color': 'm', },
        'calc_dep': { 'edge_color': 'g', },
        ## DATA-dependencies
        'file_dep': { 'edge_color': 'b', },
        'wild_dep': { 'edge_color': 'b', 'style': 'dashed'},
        'target':   { 'edge_color': 'c', },
    }

    pos = nx.spring_layout(graph, dim=2)
    for item_type, style in six.iteritems(node_type_styles):
        nodes       = find_node_attr(graph, 'type', item_type)
        nx.draw_networkx_nodes(graph, pos, nodes,
                               label=item_type, alpha=0.8,
                               **style)
    for item_type, style in six.iteritems(dep_type_styles):
        edges       = find_edge_attr(graph, 'type', item_type)
        edge_col    = nx.draw_networkx_edges(graph, pos, edges,
                               label=item_type, alpha=0.5,
                               **style)
        if edge_col:
            edge_col.set_label(None)    ## Remove duplicate label on DiGraph.

    if template is None:
        template = '{name}'
        if show_status:
            template = '({status})' + template
    labels = {n: (template.format(name=n, **d) if d['type'] == 'task' else n)
                   for n,d in graph.nodes_iter(data=True)}
    nx.draw_networkx_labels(graph, pos, labels)

    ax = plt.gca()
    ax.legend(scatterpoints=1, framealpha=0.5)
    #ax.set_frame_on(False)
    ax.get_xaxis().set_visible(False)
    ax.get_yaxis().set_visible(False)
    plt.subplots_adjust(0,0,1,1)

    plt.show()

SUPPORTED_GRAPH_TYPES = {'matplotlib': _draw_matplotlib_graph}

opt_subtasks = {
    'name': 'subtasks',
    'short':'b',
    'long': 'subtasks',
    'type': bool,
    'default': False,
    'help': "include also sub-tasks"
            "(applies when task-list given)"
    }

opt_private = {
    'name': 'private',
    'short': 'p',
    'long': 'private',
    'type': bool,
    'default': False,
    'help': "include also private tasks starting with '_'"
            " (applies when task-list given)"
    }

opt_no_children = {
    'name': 'no_children',
    'short':'c',
    'long': 'no-children',
    'type': bool,
    'default': False,
    'help': "TODO: include only selected tasks"
            " (applies when task-list given)"
    }

opt_deps = {
    'name': 'deps',
    'short': '',
    'long': 'deps',
    'type': str,
    'default': '',
    'help': "type of dependencies to include from selected tasks"
            " (list of: ALL|file|wild|task|calc|setup|none|TODO:target)"
    }

opt_show_status = {
    'name': 'show_status',
    'short': 's',
    'long': 'status',
    'type': bool,
    'default': False,
    'help': 'read task-status (R)un, (U)p-to-date, (I)gnored (see `--template`)'
    }

opt_template = {
    'name': 'template',
    'short': '',
    'long': 'template',
    'type': str,
    'default': None,
    'help': "template for task-labels (use %s to get all keywords)"
    }

opt_graph_type = {
    'name': 'graph_type',
    'short': 'g',
    'long': 'graph',
    'type': str,
    'default': 'matplotlib',
    'help': "selection of graph library"
            " (one of: %s)." % list(SUPPORTED_GRAPH_TYPES)
    }

opt_out_file = {
    'name': 'out_file',
    'short': 'O',
    'long': 'out-file',
    'type': str,
    'default': False,
    'help': "TODO: where to store graph, if textual"
    }

def my_safe_repr(obj, context, maxlevels, level):
    """pretty print supressing unicode prefix

    http://stackoverflow.com/questions/16888409/
           suppress-unicode-prefix-on-strings-when-using-pprint
    """
    typ = type(obj)
    if six.PY2 and typ is six.text_type:
        obj = str(obj)
    return pprint._safe_repr(obj, context, maxlevels, level)

class Graphx(DoitCmdBase):
    """command doit graph"""

    doc_purpose = "display a dependency-graph for all (or selected ) tasks"
    doc_usage = "[TASK ...]"
    doc_description = dedent("""\
        Without any options, includes all known taks.
        TODO: Task-selection works also with wildcards.

        Examples:
          doit graph
          doit graph --deps file,calc,target --private
          doit graph --out-file some.png
          doit graph --graph-type json --out-file some.png
        """)

    cmd_options = (opt_subtasks, opt_private, opt_no_children, opt_deps,
                   opt_show_status, opt_template, opt_graph_type,
                   opt_out_file)

    STATUS_MAP = {'ignore': 'I', 'up-to-date': 'U', 'run': 'R'}

    @staticmethod
    def _check_task_names(all_task_names, task_names):
        """repost if task 'task_names' """
        ## Note: simpler and user-friendlier than cmd_base.check_tasks_exist()
        if not set(task_names).issubset(all_task_names):
            bad_tasks = set(task_names) - all_task_names
            msg = "Task(s) not found: %s" % str(bad_tasks)
            raise InvalidCommand(msg)

    @staticmethod
    def _include_subtasks(tasks, task_names, include_subtasks):
        """append any subtasks of 'task_names' """
        # get task by name
        subtasks = []
        for name in task_names:
            subtasks.extend(cmd_base.subtasks_iter(tasks, tasks[name]))
        return subtasks

    @staticmethod
    def _get_task_status(dep_manager, task):
        """print a single task"""
        # FIXME group task status is never up-to-date
        if dep_manager.status_is_ignore(task):
            task_status = 'ignore'
        else:
            # FIXME:'ignore' handling is ugly
            task_status = dep_manager.get_status(task, None)
        return Graphx.STATUS_MAP[task_status]

    def _prepare_graph(self, all_tasks_map, filter_task_names, filter_deps, show_status):
        """
        Construct a *networkx* graph of nodes (Tasks/Files/Wildcards) and their dependencies (file/wildcard, task/setup,calc).

        :param filter_task_names: If None, graph includes all tasks
        """
        import networkx as nx

        def _filter_dependencies_to_collect(dep_attributes, filter_deps):
            filter_deps = re.sub(r'[\s,|]+', ' ', filter_deps).strip()
            if filter_deps:
                dep_attributes2 = {}
                for f_dep in filter_deps.split():
                    if 'all'.startswith(f_dep):
                        dep_attributes2 = dep_attributes
                    elif 'none'.startswith(f_dep):
                        dep_attributes2 = {}
                    else:
                        dep_attributes2.update({dep:dep_kws
                                for dep, dep_kws
                                in six.iteritems(dep_attributes)
                                if dep.startswith(f_dep)})
            print(dep_attributes2)
            return dep_attributes2

        dep_attributes = {
            'task_dep':     {'node_type':'task'},
            'setup_tasks':  {'node_type':'task', 'edge_type':'setup_dep'},
            'calc_dep':     {'node_type':'task'},
            'file_dep':     {'node_type':'file'},
            'wild_dep':     {'node_type':'wildcard'},
        }

        dep_attributes = _filter_dependencies_to_collect(dep_attributes, filter_deps)

        graph = nx.DiGraph()
        def add_graph_node(node, node_type, add_deps=False):
            if node in graph:
                return
            if node_type != 'task':
                graph.add_node(node, type=node_type)
            else:
                task = all_tasks_map[node]
                status = ''
                if show_status:
                    status = Graphx._get_task_status(self.dep_manager, task)
                graph.add_node(node, type=node_type,
                               is_subtask=task.is_subtask, status=status)
                if add_deps:
                    for dep, dep_kws in six.iteritems(dep_attributes):
                        for dname in getattr(task, dep):
                            dig_deps = filter_task_names is None or dname in filter_task_names
                            add_graph_node(dname, dep_kws['node_type'], add_deps=dig_deps)
                            graph.add_edge(node, dname, type=dep_kws.get('edge_type', dep))
                    ## Above loop cannot add targets
                    #    because they are reversed.
                    #
                    # FIX: Targets are not filtered!!
                    for dname in task.targets:
                        add_graph_node(dname, 'file')
                        graph.add_edge(dname, node, type='target')

        ## Add all named-tasks
        #    and their dependencies.
        #
        for tname in (filter_task_names or all_tasks_map.keys()):
            add_graph_node(tname, 'task', add_deps=True)

        return graph

    def _display_graph(self, graph, graph_type, template, show_status, out_file):
        for gtype in SUPPORTED_GRAPH_TYPES:
            if gtype.startswith(graph_type):
                func = SUPPORTED_GRAPH_TYPES[gtype]
                func(graph, template, show_status)
                break
        else:
            msg = "Unsupported graph-type '%s'; should be one : %s"
            raise InvalidCommand(msg % (graph_type, list(SUPPORTED_GRAPH_TYPES)))

    def _execute(self, subtasks=False, no_children=True, show_status=False,
                 private=False, deps=False, template=None,
                 graph_type='matplotlib', out_file=None, pos_args=None):
        task_names=pos_args
        tasks_map = dict([(t.name, t) for t in self.task_list])

        if task_names:
            Graphx._check_task_names(tasks_map.keys(), task_names)
            if not private:
                task_names = [t for t in task_names if not t.startswith('_')]
            if subtasks:
                task_names = Graphx._include_subtasks(tasks_map, task_names, subtasks)

        graph = self._prepare_graph(tasks_map, task_names, deps, show_status)

        self._display_graph(graph, graph_type, template, show_status, out_file)

<|endoftext|>


<commit_before>"""
mbed SDK
Copyright (c) 2016 ARM Limited

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from os.path import basename, join, dirname
from project_generator_definitions.definitions import ProGenDef

from tools.export.exporters import Exporter, ExporterTargetsProperty
from tools.targets import TARGET_MAP, TARGET_NAMES
from tools.settings import ARM_INC

# If you wish to add a new target, add it to project_generator_definitions, and then
# define progen_target name in the target class (`` self.progen_target = 'my_target_name' ``)
# There are 2 default mbed templates (predefined settings) uvision.uvproj and uvproj_microlib.uvproj.tmpl
class Uvision5(Exporter):
    """
    Exporter class for uvision5. This class uses project generator.
    """
    # These 2 are currently for exporters backward compatiblity
    NAME = 'uVision5'
    TOOLCHAIN = 'ARM'
    # PROGEN_ACTIVE contains information for exporter scripts that this is using progen
    PROGEN_ACTIVE = True

    MBED_CONFIG_HEADER_SUPPORTED = True

    # backward compatibility with our scripts
    def __init__(self):
        self._targets = []

    @ExporterTargetsProperty
    def TARGETS(cls):
        if not hasattr(cls, "_targets_supported"):
            cls._targets_supported = []
            for target in TARGET_NAMES:
                try:
                    if (ProGenDef('uvision5').is_supported(str(TARGET_MAP[target])) or
                        ProGenDef('uvision5').is_supported(TARGET_MAP[target].progen['target'])):
                        cls._targets_supported.append(target)
                except AttributeError:
                    # target is not supported yet
                    continue
        return cls._targets_supported

    def get_toolchain(self):
        return TARGET_MAP[self.target].default_toolchain

    def generate(self, progen_build=False):
        """ Generates the project files """
        project_data = self.progen_get_project_data()
        tool_specific = {}
        # Expand tool specific settings by uvision specific settings which are required
        try:
            if TARGET_MAP[self.target].progen['uvision5']['template']:
                tool_specific['uvision5'] = TARGET_MAP[self.target].progen['uvision5']
        except KeyError:
            # use default template
            # by the mbed projects
            tool_specific['uvision5'] = {
                    'template': [join(dirname(__file__),  'uvision.uvproj.tmpl')],
            }

        project_data['tool_specific'] = {}
        project_data['tool_specific'].update(tool_specific)

        # get flags from toolchain and apply
        project_data['tool_specific']['uvision5']['misc'] = {}
        # asm flags only, common are not valid within uvision project, they are armcc specific
        project_data['tool_specific']['uvision5']['misc']['asm_flags'] = list(set(self.progen_flags['asm_flags']))
        # cxx flags included, as uvision have them all in one tab
        project_data['tool_specific']['uvision5']['misc']['c_flags'] = list(set(self.progen_flags['common_flags'] + self.progen_flags['c_flags'] + self.progen_flags['cxx_flags']))
        # ARM_INC is by default as system inclusion, not required for exported project
        project_data['tool_specific']['uvision5']['misc']['c_flags'].remove("-I \""+ARM_INC+"\"")
        # not compatible with c99 flag set in the template
        project_data['tool_specific']['uvision5']['misc']['c_flags'].remove("--c99")
        # cpp is not required as it's implicit for cpp files
        project_data['tool_specific']['uvision5']['misc']['c_flags'].remove("--cpp")
        # we want no-vla for only cxx, but it's also applied for C in IDE, thus we remove it
        project_data['tool_specific']['uvision5']['misc']['c_flags'].remove("--no_vla")
        project_data['tool_specific']['uvision5']['misc']['ld_flags'] = self.progen_flags['ld_flags']

        i = 0
        for macro in project_data['common']['macros']:
            # armasm does not like floating numbers in macros, timestamp to int
            if macro.startswith('MBED_BUILD_TIMESTAMP'):
                timestamp = macro[len('MBED_BUILD_TIMESTAMP='):]
                project_data['common']['macros'][i] = 'MBED_BUILD_TIMESTAMP=' + str(int(float(timestamp)))
            # armasm does not even accept MACRO=string
            if macro.startswith('MBED_USERNAME'):
                project_data['common']['macros'].pop(i)
            i += 1
        project_data['common']['macros'].append('__ASSERT_MSG')
        if progen_build:
            self.progen_gen_file('uvision5', project_data, True)
        else:
            self.progen_gen_file('uvision5', project_data)
<commit_msg>uvision5 - remove init, not used<commit_after>"""
mbed SDK
Copyright (c) 2016 ARM Limited

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from os.path import basename, join, dirname
from project_generator_definitions.definitions import ProGenDef

from tools.export.exporters import Exporter, ExporterTargetsProperty
from tools.targets import TARGET_MAP, TARGET_NAMES
from tools.settings import ARM_INC

# If you wish to add a new target, add it to project_generator_definitions, and then
# define progen_target name in the target class (`` self.progen_target = 'my_target_name' ``)
# There are 2 default mbed templates (predefined settings) uvision.uvproj and uvproj_microlib.uvproj.tmpl
class Uvision5(Exporter):
    """
    Exporter class for uvision5. This class uses project generator.
    """
    # These 2 are currently for exporters backward compatiblity
    NAME = 'uVision5'
    TOOLCHAIN = 'ARM'
    # PROGEN_ACTIVE contains information for exporter scripts that this is using progen
    PROGEN_ACTIVE = True

    MBED_CONFIG_HEADER_SUPPORTED = True

    @ExporterTargetsProperty
    def TARGETS(cls):
        if not hasattr(cls, "_targets_supported"):
            cls._targets_supported = []
            for target in TARGET_NAMES:
                try:
                    if (ProGenDef('uvision5').is_supported(str(TARGET_MAP[target])) or
                        ProGenDef('uvision5').is_supported(TARGET_MAP[target].progen['target'])):
                        cls._targets_supported.append(target)
                except AttributeError:
                    # target is not supported yet
                    continue
        return cls._targets_supported

    def get_toolchain(self):
        return TARGET_MAP[self.target].default_toolchain

    def generate(self, progen_build=False):
        """ Generates the project files """
        project_data = self.progen_get_project_data()
        tool_specific = {}
        # Expand tool specific settings by uvision specific settings which are required
        try:
            if TARGET_MAP[self.target].progen['uvision5']['template']:
                tool_specific['uvision5'] = TARGET_MAP[self.target].progen['uvision5']
        except KeyError:
            # use default template
            # by the mbed projects
            tool_specific['uvision5'] = {
                    'template': [join(dirname(__file__),  'uvision.uvproj.tmpl')],
            }

        project_data['tool_specific'] = {}
        project_data['tool_specific'].update(tool_specific)

        # get flags from toolchain and apply
        project_data['tool_specific']['uvision5']['misc'] = {}
        # asm flags only, common are not valid within uvision project, they are armcc specific
        project_data['tool_specific']['uvision5']['misc']['asm_flags'] = list(set(self.progen_flags['asm_flags']))
        # cxx flags included, as uvision have them all in one tab
        project_data['tool_specific']['uvision5']['misc']['c_flags'] = list(set(self.progen_flags['common_flags'] + self.progen_flags['c_flags'] + self.progen_flags['cxx_flags']))
        # ARM_INC is by default as system inclusion, not required for exported project
        project_data['tool_specific']['uvision5']['misc']['c_flags'].remove("-I \""+ARM_INC+"\"")
        # not compatible with c99 flag set in the template
        project_data['tool_specific']['uvision5']['misc']['c_flags'].remove("--c99")
        # cpp is not required as it's implicit for cpp files
        project_data['tool_specific']['uvision5']['misc']['c_flags'].remove("--cpp")
        # we want no-vla for only cxx, but it's also applied for C in IDE, thus we remove it
        project_data['tool_specific']['uvision5']['misc']['c_flags'].remove("--no_vla")
        project_data['tool_specific']['uvision5']['misc']['ld_flags'] = self.progen_flags['ld_flags']

        i = 0
        for macro in project_data['common']['macros']:
            # armasm does not like floating numbers in macros, timestamp to int
            if macro.startswith('MBED_BUILD_TIMESTAMP'):
                timestamp = macro[len('MBED_BUILD_TIMESTAMP='):]
                project_data['common']['macros'][i] = 'MBED_BUILD_TIMESTAMP=' + str(int(float(timestamp)))
            # armasm does not even accept MACRO=string
            if macro.startswith('MBED_USERNAME'):
                project_data['common']['macros'].pop(i)
            i += 1
        project_data['common']['macros'].append('__ASSERT_MSG')
        if progen_build:
            self.progen_gen_file('uvision5', project_data, True)
        else:
            self.progen_gen_file('uvision5', project_data)
<|endoftext|>


<commit_before>#
# Copyright (c) 2009-2011, ETH Zurich.
# All rights reserved.
#
# This file is distributed under the terms in the attached LICENSE file.
# If you do not find this file, copies can be found by writing to:
# ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group.
#

import os
import types
import string
import datetime
import debug

RAW_FILE_NAME = 'raw.txt'

def run_test(build, machine, test, path):
    # Open files for raw output from the victim and log data from the test
    raw_file_name = os.path.join(path, RAW_FILE_NAME)
    debug.verbose('open %s for raw output' % raw_file_name)
    raw_file = open(raw_file_name, 'w')

    # run the test, dumping the output to the raw file as we go
    try:
        debug.verbose('harness: setup test')
        test.setup(build, machine, path)
        debug.verbose('harness: run test')
        starttime = datetime.datetime.now()
        for out in test.run(build, machine, path):
            # timedelta for the time this line was emitted from the start of the run
            timestamp = datetime.datetime.now() - starttime
            # format as string, discarding sub-second precision
            timestr = str(timestamp).split('.', 1)[0]
            # filter output line of control characters
            filtered_out = filter(lambda c: c in string.printable, out.rstrip())
            # debug filtered output along with timestamp
            debug.debug('[%s] %s' % (timestr, filtered_out))
            # log full raw line (without timestamp) to output file
            raw_file.write(out)
        debug.verbose('harness: output complete')
    except KeyboardInterrupt:
        # let the user know that we are on our way out
        debug.error('Interrupted! Performing cleanup...')
        raise
    finally:
        raw_file.close()
        debug.verbose('harness: cleanup test')
        test.cleanup(machine)

def process_results(test, path):
    # open raw file for input processing
    raw_file_name = os.path.join(path, RAW_FILE_NAME)
    debug.verbose('open %s for raw input' % raw_file_name)
    raw_file = open(raw_file_name, 'r')

    try:
        results = test.process_data(path, raw_file)
    finally:
        raw_file.close()
    if not results:
        debug.verbose('no results')
        return True  # no results, assume success

    retval = True  # everything OK

    # if a single result, turn it into a list
    if not isinstance(results, types.ListType):
        results = [results]
    for result in results:
        # see if it passed
        try:
            passed = result.passed()
        except NotImplementedError:
            passed = None
        if passed is False:
            debug.log('Test %s FAILED' % test.name)
            retval = False
        elif passed:
            debug.verbose('Test %s PASSED' % test.name)

        # write it to a file
        name = result.name if result.name else 'results'
        data_file_name = os.path.join(path, name + '.dat')
        debug.verbose('create %s for processed output' % data_file_name)
        data_file = open(data_file_name, 'w')
        try:
            result.to_file(data_file)
            data_file.close()
        except NotImplementedError:
            debug.verbose('no processed output, remove %s' % data_file_name)
            data_file.close()
            os.remove(data_file_name)

    return retval
<commit_msg>Harness test creates a file called bootlog.txt containing only Barrelfish console output.<commit_after>#
# Copyright (c) 2009-2011, ETH Zurich.
# All rights reserved.
#
# This file is distributed under the terms in the attached LICENSE file.
# If you do not find this file, copies can be found by writing to:
# ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group.
#

import os
import types
import string
import datetime
import debug

RAW_FILE_NAME = 'raw.txt'
BOOT_FILE_NAME = 'bootlog.txt'

def run_test(build, machine, test, path):
    # Open files for raw output from the victim and log data from the test
    raw_file_name = os.path.join(path, RAW_FILE_NAME)
    debug.verbose('open %s for raw output' % raw_file_name)
    raw_file = open(raw_file_name, 'w')

    # run the test, dumping the output to the raw file as we go
    try:
        debug.verbose('harness: setup test')
        test.setup(build, machine, path)
        debug.verbose('harness: run test')
        starttime = datetime.datetime.now()
        for out in test.run(build, machine, path):
            # timedelta for the time this line was emitted from the start of the run
            timestamp = datetime.datetime.now() - starttime
            # format as string, discarding sub-second precision
            timestr = str(timestamp).split('.', 1)[0]
            # filter output line of control characters
            filtered_out = filter(lambda c: c in string.printable, out.rstrip())
            # debug filtered output along with timestamp
            debug.debug('[%s] %s' % (timestr, filtered_out))
            # log full raw line (without timestamp) to output file
            raw_file.write(out)
        debug.verbose('harness: output complete')
    except KeyboardInterrupt:
        # let the user know that we are on our way out
        debug.error('Interrupted! Performing cleanup...')
        raise
    finally:
        raw_file.close()
        debug.verbose('harness: cleanup test')
        test.cleanup(machine)

def process_results(test, path):
    # open raw file for input processing
    raw_file_name = os.path.join(path, RAW_FILE_NAME)
    debug.verbose('open %s for raw input' % raw_file_name)
    raw_file = open(raw_file_name, 'r')

    try:
        results = test.process_data(path, raw_file)
    finally:
        raw_file.close()
    if not results:
        debug.verbose('no results')
        return True  # no results, assume success

    retval = True  # everything OK

    # Process raw.txt and make a bootlog.txt that begins with grubs
    # output, avoids having encoding issues when viewing logfiles
    boot_file_name = os.path.join(path, BOOT_FILE_NAME)
    if os.path.exists(raw_file_name):
        idx = 0
        with open(raw_file_name, 'r') as rf:
            lines = rf.readlines()
            for idx, line in enumerate(lines):
                if line.strip() == "root (nd)":
                    break
        if idx > 0:
            with open(boot_file_name, 'w') as wf:
                wf.writelines(lines[idx:])
        else:
            debug.verbose('Magic string root (nd) not found, do not write bootlog.txt')
    else:
        debug.verbose('No file named %s exists. Do not create bootlog.txt.' % raw_file_name)

    # if a single result, turn it into a list
    if not isinstance(results, types.ListType):
        results = [results]
    for result in results:
        # see if it passed
        try:
            passed = result.passed()
        except NotImplementedError:
            passed = None
        if passed is False:
            debug.log('Test %s FAILED' % test.name)
            retval = False
        elif passed:
            debug.verbose('Test %s PASSED' % test.name)

        # write it to a file
        name = result.name if result.name else 'results'
        data_file_name = os.path.join(path, name + '.dat')
        debug.verbose('create %s for processed output' % data_file_name)
        data_file = open(data_file_name, 'w')
        try:
            result.to_file(data_file)
            data_file.close()
        except NotImplementedError:
            debug.verbose('no processed output, remove %s' % data_file_name)
            data_file.close()
            os.remove(data_file_name)

    return retval
<|endoftext|>


<commit_before>import argparse
import os
import tensorflow as tf
import hypergan as hg
import hyperchamber as hc
from hypergan.loaders import *
from hypergan.samplers.common import *
from hypergan.util.globals import *

def parse_args():
    parser = argparse.ArgumentParser(description='Train a colorizer!', add_help=True)
    parser.add_argument('directory', action='store', type=str, help='The location of your data.  Subdirectories are treated as different classes.  You must have at least 1 subdirectory.')
    parser.add_argument('--batch_size', '-b', type=int, default=32, help='Number of samples to include in each batch.  If using batch norm, this needs to be preserved when in server mode')
    parser.add_argument('--crop', type=bool, default=False, help='If your images are perfectly sized you can skip cropping.')
    parser.add_argument('--device', '-d', type=str, default='/gpu:0', help='In the form "/gpu:0", "/cpu:0", etc.  Always use a GPU (or TPU) to train')
    parser.add_argument('--format', '-f', type=str, default='png', help='jpg or png')
    parser.add_argument('--sample_every', type=int, default=50, help='Samples the model every n epochs.')
    parser.add_argument('--save_every', type=int, default=30000, help='Saves the model every n epochs.')
    parser.add_argument('--size', '-s', type=str, default='64x64x3', help='Size of your data.  For images it is widthxheightxchannels.')
    parser.add_argument('--use_hc_io', type=bool, default=False, help='Set this to no unless you are feeling experimental.')
    return parser.parse_args()

x_v = None
z_v = None
def sampler(name, sess, config):
    generator = get_tensor("g")[0]
    y_t = get_tensor("y")
    z_t = get_tensor("z")
    x_t = get_tensor('x')
    fltr_x_t = get_tensor('xfiltered')
    global x_v
    global z_v
    if(x_v == None):
        x_v, z_v = sess.run([x_t, z_t])
        x_v = np.tile(x_v[0], [config['batch_size'],1,1,1])

    sample, bw_x = sess.run([generator, fltr_x_t], {x_t: x_v, z_t: z_v})
    bw = np.squeeze(np.tile(bw_x[0], [1,1,1,3]))
    stacks = []
    stacks.append([x_v[0], bw, sample[0], sample[1], sample[2], sample[3]])
    for i in range(4):
        stacks.append([sample[i*6+4+j] for j in range(6)])

    print('bwxshape', bw.shape, x_v[0].shape)
    images = np.vstack([np.hstack(s) for s in stacks])
    plot(config, images, name)

def add_bw(gan, net):
    x = get_tensor('x')
    s = [int(x) for x in net.get_shape()]
    shape = [s[1], s[2]]
    x = tf.image.resize_images(x, shape, 1)
    print("Created bw ", x)

    x = tf.image.rgb_to_grayscale(x)
    #x += tf.random_normal(x.get_shape(), mean=0, stddev=1e-1, dtype=config['dtype'])

    return x

def add_original_x(gan, net):
    x = get_tensor('x')
    s = [int(x) for x in net.get_shape()]
    shape = [s[1], s[2]]
    x = tf.image.resize_images(x, shape, 1)
    print("Created bw ", x)

    x = tf.image.rgb_to_grayscale(x)
    #x = tf.nn.dropout(x, 0.005)
    #x += tf.random_normal(x.get_shape(), mean=0, stddev=1e-1, dtype=config['dtype'])

    return x

args = parse_args()

width = int(args.size.split("x")[0])
height = int(args.size.split("x")[1])
channels = int(args.size.split("x")[2])

selector = hg.config.selector(args)

config = selector.random_config()
config_filename = os.path.expanduser('~/.hypergan/configs/colorizer.json')
config = selector.load_or_create_config(config_filename, config)

#TODO add this option to D
#TODO add this option to G
config['generator.layer_filter'] = add_bw
config['discriminators'][0]['layer_filter'] = add_original_x

# TODO refactor, shared in CLI
config['dtype']=tf.float32
config['batch_size'] = args.batch_size
x,y,f,num_labels,examples_per_epoch = image_loader.labelled_image_tensors_from_directory(
                        args.directory,
                        config['batch_size'],
                        channels=channels,
                        format=args.format,
                        crop=args.crop,
                        width=width,
                        height=height)

config['y_dims']=num_labels
config['x_dims']=[height,width]
config['channels']=channels
config['model']='colorizer'
config = hg.config.lookup_functions(config)

initial_graph = {
    'x':x,
    'y':y,
    'f':f,
    'num_labels':num_labels,
    'examples_per_epoch':examples_per_epoch
}

gan = hg.GAN(config, initial_graph)

save_file = os.path.expanduser("~/.hypergan/saves/colorizer.ckpt")
gan.load_or_initialize_graph(save_file)

tf.train.start_queue_runners(sess=gan.sess)
for i in range(10000000):
    d_loss, g_loss = gan.train()

    if i % args.save_every == 0 and i > 0:
        print("Saving " + save_file)
        gan.save(save_file)

    if i % args.sample_every == 0 and i > 0:
        print("Sampling "+str(i))
        sample_file = "samples/"+str(i)+".png"
        gan.sample_to_file(sample_file, sampler=sampler)
        if args.use_hc_io:
            hc.io.sample(config, [{"image":sample_file, "label": 'sample'}])

tf.reset_default_graph()
sess.close()
<commit_msg>[fix] remove b&w x from D in colorizer - it's more interesting without<commit_after>import argparse
import os
import tensorflow as tf
import hypergan as hg
import hyperchamber as hc
from hypergan.loaders import *
from hypergan.samplers.common import *
from hypergan.util.globals import *

def parse_args():
    parser = argparse.ArgumentParser(description='Train a colorizer!', add_help=True)
    parser.add_argument('directory', action='store', type=str, help='The location of your data.  Subdirectories are treated as different classes.  You must have at least 1 subdirectory.')
    parser.add_argument('--batch_size', '-b', type=int, default=32, help='Number of samples to include in each batch.  If using batch norm, this needs to be preserved when in server mode')
    parser.add_argument('--crop', type=bool, default=False, help='If your images are perfectly sized you can skip cropping.')
    parser.add_argument('--device', '-d', type=str, default='/gpu:0', help='In the form "/gpu:0", "/cpu:0", etc.  Always use a GPU (or TPU) to train')
    parser.add_argument('--format', '-f', type=str, default='png', help='jpg or png')
    parser.add_argument('--sample_every', type=int, default=50, help='Samples the model every n epochs.')
    parser.add_argument('--save_every', type=int, default=30000, help='Saves the model every n epochs.')
    parser.add_argument('--size', '-s', type=str, default='64x64x3', help='Size of your data.  For images it is widthxheightxchannels.')
    parser.add_argument('--use_hc_io', type=bool, default=False, help='Set this to no unless you are feeling experimental.')
    return parser.parse_args()

x_v = None
z_v = None
def sampler(name, sess, config):
    generator = get_tensor("g")[0]
    y_t = get_tensor("y")
    z_t = get_tensor("z")
    x_t = get_tensor('x')
    fltr_x_t = get_tensor('xfiltered')
    global x_v
    global z_v
    if(x_v == None):
        x_v, z_v = sess.run([x_t, z_t])
        x_v = np.tile(x_v[0], [config['batch_size'],1,1,1])

    sample, bw_x = sess.run([generator, fltr_x_t], {x_t: x_v, z_t: z_v})
    bw = np.squeeze(np.tile(bw_x[0], [1,1,1,3]))
    stacks = []
    stacks.append([x_v[0], bw, sample[0], sample[1], sample[2], sample[3]])
    for i in range(4):
        stacks.append([sample[i*6+4+j] for j in range(6)])

    print('bwxshape', bw.shape, x_v[0].shape)
    images = np.vstack([np.hstack(s) for s in stacks])
    plot(config, images, name)

def add_bw(gan, net):
    x = get_tensor('x')
    s = [int(x) for x in net.get_shape()]
    shape = [s[1], s[2]]
    x = tf.image.resize_images(x, shape, 1)
    print("Created bw ", x)

    x = tf.image.rgb_to_grayscale(x)
    #x += tf.random_normal(x.get_shape(), mean=0, stddev=1e-1, dtype=config['dtype'])

    return x

def add_original_x(gan, net):
    x = get_tensor('x')
    s = [int(x) for x in net.get_shape()]
    shape = [s[1], s[2]]
    x = tf.image.resize_images(x, shape, 1)
    print("Created bw ", x)

    x = tf.image.rgb_to_grayscale(x)
    #x = tf.nn.dropout(x, 0.005)
    #x += tf.random_normal(x.get_shape(), mean=0, stddev=1e-1, dtype=config['dtype'])

    return x

args = parse_args()

width = int(args.size.split("x")[0])
height = int(args.size.split("x")[1])
channels = int(args.size.split("x")[2])

selector = hg.config.selector(args)

config = selector.random_config()
config_filename = os.path.expanduser('~/.hypergan/configs/colorizer.json')
config = selector.load_or_create_config(config_filename, config)

#TODO add this option to D
#TODO add this option to G
config['generator.layer_filter'] = add_bw
config['discriminators'][0]['layer_filter'] = None#add_original_x

# TODO refactor, shared in CLI
config['dtype']=tf.float32
config['batch_size'] = args.batch_size
x,y,f,num_labels,examples_per_epoch = image_loader.labelled_image_tensors_from_directory(
                        args.directory,
                        config['batch_size'],
                        channels=channels,
                        format=args.format,
                        crop=args.crop,
                        width=width,
                        height=height)

config['y_dims']=num_labels
config['x_dims']=[height,width]
config['channels']=channels
config['model']='colorizer'
config = hg.config.lookup_functions(config)

initial_graph = {
    'x':x,
    'y':y,
    'f':f,
    'num_labels':num_labels,
    'examples_per_epoch':examples_per_epoch
}

gan = hg.GAN(config, initial_graph)

save_file = os.path.expanduser("~/.hypergan/saves/colorizer.ckpt")
gan.load_or_initialize_graph(save_file)

tf.train.start_queue_runners(sess=gan.sess)
for i in range(10000000):
    d_loss, g_loss = gan.train()

    if i % args.save_every == 0 and i > 0:
        print("Saving " + save_file)
        gan.save(save_file)

    if i % args.sample_every == 0 and i > 0:
        print("Sampling "+str(i))
        sample_file = "samples/"+str(i)+".png"
        gan.sample_to_file(sample_file, sampler=sampler)
        if args.use_hc_io:
            hc.io.sample(config, [{"image":sample_file, "label": 'sample'}])

tf.reset_default_graph()
sess.close()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import json
import requests
import logging

def findIndexOfSequence(data, sequence, startindex = 0):
    index = startindex
    for token in sequence:
        index = data.find(token, index)
        if index == -1:
            return -1

    return index + len(sequence[-1])

def getCardValue(cardName, setCode):
    url = "http://www.mtggoldfish.com/widgets/autocard/%s [%s]" % (cardName, setCode)
    headers = {
        'Pragma': 'no-cache',
        'Accept-Encoding': 'gzip, deflate, sdch',
        'Accept-Language': 'en-US,en;q=0.8,de;q=0.6,sv;q=0.4',
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.109 Safari/537.36',
        'Accept': 'text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01',
        'Referer': 'http://www.mtggoldfish.com/widgets/autocard/%s' % cardName,
        'X-Requested-With': 'XMLHttpRequest',
        'Connection': 'keep-alive',
        'Cache-Control': 'no-cache'
    }
    response = requests.get(url, headers=headers)
    index = findIndexOfSequence(response.content, ["tcgplayer", "btn-shop-price", "$"])
    endIndex = response.content.find("\\n", index)
    try:
        value = float(response.content[index+2:endIndex].replace(",", ""))
    except ValueError:
        value = 0

    return value

def getCard(name):
    queryUrl = "http://api.deckbrew.com/mtg/cards?name=%s" % name
    print(queryUrl)
    r = requests.get(queryUrl)
    cards = r.json()

    if len(cards) < 1:
        return None

    card = cards[0]
    bestMatch = None
    for cardIter in cards:
        pos = cardIter["name"].lower().find(name)
        if bestMatch is None or (pos != -1 and pos < bestMatch):
            bestMatch = pos
            card = cardIter

    mostRecent = card["editions"][0]
    card["value"] = getCardValue(card["name"], mostRecent["set_id"])

    return card

def getPlaneswalker(dciNumber):
    url = "http://www.wizards.com/Magic/PlaneswalkerPoints/JavaScript/GetPointsHistoryModal"
    headers = {
        'Pragma': 'no-cache',
        'Origin': 'http://www.wizards.com',
        'Accept-Encoding': 'gzip, deflate',
        'Accept-Language': 'en-US,en;q=0.8,de;q=0.6,sv;q=0.4',
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.109 Safari/537.36',
        'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
        'Accept': '*/*',
        'Cache-Control': 'no-cache',
        'X-Requested-With': 'XMLHttpRequest',
        'Cookie': 'f5_cspm=1234; BIGipServerWWWPWPPOOL01=353569034.20480.0000; __utmt=1; BIGipServerWWWPool1=3792701706.20480.0000; PlaneswalkerPointsSettings=0=0&lastviewed=9212399887; __utma=75931667.1475261136.1456488297.1456488297.1456488297.1; __utmb=75931667.5.10.1456488297; __utmc=75931667; __utmz=75931667.1456488297.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)',
        'Connection': 'keep-alive',
        'Referer': 'http://www.wizards.com/Magic/PlaneswalkerPoints/%s' % dciNumber
    }
    data = {"Parameters":{"DCINumber":dciNumber,"SelectedType":"Yearly"}}
    response = requests.post(url, headers=headers, data=json.dumps(data))

    seasons = []

    responseData = json.loads(response.content)
    markup = responseData["ModalContent"]
    searchPosition = markup.find("SeasonRange")

    while searchPosition != -1:
        pointsvalue = "PointsValue\">"
        searchPosition = markup.find(pointsvalue, searchPosition)
        searchPosition += len(pointsvalue)
        endPosition = markup.find("
", searchPosition)
        if endPosition != -1:
            value = markup[searchPosition:endPosition]
            seasons.append(int(value))
        searchPosition = markup.find("SeasonRange", searchPosition)

    return {"currentSeason": seasons[0], "lastSeason": seasons[1]}

def getPlaneswalkerByes(walker):
    if walker["currentSeason"] >= 2250 or walker["lastSeason"] >= 2250:
        return 2
    elif walker["currentSeason"] >= 1300 or walker["lastSeason"] >= 1300:
        return 1

    return 0

def emojiFilter(indata):
    ret = indata.replace("{", ":_")
    ret = ret.replace("}", "_:")
    lastpos = None
    while ret.rfind(":_", 0, lastpos) != -1:
        lastpos = ret.rfind(":_", 0, lastpos)
        start = lastpos + 2
        end = ret.rfind("_:")
        content = ret[start:end]
        content = content.lower()
        content = content.replace("/", "")
        ret = ret[:start] + content + ret[end:]

    return ret

def parseCardTags(str):
    bangTrigger = "!card "
    cardTagOpen = "[["
    cardTagEnd = "]]"

    triggerIndex = str.find(bangTrigger)
    if triggerIndex > -1:
        return [str[triggerIndex + len(bangTrigger):]]

    allTags = []
    while True:
        tagOpenIndex = str.find(cardTagOpen)
        tagEndIndex = str.find(cardTagEnd)
        if tagOpenIndex > -1 and tagEndIndex > -1 and tagEndIndex > tagOpenIndex:
            allTags.append(str[tagOpenIndex + len(cardTagOpen):tagEndIndex])
            str = str[:tagEndIndex + len(cardTagEnd)]
        else:
            break
    return allTags

def parseForCardInput(sc, indata):
    if indata.has_key("text"):
        logging.info("Shit is happening yo")
        userinput = indata["text"].lower()

        attachments = ""
        text = ""
        notFound = "No results found"

        allTags = parseCardTags(userinput)
        logging.info("parsed some tags! %s" % allTags)
        if len(allTags) > 0:
            tempText = ""
            tempAttachments = []
            for searchTerm in allTags:
                card = getCard(searchTerm)
                if not card:
                    tempText += "No result for %s\n" % searchTerm
                else:
                    mostRecentPrinting = card["editions"][0]
                    header = "Latest printing for %s is %s\n" % (card["name"], mostRecentPrinting["set"])
                    tempAttachments.append('{"image_url":"%s","title":"%s"}'
                      % (mostRecentPrinting["image_url"], card["name"].replace("\"", "\\\"")))
                    tempText += header

            text += tempText
            attachments += "[%s]" % (",".join(tempAttachments))

        oracleTrigger = "!oracle "
        if userinput.find(oracleTrigger) > -1:
            searchTerm = userinput[userinput.find(oracleTrigger) + len(oracleTrigger):]
            card = getCard(searchTerm)
            if not card:
                text += notFound
            else:
                mostRecentPrinting = card["editions"][0]
                typeline = ""
                if card.has_key("supertypes"):
                    for supertype in card["supertypes"]:
                        typeline += supertype.capitalize() + " "
                if card.has_key("types"):
                    for cardtype in card["types"]:
                        typeline += cardtype.capitalize() + " "
                    if card.has_key("subtypes"):
                        typeline += "- "
                if card.has_key("subtypes"):
                    for subtype in card["subtypes"]:
                        typeline += subtype.capitalize() + " "
                answer = "%s\t\t%s\n%s\n%s" % (card["name"], emojiFilter(card["cost"]), typeline, emojiFilter(card["text"]))
                valueinfo = ""
                if card.has_key("power") and card.has_key("toughness"):
                    answer += "\n*`%s/%s`*" % (card["power"], card["toughness"])
                if card["value"] > 0:
                    valueinfo = "\n\nCurrent market price for most recent printing (%s) - $%.1f" % (mostRecentPrinting["set"], card["value"])

                answer += valueinfo
                text += answer

        priceTrigger = "!price "
        if userinput.find(priceTrigger) > -1:
            searchTerm = userinput[userinput.find(priceTrigger) + len(priceTrigger):]
            card = getCard(searchTerm)
            if not card:
                text += notFound
            else:
                mostRecentPrinting = card["editions"][0]
                answer = "Unable to find price information for %s" % card["name"]
                if card["value"] > 0:
                    answer = "Current market price for most recent printing of %s (%s) - $%.1f" % (card["name"], mostRecentPrinting["set"], card["value"])

                text += answer

        pwpTrigger = "!pwp "
        if userinput.find(pwpTrigger) > -1:
            searchTerm = userinput[userinput.find(pwpTrigger) + len(pwpTrigger):]
            planeswalker = getPlaneswalker(searchTerm)
            answer = "DCI# %s has %s points in the current season, %s points last season\nCurrently " % (searchTerm, planeswalker["currentSeason"], planeswalker["lastSeason"])
            byes = getPlaneswalkerByes(planeswalker)
            if not byes:
                answer += "not eligible for GP byes"
            else:
                answer += "eligible for %d GP byes" % byes

            text += answer

        crTrigger = "!cr "
        if userinput.find(crTrigger) > -1:
            searchTerm = userinput[userinput.find(crTrigger) + len(crTrigger):]
            rule = getRule(searchTerm)
            if rule:
                answer = "%s - %s" % (searchTerm, rule)
            else:
                answer = "No result found"
            text += answer

        if text or attachments:
            sc.rtm.api_call(
                "chat.postMessage",
                channel=indata["channel"],
                attachments=attachments,
                text=text,
                as_user=True)

compRulesUrl = "http://media.wizards.com/2016/docs/MagicCompRules_04082016.txt"
compRulesLookup = {}

def getCompRules():
    r = requests.get(compRulesUrl)
    rules = r.text.encode("utf-8")
    for rule in rules.split("\n"):
        compRulesLookup[rule.split(" ")[0]] = (" ".join(rule.split(" ")[1:])).decode("Windows.1252").replace(u"Â","")
    return compRulesLookup

def getRule(ruleKey):
    if ruleKey not in compRulesLookup:
        if ruleKey[-1:] == ".":
            ruleKey = ruleKey[:-1]
    if ruleKey not in compRulesLookup:
        ruleKey += "."
    if ruleKey not in compRulesLookup:
        return None
    return compRulesLookup[ruleKey]
<commit_msg>more logging<commit_after># -*- coding: utf-8 -*-

import json
import requests
import logging

def findIndexOfSequence(data, sequence, startindex = 0):
    index = startindex
    for token in sequence:
        index = data.find(token, index)
        if index == -1:
            return -1

    return index + len(sequence[-1])

def getCardValue(cardName, setCode):
    url = "http://www.mtggoldfish.com/widgets/autocard/%s [%s]" % (cardName, setCode)
    headers = {
        'Pragma': 'no-cache',
        'Accept-Encoding': 'gzip, deflate, sdch',
        'Accept-Language': 'en-US,en;q=0.8,de;q=0.6,sv;q=0.4',
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.109 Safari/537.36',
        'Accept': 'text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01',
        'Referer': 'http://www.mtggoldfish.com/widgets/autocard/%s' % cardName,
        'X-Requested-With': 'XMLHttpRequest',
        'Connection': 'keep-alive',
        'Cache-Control': 'no-cache'
    }
    response = requests.get(url, headers=headers)
    index = findIndexOfSequence(response.content, ["tcgplayer", "btn-shop-price", "$"])
    endIndex = response.content.find("\\n", index)
    try:
        value = float(response.content[index+2:endIndex].replace(",", ""))
    except ValueError:
        value = 0

    return value

def getCard(name):
    queryUrl = "http://api.deckbrew.com/mtg/cards?name=%s" % name
    print(queryUrl)
    r = requests.get(queryUrl)
    cards = r.json()

    if len(cards) < 1:
        return None

    card = cards[0]
    bestMatch = None
    for cardIter in cards:
        pos = cardIter["name"].lower().find(name)
        if bestMatch is None or (pos != -1 and pos < bestMatch):
            bestMatch = pos
            card = cardIter

    mostRecent = card["editions"][0]
    card["value"] = getCardValue(card["name"], mostRecent["set_id"])

    return card

def getPlaneswalker(dciNumber):
    url = "http://www.wizards.com/Magic/PlaneswalkerPoints/JavaScript/GetPointsHistoryModal"
    headers = {
        'Pragma': 'no-cache',
        'Origin': 'http://www.wizards.com',
        'Accept-Encoding': 'gzip, deflate',
        'Accept-Language': 'en-US,en;q=0.8,de;q=0.6,sv;q=0.4',
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.109 Safari/537.36',
        'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
        'Accept': '*/*',
        'Cache-Control': 'no-cache',
        'X-Requested-With': 'XMLHttpRequest',
        'Cookie': 'f5_cspm=1234; BIGipServerWWWPWPPOOL01=353569034.20480.0000; __utmt=1; BIGipServerWWWPool1=3792701706.20480.0000; PlaneswalkerPointsSettings=0=0&lastviewed=9212399887; __utma=75931667.1475261136.1456488297.1456488297.1456488297.1; __utmb=75931667.5.10.1456488297; __utmc=75931667; __utmz=75931667.1456488297.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)',
        'Connection': 'keep-alive',
        'Referer': 'http://www.wizards.com/Magic/PlaneswalkerPoints/%s' % dciNumber
    }
    data = {"Parameters":{"DCINumber":dciNumber,"SelectedType":"Yearly"}}
    response = requests.post(url, headers=headers, data=json.dumps(data))

    seasons = []

    responseData = json.loads(response.content)
    markup = responseData["ModalContent"]
    searchPosition = markup.find("SeasonRange")

    while searchPosition != -1:
        pointsvalue = "PointsValue\">"
        searchPosition = markup.find(pointsvalue, searchPosition)
        searchPosition += len(pointsvalue)
        endPosition = markup.find("
", searchPosition)
        if endPosition != -1:
            value = markup[searchPosition:endPosition]
            seasons.append(int(value))
        searchPosition = markup.find("SeasonRange", searchPosition)

    return {"currentSeason": seasons[0], "lastSeason": seasons[1]}

def getPlaneswalkerByes(walker):
    if walker["currentSeason"] >= 2250 or walker["lastSeason"] >= 2250:
        return 2
    elif walker["currentSeason"] >= 1300 or walker["lastSeason"] >= 1300:
        return 1

    return 0

def emojiFilter(indata):
    ret = indata.replace("{", ":_")
    ret = ret.replace("}", "_:")
    lastpos = None
    while ret.rfind(":_", 0, lastpos) != -1:
        lastpos = ret.rfind(":_", 0, lastpos)
        start = lastpos + 2
        end = ret.rfind("_:")
        content = ret[start:end]
        content = content.lower()
        content = content.replace("/", "")
        ret = ret[:start] + content + ret[end:]

    return ret

def parseCardTags(str):
    bangTrigger = "!card "
    cardTagOpen = "[["
    cardTagEnd = "]]"

    triggerIndex = str.find(bangTrigger)
    if triggerIndex > -1:
        return [str[triggerIndex + len(bangTrigger):]]

    allTags = []
    while True:
        tagOpenIndex = str.find(cardTagOpen)
        tagEndIndex = str.find(cardTagEnd)
        if tagOpenIndex > -1 and tagEndIndex > -1 and tagEndIndex > tagOpenIndex:
            logging.info(str)
            tag = str[tagOpenIndex + len(cardTagOpen):tagEndIndex]
            logging.info(tag)
            allTags.append(str[tagOpenIndex + len(cardTagOpen):tagEndIndex])
            str = str[:tagEndIndex + len(cardTagEnd)]
        else:
            break
    return allTags

def parseForCardInput(sc, indata):
    if indata.has_key("text"):
        userinput = indata["text"].lower()

        attachments = ""
        text = ""
        notFound = "No results found"

        allTags = parseCardTags(userinput)
        logging.info("parsed some tags! %s" % allTags)
        if len(allTags) > 0:
            tempText = ""
            tempAttachments = []
            for searchTerm in allTags:
                card = getCard(searchTerm)
                if not card:
                    tempText += "No result for %s\n" % searchTerm
                else:
                    mostRecentPrinting = card["editions"][0]
                    header = "Latest printing for %s is %s\n" % (card["name"], mostRecentPrinting["set"])
                    tempAttachments.append('{"image_url":"%s","title":"%s"}'
                      % (mostRecentPrinting["image_url"], card["name"].replace("\"", "\\\"")))
                    tempText += header

            text += tempText
            attachments += "[%s]" % (",".join(tempAttachments))

        oracleTrigger = "!oracle "
        if userinput.find(oracleTrigger) > -1:
            searchTerm = userinput[userinput.find(oracleTrigger) + len(oracleTrigger):]
            card = getCard(searchTerm)
            if not card:
                text += notFound
            else:
                mostRecentPrinting = card["editions"][0]
                typeline = ""
                if card.has_key("supertypes"):
                    for supertype in card["supertypes"]:
                        typeline += supertype.capitalize() + " "
                if card.has_key("types"):
                    for cardtype in card["types"]:
                        typeline += cardtype.capitalize() + " "
                    if card.has_key("subtypes"):
                        typeline += "- "
                if card.has_key("subtypes"):
                    for subtype in card["subtypes"]:
                        typeline += subtype.capitalize() + " "
                answer = "%s\t\t%s\n%s\n%s" % (card["name"], emojiFilter(card["cost"]), typeline, emojiFilter(card["text"]))
                valueinfo = ""
                if card.has_key("power") and card.has_key("toughness"):
                    answer += "\n*`%s/%s`*" % (card["power"], card["toughness"])
                if card["value"] > 0:
                    valueinfo = "\n\nCurrent market price for most recent printing (%s) - $%.1f" % (mostRecentPrinting["set"], card["value"])

                answer += valueinfo
                text += answer

        priceTrigger = "!price "
        if userinput.find(priceTrigger) > -1:
            searchTerm = userinput[userinput.find(priceTrigger) + len(priceTrigger):]
            card = getCard(searchTerm)
            if not card:
                text += notFound
            else:
                mostRecentPrinting = card["editions"][0]
                answer = "Unable to find price information for %s" % card["name"]
                if card["value"] > 0:
                    answer = "Current market price for most recent printing of %s (%s) - $%.1f" % (card["name"], mostRecentPrinting["set"], card["value"])

                text += answer

        pwpTrigger = "!pwp "
        if userinput.find(pwpTrigger) > -1:
            searchTerm = userinput[userinput.find(pwpTrigger) + len(pwpTrigger):]
            planeswalker = getPlaneswalker(searchTerm)
            answer = "DCI# %s has %s points in the current season, %s points last season\nCurrently " % (searchTerm, planeswalker["currentSeason"], planeswalker["lastSeason"])
            byes = getPlaneswalkerByes(planeswalker)
            if not byes:
                answer += "not eligible for GP byes"
            else:
                answer += "eligible for %d GP byes" % byes

            text += answer

        crTrigger = "!cr "
        if userinput.find(crTrigger) > -1:
            searchTerm = userinput[userinput.find(crTrigger) + len(crTrigger):]
            rule = getRule(searchTerm)
            if rule:
                answer = "%s - %s" % (searchTerm, rule)
            else:
                answer = "No result found"
            text += answer

        if text or attachments:
            sc.rtm.api_call(
                "chat.postMessage",
                channel=indata["channel"],
                attachments=attachments,
                text=text,
                as_user=True)

compRulesUrl = "http://media.wizards.com/2016/docs/MagicCompRules_04082016.txt"
compRulesLookup = {}

def getCompRules():
    r = requests.get(compRulesUrl)
    rules = r.text.encode("utf-8")
    for rule in rules.split("\n"):
        compRulesLookup[rule.split(" ")[0]] = (" ".join(rule.split(" ")[1:])).decode("Windows.1252").replace(u"Â","")
    return compRulesLookup

def getRule(ruleKey):
    if ruleKey not in compRulesLookup:
        if ruleKey[-1:] == ".":
            ruleKey = ruleKey[:-1]
    if ruleKey not in compRulesLookup:
        ruleKey += "."
    if ruleKey not in compRulesLookup:
        return None
    return compRulesLookup[ruleKey]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Decorates a function for Bowtie.

Reference
---------
https://gist.github.com/carlsmith/800cbe3e11f630ac8aa0
"""

from __future__ import print_function

import os
import sys
import inspect
from subprocess import call

import click

from bowtie._compat import numargs
from bowtie._app import _DIRECTORY, _WEBPACK, App

class WrongNumberOfArguments(TypeError):
    """The "build" function accepts an incorrect number of arguments."""

    pass

def _build(app):
    if app is None:
        print(('No `App` instance was returned. '
               'In the function decorated with @command, '
               'return the `App` instance so it can be built.'))
    else:
        if not isinstance(app, App):
            raise Exception(('Returned value {} is of type {}, '
                             'it needs to be a bowtie.App instance.'.format(app, type(app))))
        # pylint:disable=protected-access
        app._build()

def command(func):
    """Command line interface decorator.

    Decorate a function for building a Bowtie
    application and turn it into a command line interface.
    """
    @click.group(options_metavar='[--help]')
    def cmd():
        """Bowtie CLI to help build and run your app."""
        pass

    # pylint: disable=unused-variable
    @cmd.command(add_help_option=False)
    def build():
        """Write the app, downloads the packages, and bundles it with Webpack."""
        nargs = numargs(func)
        if nargs == 0:
            app = func()
        else:
            raise WrongNumberOfArguments(
                'Decorated function "{}" should have no arguments, it has {}.'
                .format(func.__name__, nargs)
            )
        _build(app)

    @cmd.command(context_settings=dict(ignore_unknown_options=True),
                 add_help_option=False)
    @click.argument('extra', nargs=-1, type=click.UNPROCESSED)
    def run(extra):
        """Write the app, downloads the packages, and bundles it with Webpack."""
        nargs = numargs(func)
        if nargs == 0:
            app = func()
        else:
            raise WrongNumberOfArguments(
                'Decorated function "{}" should have no arguments, it has {}.'
                .format(func.__name__, nargs)
            )
        # pylint:disable=protected-access
        _build(app)
        filepath = './{}/src/server.py'.format(_DIRECTORY)
        line = (filepath,) + extra
        call(line)

    @cmd.command(context_settings=dict(ignore_unknown_options=True),
                 add_help_option=False)
    @click.argument('extra', nargs=-1, type=click.UNPROCESSED)
    def serve(extra):
        """Serve the Bowtie app."""
        filepath = './{}/src/server.py'.format(_DIRECTORY)
        if os.path.isfile(filepath):
            line = (filepath,) + extra
            call(line)
        else:
            print("Cannot find '{}'. Did you build the app?".format(filepath))

    @cmd.command(context_settings=dict(ignore_unknown_options=True),
                 add_help_option=False)
    @click.argument('extra', nargs=-1, type=click.UNPROCESSED)
    def dev(extra):
        """Recompile the app for development."""
        line = (_WEBPACK, '-d') + extra
        call(line, cwd=_DIRECTORY)

    @cmd.command(context_settings=dict(ignore_unknown_options=True),
                 add_help_option=False)
    @click.argument('extra', nargs=-1, type=click.UNPROCESSED)
    def prod(extra):
        """Recompile the app for production."""
        line = (_WEBPACK, '--define', 'process.env.NODE_ENV="production"', '--progress') + extra
        call(line, cwd=_DIRECTORY)

    locale = inspect.stack()[1][0].f_locals
    module = locale.get("__name__")

    if module == "__main__":
        try:
            arg = sys.argv[1:]
        except IndexError:
            arg = ('--help',)
        # pylint: disable=too-many-function-args
        sys.exit(cmd(arg))

    return cmd
<commit_msg>update run command docstring<commit_after># -*- coding: utf-8 -*-
"""
Decorates a function for Bowtie.

Reference
---------
https://gist.github.com/carlsmith/800cbe3e11f630ac8aa0
"""

from __future__ import print_function

import os
import sys
import inspect
from subprocess import call

import click

from bowtie._compat import numargs
from bowtie._app import _DIRECTORY, _WEBPACK, App

class WrongNumberOfArguments(TypeError):
    """The "build" function accepts an incorrect number of arguments."""

    pass

def _build(app):
    if app is None:
        print(('No `App` instance was returned. '
               'In the function decorated with @command, '
               'return the `App` instance so it can be built.'))
    else:
        if not isinstance(app, App):
            raise Exception(('Returned value {} is of type {}, '
                             'it needs to be a bowtie.App instance.'.format(app, type(app))))
        # pylint:disable=protected-access
        app._build()

def command(func):
    """Command line interface decorator.

    Decorate a function for building a Bowtie
    application and turn it into a command line interface.
    """
    @click.group(options_metavar='[--help]')
    def cmd():
        """Bowtie CLI to help build and run your app."""
        pass

    # pylint: disable=unused-variable
    @cmd.command(add_help_option=False)
    def build():
        """Write the app, downloads the packages, and bundles it with Webpack."""
        nargs = numargs(func)
        if nargs == 0:
            app = func()
        else:
            raise WrongNumberOfArguments(
                'Decorated function "{}" should have no arguments, it has {}.'
                .format(func.__name__, nargs)
            )
        _build(app)

    @cmd.command(context_settings=dict(ignore_unknown_options=True),
                 add_help_option=False)
    @click.argument('extra', nargs=-1, type=click.UNPROCESSED)
    def run(extra):
        """Build the app and serve it."""
        nargs = numargs(func)
        if nargs == 0:
            app = func()
        else:
            raise WrongNumberOfArguments(
                'Decorated function "{}" should have no arguments, it has {}.'
                .format(func.__name__, nargs)
            )
        # pylint:disable=protected-access
        _build(app)
        filepath = './{}/src/server.py'.format(_DIRECTORY)
        line = (filepath,) + extra
        call(line)

    @cmd.command(context_settings=dict(ignore_unknown_options=True),
                 add_help_option=False)
    @click.argument('extra', nargs=-1, type=click.UNPROCESSED)
    def serve(extra):
        """Serve the Bowtie app."""
        filepath = './{}/src/server.py'.format(_DIRECTORY)
        if os.path.isfile(filepath):
            line = (filepath,) + extra
            call(line)
        else:
            print("Cannot find '{}'. Did you build the app?".format(filepath))

    @cmd.command(context_settings=dict(ignore_unknown_options=True),
                 add_help_option=False)
    @click.argument('extra', nargs=-1, type=click.UNPROCESSED)
    def dev(extra):
        """Recompile the app for development."""
        line = (_WEBPACK, '-d') + extra
        call(line, cwd=_DIRECTORY)

    @cmd.command(context_settings=dict(ignore_unknown_options=True),
                 add_help_option=False)
    @click.argument('extra', nargs=-1, type=click.UNPROCESSED)
    def prod(extra):
        """Recompile the app for production."""
        line = (_WEBPACK, '--define', 'process.env.NODE_ENV="production"', '--progress') + extra
        call(line, cwd=_DIRECTORY)

    locale = inspect.stack()[1][0].f_locals
    module = locale.get("__name__")

    if module == "__main__":
        try:
            arg = sys.argv[1:]
        except IndexError:
            arg = ('--help',)
        # pylint: disable=too-many-function-args
        sys.exit(cmd(arg))

    return cmd
<|endoftext|>


<commit_before>import torch
from torch.autograd import Variable
import torch.nn
import numpy as np
import torch.nn.functional as F

class LinearModel(torch.nn.Module):
    """
    Cost sensitive linear model with label dependent features and shared parameter.
    Input: a [n, 1, d*K] tensor
    Output: [n, 1, K] tensor
    """

    def __init__(self, d, K):
        super(LinearModel, self).__init__()
        self.conv = torch.nn.Conv1d(1, 1, kernel_size=d, stride=d, bias=False)

    def forward(self,x):
        h = self.conv(x)
        z = h - torch.mean(h)
        return (z)

class TwoLayer(torch.nn.Module):
    """
    Cost sensitive two-layer NN with label dependent features and shared parameter.
    Input: a [n, 1, d*K] tensor
    Output: [n, 1, K] tensor
    """
    def __init__(self, d, K, hidden=2):
        super(TwoLayer, self).__init__()
        self.fc1 = torch.nn.Conv1d(1, hidden, kernel_size=d,stride=d, bias=False)
        self.fc2 = torch.nn.Conv1d(hidden, 1, kernel_size=1, stride=1, bias=False)

    def forward(self,x):
        h1 = F.relu(self.fc1(x))
        h = self.fc2(h1)
        z = h - torch.mean(h)
        return (z)

def model_to_action(model, x):
    X = np.zeros((1, 1, x.get_ld_dim()*x.get_K()))
    X[0,0,:] = x.get_ld_features().reshape(1,x.get_ld_dim()*x.get_K())
    X = Variable(torch.FloatTensor(X))
    output = model(X)
    # print(output.data)
    p = torch.clamp(1+output,0)/torch.sum(torch.clamp(1+output,0))
    return (np.array(p.data)[0,0])

def langevin_step(model, features, targets, lr, iters=512, noise=True, debug=False):
    X = Variable(torch.FloatTensor(features))
    Y = Variable(torch.FloatTensor(targets))
    for i in range(1,iters+1):
        model.zero_grad()
        output = model(X)
        loss = Y.dot(torch.clamp(1+output, 0.0))/Y.shape[0]
        if np.log2(i) == np.floor(np.log2(i)) and debug:
            print("Iteration %d, Loss %0.3f" % (i, loss.data[0]))
        loss.backward()
        for f in model.parameters():
            if noise:
                r = torch.FloatTensor(np.random.normal(0,1,np.shape(f.grad.data)))
            else:
                r = torch.zeros(f.grad.data.size())
            f.data.sub_(f.grad.data*lr/2 - np.sqrt(lr)*r)
            f.data = torch.clamp(f.data, -1, 1)

if __name__=='__main__':
    import Simulators
    d = 20
    K = 5
    S = Simulators.LinearBandit(d, 1, K, noise=True, pos=False, quad=False)

    n = 1

    X = np.zeros([n, 1, d*K])
    Y = np.zeros([n, 1, K])
    for i in range(n):
        x = S.get_new_context()
        X[i,0,:] = x.get_ld_features().reshape(1,d*K)
        r = np.array(S.get_all_rewards())
        Y[i,0,:] = np.array(S.get_all_rewards())

    model = LinearModel(d,K)
    print(model)
    print([param for param in model.parameters()])
    X = torch.FloatTensor(X)
    Y = torch.FloatTensor(Y)
    langevin_step(model, X, Y, 0.1, iters=2048, noise=True)

    # model = TwoLayer(d,K)
    # print(model)
    # print([param for param in model.parameters()])

    # langevin_step(model, X, Y, 0.1, iters=2048, noise=True)

    # print([np.linalg.norm(param) for param in model.parameters()])

    print(model_to_action(model,S.get_new_context()))
<commit_msg>fixed dot product upgrade issue<commit_after>import torch
from torch.autograd import Variable
import torch.nn
import numpy as np
import torch.nn.functional as F

class LinearModel(torch.nn.Module):
    """
    Cost sensitive linear model with label dependent features and shared parameter.
    Input: a [n, 1, d*K] tensor
    Output: [n, 1, K] tensor
    """

    def __init__(self, d, K):
        super(LinearModel, self).__init__()
        self.conv = torch.nn.Conv1d(1, 1, kernel_size=d, stride=d, bias=False)

    def forward(self,x):
        h = self.conv(x)
        z = h - torch.mean(h)
        return (z)

class TwoLayer(torch.nn.Module):
    """
    Cost sensitive two-layer NN with label dependent features and shared parameter.
    Input: a [n, 1, d*K] tensor
    Output: [n, 1, K] tensor
    """
    def __init__(self, d, K, hidden=2):
        super(TwoLayer, self).__init__()
        self.fc1 = torch.nn.Conv1d(1, hidden, kernel_size=d,stride=d, bias=False)
        self.fc2 = torch.nn.Conv1d(hidden, 1, kernel_size=1, stride=1, bias=False)

    def forward(self,x):
        h1 = F.relu(self.fc1(x))
        h = self.fc2(h1)
        z = h - torch.mean(h)
        return (z)

def model_to_action(model, x):
    X = np.zeros((1, 1, x.get_ld_dim()*x.get_K()))
    X[0,0,:] = x.get_ld_features().reshape(1,x.get_ld_dim()*x.get_K())
    X = Variable(torch.FloatTensor(X))
    output = model(X)
    # print(output.data)
    p = torch.clamp(1+output,0)/torch.sum(torch.clamp(1+output,0))
    return (np.array(p.data)[0,0])

def langevin_step(model, features, targets, lr, iters=512, noise=True, debug=False):
    X = Variable(torch.FloatTensor(features))
    Y = Variable(torch.FloatTensor(targets))
    for i in range(1,iters+1):
        model.zero_grad()
        output = model(X)[0][0]
        v = Y[0][0]
        loss = v.dot(torch.clamp(1+output, 0.0))/Y.shape[0]
        if np.log2(i) == np.floor(np.log2(i)) and debug:
            print("Iteration %d, Loss %0.3f" % (i, loss.data[0]))
        loss.backward()
        for f in model.parameters():
            if noise:
                r = torch.FloatTensor(np.random.normal(0,1,np.shape(f.grad.data)))
            else:
                r = torch.zeros(f.grad.data.size())
            f.data.sub_(f.grad.data*lr/2 - np.sqrt(lr)*r)
            f.data = torch.clamp(f.data, -1, 1)

if __name__=='__main__':
    import Simulators
    d = 20
    K = 5
    S = Simulators.LinearBandit(d, 1, K, noise=True, pos=False, quad=False)

    n = 1

    X = np.zeros([n, 1, d*K])
    Y = np.zeros([n, 1, K])
    for i in range(n):
        x = S.get_new_context()
        X[i,0,:] = x.get_ld_features().reshape(1,d*K)
        r = np.array(S.get_all_rewards())
        Y[i,0,:] = np.array(S.get_all_rewards())

    model = LinearModel(d,K)
    print(model)
    print([param for param in model.parameters()])
    X = torch.FloatTensor(X)
    Y = torch.FloatTensor(Y)
    langevin_step(model, X, Y, 0.1, iters=2048, noise=True)

    # model = TwoLayer(d,K)
    # print(model)
    # print([param for param in model.parameters()])

    # langevin_step(model, X, Y, 0.1, iters=2048, noise=True)

    # print([np.linalg.norm(param) for param in model.parameters()])

    print(model_to_action(model,S.get_new_context()))
<|endoftext|>