<commit_before>from kivy.app import App
from kivy.uix.screenmanager import Screen, ScreenManager
from kivy.storage.jsonstore import JsonStore

import requests

class Model(object):
    def __init__(self, name):
        self.name = name
        self.presenters = []

    # return data for id here
    def get(self, id):
        raise Exception("not implemented")

    # set data for id here
    def _set(self, id, data):
        raise Exception("not implemented")

    def set(self, id, data):
        self._set(id, data)
        for p in self.presenters:
            p.modelEvent(self, id)

# Transient Dict Model.
class DictModel(Model):
    def __init__(self, name):
        super(DictModel, self).__init__(name)
        self.data = {}

    def get(self, id):
        if id in self.data.keys():
            return self.data[id]
        else:
            return None

    def _set(self, id, data):
        self.data[id] = data

# Persistent JSON Model.
class JsonModel(Model):
    def __init__(self, name, path_prefix='storage/'):
        super(JsonModel, self).__init__(name)
        self.store = JsonStore(path_prefix + name + '.json')

    def get(self, id):
        if self.store.exists(id):
            return self.store[id]
        return None

    def _set(self, id, data):
        if not data:
            self.store.delete(id)
        else:
            self.store[id] = data

# Transient Rest HTTP Model.
class RestModel(DictModel):
    # request can be an OAuthRequest specified to your needs or a plain HTTP request.
    def __init__(self, name, request):
        super(RestModel, self).__init__(name)
        self.r = request

    # We cache data locally.
    def get(self, id, url):
        data = super(RestModel, self).get(id)
        if data is None:
            data = self._get(id, url)
            self.set(id, data)
        return data

    def _get(self, id, url):
        resp = self.r.get(url + str(id))
        if resp.status_code != requests.codes.ok:
            return None
        return resp.json()

    def post(self, url, data):
        resp = self.r.post(url, data=data)
        if resp.status_code != requests.codes.ok:
            return {}
        return resp.json()

    def put(self, id, url, data):
        if data:
            self.r.put(url + str(id), data=data)

    def delete(self, id, url):
        self.r.delete(url + str(id))

class View(Screen):
    def __init__(self, presenter, **kwargs):
        super(View, self).__init__(**kwargs)
        self.presenter = presenter

    # update view based on new data here
    def _update(self, data):
        pass

    def update(self, data):
        self._update(data)
        self.canvas.ask_update()

    def event(self, e):
        self.presenter.userEvent(e)

# A View is just a small wrapper around kivy screens; no need for lots of functionality here.

class Runnable(object):
    # hook for kivy's on_pause
    def onPause(self):
        return True

    # hook for kivy's on_resume
    def onResume(self):
        pass

    # hook for kivy's on_start
    def onStart(self):
        pass

    # hook for kivy's on_stop
    def onStop(self):
        pass

    # generic event from app controller or other presenter
    def receive(self, e):
        pass

class Presenter(Runnable):
    def __init__(self, ctrl, viewClass, models):
        self.bus = ctrl.bus
        self.view = viewClass(self, name=self._name())
        ctrl.sm.add_widget(self.view)
        self.models = {}
        for model in models:
            self.models[model.name] = model
            model.presenters.append(self)
            self.modelEvent(model)

    # provide name here
    def _name(self):
        raise Exception("not implemented")

    def emit(self, event):
        self.bus.emit(event)

    # associated view notifies us of user event, update model appropriately
    def userEvent(self, e):
        pass

    # model notfies us of update, refresh the view
    def modelEvent(self, model, e=None):
        pass

class AppController(Runnable):
    def __init__(self):
        class EventBus(object):
            def __init__(self):
                self.listeners = []

            def register(self, obj):
                self.listeners.append(obj)

            def emit(self, event):
                for listener in self.listeners:
                    listener.receive(event)

        self.bus = EventBus()
        self.bus.register(self)
        self.sm = ScreenManager()
        self.presenters = {}

        bus = self.bus
        sm = self.sm

        class KivyMVPApp(App):
            def build(self):
                return sm
            def on_pause(self):
                for listener in bus.listeners:
                    if not listener.onPause():
                        return False
                return True
            def on_resume(self):
                for listener in bus.listeners:
                    listener.onResume()
            def on_start(self):
                for listener in bus.listeners:
                    listener.onStart()
            def on_stop(self):
                for listener in bus.listeners:
                    listener.onStop()

        self.app = KivyMVPApp()

    def current(self):
        return self.sm.current

    def switch(self, name):
        self.sm.current = name

    def go(self, first):
        self.sm.current = first
        self.app.run()

    def add(self, pres):
        if pres._name() in self.presenters:
            raise Exception("presenter with name %s exists" % pres._name())
        self.presenters[pres._name()] = pres
        self.bus.register(pres)

if __name__ == '__main__':
    from kivy.graphics import Color, Rectangle
    from kivy.uix.button import Button
    from kivy.uix.floatlayout import FloatLayout
    from kivy.uix.label import Label

    # Our app controller simply listens for "switch" events and switches between
    # the two presenters, if it receives one.
    class TestAppController(AppController):
        def receive(self, e):
            if e == "switch":
                for p in self.presenters:
                    if self.current() != p:
                        self.switch(p)
                        break

    ctrl = TestAppController()

    # Our model is a simple dictionary containing a single integer at key 0.
    model = DictModel("aSingleNumber")
    model.set(0, 0)

    # This is a very basic example. Of course we should not duplicate code
    # for such a small difference in functionality. It is just to outline
    # how the framework is intended to be used.

    # The black presenter listens for two user events.
    # If it receives "done" it signals "switch" to the app controller's event bus.
    # (Note: all presenters and the app controller are registered at the event bus
    #  and can response to events if required)
    # If it receives an "add" event it retrieves the current number from the model,
    # increments by one and puts it back into the model.
    # On receiving any event from the model it simply retrieves the currently stored
    # number and instructs the view to update based on it.
    class BlackPresenter(Presenter):
        def _name(self):
            return "black"

        def userEvent(self, e):
            if e == "done":
                self.emit("switch")
            elif e == "add":
                x = self.models["aSingleNumber"].get(0)
                self.models["aSingleNumber"].set(0, x+1)

        def modelEvent(self, m, e=None):
            self.view.update(str(m.get(0)))

    # The white presenter listens for two user events.
    # If it receives "done" it signals "switch" to the app controller's event bus.
    # If it receives an "subtract" event it retrieves the current number from the model,
    # decrements by one and puts it back into the model.
    # On receiving any event from the model it simply retrieves the currently stored
    # number and instructs the view to update based on it.
    class WhitePresenter(Presenter):
        def _name(self):
            return "white"

        def userEvent(self, e):
            if e == "done":
                self.emit("switch")
            elif e == "subtract":
                x = self.models["aSingleNumber"].get(0)
                self.models["aSingleNumber"].set(0, x-1)

        def modelEvent(self, m, e=None):
            self.view.update(str(m.get(0)))

    # This is just a simple layout with a background color such that we can easily
    # distinguish our two views.
    class ColorLayout(FloatLayout):
        def __init__(self, color, **kwargs):
            super(ColorLayout, self).__init__(**kwargs)
            with self.canvas.before:
                Color(color[0], color[1], color[2], color[3])
                self.rect = Rectangle(size=self.size, pos=self.pos)
            self.bind(size=self._update_rect, pos=self._update_rect)

        def _update_rect(self, instance, value):
            self.rect.pos = instance.pos
            self.rect.size = instance.size

    # The black view has a button "add" and a button "to_white" on a black background.
    # Pressing "add" triggers emits the event "add" and pressing "to white" triggers "done".
    # When it receives an update event it updates the label text with the new data.
    # Note that all kivy events should just trigger self.event with the appropriate data to
    # integrate into the MVP workflow.
    class BlackView(View):
        def __init__(self, presenter, **kwargs):
            super(BlackView, self).__init__(presenter, **kwargs)
            with self.canvas:
                f = ColorLayout((0,0,0,1))
                self.l = Label(text="TEST", size_hint=(1, 0.25), pos_hint={ "x":0, "y":0.8 },
                     color=(0.75, 0.75, 0.75, 1), font_size=60)
                f.add_widget(self.l)
                b = Button(text='add', font_size=20, size_hint=(1, 0.25),
                    pos_hint={ "x":0, "y":0.25 })
                b.bind(on_press=lambda x: self.event("add"))
                f.add_widget(b)
                b = Button(text='to white', font_size=20, size_hint=(1, 0.25))
                b.bind(on_press=lambda x: self.event("done"))
                f.add_widget(b)
                self.add_widget(f)

        def _update(self, data):
            self.l.text = data

    # The white view has a button "add" and a button "to_black" on a white background.
    # Pressing "add" triggers emits the event "add" and pressing "to black" triggers "done".
    # When it receives an update event it updates the label text with the new data.
    class WhiteView(View):
        def __init__(self, presenter, **kwargs):
            super(WhiteView, self).__init__(presenter, **kwargs)
            with self.canvas:
                f = ColorLayout((1,1,1,1))
                self.l = Label(text="TEST", size_hint=(1, 0.25), pos_hint={ "x":0, "y":0.8 },
                    color=(0.75, 0.75, 0.75, 1), font_size=60)
                f.add_widget(self.l)
                b = Button(text='subtract', font_size=20, size_hint=(1, 0.25),
                    pos_hint={ "x":0, "y":0.25 })
                b.bind(on_press=lambda x: self.event("subtract"))
                f.add_widget(b)
                b = Button(text='to black', font_size=20, size_hint=(1, 0.25))
                b.bind(on_press=lambda x: self.event("done"))
                f.add_widget(b)
                self.add_widget(f)

        def _update(self, data):
            self.l.text = data

    black_pres = BlackPresenter(ctrl, BlackView, [model])
    white_pres = WhitePresenter(ctrl, WhiteView, [model])

    ctrl.add(white_pres)
    ctrl.add(black_pres)

    # Start black.
    ctrl.go('black')
<commit_msg>key hooks<commit_after>from kivy.app import App
from kivy.base import EventLoop
from kivy.uix.screenmanager import Screen, ScreenManager
from kivy.storage.jsonstore import JsonStore

import requests

class Model(object):
    def __init__(self, name):
        self.name = name
        self.presenters = []

    # return data for id here
    def get(self, id):
        raise Exception("not implemented")

    # set data for id here
    def _set(self, id, data):
        raise Exception("not implemented")

    def set(self, id, data):
        self._set(id, data)
        for p in self.presenters:
            p.modelEvent(self, id)

# Transient Dict Model.
class DictModel(Model):
    def __init__(self, name):
        super(DictModel, self).__init__(name)
        self.data = {}

    def get(self, id):
        if id in self.data.keys():
            return self.data[id]
        else:
            return None

    def _set(self, id, data):
        self.data[id] = data

# Persistent JSON Model.
class JsonModel(Model):
    def __init__(self, name, path_prefix='storage/'):
        super(JsonModel, self).__init__(name)
        self.store = JsonStore(path_prefix + name + '.json')

    def get(self, id):
        if self.store.exists(id):
            return self.store[id]
        return None

    def _set(self, id, data):
        if not data:
            self.store.delete(id)
        else:
            self.store[id] = data

# Transient Rest HTTP Model.
class RestModel(DictModel):
    # request can be an OAuthRequest specified to your needs or a plain HTTP request.
    def __init__(self, name, request):
        super(RestModel, self).__init__(name)
        self.r = request

    # We cache data locally.
    def get(self, id, url):
        data = super(RestModel, self).get(id)
        if data is None:
            data = self._get(id, url)
            self.set(id, data)
        return data

    def _get(self, id, url):
        resp = self.r.get(url + str(id))
        if resp.status_code != requests.codes.ok:
            return None
        return resp.json()

    def post(self, url, data):
        resp = self.r.post(url, data=data)
        if resp.status_code != requests.codes.ok:
            return {}
        return resp.json()

    def put(self, id, url, data):
        if data:
            self.r.put(url + str(id), data=data)

    def delete(self, id, url):
        self.r.delete(url + str(id))

class View(Screen):
    def __init__(self, presenter, **kwargs):
        super(View, self).__init__(**kwargs)
        self.presenter = presenter

    # update view based on new data here
    def _update(self, data):
        pass

    def update(self, data):
        self._update(data)
        self.canvas.ask_update()

    def event(self, e):
        self.presenter.userEvent(e)

# A View is just a small wrapper around kivy screens; no need for lots of functionality here.

class Runnable(object):
    # hook for kivy's on_pause
    def onPause(self):
        return True

    # hook for kivy's on_resume
    def onResume(self):
        pass

    # hook for kivy's on_start
    def onStart(self):
        pass

    # hook for kivy's on_stop
    def onStop(self):
        pass

    # generic event from app controller or other presenter
    def receive(self, e):
        pass

class Presenter(Runnable):
    def __init__(self, ctrl, viewClass, models):
        self.bus = ctrl.bus
        self.view = viewClass(self, name=self._name())
        ctrl.sm.add_widget(self.view)
        self.models = {}
        for model in models:
            self.models[model.name] = model
            model.presenters.append(self)
            self.modelEvent(model)

    # provide name here
    def _name(self):
        raise Exception("not implemented")

    def emit(self, event):
        self.bus.emit(event)

    # associated view notifies us of user event, update model appropriately
    def userEvent(self, e):
        pass

    # model notfies us of update, refresh the view
    def modelEvent(self, model, e=None):
        pass

class AppController(Runnable):
    def __init__(self):
        class EventBus(object):
            def __init__(self):
                self.listeners = []

            def register(self, obj):
                self.listeners.append(obj)

            def emit(self, event):
                for listener in self.listeners:
                    listener.receive(event)

        self.bus = EventBus()
        self.bus.register(self)
        self.sm = ScreenManager()
        self.presenters = {}
        self.key_hooks = {}

        bus = self.bus
        sm = self.sm

        class KivyMVPApp(App):
            def build(self):
                return sm
            def on_pause(self):
                for listener in bus.listeners:
                    if not listener.onPause():
                        return False
                return True
            def on_resume(self):
                for listener in bus.listeners:
                    listener.onResume()
            def on_start(self):
                EventLoop.window.bind(on_keyboard=self.hook_keyboard)
                for listener in bus.listeners:
                    listener.onStart()
            def on_stop(self):
                for listener in bus.listeners:
                    listener.onStop()
            def hook_keyboard(self, window, key, *largs):
                if key in self.key_hooks:
                    return self.key_hooks[key]()
                return True

        self.app = KivyMVPApp()

    def hook_key(self, key, func):
        self.key_hooks[key] = func

    def current(self):
        return self.sm.current

    def switch(self, name):
        self.sm.current = name

    def go(self, first):
        self.sm.current = first
        self.app.run()

    def add(self, pres):
        if pres._name() in self.presenters:
            raise Exception("presenter with name %s exists" % pres._name())
        self.presenters[pres._name()] = pres
        self.bus.register(pres)

if __name__ == '__main__':
    from kivy.graphics import Color, Rectangle
    from kivy.uix.button import Button
    from kivy.uix.floatlayout import FloatLayout
    from kivy.uix.label import Label

    # Our app controller simply listens for "switch" events and switches between
    # the two presenters, if it receives one.
    class TestAppController(AppController):
        def receive(self, e):
            if e == "switch":
                for p in self.presenters:
                    if self.current() != p:
                        self.switch(p)
                        break

    ctrl = TestAppController()

    # Our model is a simple dictionary containing a single integer at key 0.
    model = DictModel("aSingleNumber")
    model.set(0, 0)

    # This is a very basic example. Of course we should not duplicate code
    # for such a small difference in functionality. It is just to outline
    # how the framework is intended to be used.

    # The black presenter listens for two user events.
    # If it receives "done" it signals "switch" to the app controller's event bus.
    # (Note: all presenters and the app controller are registered at the event bus
    #  and can response to events if required)
    # If it receives an "add" event it retrieves the current number from the model,
    # increments by one and puts it back into the model.
    # On receiving any event from the model it simply retrieves the currently stored
    # number and instructs the view to update based on it.
    class BlackPresenter(Presenter):
        def _name(self):
            return "black"

        def userEvent(self, e):
            if e == "done":
                self.emit("switch")
            elif e == "add":
                x = self.models["aSingleNumber"].get(0)
                self.models["aSingleNumber"].set(0, x+1)

        def modelEvent(self, m, e=None):
            self.view.update(str(m.get(0)))

    # The white presenter listens for two user events.
    # If it receives "done" it signals "switch" to the app controller's event bus.
    # If it receives an "subtract" event it retrieves the current number from the model,
    # decrements by one and puts it back into the model.
    # On receiving any event from the model it simply retrieves the currently stored
    # number and instructs the view to update based on it.
    class WhitePresenter(Presenter):
        def _name(self):
            return "white"

        def userEvent(self, e):
            if e == "done":
                self.emit("switch")
            elif e == "subtract":
                x = self.models["aSingleNumber"].get(0)
                self.models["aSingleNumber"].set(0, x-1)

        def modelEvent(self, m, e=None):
            self.view.update(str(m.get(0)))

    # This is just a simple layout with a background color such that we can easily
    # distinguish our two views.
    class ColorLayout(FloatLayout):
        def __init__(self, color, **kwargs):
            super(ColorLayout, self).__init__(**kwargs)
            with self.canvas.before:
                Color(color[0], color[1], color[2], color[3])
                self.rect = Rectangle(size=self.size, pos=self.pos)
            self.bind(size=self._update_rect, pos=self._update_rect)

        def _update_rect(self, instance, value):
            self.rect.pos = instance.pos
            self.rect.size = instance.size

    # The black view has a button "add" and a button "to_white" on a black background.
    # Pressing "add" triggers emits the event "add" and pressing "to white" triggers "done".
    # When it receives an update event it updates the label text with the new data.
    # Note that all kivy events should just trigger self.event with the appropriate data to
    # integrate into the MVP workflow.
    class BlackView(View):
        def __init__(self, presenter, **kwargs):
            super(BlackView, self).__init__(presenter, **kwargs)
            with self.canvas:
                f = ColorLayout((0,0,0,1))
                self.l = Label(text="TEST", size_hint=(1, 0.25), pos_hint={ "x":0, "y":0.8 },
                     color=(0.75, 0.75, 0.75, 1), font_size=60)
                f.add_widget(self.l)
                b = Button(text='add', font_size=20, size_hint=(1, 0.25),
                    pos_hint={ "x":0, "y":0.25 })
                b.bind(on_press=lambda x: self.event("add"))
                f.add_widget(b)
                b = Button(text='to white', font_size=20, size_hint=(1, 0.25))
                b.bind(on_press=lambda x: self.event("done"))
                f.add_widget(b)
                self.add_widget(f)

        def _update(self, data):
            self.l.text = data

    # The white view has a button "add" and a button "to_black" on a white background.
    # Pressing "add" triggers emits the event "add" and pressing "to black" triggers "done".
    # When it receives an update event it updates the label text with the new data.
    class WhiteView(View):
        def __init__(self, presenter, **kwargs):
            super(WhiteView, self).__init__(presenter, **kwargs)
            with self.canvas:
                f = ColorLayout((1,1,1,1))
                self.l = Label(text="TEST", size_hint=(1, 0.25), pos_hint={ "x":0, "y":0.8 },
                    color=(0.75, 0.75, 0.75, 1), font_size=60)
                f.add_widget(self.l)
                b = Button(text='subtract', font_size=20, size_hint=(1, 0.25),
                    pos_hint={ "x":0, "y":0.25 })
                b.bind(on_press=lambda x: self.event("subtract"))
                f.add_widget(b)
                b = Button(text='to black', font_size=20, size_hint=(1, 0.25))
                b.bind(on_press=lambda x: self.event("done"))
                f.add_widget(b)
                self.add_widget(f)

        def _update(self, data):
            self.l.text = data

    black_pres = BlackPresenter(ctrl, BlackView, [model])
    white_pres = WhitePresenter(ctrl, WhiteView, [model])

    ctrl.add(white_pres)
    ctrl.add(black_pres)

    # Start black.
    ctrl.go('black')
<|endoftext|>


<commit_before>from argparse import ArgumentParser
import logging
import sys

from .manager import SerialButtonManager

logger = logging.getLogger(__name__)

def main(args=None):
    if args is None:
        args = sys.argv[1:]

    parser = ArgumentParser()
    parser.add_argument('serial_device', nargs=1)
    parser.add_argument('service_name', nargs=1)
    parser.add_argument('--loglevel', default='DEBUG')

    options = parser.parse_args(args)

    logger.basicConfig(level=logging.getLevelName(options.loglevel))

    manager = SerialButtonManager(options.serial_device, options.service_name)
    manager.run()
<commit_msg>Set logging level globally.<commit_after>from argparse import ArgumentParser
import logging
import sys

from .manager import SerialButtonManager

logger = logging.getLogger(__name__)

def main(args=None):
    if args is None:
        args = sys.argv[1:]

    parser = ArgumentParser()
    parser.add_argument('serial_device', nargs=1)
    parser.add_argument('service_name', nargs=1)
    parser.add_argument('--loglevel', default='DEBUG')

    options = parser.parse_args(args)

    logging.basicConfig(level=logging.getLevelName(options.loglevel))

    manager = SerialButtonManager(options.serial_device, options.service_name)
    manager.run()
<|endoftext|>


<commit_before>from argparse import ArgumentParser, Namespace
from typing import List, Optional, Union, Any

from fastapi import FastAPI, HTTPException, Body
from logging import getLogger

from pydantic import BaseModel
from uvicorn import run

from transformers import Pipeline
from transformers.commands import BaseTransformersCLICommand
from transformers.pipelines import SUPPORTED_TASKS, pipeline

def serve_command_factory(args: Namespace):
    """
    Factory function used to instantiate serving server from provided command line arguments.
    :return: ServeCommand
    """
    nlp = pipeline(args.task, args.model)
    return ServeCommand(nlp, args.host, args.port, args.model, args.graphql)

class ServeResult(BaseModel):
    """
    Base class for serving result
    """
    model: str

class ServeModelInfoResult(ServeResult):
    """
    Expose model information
    """
    infos: dict

class ServeTokenizeResult(ServeResult):
    """
    Tokenize result model
    """
    tokens: List[str]
    tokens_ids: Optional[List[int]]

class ServeDeTokenizeResult(ServeResult):
    """
    DeTokenize result model
    """
    text: str

class ServeForwardResult(ServeResult):
    """
    Forward result model
    """
    output: Any

class ServeCommand(BaseTransformersCLICommand):

    @staticmethod
    def register_subcommand(parser: ArgumentParser):
        """
        Register this command to argparse so it's available for the transformer-cli
        :param parser: Root parser to register command-specific arguments
        :return:
        """
        serve_parser = parser.add_parser('serve', help='CLI tool to run inference requests through REST and GraphQL endpoints.')
        serve_parser.add_argument('--task', type=str, choices=SUPPORTED_TASKS.keys(), help='The task to run the pipeline on')
        serve_parser.add_argument('--device', type=int, default=-1, help='Indicate the device to run onto, -1 indicates CPU, >= 0 indicates GPU (default: -1)')
        serve_parser.add_argument('--host', type=str, default='localhost', help='Interface the server will listen on.')
        serve_parser.add_argument('--port', type=int, default=8888, help='Port the serving will listen to.')
        serve_parser.add_argument('--model', type=str, required=True, help='Model\'s name or path to stored model to infer from.')
        serve_parser.add_argument('--graphql', action='store_true', default=False, help='Enable GraphQL endpoints.')
        serve_parser.set_defaults(func=serve_command_factory)

    def __init__(self, pipeline: Pipeline, host: str, port: int, model: str, graphql: bool):
        self._logger = getLogger('transformers-cli/serving')

        self._pipeline = pipeline

        self._logger.info('Serving model over {}:{}'.format(host, port))
        self._host = host
        self._port = port
        self._app = FastAPI()

        # Register routes
        self._app.add_api_route('/', self.model_info, response_model=ServeModelInfoResult, methods=['GET'])
        self._app.add_api_route('/tokenize', self.tokenize, response_model=ServeTokenizeResult, methods=['POST'])
        self._app.add_api_route('/detokenize', self.detokenize, response_model=ServeDeTokenizeResult, methods=['POST'])
        self._app.add_api_route('/forward', self.forward, response_model=ServeForwardResult, methods=['POST'])

    def run(self):
        run(self._app, host=self._host, port=self._port)

    def model_info(self):
        return ServeModelInfoResult(model='', infos=vars(self._pipeline.model.config))

    def tokenize(self, text_input: str = Body(None, embed=True), return_ids: bool = Body(False, embed=True)):
        """
        Tokenize the provided input and eventually returns corresponding tokens id:
        - **text_input**: String to tokenize
        - **return_ids**: Boolean flags indicating if the tokens have to be converted to their integer mapping.
        """
        try:
            tokens_txt = self._pipeline.tokenizer.tokenize(text_input)

            if return_ids:
                tokens_ids = self._pipeline.tokenizer.convert_tokens_to_ids(tokens_txt)
                return ServeTokenizeResult(model='', tokens=tokens_txt, tokens_ids=tokens_ids)
            else:
                return ServeTokenizeResult(model='', tokens=tokens_txt)

        except Exception as e:
            raise HTTPException(status_code=500, detail={"model": '', "error": str(e)})

    def detokenize(self, tokens_ids: List[int] = Body(None, embed=True),
                   skip_special_tokens: bool = Body(False, embed=True),
                   cleanup_tokenization_spaces: bool = Body(True, embed=True)):
        """
        Detokenize the provided tokens ids to readable text:
        - **tokens_ids**: List of tokens ids
        - **skip_special_tokens**: Flag indicating to not try to decode special tokens
        - **cleanup_tokenization_spaces**: Flag indicating to remove all leading/trailing spaces and intermediate ones.
        """
        try:
            decoded_str = self._pipeline.tokenizer.decode(tokens_ids, skip_special_tokens, cleanup_tokenization_spaces)
            return ServeDeTokenizeResult(model='', text=decoded_str)
        except Exception as e:
            raise HTTPException(status_code=500, detail={"model": '', "error": str(e)})

    def forward(self, inputs: Union[str, dict, List[str], List[int], List[dict]] = Body(None, embed=True)):
        """
        **inputs**:
        **attention_mask**:
        **tokens_type_ids**:
        """

        # Check we don't have empty string
        if len(inputs) == 0:
            return ServeForwardResult(model='', output=[], attention=[])

        try:
            # Forward through the model
            output = self._pipeline(inputs)
            return ServeForwardResult(
                model='', output=output
            )
        except Exception as e:
            raise HTTPException(500, {"error": str(e)})
<commit_msg>Expose all the pipeline argument on serve command.<commit_after>from argparse import ArgumentParser, Namespace
from typing import List, Optional, Union, Any

from fastapi import FastAPI, HTTPException, Body
from logging import getLogger

from pydantic import BaseModel
from uvicorn import run

from transformers import Pipeline
from transformers.commands import BaseTransformersCLICommand
from transformers.pipelines import SUPPORTED_TASKS, pipeline

def serve_command_factory(args: Namespace):
    """
    Factory function used to instantiate serving server from provided command line arguments.
    :return: ServeCommand
    """
    nlp = pipeline(task=args.task, model=args.model, config=args.config, tokenizer=args.tokenizer, device=args.device)
    return ServeCommand(nlp, args.host, args.port)

class ServeModelInfoResult(BaseModel):
    """
    Expose model information
    """
    infos: dict

class ServeTokenizeResult(BaseModel):
    """
    Tokenize result model
    """
    tokens: List[str]
    tokens_ids: Optional[List[int]]

class ServeDeTokenizeResult(BaseModel):
    """
    DeTokenize result model
    """
    text: str

class ServeForwardResult(BaseModel):
    """
    Forward result model
    """
    output: Any

class ServeCommand(BaseTransformersCLICommand):

    @staticmethod
    def register_subcommand(parser: ArgumentParser):
        """
        Register this command to argparse so it's available for the transformer-cli
        :param parser: Root parser to register command-specific arguments
        :return:
        """
        serve_parser = parser.add_parser('serve', help='CLI tool to run inference requests through REST and GraphQL endpoints.')
        serve_parser.add_argument('--task', type=str, choices=SUPPORTED_TASKS.keys(), help='The task to run the pipeline on')
        serve_parser.add_argument('--device', type=int, default=-1, help='Indicate the device to run onto, -1 indicates CPU, >= 0 indicates GPU (default: -1)')
        serve_parser.add_argument('--host', type=str, default='localhost', help='Interface the server will listen on.')
        serve_parser.add_argument('--port', type=int, default=8888, help='Port the serving will listen to.')
        serve_parser.add_argument('--model', type=str, required=True, help='Model\'s name or path to stored model.')
        serve_parser.add_argument('--config', type=str, help='Model\'s config name or path to stored model.')
        serve_parser.add_argument('--tokenizer', type=str, help='Tokenizer name to use.')
        serve_parser.set_defaults(func=serve_command_factory)

    def __init__(self, pipeline: Pipeline, host: str, port: int):
        self._logger = getLogger('transformers-cli/serving')

        self._pipeline = pipeline

        self._logger.info('Serving model over {}:{}'.format(host, port))
        self._host = host
        self._port = port
        self._app = FastAPI()

        # Register routes
        self._app.add_api_route('/', self.model_info, response_model=ServeModelInfoResult, methods=['GET'])
        self._app.add_api_route('/tokenize', self.tokenize, response_model=ServeTokenizeResult, methods=['POST'])
        self._app.add_api_route('/detokenize', self.detokenize, response_model=ServeDeTokenizeResult, methods=['POST'])
        self._app.add_api_route('/forward', self.forward, response_model=ServeForwardResult, methods=['POST'])

    def run(self):
        run(self._app, host=self._host, port=self._port)

    def model_info(self):
        return ServeModelInfoResult(infos=vars(self._pipeline.model.config))

    def tokenize(self, text_input: str = Body(None, embed=True), return_ids: bool = Body(False, embed=True)):
        """
        Tokenize the provided input and eventually returns corresponding tokens id:
        - **text_input**: String to tokenize
        - **return_ids**: Boolean flags indicating if the tokens have to be converted to their integer mapping.
        """
        try:
            tokens_txt = self._pipeline.tokenizer.tokenize(text_input)

            if return_ids:
                tokens_ids = self._pipeline.tokenizer.convert_tokens_to_ids(tokens_txt)
                return ServeTokenizeResult(tokens=tokens_txt, tokens_ids=tokens_ids)
            else:
                return ServeTokenizeResult(tokens=tokens_txt)

        except Exception as e:
            raise HTTPException(status_code=500, detail={"model": '', "error": str(e)})

    def detokenize(self, tokens_ids: List[int] = Body(None, embed=True),
                   skip_special_tokens: bool = Body(False, embed=True),
                   cleanup_tokenization_spaces: bool = Body(True, embed=True)):
        """
        Detokenize the provided tokens ids to readable text:
        - **tokens_ids**: List of tokens ids
        - **skip_special_tokens**: Flag indicating to not try to decode special tokens
        - **cleanup_tokenization_spaces**: Flag indicating to remove all leading/trailing spaces and intermediate ones.
        """
        try:
            decoded_str = self._pipeline.tokenizer.decode(tokens_ids, skip_special_tokens, cleanup_tokenization_spaces)
            return ServeDeTokenizeResult(model='', text=decoded_str)
        except Exception as e:
            raise HTTPException(status_code=500, detail={"model": '', "error": str(e)})

    def forward(self, inputs: Union[str, dict, List[str], List[int], List[dict]] = Body(None, embed=True)):
        """
        **inputs**:
        **attention_mask**:
        **tokens_type_ids**:
        """

        # Check we don't have empty string
        if len(inputs) == 0:
            return ServeForwardResult(output=[], attention=[])

        try:
            # Forward through the model
            output = self._pipeline(inputs)
            return ServeForwardResult(output=output)
        except Exception as e:
            raise HTTPException(500, {"error": str(e)})
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- encoding: utf-8 -*-
#
# Copyright (c) 2016 ASMlover. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#  * Redistributions of source code must retain the above copyright
#    notice, this list ofconditions and the following disclaimer.
#
#  * Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in
#    the documentation and/or other materialsprovided with the
#    distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import PathHelper as PH
PH.addPathes('../')

import asyncore
import inspect
import socket
import sys
import time

import AsyncIo as asio
from ChannelObjs import EchoChannelObj
from ConnectorManager import ConnectorManager
from MarsLog.LogManager import LogManager
from TcpConnector import TcpConnector

class TcpServer(asyncore.dispatcher):
    """负责监听TCP连接

    建立连接后交由上层的connector handler来处理，connector handler
    需要注册一个handlerNewConnector函数来处理新连接
    """

    def __init__(self, ip, port, connectorHandler=None, reuseAddr=False):
        super(TcpServer, self).__init__()

        self.logger = LogManager.getLogger('MarsRpc.TcpServer')
        self.ip = ip
        self.port = port
        self.connectorHandler = connectorHandler
        self.create_socket(socket.AF_INT, socket.SOCK_STREAM)
        if reuseAddr:
            self.set_reuse_addr()
        self.started = False
        self.tryBind()
        self.logger.info('TcpServer init: server liston on: %s, %d', self.ip, self.port)
        self.listen(50)

    def tryBind(self):
        while True:
            try:
                self.bind((self.ip, self.port))
                break
            except:
                self.logger.info('tryBind: server faild to bind: %s, %d, try next port', self.ip, self.port)
                self.port += 1
                if self.port > 65535:
                    self.logger.error('tryBind: server faild to find a usable port: %s, %d', self.ip, self.port)
                    raise StandardError('server faild to find a usable port')
        self.started = True

    def stop(self):
        self.close()
        self.started = False
        self.logger.info('stop')

    def close(self):
        super(TcpServer, self).close()
        self.logger.info('close: called from: %s', str(inspect.stack()))

    def getListen(self):
        return (self.ip, self.port)

    def setConnectorHandler(self, connectorHandler):
        self.connectorHandler = connectorHandler

    def handle_accept(self):
        """连接回调"""
        pass

    def handle_error(self):
        """错误回调"""
        pass

    def handle_close(self):
        """关闭连接回调"""
        pass
<commit_msg>chore(TcpServer): finished the tcp server<commit_after>#!/usr/bin/env python
# -*- encoding: utf-8 -*-
#
# Copyright (c) 2016 ASMlover. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#  * Redistributions of source code must retain the above copyright
#    notice, this list ofconditions and the following disclaimer.
#
#  * Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in
#    the documentation and/or other materialsprovided with the
#    distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import PathHelper as PH
PH.addPathes('../')

import asyncore
import inspect
import socket
import sys
import time

import AsyncIo as asio
from ChannelObjs import EchoChannelObj
from ConnectorManager import ConnectorManager
from MarsLog.LogManager import LogManager
from TcpConnector import TcpConnector

class TcpServer(asyncore.dispatcher):
    """负责监听TCP连接

    建立连接后交由上层的connector handler来处理，connector handler
    需要注册一个handlerNewConnector函数来处理新连接
    """

    def __init__(self, ip, port, connectorHandler=None, reuseAddr=False):
        super(TcpServer, self).__init__()

        self.logger = LogManager.getLogger('MarsRpc.TcpServer')
        self.ip = ip
        self.port = port
        self.connectorHandler = connectorHandler
        self.create_socket(socket.AF_INT, socket.SOCK_STREAM)
        if reuseAddr:
            self.set_reuse_addr()
        self.started = False
        self.tryBind()
        self.logger.info('TcpServer init: server liston on: %s, %d', self.ip, self.port)
        self.listen(50)

    def tryBind(self):
        while True:
            try:
                self.bind((self.ip, self.port))
                break
            except:
                self.logger.info('tryBind: server faild to bind: %s, %d, try next port', self.ip, self.port)
                self.port += 1
                if self.port > 65535:
                    self.logger.error('tryBind: server faild to find a usable port: %s, %d', self.ip, self.port)
                    raise StandardError('server faild to find a usable port')
        self.started = True

    def stop(self):
        self.close()
        self.started = False
        self.logger.info('stop')

    def close(self):
        super(TcpServer, self).close()
        self.logger.info('close: called from: %s', str(inspect.stack()))

    def getListen(self):
        return (self.ip, self.port)

    def setConnectorHandler(self, connectorHandler):
        self.connectorHandler = connectorHandler

    def handle_accept(self):
        """连接回调"""
        try:
            fd, addr = self.accept()
        except socket.error:
            self.logger.warn('server accept throw exception')
            self.logger.logLastExcept()
            return
        self.logger.info('handle_accept with peer: %s', fd.getpeername())

        if self.connectorHandler:
            connector = TcpConnector(fd, addr)
            self.connectorHandler.handlerNewConnector(connector)
        else:
            self.logger.wran('no connector manager to handle new connector')

    def handle_error(self):
        """错误回调"""
        self.logger.error('handle_error uncaptured exception')
        self.logger.error(str(inspect.stack()))

    def handle_close(self):
        """关闭连接回调"""
        self.logger.info('handle_close called from: %s', str(inspect.stack()))
<|endoftext|>


<commit_before>import sqlite3

import web
import app
import os

global appPath
appPath = os.getcwd()

conn = sqlite3.connect(appPath + '/data/gamedb.sqlite')
cur = conn.cursor()

cur.executescript('''

DROP TABLE IF EXISTS Leaderboard;
DROP TABLE IF EXISTS Player;
DROP TABLE IF EXISTS Game;
DROP TABLE IF EXISTS History;
DROP TABLE IF EXISTS sessions;

CREATE TABLE Leaderboard (
  id  INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
  score INTEGER,
  badge TEXT DEFAULT NONE,
  player_id INTEGER,
  game_id INTEGER,
  FOREIGN KEY(player_id) REFERENCES Player(id),
  FOREIGN KEY(game_id) REFERENCES Game(id)
);

CREATE TABLE Player (
  id  INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
  username  TEXT UNIQUE,
  userpassword  TEXT,
  games INTEGER DEFAULT 0,
  wins  INTEGER DEFAULT 0,
  losses   INTEGER DEFAULT 0,
  maxScore  INTEGER DEFAULT 0,
  totalScore INTEGER DEFAULT 0
);

CREATE TABLE Game (
  id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
  password TEXT,
  level INTEGER,
  won INTEGER,
  score INTEGER DEFAULT 0,
  totalRounds INTEGER DEFAULT 0,
  digits  INTEGER,
  complexity  INTEGER,
  goldCoins INTEGER,
  silverCoins INTEGER,
  goldSpent INTEGER DEFAULT 0,
  silverSpent INTEGER DEFAULT 0,
  player_id INTEGER,
  FOREIGN KEY(player_id) REFERENCES Player(id)
);

CREATE TABLE History(
  id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
  round INTEGER,
  guess TEXT,
  goldReceived INTEGER,
  silverReceived INTEGER,
  goldInBag INTEGER,
  silverInBag INTEGER,
  game_id INTEGER,
  FOREIGN KEY(game_id) REFERENCES Game(id)
);

CREATE TABLE sessions (
    session_id char(128) UNIQUE NOT NULL,
    atime timestamp NOT NULL default current_timestamp,
    data TEXT
)

''')

conn.commit()
conn.close()
<commit_msg>added security question and answer colunms to db<commit_after>import sqlite3

import web
import app
import os

global appPath
appPath = os.getcwd()

conn = sqlite3.connect(appPath + '/data/gamedb.sqlite')
cur = conn.cursor()

cur.executescript('''

DROP TABLE IF EXISTS Leaderboard;
DROP TABLE IF EXISTS Player;
DROP TABLE IF EXISTS Game;
DROP TABLE IF EXISTS History;
DROP TABLE IF EXISTS sessions;

CREATE TABLE Leaderboard (
  id  INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
  score INTEGER,
  badge TEXT DEFAULT NONE,
  player_id INTEGER,
  game_id INTEGER,
  FOREIGN KEY(player_id) REFERENCES Player(id),
  FOREIGN KEY(game_id) REFERENCES Game(id)
);

CREATE TABLE Player (
  id  INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
  username  TEXT UNIQUE,
  userpassword  TEXT,
  secretQuestion TEXT,
  secretAnswer TEXT,
  games INTEGER DEFAULT 0,
  wins  INTEGER DEFAULT 0,
  losses   INTEGER DEFAULT 0,
  maxScore  INTEGER DEFAULT 0,
  totalScore INTEGER DEFAULT 0
);

CREATE TABLE Game (
  id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
  password TEXT,
  level INTEGER,
  won INTEGER,
  score INTEGER DEFAULT 0,
  totalRounds INTEGER DEFAULT 0,
  digits  INTEGER,
  complexity  INTEGER,
  goldCoins INTEGER,
  silverCoins INTEGER,
  goldSpent INTEGER DEFAULT 0,
  silverSpent INTEGER DEFAULT 0,
  player_id INTEGER,
  FOREIGN KEY(player_id) REFERENCES Player(id)
);

CREATE TABLE History(
  id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
  round INTEGER,
  guess TEXT,
  goldReceived INTEGER,
  silverReceived INTEGER,
  goldInBag INTEGER,
  silverInBag INTEGER,
  game_id INTEGER,
  FOREIGN KEY(game_id) REFERENCES Game(id)
);

CREATE TABLE sessions (
    session_id char(128) UNIQUE NOT NULL,
    atime timestamp NOT NULL default current_timestamp,
    data TEXT
)

''')

conn.commit()
conn.close()
<|endoftext|>


<commit_before>import torch
import torch.nn as nn
import torch.nn.functional as F

class PReLUConv(nn.Module):
    def __init__(self, nchan, inplace):
        super(PReLUConv, self).__init__()
        if inplace:
            self.prelu = nn.ReLU(inplace=inplace)
        else:
            self.prelu = nn.PReLU()
        self.conv = nn.Conv3d(nchan, nchan, kernel_size=5, padding=2)

    def forward(self, x):
        out = self.prelu(self.conv(x))
        return out

def _make_nConv(nchan, depth, inplace):
    layers = []
    for _ in range(depth):
        layers.append(PReLUConv(nchan, inplace))
    return nn.Sequential(*layers)

class InputTransition(nn.Module):
    def __init__(self, outChans, inplace):
        super(InputTransition, self).__init__()
        self.conv1 = nn.Conv3d(1, 16, kernel_size=5, padding=2)
        if inplace:
            self.relu1 = nn.ReLU(inplace=inplace)
        else:
            self.relu1 = nn.PReLU()

    def forward(self, x):
        # do we want a PRELU here as well?
        out = self.conv1(x)
        # split input in to 16 channels
        x16 = torch.cat((x, x, x, x, x, x, x, x,
                         x, x, x, x, x, x, x, x), 0)
        out = self.relu1(torch.add(out, x16))
        return out

class DownTransition(nn.Module):
    def __init__(self, inChans, nConvs, inplace):
        super(DownTransition, self).__init__()
        outChans = 2*inChans
        self.down_conv = nn.Conv3d(inChans, outChans, kernel_size=2, stride=2)
        if inplace:
            self.relu1 = nn.ReLU(inplace=inplace)
            self.relu2 = nn.ReLU(inplace=inplace)
        else:
            self.relu1 = nn.PReLU()
            self.relu2 = nn.PReLU()
        self.ops = _make_nConv(outChans, nConvs, inplace)

    def forward(self, x):
        down = self.relu1(self.down_conv(x))
        out = self.ops(down)
        out = self.relu2(torch.add(out, down))
        return out

class UpTransition(nn.Module):
    def __init__(self, inChans, outChans, nConvs, inplace):
        super(UpTransition, self).__init__()
        self.up_conv = nn.ConvTranspose3d(inChans, outChans // 2, kernel_size=2, stride=2)
        if inplace:
            self.relu1 = nn.ReLU(inplace=inplace)
            self.relu2 = nn.ReLU(inplace=inplace)
        else:
            self.relu1 = nn.PReLU()
            self.relu2 = nn.PReLU()
        self.ops = _make_nConv(outChans, nConvs, inplace)

    def forward(self, x, skipx):
        out = self.relu1(self.up_conv(x))
        xcat = torch.cat((out, skipx), 1)
        out = self.ops(xcat)
        out = self.relu2(torch.add(out, xcat))
        return out

class OutputTransition(nn.Module):
    def __init__(self, inChans, batchSize, inplace):
        super(OutputTransition, self).__init__()
        self.conv1 = nn.Conv3d(inChans, 2, kernel_size=5, padding=2)
        self.conv2 = nn.Conv3d(2, 2, kernel_size=1)
        if inplace:
            self.relu1 = nn.ReLU(inplace=inplace)
        else:
            self.relu1 = nn.PReLU()

    def forward(self, x):
        # convolve 32 down to 2 channels
        out = self.relu1(self.conv1(x))
        out = self.conv2(out)
        # make channels the first axis
        out = out.permute(1, 0, 2, 3, 4)
        # flatten to cope with limited number
        # of dimensions softmax will take
        out = out.view(2, out.numel() // 2)
        # make channels last so that softmax DTRT
        out = out.permute(1, 0)
        # put channels first
        out = F.softmax(out)
        # treat channel 0 as the predicted output
        return out

class VNet(nn.Module):
    # the number of convolutions in each layer corresponds
    # to what is in the actual prototxt, not the intent
    def __init__(self, batchSize, inplace=True):
        super(VNet, self).__init__()
        self.in_tr = InputTransition(16, inplace)
        self.down_tr32 = DownTransition(16, 1, inplace)
        self.down_tr64 = DownTransition(32, 2, inplace)
        self.down_tr128 = DownTransition(64, 3, inplace)
        self.down_tr256 = DownTransition(128, 3, inplace)
        self.up_tr256 = UpTransition(256, 256, 3, inplace)
        self.up_tr128 = UpTransition(256, 128, 2, inplace)
        self.up_tr64 = UpTransition(128, 64, 1, inplace)
        self.up_tr32 = UpTransition(64, 32, 1, inplace)
        self.out_tr = OutputTransition(32, batchSize, inplace)

    # The network topology as described in the diagram
    # in the VNet paper
    # def __init__(self):
    #     super(VNet, self).__init__()
    #     self.in_tr =  InputTransition(16)
    #     # the number of convolutions in each layer corresponds
    #     # to what is in the actual prototxt, not the intent
    #     self.down_tr32 = DownTransition(16, 2)
    #     self.down_tr64 = DownTransition(32, 3)
    #     self.down_tr128 = DownTransition(64, 3)
    #     self.down_tr256 = DownTransition(128, 3)
    #     self.up_tr256 = UpTransition(256, 3)
    #     self.up_tr128 = UpTransition(128, 3)
    #     self.up_tr64 = UpTransition(64, 2)
    #     self.up_tr32 = UpTransition(32, 1)
    #     self.out_tr = OutputTransition(16)
    def forward(self, x):
        out16 = self.in_tr(x)
        out32 = self.down_tr32(out16)
        out64 = self.down_tr64(out32)
        out128 = self.down_tr128(out64)
        out256 = self.down_tr256(out128)
        out = self.up_tr256(out256, out128)
        out = self.up_tr128(out, out64)
        out = self.up_tr64(out, out32)
        out = self.up_tr32(out, out16)
        out = self.out_tr(out)
        return out
<commit_msg>skip additional permuting in output, make contiguous for the minibatch > 1 case<commit_after>import torch
import torch.nn as nn
import torch.nn.functional as F

class PReLUConv(nn.Module):
    def __init__(self, nchan, inplace):
        super(PReLUConv, self).__init__()
        if inplace:
            self.prelu = nn.ReLU(inplace=inplace)
        else:
            self.prelu = nn.PReLU()
        self.conv = nn.Conv3d(nchan, nchan, kernel_size=5, padding=2)

    def forward(self, x):
        out = self.prelu(self.conv(x))
        return out

def _make_nConv(nchan, depth, inplace):
    layers = []
    for _ in range(depth):
        layers.append(PReLUConv(nchan, inplace))
    return nn.Sequential(*layers)

class InputTransition(nn.Module):
    def __init__(self, outChans, inplace):
        super(InputTransition, self).__init__()
        self.conv1 = nn.Conv3d(1, 16, kernel_size=5, padding=2)
        if inplace:
            self.relu1 = nn.ReLU(inplace=inplace)
        else:
            self.relu1 = nn.PReLU()

    def forward(self, x):
        # do we want a PRELU here as well?
        out = self.conv1(x)
        # split input in to 16 channels
        x16 = torch.cat((x, x, x, x, x, x, x, x,
                         x, x, x, x, x, x, x, x), 0)
        out = self.relu1(torch.add(out, x16))
        return out

class DownTransition(nn.Module):
    def __init__(self, inChans, nConvs, inplace):
        super(DownTransition, self).__init__()
        outChans = 2*inChans
        self.down_conv = nn.Conv3d(inChans, outChans, kernel_size=2, stride=2)
        if inplace:
            self.relu1 = nn.ReLU(inplace=inplace)
            self.relu2 = nn.ReLU(inplace=inplace)
        else:
            self.relu1 = nn.PReLU()
            self.relu2 = nn.PReLU()
        self.ops = _make_nConv(outChans, nConvs, inplace)

    def forward(self, x):
        down = self.relu1(self.down_conv(x))
        out = self.ops(down)
        out = self.relu2(torch.add(out, down))
        return out

class UpTransition(nn.Module):
    def __init__(self, inChans, outChans, nConvs, inplace):
        super(UpTransition, self).__init__()
        self.up_conv = nn.ConvTranspose3d(inChans, outChans // 2, kernel_size=2, stride=2)
        if inplace:
            self.relu1 = nn.ReLU(inplace=inplace)
            self.relu2 = nn.ReLU(inplace=inplace)
        else:
            self.relu1 = nn.PReLU()
            self.relu2 = nn.PReLU()
        self.ops = _make_nConv(outChans, nConvs, inplace)

    def forward(self, x, skipx):
        out = self.relu1(self.up_conv(x))
        xcat = torch.cat((out, skipx), 1)
        out = self.ops(xcat)
        out = self.relu2(torch.add(out, xcat))
        return out

class OutputTransition(nn.Module):
    def __init__(self, inChans, batchSize, inplace):
        super(OutputTransition, self).__init__()
        self.conv1 = nn.Conv3d(inChans, 2, kernel_size=5, padding=2)
        self.conv2 = nn.Conv3d(2, 2, kernel_size=1)
        self.batchSize = batchSize
        if inplace:
            self.relu1 = nn.ReLU(inplace=inplace)
        else:
            self.relu1 = nn.PReLU()

    def forward(self, x):
        # convolve 32 down to 2 channels
        out = self.relu1(self.conv1(x))
        out = self.conv2(out)
        # make channels the first axis
        out = out.permute(1, 0, 2, 3, 4).contiguous()
        out = out.view(2, out.numel() // 2)
        # flatten to cope with limited number
        # of dimensions softmax will take
        # make channels last so that softmax DTRT
        # put channels first
        out = F.softmax(out)
        # treat channel 0 as the predicted output
        return out

class VNet(nn.Module):
    # the number of convolutions in each layer corresponds
    # to what is in the actual prototxt, not the intent
    def __init__(self, batchSize, inplace=True):
        super(VNet, self).__init__()
        self.in_tr = InputTransition(16, inplace)
        self.down_tr32 = DownTransition(16, 1, inplace)
        self.down_tr64 = DownTransition(32, 2, inplace)
        self.down_tr128 = DownTransition(64, 3, inplace)
        self.down_tr256 = DownTransition(128, 3, inplace)
        self.up_tr256 = UpTransition(256, 256, 3, inplace)
        self.up_tr128 = UpTransition(256, 128, 2, inplace)
        self.up_tr64 = UpTransition(128, 64, 1, inplace)
        self.up_tr32 = UpTransition(64, 32, 1, inplace)
        self.out_tr = OutputTransition(32, batchSize, inplace)

    # The network topology as described in the diagram
    # in the VNet paper
    # def __init__(self):
    #     super(VNet, self).__init__()
    #     self.in_tr =  InputTransition(16)
    #     # the number of convolutions in each layer corresponds
    #     # to what is in the actual prototxt, not the intent
    #     self.down_tr32 = DownTransition(16, 2)
    #     self.down_tr64 = DownTransition(32, 3)
    #     self.down_tr128 = DownTransition(64, 3)
    #     self.down_tr256 = DownTransition(128, 3)
    #     self.up_tr256 = UpTransition(256, 3)
    #     self.up_tr128 = UpTransition(128, 3)
    #     self.up_tr64 = UpTransition(64, 2)
    #     self.up_tr32 = UpTransition(32, 1)
    #     self.out_tr = OutputTransition(16)
    def forward(self, x):
        out16 = self.in_tr(x)
        out32 = self.down_tr32(out16)
        out64 = self.down_tr64(out32)
        out128 = self.down_tr128(out64)
        out256 = self.down_tr256(out128)
        out = self.up_tr256(out256, out128)
        out = self.up_tr128(out, out64)
        out = self.up_tr64(out, out32)
        out = self.up_tr32(out, out16)
        out = self.out_tr(out)
        return out
<|endoftext|>


<commit_before>print ("Hello World!")
thetext = input("Enter some text ")
print ("This is what you entered:")
print (thetext)<commit_msg>Updated from Brython Server: 2016/9/26 下午2:53:37<commit_after>#print ("Hello World!")
thetext = input("Enter some text ")
print ("This is what you entered:")
print (thetext)<|endoftext|>


<commit_before>import collections
import io
import os
import unittest
import familyanalyzer.familyanalyzer as fa
import json
import itertools
import tempfile
import shutil
from string import Template
import re
try:
    no_mock = False
    from unittest.mock import MagicMock
except ImportError:
    try:
        from mock import MagicMock
    except ImportError:
        no_mock = True

__author__ = 'adriaal'

class OGLevelMapper(object):
    def __init__(self, ogs):
        self.levels = collections.defaultdict(list)
        self.id2pos = {}
        for og in ogs:
            for lev in fa.OrthoXMLQuery.getLevels(og):
                self.levels[lev].append(og)
                pos = len(self.levels[lev]) - 1
                try:
                    pos_before = self.id2pos[og.get('og')]
                    if pos != pos_before:
                        raise HOGError(
                            'HOG {:!r} with several levels has inconsistent positions. This should not happen')
                except KeyError:
                    self.id2pos[og.get('og')] = pos

    def index(self, og):
        return self.id2pos[og.get('og')]

    def nr_subhogs_on_level(self, level):
        return len(self.levels[level])

class HOGVisExtractor(object):
    def __init__(self, fname):
        self.parser = fa.OrthoXMLParser(fname)
        self.re_non_char = re.compile(r'\W')  # matches non-standard chars (not A-Za-z0-9 or _)

    def setup_extractor(self):
        self.tax = fa.TaxRangeOrthoXMLTaxonomy(self.parser)
        self.parser.augmentTaxonomyInfo(self.tax)
        anno = fa.GroupAnnotator(self.parser)
        anno.annotateDoc()

    def get_per_species_structure(self, fam):
        genes_per_species = self.genenodes_per_species(fam)
        ogs_mapper = self.get_groups_to_level_mapper(fam)
        per_species = {}
        for org, genelist in genes_per_species.items():
            per_species[org] = cur_map = {}
            cur_map[org] = [[] for _ in range(len(genelist))]
            for pos, gene in enumerate(genelist):
                cur_map[org][pos].append(int(gene.get('id')))
                parent = gene.getparent()
                while self.parser.is_evolutionary_node(parent):
                    if self.parser.is_ortholog_group(parent):
                        for lev in fa.OrthoXMLQuery.getLevels(parent):
                            lev_repr = lev
                            if self.re_non_char.search(lev):
                                lev_repr = '"{}"'.format(lev)
                            if lev_repr not in cur_map:
                                cur_map[lev_repr] = [[] for _ in range(ogs_mapper.nr_subhogs_on_level(lev))]
                            if lev != org:
                                cur_map[lev_repr][ogs_mapper.index(parent)].append(int(gene.get('id')))
                    parent = parent.getparent()
        return per_species

    def get_groups_to_level_mapper(self, fam):
        ogs = fa.OrthoXMLQuery.getSubNodes('orthologGroup', fam)
        ogs.append(fam)
        return OGLevelMapper(ogs)

    def genenodes_per_species(self, fam):
        genes = collections.defaultdict(list)
        geneRefs = fa.OrthoXMLQuery.getSubNodes("geneRef", fam)
        for gref in geneRefs:
            gid = gref.get('id')
            sp = self.parser.mapGeneToSpecies(gid)
            genes[sp].append(gref)
        return genes

    def iter_families(self):
        for fam in self.parser.getToplevelGroups():
            topology = self.get_topology(fam)
            per_species = self.get_per_species_structure(fam)
            xrefs = self.get_xrefs(fam)
            yield topology, per_species, xrefs, int(fam.get('id'))

    def get_topology(self, fam):
        lev_of_famroot = fa.OrthoXMLQuery.getLevels(fam)[0]
        return str(self.tax.hierarchy[lev_of_famroot])+';'

    def get_xrefs(self, fam):
        geneRefs = fa.OrthoXMLQuery.getSubNodes("geneRef", fam)
        xrefs = {}
        for gene in geneRefs:
            gid = gene.get('id')
            xrefs[int(gid)] = dict(fa.OrthoXMLQuery.getGeneFromId(gid, self.parser.root).attrib)
        return xrefs

class HOGError(Exception):
    pass

class Writer(object):
    html_template = Template("""<!DOCTYPE html>

 <title>$name</title>

 <link rel="stylesheet" href="http://cbrg-oma.ethz.ch/static/css/tnt.css" type="text/css" />

HOG Viewer for $name

""")

    def __init__(self, folder, extractor):
        if os.path.exists(folder):
            if not os.path.isdir(folder):
                raise FileExistsError(folder)
        else:
            os.mkdir(folder)
        self.outdir = folder
        self.extractor = extractor

    def dump_hogs(self):
        for famdata in self.extractor.iter_families():
            newick, per_species, xrefs, fam_id = famdata
            self.write_hog(fam_id, newick, per_species, xrefs)

    def write_hog(self, fam_id, species_tree, per_species, xrefs):
        with open(os.path.join(self.outdir, "hog{:06d}.html".format(fam_id)), 'w') as fh:
            fh.write(self.html_template.safe_substitute({'name': "HOG{:06d}".format(fam_id),
                                                         'species_tree': species_tree,
                                                         'xrefs': json.dumps(xrefs),
                                                         'per_species': json.dumps(per_species)}))

class HOGVisExtractorTest(unittest.TestCase):
    def setUp(self):
        file = io.BytesIO("""<?xml version="1.0" encoding="UTF-8"?>
<orthoXML xmlns="http://orthoXML.org/2011/" version="0.3" origin="Family Analyzer Testcase" originVersion="0.2">
 <notes>
Example Notes without meaning
 </notes>
 <species name="HUMAN" NCBITaxId="9601">
  <database name="HUMANfake" version="0.1">
   <genes>
    <gene id="3" protId="HUMAN3" geneId="HUMANg3" />
    <gene id="5" protId="HUMAN5" geneId="HUMANg5" />
   </genes>
  </database>
 </species>
 <species name="PANTR" NCBITaxId="9483">
  <database name="PANTRfake" version="0.1">
   <genes>
    <gene id="13" protId="PANTR3" geneId="PANTRg3" />
    <gene id="14" protId="PANTR4" geneId="PANTRg4" />
   </genes>
  </database>
 </species>
 <species name="CANFA" NCBITaxId="9615">
  <database name="CANFAfake" version="0.1">
   <genes>
    <gene id="23" protId="CANFA3" geneId="CANFAg3" />
   </genes>
  </database>
 </species>
 <species name="MOUSE" NCBITaxId="10090">
  <database name="MOUSEfake" version="0.1">
   <genes>
    <gene id="33" protId="MOUSE3" geneId="MOUSEg3" />
    <gene id="34" protId="MOUSE4" geneId="MOUSEg4" />
   </genes>
  </database>
 </species>
 <species name="RATNO" NCBITaxId="10116">
  <database name="RATNOfake" version="0.1">
   <genes>
    <gene id="41" protId="RATNO1" geneId="RATNOg1" />
    <gene id="43" protId="RATNO3" geneId="RATNOg3" />
   </genes>
  </database>
 </species>
 <species name="XENTR" NCBITaxId="1">
  <database name="XENTRfake" version="0.1">
   <genes>
    <gene id="51" protId="XENTR1" geneId="XENTRg1" />
    <gene id="53" protId="XENTR3" geneId="XENTRg3" />
   </genes>
  </database>
 </species>
 <groups>
  <orthologGroup id="3">
   <property name="TaxRange" value="Vertebrata"/>
   <geneRef id="53"/>
   <orthologGroup >
    <property name="TaxRange" value="Mammalia"/>
    <geneRef id="23"/>
    <paralogGroup>
     <orthologGroup >
      <property name="TaxRange" value="Euarchontoglires" />
      <geneRef id="33"/>
      <orthologGroup>
       <property name="TaxRange" value="Primates"/>
       <paralogGroup>
        <geneRef id="3"/>
        <geneRef id="5"/>
       </paralogGroup>
       <geneRef id="13"/>
      </orthologGroup>
     </orthologGroup>
     <orthologGroup >
      <property name="TaxRange" value="Euarchontoglires" />
      <geneRef id="34" />
      <geneRef id="14" />
     </orthologGroup>
    </paralogGroup>
   </orthologGroup>
  </orthologGroup>
 </groups>
</orthoXML>""".encode('utf-8'))
        self.hog_extractor = HOGVisExtractor(file)
        self.hog_extractor.setup_extractor()
        self.fam = self.hog_extractor.parser.getToplevelGroups()[0]
        self.map = self.hog_extractor.get_groups_to_level_mapper(self.fam)

    def test_levelMap(self):
        self.assertEqual(self.map.nr_subhogs_on_level('Euarchontoglires'), 2)
        self.assertEqual(self.map.nr_subhogs_on_level('Primates'), 2)
        self.assertEqual(self.map.nr_subhogs_on_level('Mammalia'), 1)
        self.assertEqual(self.map.nr_subhogs_on_level('LUCA'), 0)

    def test_og_id_equality(self):
        primate_node = self.hog_extractor.parser.getSubFamilies('Primates')[0]
        primate_from_map = self.map.levels['Primates'][0]
        glires = self.map.levels['Euarchontoglires']
        self.assertEqual(primate_node.get('og'), primate_from_map.get('og'))
        self.assertNotEqual(glires[0].get('og'), glires[1].get('og'))

    def test_per_species_mapping(self):
        mapping = self.hog_extractor.genenodes_per_species(self.fam)
        mouse_geneids = [int(z.get('id')) for z in mapping['MOUSE']]
        self.assertEqual([33, 34], mouse_geneids)

    def test_mapping(self):
        expected_HUMAN = {'HUMAN': [[3], [5]], 'Primates': [[3, 5],[]],
                          'Euarchontoglires': [[3, 5], []],
                          'Mammalia': [[3, 5]], 'Vertebrata': [[3, 5]]}
        expected_MOUSE = {'MOUSE': [[33], [34]], 'Euarchontoglires': [[33], [34]],
                          'Mammalia': [[33, 34]], 'Vertebrata': [[33, 34]]}
        per_species = self.hog_extractor.get_per_species_structure(self.fam)
        self.assertDictEqual(expected_HUMAN, per_species['HUMAN'])
        self.assertDictEqual(expected_MOUSE, per_species['MOUSE'])

    def test_xrefs(self):
        exprected_of_human5 = {'id': "5", 'protId': "HUMAN5", 'geneId': "HUMANg5"}
        xrefs = self.hog_extractor.get_xrefs(self.fam)
        self.assertEqual(exprected_of_human5, xrefs[5])

@unittest.skipIf(no_mock, 'no mock module available')
class WriterTest(unittest.TestCase):
    def setUp(self):
        self.dir = tempfile.mkdtemp()
        self.extractor_mock = HOGVisExtractor(io.BytesIO(b"""<?xml version="1.0" encoding="UTF-8"?>
<orthoXML xmlns="http://orthoXML.org/2011/" version="0.3" origin="Family Analyzer Testcase" originVersion="0.2"></orthoXML>"""))
        self.extractor_mock.iter_families = MagicMock(return_value=(('(HUMAN,MOUSE)CANFA;',
                                                                     {'HUMAN': [2, 4]}, {2: {'id': 2}}, 1),))

    def test_write(self):
        writer = Writer(self.dir, self.extractor_mock)
        writer.dump_hogs()
        expected_filename = os.path.join(self.dir, 'hog{:06d}.html'.format(1))
        self.assertTrue(os.path.exists(expected_filename))
        with open(expected_filename) as fh:
            data = fh.read()
        self.assertIn('<title>HOG000001</title>', data)
        self.assertIn('(HUMAN,MOUSE)CANFA;', data)

    def tearDown(self):
        shutil.rmtree(self.dir)

def handle_args():
    import argparse
    parser = argparse.ArgumentParser(prog='StandaloneHogVis',
                                     description="Tool to prepare HogVis html pages to analyze the HOGs graphically")
    parser.add_argument('-o', '--outdir', default='hogvis', help="directory where to store the html files."
                                                                 "(defaults: 'vis/')")
    parser.add_argument('orthoxml', help="path to the orthoxml file that should be converted")

    conf = parser.parse_args()

    ex = HOGVisExtractor(conf.orthoxml)
    ex.setup_extractor()
    wr = Writer(conf.outdir, ex)
    wr.dump_hogs()

if __name__ == "__main__":
    handle_args()
<commit_msg>fixing problem with incosistent positions. Not sure whether bug or not<commit_after>import collections
import io
import logging
import os
import unittest
import familyanalyzer.familyanalyzer as fa
import json
import itertools
import tempfile
import shutil
from string import Template
import re
try:
    no_mock = False
    from unittest.mock import MagicMock
except ImportError:
    try:
        from mock import MagicMock
    except ImportError:
        no_mock = True

__author__ = 'adriaal'

class OGLevelMapper(object):
    def __init__(self, ogs):
        self.levels = collections.defaultdict(list)
        self.id2pos = {}
        for og in ogs:
            for lev in fa.OrthoXMLQuery.getLevels(og):
                self.levels[lev].append(og)
                pos = len(self.levels[lev]) - 1
                try:
                    pos_before = self.id2pos[og.get('og')]
                    if pos != pos_before:
                        logging.warn(
                            "HOG {} with several levels has inconsistent positions."
                            "Lev: {}, pos_before, pos: {}/{}".format(og, lev, pos_before, pos))
                        for _ in range(pos_before-pos):
                            self.levels[lev].insert(pos, None)
                except KeyError:
                    self.id2pos[og.get('og')] = pos

    def index(self, og):
        return self.id2pos[og.get('og')]

    def nr_subhogs_on_level(self, level):
        return len(self.levels[level])

class HOGVisExtractor(object):
    def __init__(self, fname):
        self.parser = fa.OrthoXMLParser(fname)
        self.re_non_char = re.compile(r'\W')  # matches non-standard chars (not A-Za-z0-9 or _)

    def setup_extractor(self):
        self.tax = fa.TaxRangeOrthoXMLTaxonomy(self.parser)
        self.parser.augmentTaxonomyInfo(self.tax)
        anno = fa.GroupAnnotator(self.parser)
        anno.annotateDoc()

    def get_per_species_structure(self, fam):
        genes_per_species = self.genenodes_per_species(fam)
        ogs_mapper = self.get_groups_to_level_mapper(fam)
        per_species = {}
        for org, genelist in genes_per_species.items():
            per_species[org] = cur_map = {}
            cur_map[org] = [[] for _ in range(len(genelist))]
            for pos, gene in enumerate(genelist):
                cur_map[org][pos].append(int(gene.get('id')))
                parent = gene.getparent()
                while self.parser.is_evolutionary_node(parent):
                    if self.parser.is_ortholog_group(parent):
                        for lev in fa.OrthoXMLQuery.getLevels(parent):
                            lev_repr = lev
                            if self.re_non_char.search(lev):
                                lev_repr = '"{}"'.format(lev)
                            if lev_repr not in cur_map:
                                cur_map[lev_repr] = [[] for _ in range(ogs_mapper.nr_subhogs_on_level(lev))]
                            if lev != org:
                                cur_map[lev_repr][ogs_mapper.index(parent)].append(int(gene.get('id')))
                    parent = parent.getparent()
        return per_species

    def get_groups_to_level_mapper(self, fam):
        ogs = fa.OrthoXMLQuery.getSubNodes('orthologGroup', fam)
        ogs.append(fam)
        return OGLevelMapper(ogs)

    def genenodes_per_species(self, fam):
        genes = collections.defaultdict(list)
        geneRefs = fa.OrthoXMLQuery.getSubNodes("geneRef", fam)
        for gref in geneRefs:
            gid = gref.get('id')
            sp = self.parser.mapGeneToSpecies(gid)
            genes[sp].append(gref)
        return genes

    def iter_families(self):
        for fam in self.parser.getToplevelGroups():
            topology = self.get_topology(fam)
            per_species = self.get_per_species_structure(fam)
            xrefs = self.get_xrefs(fam)
            yield topology, per_species, xrefs, int(fam.get('id'))

    def get_topology(self, fam):
        lev_of_famroot = fa.OrthoXMLQuery.getLevels(fam)[0]
        return str(self.tax.hierarchy[lev_of_famroot])+';'

    def get_xrefs(self, fam):
        geneRefs = fa.OrthoXMLQuery.getSubNodes("geneRef", fam)
        xrefs = {}
        for gene in geneRefs:
            gid = gene.get('id')
            xrefs[int(gid)] = dict(fa.OrthoXMLQuery.getGeneFromId(gid, self.parser.root).attrib)
        return xrefs

class HOGError(Exception):
    pass

class Writer(object):
    html_template = Template("""<!DOCTYPE html>

 <title>$name</title>

 <link rel="stylesheet" href="http://cbrg-oma.ethz.ch/static/css/tnt.css" type="text/css" />

HOG Viewer for $name

""")

    def __init__(self, folder, extractor):
        if os.path.exists(folder):
            if not os.path.isdir(folder):
                raise FileExistsError(folder)
        else:
            os.mkdir(folder)
        self.outdir = folder
        self.extractor = extractor

    def dump_hogs(self):
        for famdata in self.extractor.iter_families():
            newick, per_species, xrefs, fam_id = famdata
            self.write_hog(fam_id, newick, per_species, xrefs)

    def write_hog(self, fam_id, species_tree, per_species, xrefs):
        with open(os.path.join(self.outdir, "hog{:06d}.html".format(fam_id)), 'w') as fh:
            fh.write(self.html_template.safe_substitute({'name': "HOG{:06d}".format(fam_id),
                                                         'species_tree': species_tree,
                                                         'xrefs': json.dumps(xrefs),
                                                         'per_species': json.dumps(per_species)}))

class HOGVisExtractorTest(unittest.TestCase):
    def setUp(self):
        file = io.BytesIO("""<?xml version="1.0" encoding="UTF-8"?>
<orthoXML xmlns="http://orthoXML.org/2011/" version="0.3" origin="Family Analyzer Testcase" originVersion="0.2">
 <notes>
Example Notes without meaning
 </notes>
 <species name="HUMAN" NCBITaxId="9601">
  <database name="HUMANfake" version="0.1">
   <genes>
    <gene id="3" protId="HUMAN3" geneId="HUMANg3" />
    <gene id="5" protId="HUMAN5" geneId="HUMANg5" />
   </genes>
  </database>
 </species>
 <species name="PANTR" NCBITaxId="9483">
  <database name="PANTRfake" version="0.1">
   <genes>
    <gene id="13" protId="PANTR3" geneId="PANTRg3" />
    <gene id="14" protId="PANTR4" geneId="PANTRg4" />
   </genes>
  </database>
 </species>
 <species name="CANFA" NCBITaxId="9615">
  <database name="CANFAfake" version="0.1">
   <genes>
    <gene id="23" protId="CANFA3" geneId="CANFAg3" />
   </genes>
  </database>
 </species>
 <species name="MOUSE" NCBITaxId="10090">
  <database name="MOUSEfake" version="0.1">
   <genes>
    <gene id="33" protId="MOUSE3" geneId="MOUSEg3" />
    <gene id="34" protId="MOUSE4" geneId="MOUSEg4" />
   </genes>
  </database>
 </species>
 <species name="RATNO" NCBITaxId="10116">
  <database name="RATNOfake" version="0.1">
   <genes>
    <gene id="41" protId="RATNO1" geneId="RATNOg1" />
    <gene id="43" protId="RATNO3" geneId="RATNOg3" />
   </genes>
  </database>
 </species>
 <species name="XENTR" NCBITaxId="1">
  <database name="XENTRfake" version="0.1">
   <genes>
    <gene id="51" protId="XENTR1" geneId="XENTRg1" />
    <gene id="53" protId="XENTR3" geneId="XENTRg3" />
   </genes>
  </database>
 </species>
 <groups>
  <orthologGroup id="3">
   <property name="TaxRange" value="Vertebrata"/>
   <geneRef id="53"/>
   <orthologGroup >
    <property name="TaxRange" value="Mammalia"/>
    <geneRef id="23"/>
    <paralogGroup>
     <orthologGroup >
      <property name="TaxRange" value="Euarchontoglires" />
      <geneRef id="33"/>
      <orthologGroup>
       <property name="TaxRange" value="Primates"/>
       <paralogGroup>
        <geneRef id="3"/>
        <geneRef id="5"/>
       </paralogGroup>
       <geneRef id="13"/>
      </orthologGroup>
     </orthologGroup>
     <orthologGroup >
      <property name="TaxRange" value="Euarchontoglires" />
      <geneRef id="34" />
      <geneRef id="14" />
     </orthologGroup>
    </paralogGroup>
   </orthologGroup>
  </orthologGroup>
 </groups>
</orthoXML>""".encode('utf-8'))
        self.hog_extractor = HOGVisExtractor(file)
        self.hog_extractor.setup_extractor()
        self.fam = self.hog_extractor.parser.getToplevelGroups()[0]
        self.map = self.hog_extractor.get_groups_to_level_mapper(self.fam)

    def test_levelMap(self):
        self.assertEqual(self.map.nr_subhogs_on_level('Euarchontoglires'), 2)
        self.assertEqual(self.map.nr_subhogs_on_level('Primates'), 2)
        self.assertEqual(self.map.nr_subhogs_on_level('Mammalia'), 1)
        self.assertEqual(self.map.nr_subhogs_on_level('LUCA'), 0)

    def test_og_id_equality(self):
        primate_node = self.hog_extractor.parser.getSubFamilies('Primates')[0]
        primate_from_map = self.map.levels['Primates'][0]
        glires = self.map.levels['Euarchontoglires']
        self.assertEqual(primate_node.get('og'), primate_from_map.get('og'))
        self.assertNotEqual(glires[0].get('og'), glires[1].get('og'))

    def test_per_species_mapping(self):
        mapping = self.hog_extractor.genenodes_per_species(self.fam)
        mouse_geneids = [int(z.get('id')) for z in mapping['MOUSE']]
        self.assertEqual([33, 34], mouse_geneids)

    def test_mapping(self):
        expected_HUMAN = {'HUMAN': [[3], [5]], 'Primates': [[3, 5],[]],
                          'Euarchontoglires': [[3, 5], []],
                          'Mammalia': [[3, 5]], 'Vertebrata': [[3, 5]]}
        expected_MOUSE = {'MOUSE': [[33], [34]], 'Euarchontoglires': [[33], [34]],
                          'Mammalia': [[33, 34]], 'Vertebrata': [[33, 34]]}
        per_species = self.hog_extractor.get_per_species_structure(self.fam)
        self.assertDictEqual(expected_HUMAN, per_species['HUMAN'])
        self.assertDictEqual(expected_MOUSE, per_species['MOUSE'])

    def test_xrefs(self):
        exprected_of_human5 = {'id': "5", 'protId': "HUMAN5", 'geneId': "HUMANg5"}
        xrefs = self.hog_extractor.get_xrefs(self.fam)
        self.assertEqual(exprected_of_human5, xrefs[5])

@unittest.skipIf(no_mock, 'no mock module available')
class WriterTest(unittest.TestCase):
    def setUp(self):
        self.dir = tempfile.mkdtemp()
        self.extractor_mock = HOGVisExtractor(io.BytesIO(b"""<?xml version="1.0" encoding="UTF-8"?>
<orthoXML xmlns="http://orthoXML.org/2011/" version="0.3" origin="Family Analyzer Testcase" originVersion="0.2"></orthoXML>"""))
        self.extractor_mock.iter_families = MagicMock(return_value=(('(HUMAN,MOUSE)CANFA;',
                                                                     {'HUMAN': [2, 4]}, {2: {'id': 2}}, 1),))

    def test_write(self):
        writer = Writer(self.dir, self.extractor_mock)
        writer.dump_hogs()
        expected_filename = os.path.join(self.dir, 'hog{:06d}.html'.format(1))
        self.assertTrue(os.path.exists(expected_filename))
        with open(expected_filename) as fh:
            data = fh.read()
        self.assertIn('<title>HOG000001</title>', data)
        self.assertIn('(HUMAN,MOUSE)CANFA;', data)

    def tearDown(self):
        shutil.rmtree(self.dir)

def handle_args():
    import argparse
    parser = argparse.ArgumentParser(prog='StandaloneHogVis',
                                     description="Tool to prepare HogVis html pages to analyze the HOGs graphically")
    parser.add_argument('-o', '--outdir', default='hogvis', help="directory where to store the html files."
                                                                 "(defaults: 'vis/')")
    parser.add_argument('orthoxml', help="path to the orthoxml file that should be converted")

    conf = parser.parse_args()

    ex = HOGVisExtractor(conf.orthoxml)
    ex.setup_extractor()
    wr = Writer(conf.outdir, ex)
    wr.dump_hogs()

if __name__ == "__main__":
    handle_args()
<|endoftext|>


<commit_before>from django.contrib import admin
from django.contrib.auth.admin import UserAdmin
from django.contrib.auth.models import User
from django.core.mail import send_mail
from django.template.loader import get_template
from django.template import Context
from knowledge.models import Question, Response, Category, Company, Author

def make_public(modeladmin, request, queryset):
    queryset.update(status='public')
make_public.short_description = "Mark selected articles public"

def make_rejected(modeladmin, request, queryset):
    queryset.update(status='rejected')
    for q in queryset:
        ctx = {
            'article': q.title,
            'email': q.email,
            'comment': q.comment,
        }
        #message = 'Your article: '+q.title+' has been rejected, Visit Portalpractices for more information'
        message = get_template('registration/article_rejected_template_email.html').render(Context(ctx))
        send_mail('Portalpractices: Article rejected', message, 'no-reply@cantemo.com', [q.email])
make_rejected.short_description = "Mark selected articles rejected"

def make_draft(modeladmin, request, queryset):
    queryset.update(status='draft')
make_draft.short_description = "Mark selected articles draft"

def make_review(modeladmin, request, queryset):
    queryset.update(status='review')
make_review.short_description = "Mark selected articles review"

def make_active(modeladmin, reqeust, queryset):
    queryset.update(is_active=True)
    for q in queryset:
        ctx = {
            'username': q.username,
            'email': q.email,
        }
        message = get_template('registration/activate_user_template_email.html').render(Context(ctx))
        send_mail('Portalpractices: Account activated', message, 'no-reply@cantemo.com', [q.email])
make_active.short_description = "Mark selected users active"

def make_author_active(modeladmin, reqeust, queryset):
    for query in queryset:
        query.user.is_active=True
        query.user.save(update_fields=['is_active'])
        ctx = {
            'username': query.user.username,
            'email': query.user.email,
        }
        message = get_template('registration/activate_user_template_email.html').render(Context(ctx))
        send_mail('Portalpractices: Account activated', message, 'no-reply@cantemo.com', [query.user.email])
make_active.short_description = "Mark selected users active"

class CategoryAdmin(admin.ModelAdmin):
    list_display = [f.name for f in Category._meta.fields]
    prepopulated_fields = {'slug': ('title', )}
admin.site.register(Category, CategoryAdmin)

class QuestionAdmin(admin.ModelAdmin):
    list_display = ('id', 'added', 'lastchanged', 'user', 'alert', 'name', 'email', 'title', 'comment', 'status', 'locked', 'recommended', 'hits'  )
    list_select_related = True
    list_filter = ['status']
    raw_id_fields = ['user']
    actions = [make_public, make_draft, make_review, make_rejected]
admin.site.register(Question, QuestionAdmin)

class ResponseAdmin(admin.ModelAdmin):
    list_display = [f.name for f in Response._meta.fields]
    list_select_related = True
    raw_id_fields = ['user', 'question']
admin.site.register(Response, ResponseAdmin)

class CompanyAdmin(admin.ModelAdmin):
    list_display = [f.name for f in Company._meta.fields]
    list_select_related = True
    raw_id_fields = ['external_id']
admin.site.register(Company, CompanyAdmin)

class AuthorAdmin(admin.ModelAdmin):
    def user_email(self, instance):
        return instance.user.email
    def user_first_name(self, instance):
        return instance.user.first_name
    def user_last_name(self, instance):
        return instance.user.first_name
    def user_active(self, instance):
        return instance.user.is_active
    list_display = ('id', 'user', 'company', 'title', 'about', 'avatar', 'user_first_name', 'user_last_name', 'user_email','user_active' )
    list_select_related = True
    raw_id_fields = ['company']
    actions = [make_author_active]
admin.site.register(Author, AuthorAdmin)

UserAdmin.list_display = ('email', 'first_name', 'last_name', 'date_joined', 'is_active', 'is_staff', 'is_superuser')
UserAdmin.actions = [make_active]

admin.site.unregister(User)
admin.site.register(User, UserAdmin)<commit_msg>One page for managing registrations<commit_after>from django.contrib import admin
from django.db.models.signals import pre_save
from django.dispatch import receiver
from django.contrib.auth.admin import UserAdmin
from django.contrib.auth.models import User
from django.core.mail import send_mail
from django.template.loader import get_template
from django.template import Context
from knowledge.models import Question, Response, Category, Company, Author

def make_public(modeladmin, request, queryset):
    queryset.update(status='public')
make_public.short_description = "Mark selected articles public"

def make_rejected(modeladmin, request, queryset):
    queryset.update(status='rejected')
    for q in queryset:
        ctx = {
            'article': q.title,
            'email': q.email,
            'comment': q.comment,
        }
        #message = 'Your article: '+q.title+' has been rejected, Visit Portalpractices for more information'
        message = get_template('registration/article_rejected_template_email.html').render(Context(ctx))
        send_mail('Portalpractices: Article rejected', message, 'no-reply@cantemo.com', [q.email])
make_rejected.short_description = "Mark selected articles rejected"

def make_draft(modeladmin, request, queryset):
    queryset.update(status='draft')
make_draft.short_description = "Mark selected articles draft"

def make_review(modeladmin, request, queryset):
    queryset.update(status='review')
make_review.short_description = "Mark selected articles review"

def make_active(modeladmin, reqeust, queryset):
    queryset.update(is_active=True)
    for q in queryset:
        ctx = {
            'username': q.username,
            'email': q.email,
        }
        message = get_template('registration/activate_user_template_email.html').render(Context(ctx))
        send_mail('Portalpractices: Account activated', message, 'no-reply@cantemo.com', [q.email])
make_active.short_description = "Mark selected users active"

def make_author_active(modeladmin, reqeust, queryset):
    for query in queryset:
        query.user.is_active=True
        query.user.save(update_fields=['is_active'])
        ctx = {
            'username': query.user.username,
            'email': query.user.email,
        }
        message = get_template('registration/activate_user_template_email.html').render(Context(ctx))
        send_mail('Portalpractices: Account activated', message, 'no-reply@cantemo.com', [query.user.email])
make_active.short_description = "Mark selected users active"

class CategoryAdmin(admin.ModelAdmin):
    list_display = [f.name for f in Category._meta.fields]
    prepopulated_fields = {'slug': ('title', )}
admin.site.register(Category, CategoryAdmin)

class QuestionAdmin(admin.ModelAdmin):
    list_display = ('id', 'added', 'lastchanged', 'user', 'alert', 'name', 'email', 'title', 'comment', 'status', 'locked', 'recommended', 'hits'  )
    list_select_related = True
    list_filter = ['status']
    raw_id_fields = ['user']
    actions = [make_public, make_draft, make_review, make_rejected]
admin.site.register(Question, QuestionAdmin)

class ResponseAdmin(admin.ModelAdmin):
    list_display = [f.name for f in Response._meta.fields]
    list_select_related = True
    raw_id_fields = ['user', 'question']
admin.site.register(Response, ResponseAdmin)

class CompanyAdmin(admin.ModelAdmin):
    list_display = [f.name for f in Company._meta.fields]
    list_select_related = True
    raw_id_fields = ['external_id']
admin.site.register(Company, CompanyAdmin)

class AuthorAdmin(admin.TabularInline):
    model = Author
#admin.site.register(Author, AuthorAdmin)

@receiver(pre_save, sender=User)
def send_user_email(sender, instance=None, **kwargs):
    old_instance = User.objects.get(pk=instance.pk)
    if old_instance.is_active == False and instance.is_active == True:
        ctx = {
            'username': instance.username,
            'email': instance.email,
        }
        message = get_template('registration/activate_user_template_email.html').render(Context(ctx))
        send_mail('Portalpractices: Account activated', message, 'no-reply@cantemo.com', [instance.email])

pre_save.connect(send_user_email, sender=User)

class UserAdmin(admin.ModelAdmin):
    inlines = [AuthorAdmin]
    actions = [make_active]
    list_filter = ['is_active']
    list_display = ('email', 'first_name', 'last_name', 'date_joined', 'is_active', 'is_staff', 'is_superuser')
admin.site.unregister(User)
admin.site.register(User, UserAdmin)<|endoftext|>


<commit_before>from __future__ import absolute_import

from .celery import app as celery_app  # noqa: F401
<commit_msg>Acknowledge and document ignored violations<commit_after>from __future__ import absolute_import

# Ensure app is imported for Celery's `shared_task` when Django starts
from .celery import app as celery_app  # noqa: F401
<|endoftext|>


<commit_before>import logging
import os
import hmac
from hashlib import sha1
import json
import re
import boto3

logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
logging.getLogger('boto3').setLevel(logging.CRITICAL)
logging.getLogger('botocore').setLevel(logging.CRITICAL)

def lambda_handler(event, context):
    logger.info(json.dumps(event, indent=4))
    try:
        # Get the sha_name and sig from the headers
        sha_name, sig = event['headers']['X-Hub-Signature'].split('=')
        # Get the body of the message
        body = event['body']
        # Only support sha1
        if sha_name != 'sha1':
            return {
                "statusCode": 501,
                "body": "Something is wrong, unsupported sha type?\n"
            }
    except Exception as e:
        logger.exception(e)
        return {
            "statusCode": 501,
            "body": "Something is wrong, not from GitHub?\n"
        }

    secret = os.environ['GHSECRET']
    # HMAC requires the key to be bytes, but data is string
    mac = hmac.new(secret, body, sha1)
    if not hmac.compare_digest(unicode(mac.hexdigest()), unicode(sig)):
        logger.critical("Signature doesn't match")
        return {
            'body': "Signature doesn't match",
            'statusCode': 403
            }

    githookbody = json.loads(body)
    logger.debug(json.dumps(githookbody, indent=4))
    # some sanity checking, only support pushes
    if event['headers']['X-GitHub-Event'] != "push":
        logger.critical("hook event is not supported")
        return {
            'body': "hook event is not supported",
            'statusCode': 501
            }

    repo_url = githookbody['repository']['url'] + ".git"
    username = githookbody['repository']['owner']['name']
    builds = []
    for i in githookbody['commits']:
        for a in i['added']:
            if re.search("/", a):
                builds.append(a.split("/")[0])
        for m in i['modified']:
            if re.search("/", m):
                builds.append(m.split("/")[0])

    # Spawn the CodeBuild Job
    lambdac = boto3.client('lambda')
    message_input = {
        'repo_url': repo_url,
        'builds': builds,
        'username': username,
        }
    print message_input
    if builds: # False if empty
        response = lambdac.invoke(
            FunctionName=os.environ['SpawnCodeBuildFunctionArn'],
            InvocationType='Event', # async
            LogType='None',
            Payload=json.dumps(message_input)
        )
    else:
        logger.info("Not spawning a codebuild job")

    # Everything is good
    return {
        "statusCode": 200,
        "body": "worked"
    }
<commit_msg>Remove duplicates from commit list<commit_after>import logging
import os
import hmac
from hashlib import sha1
import json
import re
import boto3

logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
logging.getLogger('boto3').setLevel(logging.CRITICAL)
logging.getLogger('botocore').setLevel(logging.CRITICAL)

def lambda_handler(event, context):
    logger.info(json.dumps(event, indent=4))
    try:
        # Get the sha_name and sig from the headers
        sha_name, sig = event['headers']['X-Hub-Signature'].split('=')
        # Get the body of the message
        body = event['body']
        # Only support sha1
        if sha_name != 'sha1':
            return {
                "statusCode": 501,
                "body": "Something is wrong, unsupported sha type?\n"
            }
    except Exception as e:
        logger.exception(e)
        return {
            "statusCode": 501,
            "body": "Something is wrong, not from GitHub?\n"
        }

    secret = os.environ['GHSECRET']
    # HMAC requires the key to be bytes, but data is string
    mac = hmac.new(secret, body, sha1)
    if not hmac.compare_digest(unicode(mac.hexdigest()), unicode(sig)):
        logger.critical("Signature doesn't match")
        return {
            'body': "Signature doesn't match",
            'statusCode': 403
            }

    githookbody = json.loads(body)
    logger.debug(json.dumps(githookbody, indent=4))
    # some sanity checking, only support pushes
    if event['headers']['X-GitHub-Event'] != "push":
        logger.critical("hook event is not supported")
        return {
            'body': "hook event is not supported",
            'statusCode': 501
            }

    repo_url = githookbody['repository']['url'] + ".git"
    username = githookbody['repository']['owner']['name']
    builds_list = []
    for i in githookbody['commits']:
        for a in i['added']:
            if re.search("/", a):
                builds_list.append(a.split("/")[0])
        for m in i['modified']:
            if re.search("/", m):
                builds_list.append(m.split("/")[0])
    # This is slow but don't care for our list size
    # Removes dupes to prevent spawning duplicate jobs
    builds = sorted(set(builds_list),key=builds_list.index)

    # Spawn the CodeBuild Job
    lambdac = boto3.client('lambda')
    message_input = {
        'repo_url': repo_url,
        'builds': builds,
        'username': username,
        }
    print message_input
    if builds: # False if empty
        response = lambdac.invoke(
            FunctionName=os.environ['SpawnCodeBuildFunctionArn'],
            InvocationType='Event', # async
            LogType='None',
            Payload=json.dumps(message_input)
        )
    else:
        logger.info("Not spawning a codebuild job")

    # Everything is good
    return {
        "statusCode": 200,
        "body": "worked"
    }
<|endoftext|>


<commit_before>#!/usr/bin/env python

import re
import sys
import os
import platform
import subprocess

distro = platform.linux_distribution()[0]

args = {'act': sys.argv[1], 'service': sys.argv[2]}

if re.search(distro, 'Ubuntu'):
    if os.path.isfile("/etc/init/%s.conf" % args['service']):
        cmd = args['act'] + " " + args['service']
    elif os.path.isfile("/etc/init.d/%s" % args['service']):
        cmd = "/etc/init.d/%s %s" % (args['service'], args['act'])
    else:
        print("Unknown service")
        sys.exit(2)
elif re.search(distro, 'Redhat') or re.search(distro, 'Fedora'):
    cmd = "systemctl %s %s" % (args['act'], args['service'])

subprocess.call(cmd, shell=True)
<commit_msg>Update service.py<commit_after>#!/usr/bin/env python

import re
import sys
import os
import platform
import subprocess

distro = platform.linux_distribution()[0]

args = {'act': sys.argv[1], 'service': sys.argv[2]}

if re.search(distro, 'Ubuntu'):
    if os.path.isfile("/etc/init/%s.conf" % args['service']):
        cmd = args['act'] + " " + args['service']
    elif os.path.isfile("/etc/init.d/%s" % args['service']):
        cmd = "/etc/init.d/%s %s" % (args['service'], args['act'])
    else:
        print("Unknown service")
        sys.exit(2)
elif re.search(distro, 'Redhat') or re.search(distro, 'Fedora') or re.search(distro, 'CentOS Linux'):
    cmd = "systemctl %s %s" % (args['act'], args['service'])

subprocess.call(cmd, shell=True)
<|endoftext|>