<commit_before>#!/usr/bin/env python3
# Coin Price indicator
#
# Nil Gradisnik <nil.gradisnik@gmail.com>
# Sander Van de Moortel <sander.vandemoortel@gmail.com>
#

import signal
import yaml
import logging
import glob
import dbus
import importlib
import notify2

from os.path import abspath, dirname, isfile, basename
from indicator import Indicator
from plugin_selection import PluginSelectionWindow
from async_downloader import AsyncDownloadService
from dbus.mainloop.glib import DBusGMainLoop
from gi.repository import Gtk, GdkPixbuf

try:
    from gi.repository import AppIndicator3 as AppIndicator
except ImportError:
    from gi.repository import AppIndicator

PROJECT_ROOT = abspath(dirname(dirname(__file__)))
SETTINGS_FILE = PROJECT_ROOT + '/user.conf'

class Coin():
    config = yaml.load(open(PROJECT_ROOT + '/config.yaml', 'r'))
    config['project_root'] = PROJECT_ROOT

    def __init__(self):
        self.downloader = AsyncDownloadService()
        self.unique_id = 0

        self._load_exchanges()
        self._load_assets()
        self._load_settings()
        self._start_main()

        self.instances = []
        self.discoveries = 0
        self._add_many_indicators(self.settings.get('tickers'))
        self._start_gui()

    # Load exchange 'plug-ins' from exchanges dir
    def _load_exchanges(self):
        dirfiles = glob.glob(dirname(__file__) + "/exchanges/*.py")
        plugins = [basename(f)[:-3] for f in dirfiles if isfile(f) and not f.endswith('__init__.py')]
        plugins.sort()

        self.EXCHANGES = []
        for plugin in plugins:
            class_name = plugin.capitalize()
            class_ = getattr(importlib.import_module('exchanges.' + plugin), class_name)
            self.EXCHANGES.append(class_)

    # Find an exchange
    def find_exchange_by_code(self, code):
        for exchange in self.EXCHANGES:
            if exchange.get_code() == code.lower():
                return exchange

    # Creates a structure of available assets (from_currency > to_currency > exchange)
    def _load_assets(self):
        self.assets = {}

        for exchange in self.EXCHANGES:
            if exchange.active:
                self.assets[exchange.get_code()] = exchange.get_asset_pairs()

        # inverse the hierarchy for easier asset selection
        bases = {}
        for exchange in self.assets:
            for asset_pair in self.assets.get(exchange):
                base = asset_pair.get('base')
                quote = asset_pair.get('quote')

                if base not in bases:
                    bases[base] = {}

                if quote not in bases[base]:
                    bases[base][quote] = []

                bases[base][quote].append(self.find_exchange_by_code(exchange))

        self.bases = bases

    # load instances
    def _load_settings(self):
        self.settings = {}
        # load from file
        if isfile(SETTINGS_FILE):
            self.settings = yaml.load(open(SETTINGS_FILE, 'r'))

        for plugin in self.settings.get('plugins', {}):
            for code, active in plugin.items():
                self.find_exchange_by_code(code).active = active

        # set defaults if settings not defined
        if not self.settings.get('tickers'):
            self.settings['tickers'] = [{
                'exchange': self.EXCHANGES[0].get_code(),
                'asset_pair': self.assets[self.EXCHANGES[0].get_code()][0].get('pair'),
                'refresh': 3,
                'default_label': self.EXCHANGES[0].get_default_label()
            }]

        if not self.settings.get('recent'):
            self.settings['recent'] = []

    # saves settings for each ticker
    def save_settings(self):
        tickers = []
        for instance in self.instances:
            ticker = {
                'exchange': instance.exchange.get_code(),
                'asset_pair': instance.exchange.asset_pair.get('pair'),
                'refresh': instance.refresh_frequency,
                'default_label': instance.default_label
            }
            tickers.append(ticker)
            self.settings['tickers'] = tickers

        plugins = []
        for exchange in self.EXCHANGES:
            plugin = {exchange.get_code(): exchange.active}
            plugins.append(plugin)

        self.settings['plugins'] = plugins

        try:
            with open(SETTINGS_FILE, 'w') as handle:
                yaml.dump(self.settings, handle, default_flow_style=False)
        except IOError:
            logging.error('Settings file not writable')

    # Add a new base to the recents settings, and push the last one off the edge
    def add_new_recent(self, asset_pair, exchange_code):
        for recent in self.settings['recent']:
            if recent.get('asset_pair') == asset_pair and recent.get('exchange') == exchange_code:
                self.settings['recent'].remove(recent)

        self.settings['recent'] = self.settings['recent'][0:4]

        new_recent = {
            'asset_pair': asset_pair,
            'exchange': exchange_code
        }

        self.settings['recent'].insert(0, new_recent)

        for instance in self.instances:
            instance.rebuild_recents_menu()

    # Start the main indicator icon and its menu
    def _start_main(self):
        print(self.config.get('app').get('name') + ' v' + self.config.get('app').get('version') + " running!")

        self.icon = self.config.get('project_root') + '/resources/icon_32px.png'
        self.main_item = AppIndicator.Indicator.new(
            self.config.get('app').get('name'), self.icon, AppIndicator.IndicatorCategory.APPLICATION_STATUS)
        self.main_item.set_status(AppIndicator.IndicatorStatus.ACTIVE)
        self.main_item.set_ordering_index(0)
        self.main_item.set_menu(self._menu())

    def _start_gui(self):
        signal.signal(signal.SIGINT, Gtk.main_quit)  # ctrl+c exit
        DBusGMainLoop(set_as_default=True)
        bus = dbus.SystemBus()
        bus.add_signal_receiver(
            self.handle_resume,
            None,
            'org.freedesktop.login1.Manager',
            'org.freedesktop.login1'
        )
        Gtk.main()

    # Program main menu
    def _menu(self):
        menu = Gtk.Menu()

        self.add_item = Gtk.MenuItem("Add Ticker")
        self.discover_item = Gtk.MenuItem("Discover Assets")
        self.plugin_item = Gtk.MenuItem("Plugins" + u"\u2026")
        self.about_item = Gtk.MenuItem("About")
        self.quit_item = Gtk.MenuItem("Quit")

        self.add_item.connect("activate", self._add_ticker)
        self.discover_item.connect("activate", self._discover_assets)
        self.plugin_item.connect("activate", self._select_plugins)
        self.about_item.connect("activate", self._about)
        self.quit_item.connect("activate", self._quit_all)

        menu.append(self.add_item)
        menu.append(self.discover_item)
        menu.append(self.plugin_item)
        menu.append(self.about_item)
        menu.append(Gtk.SeparatorMenuItem())
        menu.append(self.quit_item)
        menu.show_all()

        return menu

    # Adds a ticker and starts it
    def _add_indicator(self, settings):
        exchange = settings.get('exchange')
        refresh = settings.get('refresh')
        asset_pair = settings.get('asset_pair')
        default_label = settings.get('default_label')
        self.unique_id += 1
        indicator = Indicator(self, self.unique_id, exchange, asset_pair, refresh, default_label)
        self.instances.append(indicator)
        indicator.start()
        return indicator

    # adds many tickers
    def _add_many_indicators(self, tickers):
        for ticker in tickers:
            self._add_indicator(ticker)

    # Menu item to add a ticker
    def _add_ticker(self, widget):
        i = self._add_indicator(self.settings.get('tickers')[len(self.settings.get('tickers')) - 1])
        i._settings(None)
        self.save_settings()

    # Remove ticker
    def remove_ticker(self, indicator):
        if len(self.instances) == 1:  # is it the last ticker?
            Gtk.main_quit()  # then quit entirely
        else:  # otherwise just remove this one
            indicator.exchange.stop()
            del indicator.indicator_widget
            self.instances.remove(indicator)
            self.save_settings()

    # Menu item to download any new assets from the exchanges
    def _discover_assets(self, widget):
        # Don't do anything if there are no active exchanges with discovery
        if len([ex for ex in self.EXCHANGES if ex.active and ex.discovery]) == 0:
            return

        self.main_item.set_icon(self.config.get('project_root') + '/resources/loading.png')

        for indicator in self.instances:
            if indicator.asset_selection_window:
                indicator.asset_selection_window.destroy()

        for exchange in self.EXCHANGES:
            if exchange.active and exchange.discovery:
                exchange.discover_assets(self.downloader, self.update_assets)

    # When discovery completes, reload currencies and rebuild menus of all instances
    def update_assets(self):
        self.discoveries += 1
        if self.discoveries < len([ex for ex in self.EXCHANGES if ex.active and ex.discovery]):
            return  # wait until all active exchanges with discovery finish discovery

        self.discoveries = 0
        self._load_assets()

        if notify2.init(self.config.get('app').get('name')):
            n = notify2.Notification(self.config.get('app').get('name'), "Finished discovering new assets", self.icon)
            n.set_urgency(1)
            n.timeout = 2000
            n.show()

        self.main_item.set_icon(self.icon)

    # Handle system resume by refreshing all tickers
    def handle_resume(self, sleeping, *args):
        if not sleeping:
            for instance in self.instances:
                instance.exchange.stop().start()

    # Shows an About dialog
    def _about(self, widget):
        logo_124px = GdkPixbuf.Pixbuf.new_from_file(self.config.get('project_root') + '/resources/icon_32px.png')
        about = Gtk.AboutDialog()
        about.set_program_name(self.config.get('app').get('name'))
        about.set_comments(self.config.get('app').get('description'))
        about.set_version(self.config.get('app').get('version'))
        about.set_website(self.config.get('app').get('url'))
        authors = []
        for author in self.config.get('authors'):
            authors.append(author.get('name') + ' <' + author.get('email') + '>')
        about.set_authors(authors)
        contributors = []
        for contributor in self.config.get('contributors'):
            contributors.append(contributor.get('name') + ' <' + contributor.get('email') + '>')
        about.add_credit_section('Exchange plugins', contributors)
        about.set_artists([self.config.get('artist').get('name') + ' <' + self.config.get('artist').get('email') + '>'])
        about.set_license_type(Gtk.License.MIT_X11)
        about.set_logo(logo_124px)
        about.set_keep_above(True)
        res = about.run()
        if res == -4 or -6:  # close events
            about.destroy()

    def _select_plugins(self, widget):
        PluginSelectionWindow(self)

    # Menu item to remove all tickers and quits the application
    def _quit_all(self, widget):
        Gtk.main_quit()

    def plugins_updated(self):
        self._load_assets()
        for instance in self.instances:
            instance.start()  # will stop exchange if inactive

        self.save_settings()

coin = Coin()
<commit_msg>open asset selection when adding new ticker<commit_after>#!/usr/bin/env python3
# Coin Price indicator
#
# Nil Gradisnik <nil.gradisnik@gmail.com>
# Sander Van de Moortel <sander.vandemoortel@gmail.com>
#

import signal
import yaml
import logging
import glob
import dbus
import importlib
import notify2

from os.path import abspath, dirname, isfile, basename
from indicator import Indicator
from plugin_selection import PluginSelectionWindow
from async_downloader import AsyncDownloadService
from dbus.mainloop.glib import DBusGMainLoop
from gi.repository import Gtk, GdkPixbuf

try:
    from gi.repository import AppIndicator3 as AppIndicator
except ImportError:
    from gi.repository import AppIndicator

PROJECT_ROOT = abspath(dirname(dirname(__file__)))
SETTINGS_FILE = PROJECT_ROOT + '/user.conf'

class Coin():
    config = yaml.load(open(PROJECT_ROOT + '/config.yaml', 'r'))
    config['project_root'] = PROJECT_ROOT

    def __init__(self):
        self.downloader = AsyncDownloadService()
        self.unique_id = 0

        self._load_exchanges()
        self._load_assets()
        self._load_settings()
        self._start_main()

        self.instances = []
        self.discoveries = 0
        self._add_many_indicators(self.settings.get('tickers'))
        self._start_gui()

    # Load exchange 'plug-ins' from exchanges dir
    def _load_exchanges(self):
        dirfiles = glob.glob(dirname(__file__) + "/exchanges/*.py")
        plugins = [basename(f)[:-3] for f in dirfiles if isfile(f) and not f.endswith('__init__.py')]
        plugins.sort()

        self.EXCHANGES = []
        for plugin in plugins:
            class_name = plugin.capitalize()
            class_ = getattr(importlib.import_module('exchanges.' + plugin), class_name)
            self.EXCHANGES.append(class_)

    # Find an exchange
    def find_exchange_by_code(self, code):
        for exchange in self.EXCHANGES:
            if exchange.get_code() == code.lower():
                return exchange

    # Creates a structure of available assets (from_currency > to_currency > exchange)
    def _load_assets(self):
        self.assets = {}

        for exchange in self.EXCHANGES:
            if exchange.active:
                self.assets[exchange.get_code()] = exchange.get_asset_pairs()

        # inverse the hierarchy for easier asset selection
        bases = {}
        for exchange in self.assets:
            for asset_pair in self.assets.get(exchange):
                base = asset_pair.get('base')
                quote = asset_pair.get('quote')

                if base not in bases:
                    bases[base] = {}

                if quote not in bases[base]:
                    bases[base][quote] = []

                bases[base][quote].append(self.find_exchange_by_code(exchange))

        self.bases = bases

    # load instances
    def _load_settings(self):
        self.settings = {}
        # load from file
        if isfile(SETTINGS_FILE):
            self.settings = yaml.load(open(SETTINGS_FILE, 'r'))

        for plugin in self.settings.get('plugins', {}):
            for code, active in plugin.items():
                self.find_exchange_by_code(code).active = active

        # set defaults if settings not defined
        if not self.settings.get('tickers'):
            self.settings['tickers'] = [{
                'exchange': self.EXCHANGES[0].get_code(),
                'asset_pair': self.assets[self.EXCHANGES[0].get_code()][0].get('pair'),
                'refresh': 3,
                'default_label': self.EXCHANGES[0].get_default_label()
            }]

        if not self.settings.get('recent'):
            self.settings['recent'] = []

    # saves settings for each ticker
    def save_settings(self):
        tickers = []
        for instance in self.instances:
            ticker = {
                'exchange': instance.exchange.get_code(),
                'asset_pair': instance.exchange.asset_pair.get('pair'),
                'refresh': instance.refresh_frequency,
                'default_label': instance.default_label
            }
            tickers.append(ticker)
            self.settings['tickers'] = tickers

        plugins = []
        for exchange in self.EXCHANGES:
            plugin = {exchange.get_code(): exchange.active}
            plugins.append(plugin)

        self.settings['plugins'] = plugins

        try:
            with open(SETTINGS_FILE, 'w') as handle:
                yaml.dump(self.settings, handle, default_flow_style=False)
        except IOError:
            logging.error('Settings file not writable')

    # Add a new base to the recents settings, and push the last one off the edge
    def add_new_recent(self, asset_pair, exchange_code):
        for recent in self.settings['recent']:
            if recent.get('asset_pair') == asset_pair and recent.get('exchange') == exchange_code:
                self.settings['recent'].remove(recent)

        self.settings['recent'] = self.settings['recent'][0:4]

        new_recent = {
            'asset_pair': asset_pair,
            'exchange': exchange_code
        }

        self.settings['recent'].insert(0, new_recent)

        for instance in self.instances:
            instance.rebuild_recents_menu()

    # Start the main indicator icon and its menu
    def _start_main(self):
        print(self.config.get('app').get('name') + ' v' + self.config.get('app').get('version') + " running!")

        self.icon = self.config.get('project_root') + '/resources/icon_32px.png'
        self.main_item = AppIndicator.Indicator.new(
            self.config.get('app').get('name'), self.icon, AppIndicator.IndicatorCategory.APPLICATION_STATUS)
        self.main_item.set_status(AppIndicator.IndicatorStatus.ACTIVE)
        self.main_item.set_ordering_index(0)
        self.main_item.set_menu(self._menu())

    def _start_gui(self):
        signal.signal(signal.SIGINT, Gtk.main_quit)  # ctrl+c exit
        DBusGMainLoop(set_as_default=True)
        bus = dbus.SystemBus()
        bus.add_signal_receiver(
            self.handle_resume,
            None,
            'org.freedesktop.login1.Manager',
            'org.freedesktop.login1'
        )
        Gtk.main()

    # Program main menu
    def _menu(self):
        menu = Gtk.Menu()

        self.add_item = Gtk.MenuItem("Add Ticker")
        self.discover_item = Gtk.MenuItem("Discover Assets")
        self.plugin_item = Gtk.MenuItem("Plugins" + u"\u2026")
        self.about_item = Gtk.MenuItem("About")
        self.quit_item = Gtk.MenuItem("Quit")

        self.add_item.connect("activate", self._add_ticker)
        self.discover_item.connect("activate", self._discover_assets)
        self.plugin_item.connect("activate", self._select_plugins)
        self.about_item.connect("activate", self._about)
        self.quit_item.connect("activate", self._quit_all)

        menu.append(self.add_item)
        menu.append(self.discover_item)
        menu.append(self.plugin_item)
        menu.append(self.about_item)
        menu.append(Gtk.SeparatorMenuItem())
        menu.append(self.quit_item)
        menu.show_all()

        return menu

    # Adds a ticker and starts it
    def _add_indicator(self, settings):
        exchange = settings.get('exchange')
        refresh = settings.get('refresh')
        asset_pair = settings.get('asset_pair')
        default_label = settings.get('default_label')
        self.unique_id += 1
        indicator = Indicator(self, self.unique_id, exchange, asset_pair, refresh, default_label)
        self.instances.append(indicator)
        indicator.start()
        return indicator

    # adds many tickers
    def _add_many_indicators(self, tickers):
        for ticker in tickers:
            self._add_indicator(ticker)

    # Menu item to add a ticker
    def _add_ticker(self, widget):
        i = self._add_indicator(self.settings.get('tickers')[len(self.settings.get('tickers')) - 1])
        i._settings(widget)
        self.save_settings()

    # Remove ticker
    def remove_ticker(self, indicator):
        if len(self.instances) == 1:  # is it the last ticker?
            Gtk.main_quit()  # then quit entirely
        else:  # otherwise just remove this one
            indicator.exchange.stop()
            del indicator.indicator_widget
            self.instances.remove(indicator)
            self.save_settings()

    # Menu item to download any new assets from the exchanges
    def _discover_assets(self, widget):
        # Don't do anything if there are no active exchanges with discovery
        if len([ex for ex in self.EXCHANGES if ex.active and ex.discovery]) == 0:
            return

        self.main_item.set_icon(self.config.get('project_root') + '/resources/loading.png')

        for indicator in self.instances:
            if indicator.asset_selection_window:
                indicator.asset_selection_window.destroy()

        for exchange in self.EXCHANGES:
            if exchange.active and exchange.discovery:
                exchange.discover_assets(self.downloader, self.update_assets)

    # When discovery completes, reload currencies and rebuild menus of all instances
    def update_assets(self):
        self.discoveries += 1
        if self.discoveries < len([ex for ex in self.EXCHANGES if ex.active and ex.discovery]):
            return  # wait until all active exchanges with discovery finish discovery

        self.discoveries = 0
        self._load_assets()

        if notify2.init(self.config.get('app').get('name')):
            n = notify2.Notification(self.config.get('app').get('name'), "Finished discovering new assets", self.icon)
            n.set_urgency(1)
            n.timeout = 2000
            n.show()

        self.main_item.set_icon(self.icon)

    # Handle system resume by refreshing all tickers
    def handle_resume(self, sleeping, *args):
        if not sleeping:
            for instance in self.instances:
                instance.exchange.stop().start()

    # Shows an About dialog
    def _about(self, widget):
        logo_124px = GdkPixbuf.Pixbuf.new_from_file(self.config.get('project_root') + '/resources/icon_32px.png')
        about = Gtk.AboutDialog()
        about.set_program_name(self.config.get('app').get('name'))
        about.set_comments(self.config.get('app').get('description'))
        about.set_version(self.config.get('app').get('version'))
        about.set_website(self.config.get('app').get('url'))
        authors = []
        for author in self.config.get('authors'):
            authors.append(author.get('name') + ' <' + author.get('email') + '>')
        about.set_authors(authors)
        contributors = []
        for contributor in self.config.get('contributors'):
            contributors.append(contributor.get('name') + ' <' + contributor.get('email') + '>')
        about.add_credit_section('Exchange plugins', contributors)
        about.set_artists([self.config.get('artist').get('name') + ' <' + self.config.get('artist').get('email') + '>'])
        about.set_license_type(Gtk.License.MIT_X11)
        about.set_logo(logo_124px)
        about.set_keep_above(True)
        res = about.run()
        if res == -4 or -6:  # close events
            about.destroy()

    def _select_plugins(self, widget):
        PluginSelectionWindow(self)

    # Menu item to remove all tickers and quits the application
    def _quit_all(self, widget):
        Gtk.main_quit()

    def plugins_updated(self):
        self._load_assets()
        for instance in self.instances:
            instance.start()  # will stop exchange if inactive

        self.save_settings()

coin = Coin()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""subcmd_anib.py

Provides the anib subcommand for pyani

(c) The James Hutton Institute 2017-18

Author: Leighton Pritchard
Contact: leighton.pritchard@hutton.ac.uk

Leighton Pritchard,
Information and Computing Sciences,
James Hutton Institute,
Errol Road,
Invergowrie,
Dundee,
DD6 9LH,
Scotland,
UK

The MIT License

Copyright (c) 2017-18 The James Hutton Institute

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

def subcmd_anib(args, logger):
    """Perform ANIb on all genome files in an input directory."""
    raise NotImplementedError
<commit_msg>update copyright notice<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""subcmd_anib.py

Provides the anib subcommand for pyani

(c) The James Hutton Institute 2017-2019

Author: Leighton Pritchard
Contact: leighton.pritchard@hutton.ac.uk

Leighton Pritchard,
Information and Computing Sciences,
James Hutton Institute,
Errol Road,
Invergowrie,
Dundee,
DD6 9LH,
Scotland,
UK

The MIT License

Copyright (c) 2017-2019 The James Hutton Institute

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

def subcmd_anib(args, logger):
    """Perform ANIb on all genome files in an input directory."""
    raise NotImplementedError
<|endoftext|>


<commit_before>"""
Deploy this project in dev/stage/production.

Requires commander_ which is installed on the systems that need it.

.. _commander: https://github.com/oremj/commander
"""

import os
import sys

sys.path.append(os.path.dirname(os.path.abspath(__file__)))

from commander.deploy import task, hostgroups
import commander_settings as settings

th_service_src = os.path.join(settings.SRC_DIR, 'treeherder-service')
th_ui_src = os.path.join(settings.SRC_DIR, 'treeherder-ui')

th_settings = 'treeherder.settings'

sys.path.append(th_service_src)

@task
def pre_update(ctx, ref=settings.UPDATE_REF):
    """Update the code to a specific git reference (tag/sha/etc)."""
    with ctx.lcd(th_service_src):
        ctx.local('git checkout %s' % ref)
        ctx.local('git pull -f')
        ctx.local('git submodule sync')
        ctx.local('git submodule update --init --recursive')
        ctx.local("find . -type f -name '*.pyc' -delete")

    with ctx.lcd(th_ui_src):
        ctx.local('git checkout %s' % ref)
        ctx.local('git pull -f')
        ctx.local('git submodule sync')
        ctx.local('git submodule update --init --recursive')
        ctx.local("find . -type f -name '*.pyc' -delete")

@task
def update(ctx):
    with ctx.lcd(th_service_src):
        # Collect the static files (eg for the Persona or Django admin UI)
        ctx.local("python2.6 manage.py collectstatic --noinput --settings {0}".format(th_settings))

        # Rebuild the Cython code (eg the log parser)
        ctx.local("python2.6 setup.py build_ext --inplace")

        # Update the database schema, if necessary.
        ctx.local('python2.6 manage.py syncdb --settings {0}'.format(th_settings))
        ctx.local('python2.6 manage.py migrate --settings {0}'.format(th_settings))

        # Update oauth credentials.
        ctx.local(
            "python2.6 manage.py export_project_credentials --settings {0}".format(th_settings))

@task
def deploy(ctx):
    # Use the local, IT-written deploy script to check in changes.
    ctx.local(settings.DEPLOY_SCRIPT)

    # Restart celerybeat on the admin node.
    ctx.local(
        '{0}/service celerybeat restart'.format(settings.SBIN_DIR))

    @hostgroups(
        settings.WEB_HOSTGROUP, remote_kwargs={'ssh_key': settings.SSH_KEY})
    def deploy_web_app(ctx):
        # Call the remote update script to push changes to webheads.
        ctx.remote(settings.REMOTE_UPDATE_SCRIPT)
        ctx.remote('{0}/service httpd graceful'.format(settings.SBIN_DIR))
        ctx.remote('{0}/service gunicorn restart'.format(settings.SBIN_DIR))
        ctx.remote('{0}/service socketio-server restart'.format(settings.SBIN_DIR))

    deploy_web_app()

    @hostgroups(
        settings.CELERY_HOSTGROUP, remote_kwargs={'ssh_key': settings.SSH_KEY})
    def deploy_workers(ctx):
        # Call the remote update script to push changes to workers.
        ctx.remote(settings.REMOTE_UPDATE_SCRIPT)

    deploy_workers()

    with ctx.lcd(th_service_src):
        # Send a warm shutdown event to all the celery workers in the cluster.
        # The workers will finish their current tasks and safely shutdown.
        # Supervisord will then start new workers to replace them.
        # We need to do this because supervisorctl generates zombies
        # every time you ask it to restart a worker.
        ctx.local("python2.6 manage.py shutdown_workers --settings {0}".format(th_settings))

        # Write info about the current repository state to a publicly visible file.
        ctx.local('date')
        ctx.local('git branch')
        ctx.local('git log -3')
        ctx.local('git status')
        ctx.local('git submodule status')
        ctx.local('git rev-parse HEAD > treeherder/webapp/media/revision')
<commit_msg>Bug 1066264 - Refactor update.py: Remove unnecessary wrapping<commit_after>"""
Deploy this project in dev/stage/production.

Requires commander_ which is installed on the systems that need it.

.. _commander: https://github.com/oremj/commander
"""

import os
import sys

sys.path.append(os.path.dirname(os.path.abspath(__file__)))

from commander.deploy import task, hostgroups
import commander_settings as settings

th_service_src = os.path.join(settings.SRC_DIR, 'treeherder-service')
th_ui_src = os.path.join(settings.SRC_DIR, 'treeherder-ui')

th_settings = 'treeherder.settings'

sys.path.append(th_service_src)

@task
def pre_update(ctx, ref=settings.UPDATE_REF):
    """Update the code to a specific git reference (tag/sha/etc)."""
    with ctx.lcd(th_service_src):
        ctx.local('git checkout %s' % ref)
        ctx.local('git pull -f')
        ctx.local('git submodule sync')
        ctx.local('git submodule update --init --recursive')
        ctx.local("find . -type f -name '*.pyc' -delete")

    with ctx.lcd(th_ui_src):
        ctx.local('git checkout %s' % ref)
        ctx.local('git pull -f')
        ctx.local('git submodule sync')
        ctx.local('git submodule update --init --recursive')
        ctx.local("find . -type f -name '*.pyc' -delete")

@task
def update(ctx):
    with ctx.lcd(th_service_src):
        # Collect the static files (eg for the Persona or Django admin UI)
        ctx.local("python2.6 manage.py collectstatic --noinput --settings {0}".format(th_settings))

        # Rebuild the Cython code (eg the log parser)
        ctx.local("python2.6 setup.py build_ext --inplace")

        # Update the database schema, if necessary.
        ctx.local('python2.6 manage.py syncdb --settings {0}'.format(th_settings))
        ctx.local('python2.6 manage.py migrate --settings {0}'.format(th_settings))

        # Update oauth credentials.
        ctx.local("python2.6 manage.py export_project_credentials --settings {0}".format(th_settings))

@task
def deploy(ctx):
    # Use the local, IT-written deploy script to check in changes.
    ctx.local(settings.DEPLOY_SCRIPT)

    # Restart celerybeat on the admin node.
    ctx.local('{0}/service celerybeat restart'.format(settings.SBIN_DIR))

    @hostgroups(settings.WEB_HOSTGROUP, remote_kwargs={'ssh_key': settings.SSH_KEY})
    def deploy_web_app(ctx):
        # Call the remote update script to push changes to webheads.
        ctx.remote(settings.REMOTE_UPDATE_SCRIPT)
        ctx.remote('{0}/service httpd graceful'.format(settings.SBIN_DIR))
        ctx.remote('{0}/service gunicorn restart'.format(settings.SBIN_DIR))
        ctx.remote('{0}/service socketio-server restart'.format(settings.SBIN_DIR))

    deploy_web_app()

    @hostgroups(settings.CELERY_HOSTGROUP, remote_kwargs={'ssh_key': settings.SSH_KEY})
    def deploy_workers(ctx):
        # Call the remote update script to push changes to workers.
        ctx.remote(settings.REMOTE_UPDATE_SCRIPT)

    deploy_workers()

    with ctx.lcd(th_service_src):
        # Send a warm shutdown event to all the celery workers in the cluster.
        # The workers will finish their current tasks and safely shutdown.
        # Supervisord will then start new workers to replace them.
        # We need to do this because supervisorctl generates zombies
        # every time you ask it to restart a worker.
        ctx.local("python2.6 manage.py shutdown_workers --settings {0}".format(th_settings))

        # Write info about the current repository state to a publicly visible file.
        ctx.local('date')
        ctx.local('git branch')
        ctx.local('git log -3')
        ctx.local('git status')
        ctx.local('git submodule status')
        ctx.local('git rev-parse HEAD > treeherder/webapp/media/revision')
<|endoftext|>


<commit_before># vim:ts=4:sts=4:sw=4:expandtab

from django.db import models

from satori.ars.server import server_info
from satori.core.dbev  import Events

from satori.core.models import Entity

ResultToRender = Struct('ResultToRender', (
    ('submit', DjangoStruct('Submit'), True),
    ('problem', unicode, True),
    ('contestant', unicode, True),
    ('status', unicode, True),
    ('details', unicode, True),
))
ResultsToRender = Struct('ResultsToRender', (
    ('count', int, True),
    ('results', TypedList(ResultToRender), True),
))
ContestantToRender = Struct('ContestantToRender', (
    ('contestant', DjangoStruct('Contestant'), True),
    ('name', unicode, True),
    ('members', TypedList(DjangoStruct('User')), True),
    ('admin', bool, True),
))
ContestantsToRender = Struct('ContestantsToRender', (
    ('count', int, True),
    ('contestants', TypedList(ContestantToRender), True),
))
AlreadyRegistered = DefineException('AlreadyRegisteres', 'The specified user \'{login}\' is already registered',
    [('login', unicode, False)])

@ExportModel
class Contest(Entity):
    """Model. Description of a contest.

    rights:
        VIEW_INTRA_FILES
        VIEW_TASKS
        APPLY
        JOIN
        OBSERVE
    """
    parent_entity = models.OneToOneField(Entity, parent_link=True, related_name='cast_contest')

    name            = models.CharField(max_length=50, unique=True)
    problems        = models.ManyToManyField('Problem', through='ProblemMapping', related_name='contests')
    contestant_role = models.ForeignKey('Role', related_name='contest_contestants+')
    admin_role      = models.ForeignKey('Role', related_name='contest_admins+')
    archived        = models.BooleanField(default=False)

    lock_start   = models.DateTimeField(null=True)
    lock_finish  = models.DateTimeField(null=True)
    lock_address = models.IPAddressField(default='0.0.0.0')
    lock_netmask = models.IPAddressField(default='255.255.255.255')

    public_files = AttributeGroupField(PCArg('self', 'VIEW'), PCArg('self', 'MANAGE'), '')
    intra_files  = AttributeGroupField(PCArg('self', 'VIEW_INTRA_FILES'), PCArg('self', 'MANAGE'), '')

    class ExportMeta(object):
        fields = [('name', 'VIEW'), ('contestant_role', 'MANAGE'), ('admin_role', 'MANAGE'), ('lock_start', 'MANAGE'), ('lock_finish', 'MANAGE'), ('lock_address', 'MANAGE'), ('lock_netmask', 'MANAGE')]

    @classmethod
    def inherit_rights(cls):
        inherits = super(Contest, cls).inherit_rights()
        cls._inherit_add(inherits, 'APPLY', 'id', 'JOIN')
        cls._inherit_add(inherits, 'JOIN', 'id', 'MANAGE')
        cls._inherit_add(inherits, 'SUBMIT', 'id', 'MANAGE')
        cls._inherit_add(inherits, 'OBSERVE', 'id', 'MANAGE')
        cls._inherit_add(inherits, 'VIEW_TASKS', 'id', 'MANAGE')
        cls._inherit_add(inherits, 'ASK_QUESTIONS', 'id', 'MANAGE')
        cls._inherit_add(inherits, 'VIEW_INTRA_FILES', 'id', 'MANAGE')
        return inherits

    def save(self, *args, **kwargs):
        self.fixup_public_files()
        self.fixup_intra_files()
        super(Contest, self).save(*args, **kwargs)

    def __str__(self):
        return self.name
    # TODO: add presentation options

    @ExportMethod(DjangoStruct('Contest'), [], PCPermit())
    @staticmethod
    def get_current_lock():
        contests = Contest.objects.filter(lock_start__lte=datetime.now(), lock_finish__gte=datetime.now())
        contests = [contest for contest in contests
            if (ipaddr.IPv4Address(server_info.client_ip) in ipaddr.IPv4Network(contest.lock_address + '/' + contest.lock_netmask))
        ]

        if len(contests) == 0:
            return None
        if len(contests) > 1:
            raise Contest.MultipleObjectsReturned()
        else:
            return contests[0]

    @ExportMethod(DjangoStruct('Contest'), [DjangoStruct('Contest')], PCGlobal('MANAGE_CONTESTS'), [CannotSetField])
    @staticmethod
    def create(fields):
        contest = Contest()
        contest.forbid_fields(fields, ['id', 'contestant_role', 'admin_role'])
        contest.update_fields(fields, ['name', 'archived', 'lock_start', 'lock_finish', 'lock_address', 'lock_netmask'])
        contestant_role = Role(name='Contestant of ' + contest.name)
        contestant_role.save()
        admin_role = Role(name='Administrator of ' + contest.name)
        admin_role.save()
        contest.contestant_role = contestant_role
        contest.admin_role = admin_role
        contest.save()
        contest.admin_role.add_member(token_container.token.role)
        Privilege.grant(contest.admin_role, contest, 'MANAGE')
        Privilege.grant(contest.admin_role, contest.admin_role, 'MANAGE')
        Privilege.grant(contest.admin_role, contest.contestant_role, 'MANAGE')
        Privilege.grant(contest.contestant_role, contest, 'VIEW')
        return contest

    @ExportMethod(DjangoStruct('Contest'), [DjangoId('Contest'), DjangoStruct('Contest')], PCArg('self', 'MANAGE'), [CannotSetField])
    def modify(self, fields):
        self.forbid_fields(fields, ['id', 'contestant_role', 'admin_role'])
        modified = self.update_fields(fields, ['name', 'archived', 'lock_start', 'lock_finish', 'lock_address', 'lock_netmask'])
        self.save()
        if 'name' in modified:
            role = self.contestant_role
            role.name='Contestant of ' + self.name
            role.save()
            role = self.admin_role
            role.name='Administrator of ' + self.name
            role.save()
        return self

    @ExportMethod(NoneType, [DjangoId('Contest')], PCArg('self', 'MANAGE'))
    def disable_lock(self):
        self.lock_start = None
        self.lock_finish = None
        self.lock_address = '0.0.0.0'
        self.lock_netmask = '255.255.255.255'
        self.save()

    @ExportMethod(DjangoStruct('Contestant'), [DjangoId('Contest'), DjangoId('Role')], PCOr(PCTokenUser('user'), PCArg('self', 'MANAGE')))
    def find_contestant(self, user):
        try:
            return Contestant.objects.get(contest=self, children__id=user.id)
        except Contestant.DoesNotExist:
            try:
                return Contestant.objects.get(contest=self, id=user.id)
            except Contestant.DoesNotExist:
               return None

    @ExportMethod(DjangoStruct('Contestant'), [DjangoId('Contest')], PCAnd(PCTokenIsUser(), PCArg('self', 'APPLY')), [AlreadyRegistered])
    def join(self):
        return Contestant.create(fields=DjangoStruct('Contestant')(contest=self, accepted=bool(Privilege.demand(self, 'JOIN')), name=token_container.token.user.login), user_list=[token_container.token.user])

    @ExportMethod(NoneType, [DjangoId('Contest'), DjangoId('User')], PCArg('self', 'MANAGE'))
    def add_admin(self, user):
        contestant = self.find_contestant(user)
        if contestant is None:
            Contestant.create(fields=DjangoStruct('Contestant')(contest=self, accepted=True, invisible=True, name=user.login), user_list=[user])
        else:
            contestant.invisible = True
            contestant.save()
#TODO: REJUDGE!
        self.admin_role.add_member(user)

    @staticmethod
    def submit_to_result_to_render(submit):
        return ResultToRender(
        	submit=submit,
            problem=submit.problem.code,
            contestant=submit.contestant.usernames,
            status=submit.get_test_suite_status(),
            details=submit.get_test_suite_report()
            )

    @ExportMethod(ResultsToRender, [DjangoId('Contest'), DjangoId('ProblemMapping'), int, int], PCPermit())
    def get_all_results(self, problem=None, limit=20, offset=0):
        res = []
        q = Privilege.where_can(Submit.objects.filter(contestant__contest=self), 'OBSERVE')
        if problem:
        	q = q.filter(problem=problem)
        for submit in q.order_by('-id')[offset:offset+limit]:
        	res.append(Contest.submit_to_result_to_render(submit))
        return ResultsToRender(
            count=len(q),
            results=res
            )

    @ExportMethod(ResultsToRender, [DjangoId('Contest'), DjangoId('Contestant'), DjangoId('ProblemMapping'), int, int], PCPermit())
    def get_results(self, contestant, problem=None, limit=20, offset=0):
        res = []
        q = Privilege.where_can(Submit.objects.filter(contestant=contestant), 'OBSERVE')
        if problem:
        	q = q.filter(problem=problem)
        for submit in q.order_by('-id')[offset:offset+limit]:
        	res.append(Contest.submit_to_result_to_render(submit))
        return ResultsToRender(
            count=len(q),
            results=res
            )

    @staticmethod
    def contestant_to_contestant_to_render(contestant):
        return ContestantToRender(
        	contestant=contestant,
            name=contestant.usernames,
            members=contestant.get_member_users(),
            admin=any([Privilege.get(member, contestant.contest, 'MANAGE') for member in contestant.get_member_users()]),
            )

    @ExportMethod(ContestantsToRender, [DjangoId('Contest'), int, int], PCArg('self', 'VIEW'))
    def get_contestants(self, limit=20, offset=0):
        res = []
        q = Contestant.objects.filter(contest=self, accepted=True)
        for contestant in q.order_by('name')[offset:offset+limit]:
        	res.append(Contest.contestant_to_contestant_to_render(contestant))
        return ContestantsToRender(
            count=len(q),
            contestants=res
            )

    @ExportMethod(ContestantsToRender, [DjangoId('Contest'), int, int], PCArg('self', 'MANAGE'))
    def get_pending_contestants(self, limit=20, offset=0):
        res = []
        q = Contestant.objects.filter(contest=self, accepted=False)
        for contestant in q.order_by('name')[offset:offset+limit]:
        	res.append(Contest.contestant_to_contestant_to_render(contestant))
        return ContestantsToRender(
            count=len(q),
            contestants=res
            )

class ContestEvents(Events):
    model = Contest
    on_insert = on_update = ['name']
    on_delete = []
<commit_msg>Contestant role has VIEW_INTRA_FILES permission on contest.<commit_after># vim:ts=4:sts=4:sw=4:expandtab

from django.db import models

from satori.ars.server import server_info
from satori.core.dbev  import Events

from satori.core.models import Entity

ResultToRender = Struct('ResultToRender', (
    ('submit', DjangoStruct('Submit'), True),
    ('problem', unicode, True),
    ('contestant', unicode, True),
    ('status', unicode, True),
    ('details', unicode, True),
))
ResultsToRender = Struct('ResultsToRender', (
    ('count', int, True),
    ('results', TypedList(ResultToRender), True),
))
ContestantToRender = Struct('ContestantToRender', (
    ('contestant', DjangoStruct('Contestant'), True),
    ('name', unicode, True),
    ('members', TypedList(DjangoStruct('User')), True),
    ('admin', bool, True),
))
ContestantsToRender = Struct('ContestantsToRender', (
    ('count', int, True),
    ('contestants', TypedList(ContestantToRender), True),
))
AlreadyRegistered = DefineException('AlreadyRegisteres', 'The specified user \'{login}\' is already registered',
    [('login', unicode, False)])

@ExportModel
class Contest(Entity):
    """Model. Description of a contest.

    rights:
        VIEW_INTRA_FILES
        VIEW_TASKS
        APPLY
        JOIN
        OBSERVE
    """
    parent_entity = models.OneToOneField(Entity, parent_link=True, related_name='cast_contest')

    name            = models.CharField(max_length=50, unique=True)
    problems        = models.ManyToManyField('Problem', through='ProblemMapping', related_name='contests')
    contestant_role = models.ForeignKey('Role', related_name='contest_contestants+')
    admin_role      = models.ForeignKey('Role', related_name='contest_admins+')
    archived        = models.BooleanField(default=False)

    lock_start   = models.DateTimeField(null=True)
    lock_finish  = models.DateTimeField(null=True)
    lock_address = models.IPAddressField(default='0.0.0.0')
    lock_netmask = models.IPAddressField(default='255.255.255.255')

    public_files = AttributeGroupField(PCArg('self', 'VIEW'), PCArg('self', 'MANAGE'), '')
    intra_files  = AttributeGroupField(PCArg('self', 'VIEW_INTRA_FILES'), PCArg('self', 'MANAGE'), '')

    class ExportMeta(object):
        fields = [('name', 'VIEW'), ('contestant_role', 'MANAGE'), ('admin_role', 'MANAGE'), ('lock_start', 'MANAGE'), ('lock_finish', 'MANAGE'), ('lock_address', 'MANAGE'), ('lock_netmask', 'MANAGE')]

    @classmethod
    def inherit_rights(cls):
        inherits = super(Contest, cls).inherit_rights()
        cls._inherit_add(inherits, 'APPLY', 'id', 'JOIN')
        cls._inherit_add(inherits, 'JOIN', 'id', 'MANAGE')
        cls._inherit_add(inherits, 'SUBMIT', 'id', 'MANAGE')
        cls._inherit_add(inherits, 'OBSERVE', 'id', 'MANAGE')
        cls._inherit_add(inherits, 'VIEW_TASKS', 'id', 'MANAGE')
        cls._inherit_add(inherits, 'ASK_QUESTIONS', 'id', 'MANAGE')
        cls._inherit_add(inherits, 'VIEW_INTRA_FILES', 'id', 'MANAGE')
        return inherits

    def save(self, *args, **kwargs):
        self.fixup_public_files()
        self.fixup_intra_files()
        super(Contest, self).save(*args, **kwargs)

    def __str__(self):
        return self.name
    # TODO: add presentation options

    @ExportMethod(DjangoStruct('Contest'), [], PCPermit())
    @staticmethod
    def get_current_lock():
        contests = Contest.objects.filter(lock_start__lte=datetime.now(), lock_finish__gte=datetime.now())
        contests = [contest for contest in contests
            if (ipaddr.IPv4Address(server_info.client_ip) in ipaddr.IPv4Network(contest.lock_address + '/' + contest.lock_netmask))
        ]

        if len(contests) == 0:
            return None
        if len(contests) > 1:
            raise Contest.MultipleObjectsReturned()
        else:
            return contests[0]

    @ExportMethod(DjangoStruct('Contest'), [DjangoStruct('Contest')], PCGlobal('MANAGE_CONTESTS'), [CannotSetField])
    @staticmethod
    def create(fields):
        contest = Contest()
        contest.forbid_fields(fields, ['id', 'contestant_role', 'admin_role'])
        contest.update_fields(fields, ['name', 'archived', 'lock_start', 'lock_finish', 'lock_address', 'lock_netmask'])
        contestant_role = Role(name='Contestant of ' + contest.name)
        contestant_role.save()
        admin_role = Role(name='Administrator of ' + contest.name)
        admin_role.save()
        contest.contestant_role = contestant_role
        contest.admin_role = admin_role
        contest.save()
        contest.admin_role.add_member(token_container.token.role)
        Privilege.grant(contest.admin_role, contest, 'MANAGE')
        Privilege.grant(contest.admin_role, contest.admin_role, 'MANAGE')
        Privilege.grant(contest.admin_role, contest.contestant_role, 'MANAGE')
        Privilege.grant(contest.contestant_role, contest, 'VIEW')
        Privilege.grant(contest.contestant_role, contest, 'VIEW_INTRA_FILES')
        return contest

    @ExportMethod(DjangoStruct('Contest'), [DjangoId('Contest'), DjangoStruct('Contest')], PCArg('self', 'MANAGE'), [CannotSetField])
    def modify(self, fields):
        self.forbid_fields(fields, ['id', 'contestant_role', 'admin_role'])
        modified = self.update_fields(fields, ['name', 'archived', 'lock_start', 'lock_finish', 'lock_address', 'lock_netmask'])
        self.save()
        if 'name' in modified:
            role = self.contestant_role
            role.name='Contestant of ' + self.name
            role.save()
            role = self.admin_role
            role.name='Administrator of ' + self.name
            role.save()
        return self

    @ExportMethod(NoneType, [DjangoId('Contest')], PCArg('self', 'MANAGE'))
    def disable_lock(self):
        self.lock_start = None
        self.lock_finish = None
        self.lock_address = '0.0.0.0'
        self.lock_netmask = '255.255.255.255'
        self.save()

    @ExportMethod(DjangoStruct('Contestant'), [DjangoId('Contest'), DjangoId('Role')], PCOr(PCTokenUser('user'), PCArg('self', 'MANAGE')))
    def find_contestant(self, user):
        try:
            return Contestant.objects.get(contest=self, children__id=user.id)
        except Contestant.DoesNotExist:
            try:
                return Contestant.objects.get(contest=self, id=user.id)
            except Contestant.DoesNotExist:
               return None

    @ExportMethod(DjangoStruct('Contestant'), [DjangoId('Contest')], PCAnd(PCTokenIsUser(), PCArg('self', 'APPLY')), [AlreadyRegistered])
    def join(self):
        return Contestant.create(fields=DjangoStruct('Contestant')(contest=self, accepted=bool(Privilege.demand(self, 'JOIN')), name=token_container.token.user.login), user_list=[token_container.token.user])

    @ExportMethod(NoneType, [DjangoId('Contest'), DjangoId('User')], PCArg('self', 'MANAGE'))
    def add_admin(self, user):
        contestant = self.find_contestant(user)
        if contestant is None:
            Contestant.create(fields=DjangoStruct('Contestant')(contest=self, accepted=True, invisible=True, name=user.login), user_list=[user])
        else:
            contestant.invisible = True
            contestant.save()
#TODO: REJUDGE!
        self.admin_role.add_member(user)

    @staticmethod
    def submit_to_result_to_render(submit):
        return ResultToRender(
        	submit=submit,
            problem=submit.problem.code,
            contestant=submit.contestant.usernames,
            status=submit.get_test_suite_status(),
            details=submit.get_test_suite_report()
            )

    @ExportMethod(ResultsToRender, [DjangoId('Contest'), DjangoId('ProblemMapping'), int, int], PCPermit())
    def get_all_results(self, problem=None, limit=20, offset=0):
        res = []
        q = Privilege.where_can(Submit.objects.filter(contestant__contest=self), 'OBSERVE')
        if problem:
        	q = q.filter(problem=problem)
        for submit in q.order_by('-id')[offset:offset+limit]:
        	res.append(Contest.submit_to_result_to_render(submit))
        return ResultsToRender(
            count=len(q),
            results=res
            )

    @ExportMethod(ResultsToRender, [DjangoId('Contest'), DjangoId('Contestant'), DjangoId('ProblemMapping'), int, int], PCPermit())
    def get_results(self, contestant, problem=None, limit=20, offset=0):
        res = []
        q = Privilege.where_can(Submit.objects.filter(contestant=contestant), 'OBSERVE')
        if problem:
        	q = q.filter(problem=problem)
        for submit in q.order_by('-id')[offset:offset+limit]:
        	res.append(Contest.submit_to_result_to_render(submit))
        return ResultsToRender(
            count=len(q),
            results=res
            )

    @staticmethod
    def contestant_to_contestant_to_render(contestant):
        return ContestantToRender(
        	contestant=contestant,
            name=contestant.usernames,
            members=contestant.get_member_users(),
            admin=any([Privilege.get(member, contestant.contest, 'MANAGE') for member in contestant.get_member_users()]),
            )

    @ExportMethod(ContestantsToRender, [DjangoId('Contest'), int, int], PCArg('self', 'VIEW'))
    def get_contestants(self, limit=20, offset=0):
        res = []
        q = Contestant.objects.filter(contest=self, accepted=True)
        for contestant in q.order_by('name')[offset:offset+limit]:
        	res.append(Contest.contestant_to_contestant_to_render(contestant))
        return ContestantsToRender(
            count=len(q),
            contestants=res
            )

    @ExportMethod(ContestantsToRender, [DjangoId('Contest'), int, int], PCArg('self', 'MANAGE'))
    def get_pending_contestants(self, limit=20, offset=0):
        res = []
        q = Contestant.objects.filter(contest=self, accepted=False)
        for contestant in q.order_by('name')[offset:offset+limit]:
        	res.append(Contest.contestant_to_contestant_to_render(contestant))
        return ContestantsToRender(
            count=len(q),
            contestants=res
            )

class ContestEvents(Events):
    model = Contest
    on_insert = on_update = ['name']
    on_delete = []
<|endoftext|>


<commit_before><commit_msg>[ADD] minor change to specify that partition is a dataset partition, perhaps that's not obvious<commit_after><|endoftext|>


<commit_before>import os

class Tree(object):
    def __init__(self, name='root', archives=None, children=None):
        self.name = name
        self.archives = []
        self.children = []
        if archives is not None:
            for archive in archives:
                self.archives.append(archive)
        if children is not None:
            for child in children:
                self.add_child(child)
    def __repr__(self):
        return self.name
    def add_child(self, node):
        assert isinstance(node, Tree)
        self.children.append(node)
    def add_archive(self, archive):
        self.archives.append(archive)
    def dump(self):
        print self.name
        print self.archives
        if self.children is not None:
            for child in self.children:
                child.dump()

def create (path): #funcao que cria o arquivo .sar
    dir = open(path+'.sar', 'w')

    directory = Tree(path)
    isRoot = True
    for dirname, dirnames, filenames in os.walk('.', topdown = True): # dirname: pasta atual, dirnames: subpastas, filenames: nomes de arquivo
        if '.git' in dirnames:
            # isso aqui eh soh pra ele nao entrar na pasta .git e imprimir um monte de coisa chata
            dirnames.remove('.git')

        for subdirname in dirnames:
            subdir = Tree(subdirname)
            if isRoot:
                directory.add_child(subdir)
            else:
                cont = 0
                index = 0
                for pasta in directory.children: #navego por todos os filhos
                    if (dirname != pasta.name): #se nao for o que eu to procurando, continuo procurando
                        cont = cont + 1
                    else:
                        index = cont # se for, guardo o indice
                directory.children[index].add_child(subdir) #insiro no indice certo

            #print("subdirname "+ subdirname)

        for filename in filenames:
            if isRoot: #quer dizer que ta na pasta raiz
                directory.add_archive(filename)

            else: #senao ta em subpasta
                cont = 0
                index = 0
                for pasta in directory.children: #navego por todos os filhos
                    if (dirname != pasta.name): #se nao for o que eu to procurando, continuo procurando
                        cont = cont + 1
                    else:
                        index = cont # se for, guardo o indice
                directory.children[index].add_archive(filename) #insiro no indice certo
            #print("filename "+ filename)
            #directory[dirname].append(filename)
        isRoot = False
    directory.dump()

def list (archive): #funcao que lista os diretorios do arquivo .sar
	pass

def extract (archive): #funcao que extrai os arquivos do arquivo .sar
	pass

if __name__ == '__main__': #define a funcao main
    create('abc')
<commit_msg>Mostra quem é o pai na hora de imprimir<commit_after>import os

class Tree(object):
    def __init__(self, name='root', archives=None, children=None):
        self.name = name
        self.archives = []
        self.children = []
        if archives is not None:
            for archive in archives:
                self.archives.append(archive)
        if children is not None:
            for child in children:
                self.add_child(child)
    def __repr__(self):
        return self.name
    def add_child(self, node):
        assert isinstance(node, Tree)
        self.children.append(node)
    def add_archive(self, archive):
        self.archives.append(archive)
    def dump(self):
        print self.name
        print self.archives
        if self.children is not None:
            for child in self.children:
                print ('\npai: '+self.name)
                child.dump()

def create (path): #funcao que cria o arquivo .sar
    dir = open(path+'.sar', 'w')

    directory = Tree(path)
    isRoot = True
    for dirname, dirnames, filenames in os.walk('.', topdown = True): # dirname: pasta atual, dirnames: subpastas, filenames: nomes de arquivo
        if '.git' in dirnames:
            # isso aqui eh soh pra ele nao entrar na pasta .git e imprimir um monte de coisa chata
            dirnames.remove('.git')

        for subdirname in dirnames:
            subdir = Tree(subdirname)
            if isRoot:
                directory.add_child(subdir)
            else:
                cont = 0
                index = 0
                for pasta in directory.children: #navego por todos os filhos
                    if (dirname != pasta.name): #se nao for o que eu to procurando, continuo procurando
                        cont = cont + 1
                    else:
                        index = cont # se for, guardo o indice
                directory.children[index].add_child(subdir) #insiro no indice certo

            #print("subdirname "+ subdirname)

        for filename in filenames:
            if isRoot: #quer dizer que ta na pasta raiz
                directory.add_archive(filename)

            else: #senao ta em subpasta
                cont = 0
                index = 0
                for pasta in directory.children: #navego por todos os filhos
                    if (dirname != pasta.name): #se nao for o que eu to procurando, continuo procurando
                        cont = cont + 1
                    else:
                        index = cont # se for, guardo o indice
                directory.children[index].add_archive(filename) #insiro no indice certo
            #print("filename "+ filename)
            #directory[dirname].append(filename)
        isRoot = False
    directory.dump()

def list (archive): #funcao que lista os diretorios do arquivo .sar
	pass

def extract (archive): #funcao que extrai os arquivos do arquivo .sar
	pass

if __name__ == '__main__': #define a funcao main
    create('abc')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Coin Price indicator
# Nil Gradisnik <nil.gradisnik@gmail.com>

import os, signal, yaml, sys, logging, gi
gi.require_version('Gtk', '3.0')
gi.require_version('AppIndicator3', '0.1')

from gi.repository import Gtk, GdkPixbuf, GObject, GLib

try:
    from gi.repository import AppIndicator3 as AppIndicator
except ImportError:
    from gi.repository import AppIndicator

from indicator import Indicator
from exchange.kraken import Kraken
from exchange.bitstamp import Bitstamp
from exchange.bityep import BitYep
from exchange.gdax import Gdax

import threading

PROJECT_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))

signal.signal(signal.SIGINT, signal.SIG_DFL)  # ctrl+c exit

class Coin(object):
    config = yaml.load(open(PROJECT_ROOT + '/config.yaml', 'r'))
    config['project_root'] = PROJECT_ROOT

    def __init__(self):
        self.gui_ready = threading.Event()
        self.start_main()
        self.instances = []
        logging.info("Coin Price indicator v" + self.config['app']['version'])
        usage_error = 'Usage: coin.py [flags]\nasset\texchange:asset_pair:refresh_rate\nfile\tLoads various asset pairs from YAML file in ./coin directory'
        if len(sys.argv) > 2:
            quit('Too many parameters\n' + usage_error)

        if len(sys.argv) == 2:
            if '=' in sys.argv[1]:
                args = sys.argv[1].split('=')
                if args[0] == 'file':
                    try:
                        cp_instances = yaml.load(open(PROJECT_ROOT + '/coin/' + args[1], 'r'))
                    except:
                        quit('Error opening assets file')

                    self.add_many_indicators(cp_instances)

                elif args[0] == 'asset':
                    self.add_indicator(args[1])

                else:
                    quit('Invalid parameter\n' + usage_error)

            else:
                quit('Invalid parameter\n' + usage_error)

        else:
            self.add_indicator()

    # Start the main indicator icon and its menu
    def start_main(self):
        icon = self.config['project_root'] + '/resources/icon_32px.png'
        self.logo_124px = GdkPixbuf.Pixbuf.new_from_file(self.config['project_root'] + '/resources/icon_32px.png')
        self.main_item = AppIndicator.Indicator.new(self.config['app']['name'], icon, AppIndicator.IndicatorCategory.APPLICATION_STATUS)
        self.main_item.set_status(AppIndicator.IndicatorStatus.ACTIVE)
        self.main_item.set_menu(self._menu())

        def start_gui_thread():
            self.gui_ready.wait()
            Gtk.main()

        self.gui_thread = threading.Thread(target=start_gui_thread)
        self.gui_thread.start()

    # Program main menu
    def _menu(self):
        menu = Gtk.Menu()

        self.add_item = Gtk.MenuItem("Add Ticker")
        self.about_item = Gtk.MenuItem("About")
        self.quit_item = Gtk.MenuItem("Quit")

        self.add_item.connect("activate", self._add_ticker)
        self.about_item.connect("activate", self._about)
        self.quit_item.connect("activate", self._quit_all)

        menu.append(self.add_item)
        menu.append(self.about_item)
        menu.append(Gtk.SeparatorMenuItem())
        menu.append(self.quit_item)
        menu.show_all()

        return menu

    # Adds a ticker and starts it
    def add_indicator(self, settings=None):
        indicator = Indicator(self, len(self.instances), self.config, settings)
        self.instances.append(indicator)
        indicator.start()
        self.gui_ready.set()

    # adds many tickers
    def add_many_indicators(self, cp_instances):
        for cp_instance in cp_instances:
            settings = cp_instance['exchange'] + ':' + cp_instance['asset_pair'] + ':' + str(cp_instance['refresh'])
            nt = threading.Thread(target=self.add_indicator(settings))

    # Menu item to add a ticker
    def _add_ticker(self, widget):
        self.add_indicator()

    # Shows an About dialog
    def _about(self, widget):
        about = Gtk.AboutDialog()
        about.set_program_name(self.config['app']['name'])
        about.set_comments(self.config['app']['description'])
        about.set_copyright(self.config['author']['copyright'])
        about.set_version(self.config['app']['version'])
        about.set_website(self.config['app']['url'])
        about.set_authors([self.config['author']['name'] + ' <' + self.config['author']['email'] + '>'])
        about.set_artists([self.config['artist']['name'] + ' <' + self.config['artist']['email'] + '>'])
        about.set_license_type(Gtk.License.MIT_X11)
        about.set_logo(self.logo_124px)
        res = about.run()
        if res == -4 or -6:  # close events
            about.destroy()

    # Menu item to remove all tickers and quits the application
    def _quit_all(self, widget):
        logging.info("Exiting")
        Gtk.main_quit()

coin = Coin()<commit_msg>conflict resolving<commit_after># -*- coding: utf-8 -*-
# Coin Price indicator
# Nil Gradisnik <nil.gradisnik@gmail.com>

import os, signal, yaml, sys, logging, gi
gi.require_version('Gtk', '3.0')
gi.require_version('AppIndicator3', '0.1')

from gi.repository import Gtk, GdkPixbuf, GObject, GLib

try:
    from gi.repository import AppIndicator3 as AppIndicator
except ImportError:
    from gi.repository import AppIndicator

from indicator import Indicator
from exchange.kraken import Kraken
from exchange.bitstamp import Bitstamp
from exchange.bityep import BitYep
from exchange.gdax import Gdax

import threading

PROJECT_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))

signal.signal(signal.SIGINT, signal.SIG_DFL)  # ctrl+c exit

class Coin(object):
    config = yaml.load(open(PROJECT_ROOT + '/config.yaml', 'r'))
    config['project_root'] = PROJECT_ROOT

    def __init__(self):
        self.gui_ready = threading.Event()
        self.start_main()
        self.instances = []
        logging.info("Coin Price indicator v" + self.config['app']['version'])
        usage_error = 'Usage: coin.py [flags]\nasset\texchange:asset_pair:refresh_rate\nfile\tLoads various asset pairs from YAML file in ./coin directory'
        if len(sys.argv) > 2:
            quit('Too many parameters\n' + usage_error)

        self.start_main()
        if len(sys.argv) == 2:
            if '=' in sys.argv[1]:
                args = sys.argv[1].split('=')
                if args[0] == 'file':
                    try:
                        cp_instances = yaml.load(open(PROJECT_ROOT + '/coin/' + args[1], 'r'))
                    except:
                        quit('Error opening assets file')

                    self.add_many_indicators(cp_instances)

                elif args[0] == 'asset':
                    self.add_indicator(args[1])

                else:
                    quit('Invalid parameter\n' + usage_error)

            else:
                quit('Invalid parameter\n' + usage_error)

        else:
            self.add_indicator()

    # Start the main indicator icon and its menu
    def start_main(self):
        icon = self.config['project_root'] + '/resources/icon_32px.png'
        self.logo_124px = GdkPixbuf.Pixbuf.new_from_file(self.config['project_root'] + '/resources/icon_32px.png')
        self.main_item = AppIndicator.Indicator.new(self.config['app']['name'], icon, AppIndicator.IndicatorCategory.APPLICATION_STATUS)
        self.main_item.set_status(AppIndicator.IndicatorStatus.ACTIVE)
        self.main_item.set_menu(self._menu())

        def start_gui_thread():
            self.gui_ready.wait()
            Gtk.main()

        self.gui_thread = threading.Thread(target=start_gui_thread)
        self.gui_thread.start()

    # Program main menu
    def _menu(self):
        menu = Gtk.Menu()

        self.add_item = Gtk.MenuItem("Add Ticker")
        self.about_item = Gtk.MenuItem("About")
        self.quit_item = Gtk.MenuItem("Quit")

        self.add_item.connect("activate", self._add_ticker)
        self.about_item.connect("activate", self._about)
        self.quit_item.connect("activate", self._quit_all)

        menu.append(self.add_item)
        menu.append(self.about_item)
        menu.append(Gtk.SeparatorMenuItem())
        menu.append(self.quit_item)
        menu.show_all()

        return menu

    # Adds a ticker and starts it
    def add_indicator(self, settings=None):
        indicator = Indicator(self, len(self.instances), self.config, settings)
        self.instances.append(indicator)
        indicator.start()
        self.gui_ready.set()

    # adds many tickers
    def add_many_indicators(self, cp_instances):
        for cp_instance in cp_instances:
            settings = cp_instance['exchange'] + ':' + cp_instance['asset_pair'] + ':' + str(cp_instance['refresh'])
            nt = threading.Thread(target=self.add_indicator(settings))

    # Menu item to add a ticker
    def _add_ticker(self, widget):
        self.add_indicator()

    # Shows an About dialog
    def _about(self, widget):
        about = Gtk.AboutDialog()
        about.set_program_name(self.config['app']['name'])
        about.set_comments(self.config['app']['description'])
        about.set_copyright(self.config['author']['copyright'])
        about.set_version(self.config['app']['version'])
        about.set_website(self.config['app']['url'])
        about.set_authors([self.config['author']['name'] + ' <' + self.config['author']['email'] + '>'])
        about.set_artists([self.config['artist']['name'] + ' <' + self.config['artist']['email'] + '>'])
        about.set_license_type(Gtk.License.MIT_X11)
        about.set_logo(self.logo_124px)
        res = about.run()
        if res == -4 or -6:  # close events
            about.destroy()

    # Menu item to remove all tickers and quits the application
    def _quit_all(self, widget):
        logging.info("Exiting")
        Gtk.main_quit()

coin = Coin()
<|endoftext|>


<commit_before># Copyright 2019 The Magenta Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""A WaveNet-style AutoEncoder Configuration and FastGeneration Config."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from magenta.models.nsynth import reader
from magenta.models.nsynth import utils
from magenta.models.nsynth.wavenet import masked
from six.moves import range  # pylint: disable=redefined-builtin
import tensorflow as tf

class FastGenerationConfig(object):
  """Configuration object that helps manage the graph."""

  def __init__(self, batch_size=1):
    """."""
    self.batch_size = batch_size

  def build(self, inputs):
    """Build the graph for this configuration.

    Args:
      inputs: A dict of inputs. For training, should contain 'wav'.

    Returns:
      A dict of outputs that includes the 'predictions',
      'init_ops', the 'push_ops', and the 'quantized_input'.
    """
    num_stages = 10
    num_layers = 30
    filter_length = 3
    width = 512
    skip_width = 256
    num_z = 16

    # Encode the source with 8-bit Mu-Law.
    x = inputs['wav']
    batch_size = self.batch_size
    x_quantized = utils.mu_law(x)
    x_scaled = tf.cast(x_quantized, tf.float32) / 128.0
    x_scaled = tf.expand_dims(x_scaled, 2)

    encoding = tf.placeholder(
        name='encoding', shape=[batch_size, num_z], dtype=tf.float32)
    en = tf.expand_dims(encoding, 1)

    init_ops, push_ops = [], []

    ###
    # The WaveNet Decoder.
    ###
    l = x_scaled
    l, inits, pushs = utils.causal_linear(
        x=l,
        n_inputs=1,
        n_outputs=width,
        name='startconv',
        rate=1,
        batch_size=batch_size,
        filter_length=filter_length)

    for init in inits:
      init_ops.append(init)
    for push in pushs:
      push_ops.append(push)

    # Set up skip connections.
    s = utils.linear(l, width, skip_width, name='skip_start')

    # Residual blocks with skip connections.
    for i in range(num_layers):
      dilation = 2**(i % num_stages)

      # dilated masked cnn
      d, inits, pushs = utils.causal_linear(
          x=l,
          n_inputs=width,
          n_outputs=width * 2,
          name='dilatedconv_%d' % (i + 1),
          rate=dilation,
          batch_size=batch_size,
          filter_length=filter_length)

      for init in inits:
        init_ops.append(init)
      for push in pushs:
        push_ops.append(push)

      # local conditioning
      d += utils.linear(en, num_z, width * 2, name='cond_map_%d' % (i + 1))

      # gated cnn
      assert d.get_shape().as_list()[2] % 2 == 0
      m = d.get_shape().as_list()[2] // 2
      d = tf.sigmoid(d[:, :, :m]) * tf.tanh(d[:, :, m:])

      # residuals
      l += utils.linear(d, width, width, name='res_%d' % (i + 1))

      # skips
      s += utils.linear(d, width, skip_width, name='skip_%d' % (i + 1))

    s = tf.nn.relu(s)
    s = (utils.linear(s, skip_width, skip_width, name='out1') + utils.linear(
        en, num_z, skip_width, name='cond_map_out1'))
    s = tf.nn.relu(s)

    ###
    # Compute the logits and get the loss.
    ###
    logits = utils.linear(s, skip_width, 256, name='logits')
    logits = tf.reshape(logits, [-1, 256])
    probs = tf.nn.softmax(logits, name='softmax')

    return {
        'init_ops': init_ops,
        'push_ops': push_ops,
        'predictions': probs,
        'encoding': encoding,
        'quantized_input': x_quantized,
    }

class Config(object):
  """Configuration object that helps manage the graph."""

  def __init__(self, train_path=None):
    self.num_iters = 200000
    self.learning_rate_schedule = {
        0: 2e-4,
        90000: 4e-4 / 3,
        120000: 6e-5,
        150000: 4e-5,
        180000: 2e-5,
        210000: 6e-6,
        240000: 2e-6,
    }
    self.ae_hop_length = 512
    self.ae_bottleneck_width = 16
    self.train_path = train_path

  def get_batch(self, batch_size):
    assert self.train_path is not None
    data_train = reader.NSynthDataset(self.train_path, is_training=True)
    return data_train.get_wavenet_batch(batch_size, length=6144)

  @staticmethod
  def _condition(x, encoding):
    """Condition the input on the encoding.

    Args:
      x: The [mb, length, channels] float tensor input.
      encoding: The [mb, encoding_length, channels] float tensor encoding.

    Returns:
      The output after broadcasting the encoding to x's shape and adding them.
    """
    mb, length, channels = x.get_shape().as_list()
    enc_mb, enc_length, enc_channels = encoding.get_shape().as_list()
    assert enc_mb == mb
    assert enc_channels == channels

    encoding = tf.reshape(encoding, [mb, enc_length, 1, channels])
    x = tf.reshape(x, [mb, enc_length, -1, channels])
    x += encoding
    x = tf.reshape(x, [mb, length, channels])
    x.set_shape([mb, length, channels])
    return x

  def build(self, inputs, is_training, rescale_inputs=True):
    """Build the graph for this configuration.

    Args:
      inputs: A dict of inputs. For training, should contain 'wav'.
      is_training: Whether we are training or not. Not used in this config.
      rescale_inputs: Whether to convert inputs to mu-law and back to unit
        scaling before passing through the model (loses gradients).
    Returns:
      A dict of outputs that includes the 'predictions', 'loss', the 'encoding',
      the 'quantized_input', and whatever metrics we want to track for eval.
    """
    num_stages = 10
    num_layers = 30
    filter_length = 3
    width = 512
    skip_width = 256
    ae_num_stages = 10
    ae_num_layers = 30
    ae_filter_length = 3
    ae_width = 128

    # Encode the source with 8-bit Mu-Law.
    x = inputs['wav']
    x_quantized = utils.mu_law(x)
    x_scaled = tf.cast(x_quantized, tf.float32) / 128.0
    x_scaled = tf.expand_dims(x_scaled, 2)
    x = tf.expand_dims(x, 2)

    ###
    # The Non-Causal Temporal Encoder.
    ###
    en = masked.conv1d(
        x_scaled if rescale_inputs else x,
        causal=False,
        num_filters=ae_width,
        filter_length=ae_filter_length,
        name='ae_startconv',
        is_training=is_training)

    for num_layer in range(ae_num_layers):
      dilation = 2**(num_layer % ae_num_stages)
      d = tf.nn.relu(en)
      d = masked.conv1d(
          d,
          causal=False,
          num_filters=ae_width,
          filter_length=ae_filter_length,
          dilation=dilation,
          name='ae_dilatedconv_%d' % (num_layer + 1),
          is_training=is_training)
      d = tf.nn.relu(d)
      en += masked.conv1d(
          d,
          num_filters=ae_width,
          filter_length=1,
          name='ae_res_%d' % (num_layer + 1),
          is_training=is_training)

    en = masked.conv1d(
        en,
        num_filters=self.ae_bottleneck_width,
        filter_length=1,
        name='ae_bottleneck',
        is_training=is_training)
    en = masked.pool1d(en, self.ae_hop_length, name='ae_pool', mode='avg')
    encoding = en

    ###
    # The WaveNet Decoder.
    ###
    l = masked.shift_right(x_scaled if rescale_inputs else x)
    l = masked.conv1d(
        l,
        num_filters=width,
        filter_length=filter_length,
        name='startconv',
        is_training=is_training)

    # Set up skip connections.
    s = masked.conv1d(
        l,
        num_filters=skip_width,
        filter_length=1,
        name='skip_start',
        is_training=is_training)

    # Residual blocks with skip connections.
    for i in range(num_layers):
      dilation = 2**(i % num_stages)
      d = masked.conv1d(
          l,
          num_filters=2 * width,
          filter_length=filter_length,
          dilation=dilation,
          name='dilatedconv_%d' % (i + 1),
          is_training=is_training)
      d = self._condition(d,
                          masked.conv1d(
                              en,
                              num_filters=2 * width,
                              filter_length=1,
                              name='cond_map_%d' % (i + 1),
                              is_training=is_training))

      assert d.get_shape().as_list()[2] % 2 == 0
      m = d.get_shape().as_list()[2] // 2
      d_sigmoid = tf.sigmoid(d[:, :, :m])
      d_tanh = tf.tanh(d[:, :, m:])
      d = d_sigmoid * d_tanh

      l += masked.conv1d(
          d,
          num_filters=width,
          filter_length=1,
          name='res_%d' % (i + 1),
          is_training=is_training)
      s += masked.conv1d(
          d,
          num_filters=skip_width,
          filter_length=1,
          name='skip_%d' % (i + 1),
          is_training=is_training)

    s = tf.nn.relu(s)
    s = masked.conv1d(
        s,
        num_filters=skip_width,
        filter_length=1,
        name='out1',
        is_training=is_training)
    s = self._condition(s,
                        masked.conv1d(
                            en,
                            num_filters=skip_width,
                            filter_length=1,
                            name='cond_map_out1',
                            is_training=is_training))
    s = tf.nn.relu(s)

    ###
    # Compute the logits and get the loss.
    ###
    logits = masked.conv1d(
        s,
        num_filters=256,
        filter_length=1,
        name='logits',
        is_training=is_training)
    logits = tf.reshape(logits, [-1, 256])
    probs = tf.nn.softmax(logits, name='softmax')
    x_indices = tf.cast(tf.reshape(x_quantized, [-1]), tf.int32) + 128
    loss = tf.reduce_mean(
        tf.nn.sparse_softmax_cross_entropy_with_logits(
            logits=logits, labels=x_indices, name='nll'),
        0,
        name='loss')

    return {
        'predictions': probs,
        'loss': loss,
        'eval': {
            'nll': loss
        },
        'quantized_input': x_quantized,
        'encoding': encoding,
    }
<commit_msg>Modified h512_bo16.py to have the option of building only the encoder. Also added a reduce_mean implementation of pool1d to make it more TPU-friendly.<commit_after># Copyright 2019 The Magenta Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""A WaveNet-style AutoEncoder Configuration and FastGeneration Config."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from magenta.models.nsynth import reader
from magenta.models.nsynth import utils
from magenta.models.nsynth.wavenet import masked
from six.moves import range  # pylint: disable=redefined-builtin
import tensorflow as tf

class FastGenerationConfig(object):
  """Configuration object that helps manage the graph."""

  def __init__(self, batch_size=1):
    """."""
    self.batch_size = batch_size

  def build(self, inputs):
    """Build the graph for this configuration.

    Args:
      inputs: A dict of inputs. For training, should contain 'wav'.

    Returns:
      A dict of outputs that includes the 'predictions',
      'init_ops', the 'push_ops', and the 'quantized_input'.
    """
    num_stages = 10
    num_layers = 30
    filter_length = 3
    width = 512
    skip_width = 256
    num_z = 16

    # Encode the source with 8-bit Mu-Law.
    x = inputs['wav']
    batch_size = self.batch_size
    x_quantized = utils.mu_law(x)
    x_scaled = tf.cast(x_quantized, tf.float32) / 128.0
    x_scaled = tf.expand_dims(x_scaled, 2)

    encoding = tf.placeholder(
        name='encoding', shape=[batch_size, num_z], dtype=tf.float32)
    en = tf.expand_dims(encoding, 1)

    init_ops, push_ops = [], []

    ###
    # The WaveNet Decoder.
    ###
    l = x_scaled
    l, inits, pushs = utils.causal_linear(
        x=l,
        n_inputs=1,
        n_outputs=width,
        name='startconv',
        rate=1,
        batch_size=batch_size,
        filter_length=filter_length)

    for init in inits:
      init_ops.append(init)
    for push in pushs:
      push_ops.append(push)

    # Set up skip connections.
    s = utils.linear(l, width, skip_width, name='skip_start')

    # Residual blocks with skip connections.
    for i in range(num_layers):
      dilation = 2**(i % num_stages)

      # dilated masked cnn
      d, inits, pushs = utils.causal_linear(
          x=l,
          n_inputs=width,
          n_outputs=width * 2,
          name='dilatedconv_%d' % (i + 1),
          rate=dilation,
          batch_size=batch_size,
          filter_length=filter_length)

      for init in inits:
        init_ops.append(init)
      for push in pushs:
        push_ops.append(push)

      # local conditioning
      d += utils.linear(en, num_z, width * 2, name='cond_map_%d' % (i + 1))

      # gated cnn
      assert d.get_shape().as_list()[2] % 2 == 0
      m = d.get_shape().as_list()[2] // 2
      d = tf.sigmoid(d[:, :, :m]) * tf.tanh(d[:, :, m:])

      # residuals
      l += utils.linear(d, width, width, name='res_%d' % (i + 1))

      # skips
      s += utils.linear(d, width, skip_width, name='skip_%d' % (i + 1))

    s = tf.nn.relu(s)
    s = (utils.linear(s, skip_width, skip_width, name='out1') + utils.linear(
        en, num_z, skip_width, name='cond_map_out1'))
    s = tf.nn.relu(s)

    ###
    # Compute the logits and get the loss.
    ###
    logits = utils.linear(s, skip_width, 256, name='logits')
    logits = tf.reshape(logits, [-1, 256])
    probs = tf.nn.softmax(logits, name='softmax')

    return {
        'init_ops': init_ops,
        'push_ops': push_ops,
        'predictions': probs,
        'encoding': encoding,
        'quantized_input': x_quantized,
    }

class Config(object):
  """Configuration object that helps manage the graph."""

  def __init__(self, train_path=None):
    self.num_iters = 200000
    self.learning_rate_schedule = {
        0: 2e-4,
        90000: 4e-4 / 3,
        120000: 6e-5,
        150000: 4e-5,
        180000: 2e-5,
        210000: 6e-6,
        240000: 2e-6,
    }
    self.ae_hop_length = 512
    self.ae_bottleneck_width = 16
    self.train_path = train_path

  def get_batch(self, batch_size):
    assert self.train_path is not None
    data_train = reader.NSynthDataset(self.train_path, is_training=True)
    return data_train.get_wavenet_batch(batch_size, length=6144)

  @staticmethod
  def _condition(x, encoding):
    """Condition the input on the encoding.

    Args:
      x: The [mb, length, channels] float tensor input.
      encoding: The [mb, encoding_length, channels] float tensor encoding.

    Returns:
      The output after broadcasting the encoding to x's shape and adding them.
    """
    mb, length, channels = x.get_shape().as_list()
    enc_mb, enc_length, enc_channels = encoding.get_shape().as_list()
    assert enc_mb == mb
    assert enc_channels == channels

    encoding = tf.reshape(encoding, [mb, enc_length, 1, channels])
    x = tf.reshape(x, [mb, enc_length, -1, channels])
    x += encoding
    x = tf.reshape(x, [mb, length, channels])
    x.set_shape([mb, length, channels])
    return x

  def build(self, inputs, is_training, rescale_inputs=True,
            include_decoder=True, use_reduce_mean_to_pool=False):
    """Build the graph for this configuration.

    Args:
      inputs: A dict of inputs. For training, should contain 'wav'.
      is_training: Whether we are training or not. Not used in this config.
      rescale_inputs: Whether to convert inputs to mu-law and back to unit
        scaling before passing through the model (loses gradients).
      include_decoder: bool, whether to include the decoder in the build().
      use_reduce_mean_to_pool: whether to use reduce_mean (instead of pool1d)
        for pooling.
    Returns:
      A dict of outputs that includes the 'predictions', 'loss', the 'encoding',
      the 'quantized_input', and whatever metrics we want to track for eval.
    """
    num_stages = 10
    num_layers = 30
    filter_length = 3
    width = 512
    skip_width = 256
    ae_num_stages = 10
    ae_num_layers = 30
    ae_filter_length = 3
    ae_width = 128

    # Encode the source with 8-bit Mu-Law.
    x = inputs['wav']
    x_quantized = utils.mu_law(x)
    x_scaled = tf.cast(x_quantized, tf.float32) / 128.0
    x_scaled = tf.expand_dims(x_scaled, 2)
    x = tf.expand_dims(x, 2)

    ###
    # The Non-Causal Temporal Encoder.
    ###
    en = masked.conv1d(
        x_scaled if rescale_inputs else x,
        causal=False,
        num_filters=ae_width,
        filter_length=ae_filter_length,
        name='ae_startconv',
        is_training=is_training)

    for num_layer in range(ae_num_layers):
      dilation = 2**(num_layer % ae_num_stages)
      d = tf.nn.relu(en)
      d = masked.conv1d(
          d,
          causal=False,
          num_filters=ae_width,
          filter_length=ae_filter_length,
          dilation=dilation,
          name='ae_dilatedconv_%d' % (num_layer + 1),
          is_training=is_training)
      d = tf.nn.relu(d)
      en += masked.conv1d(
          d,
          num_filters=ae_width,
          filter_length=1,
          name='ae_res_%d' % (num_layer + 1),
          is_training=is_training)

    en = masked.conv1d(
        en,
        num_filters=self.ae_bottleneck_width,
        filter_length=1,
        name='ae_bottleneck',
        is_training=is_training)

    if use_reduce_mean_to_pool:
      # Depending on the accelerator used for training, masked.pool1d may
      # lead to out of memory error.
      # reduce_mean is equivalent to masked.pool1d when the stride is the same
      # as the window length (which is the case here).
      batch_size, unused_length, depth = en.shape.as_list()
      en = tf.reshape(en, [batch_size, -1, self.ae_hop_length, depth])
      en = tf.reduce_mean(en, axis=2)
    else:
      en = masked.pool1d(en, self.ae_hop_length, name='ae_pool', mode='avg')
    encoding = en

    if not include_decoder:
      return {'encoding': encoding}

    ###
    # The WaveNet Decoder.
    ###
    l = masked.shift_right(x_scaled if rescale_inputs else x)
    l = masked.conv1d(
        l,
        num_filters=width,
        filter_length=filter_length,
        name='startconv',
        is_training=is_training)

    # Set up skip connections.
    s = masked.conv1d(
        l,
        num_filters=skip_width,
        filter_length=1,
        name='skip_start',
        is_training=is_training)

    # Residual blocks with skip connections.
    for i in range(num_layers):
      dilation = 2**(i % num_stages)
      d = masked.conv1d(
          l,
          num_filters=2 * width,
          filter_length=filter_length,
          dilation=dilation,
          name='dilatedconv_%d' % (i + 1),
          is_training=is_training)
      d = self._condition(d,
                          masked.conv1d(
                              en,
                              num_filters=2 * width,
                              filter_length=1,
                              name='cond_map_%d' % (i + 1),
                              is_training=is_training))

      assert d.get_shape().as_list()[2] % 2 == 0
      m = d.get_shape().as_list()[2] // 2
      d_sigmoid = tf.sigmoid(d[:, :, :m])
      d_tanh = tf.tanh(d[:, :, m:])
      d = d_sigmoid * d_tanh

      l += masked.conv1d(
          d,
          num_filters=width,
          filter_length=1,
          name='res_%d' % (i + 1),
          is_training=is_training)
      s += masked.conv1d(
          d,
          num_filters=skip_width,
          filter_length=1,
          name='skip_%d' % (i + 1),
          is_training=is_training)

    s = tf.nn.relu(s)
    s = masked.conv1d(
        s,
        num_filters=skip_width,
        filter_length=1,
        name='out1',
        is_training=is_training)
    s = self._condition(s,
                        masked.conv1d(
                            en,
                            num_filters=skip_width,
                            filter_length=1,
                            name='cond_map_out1',
                            is_training=is_training))
    s = tf.nn.relu(s)

    ###
    # Compute the logits and get the loss.
    ###
    logits = masked.conv1d(
        s,
        num_filters=256,
        filter_length=1,
        name='logits',
        is_training=is_training)
    logits = tf.reshape(logits, [-1, 256])
    probs = tf.nn.softmax(logits, name='softmax')
    x_indices = tf.cast(tf.reshape(x_quantized, [-1]), tf.int32) + 128
    loss = tf.reduce_mean(
        tf.nn.sparse_softmax_cross_entropy_with_logits(
            logits=logits, labels=x_indices, name='nll'),
        0,
        name='loss')

    return {
        'predictions': probs,
        'loss': loss,
        'eval': {
            'nll': loss
        },
        'quantized_input': x_quantized,
        'encoding': encoding,
    }
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""

"""

__author__ = "Evgeny Boykov. <artscan@list.ru>"

import math
import cmath
import pickle
import shelve
import sys,os
from numpy import *

from integ import modinteg as integ

class CoverElement(object):
    def initcover(self, c):
        self.numnodes = c.numnodes
        self.h = c.h
        self.node_coordinates = c.node_coordinates
        self.intphi_over = c.intphi_over
        self.name_matrixa = c.name_matrixa
        self.eps_gmres_ = c.eps_gmres_
        self.k_wave = c.k_wave
        self.gauss = c.gauss
        self.sigma = c.sigma
        self.dim_3d = c.dim_3d

class TaskElement(CoverElement):
    name_savearrays = 'savearrays.dat'

    def save(self,fname):
        f = open(fname,'w')
        pickle.dump(self,f)
        f.close()

    @classmethod
    def load(self,fname):
        f = open(fname,'r')
        ce = pickle.load(f)
        f.close()
        return ce

    def setupkernel(self):
        """
        Linking with fortran object
        """
        self.k = integ

        self.k.set_i_ptr("numnodes", self.numnodes)
        self.k.set_i_ptr("dim_3d", self.dim_3d)
        self.k.set_dp_ptr("h", self.h)
        self.k.set_dp_ptr("k_wave", self.k_wave)
        self.k.set_dc2d_ptr("gauss", self.gauss)
        self.k.set_dp2d_ptr("node_coordinates", self.node_coordinates)
        self.k.set_dp1d_ptr("intphi_over", self.intphi_over)
        self.k.set_dp1d_ptr("sigma", self.sigma)

    def killfortran(self):
        if (hasattr(self,'k')):
            del self.k
<commit_msg>fix bug in k_wave: dp instead dc<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""

"""

__author__ = "Evgeny Boykov. <artscan@list.ru>"

import math
import cmath
import pickle
import shelve
import sys,os
from numpy import *

from integ import modinteg as integ

class CoverElement(object):
    def initcover(self, c):
        self.numnodes = c.numnodes
        self.h = c.h
        self.node_coordinates = c.node_coordinates
        self.intphi_over = c.intphi_over
        self.name_matrixa = c.name_matrixa
        self.eps_gmres_ = c.eps_gmres_
        self.k_wave = c.k_wave
        self.gauss = c.gauss
        self.sigma = c.sigma
        self.dim_3d = c.dim_3d

class TaskElement(CoverElement):
    name_savearrays = 'savearrays.dat'

    def save(self,fname):
        f = open(fname,'w')
        pickle.dump(self,f)
        f.close()

    @classmethod
    def load(self,fname):
        f = open(fname,'r')
        ce = pickle.load(f)
        f.close()
        return ce

    def setupkernel(self):
        """
        Linking with fortran object
        """
        self.k = integ

        self.k.set_i_ptr("numnodes", self.numnodes)
        self.k.set_i_ptr("dim_3d", self.dim_3d)
        self.k.set_dp_ptr("h", self.h)
        self.k.set_dc_ptr("k_wave", self.k_wave)
        self.k.set_dc2d_ptr("gauss", self.gauss)
        self.k.set_dp2d_ptr("node_coordinates", self.node_coordinates)
        self.k.set_dp1d_ptr("intphi_over", self.intphi_over)
        self.k.set_dp1d_ptr("sigma", self.sigma)

    def killfortran(self):
        if (hasattr(self,'k')):
            del self.k
<|endoftext|>