<commit_before>"""
Overview
========

This plugin implements Key-Commands to redirect the default python sys.stdout object
to an AreaVi instance. As it is possible to drop python code to vy in order
to affect the editor state, the python interpreter writes output to
sys.stdout.

With this plugin it is possible to drop the output to a given
Line.Col inside an AreaVi instance.

Key-Commands
============

Namespace: outputs

Mode: Global
Event: <Control-Alt-bracketleft>
Description: it restores sys.stdout.

Mode: Global
Event: <Alt-braceleft>
Description: It removes a given AreaVi instance from
having output written in.

Mode: Global
Event: <Alt-bracketleft>
Description: It redirects output from sys.stdout to
a given AreaVi instance.
"""

from vyapp.app import root
from vyapp.widgets import TextWindow
import sys

class Stdout:
    """
    This class is used to wrap an AreaVi widget to be
    used as stdout channel.
    """

    def __init__(self, area):
        self.area = area
        self.area.mark_set('(CODE_MARK)', 'insert')

    def write(self, data):
        """
        This method is called to write data to the AreaVi widget.
        """

        index0 = self.area.index('(CODE_MARK)')
        self.area.insert('(CODE_MARK)', data)
        self.area.see('insert')

    def __eq__(self, other):
        return self.area == other

class Transmitter:
    """
    This class would replace sys.stdout. It is used to dispatch
    data that is written to sys.stdout, the data is dispatched to its
    child objects that have a write method.
    """

    def __init__(self, default):
        self.base    = [default]
        self.default = default

    def flush(self):
        pass

    def restore(self):
        """
        It clears all children and restores the default
        child.
        """
        self.base.clear()
        self.base.append(self.default)

    def append(self, fd):
        try:
            self.base.remove(fd)
        except ValueError:
            pass
        finally:
            self.base.append(fd)

    def discard(self, fd):
        try:
            self.base.remove(fd)
        except ValueError:
            pass

    def write(self, data):
        """
        This method is used to dispatch data to the
        children.

        If it occurs an exception when writing to some
        of the supposed fd's then it is removed from
        the list of fd's.

        Consider the situation
        that an areavi instance is added then
        destroyed. It will throw an exception
        when writing to that areavi.
        """
        root.status.set_msg(('Code output written to sys.stdout.. '
        'Press <Alt-bracketright> to check.'))

        for ind in self.base[:]:
            try:
                ind.write(data)
            except Exception:
                self.base.discard(ind)

class CmdOutput:
    """
    """

    def __init__(self, win):
        self.win = win

    def write(self, data):
        self.win.text.insert('end', data)
        self.win.text.see('end')

    def __eq__(self, other):
        return self.win.text == other

class OutputController:
    def __init__(self, area):
        self.area = area

        area.install('outputs', (-1, '<Alt-bracketleft>', self.add_output),
        (-1, '<Alt-braceleft>', self.rm_output),
        (-1, '<Alt-bracketright>', self.view_log),
        (-1, '<Control-Alt-bracketleft>',  self.restore_output))

    def view_log(self, event):
        code_output.display()

    def add_output(self, event):
        sys.stdout.append(Stdout(self.area))
        root.status.set_msg('Output set on: %s' % self.area.index('insert'))
        return 'break'

    def rm_output(self, event):
        sys.stdout.discard(self.area)
        root.status.set_msg('Output removed!')
        return 'break'

    def restore_output(self, event):
        sys.stdout.restore()
        root.status.set_msg('Stdout restored!')
        return 'break'

sys.stdout  = Transmitter(sys.__stdout__)
code_output = TextWindow('', title='Cmd Output')
code_output.withdraw()
sys.stdout.append(CmdOutput(code_output))
install = OutputController

<commit_msg>Removing unecessary statement.<commit_after>"""
Overview
========

This plugin implements Key-Commands to redirect the default python sys.stdout object
to an AreaVi instance. As it is possible to drop python code to vy in order
to affect the editor state, the python interpreter writes output to
sys.stdout.

With this plugin it is possible to drop the output to a given
Line.Col inside an AreaVi instance.

Key-Commands
============

Namespace: outputs

Mode: Global
Event: <Control-Alt-bracketleft>
Description: it restores sys.stdout.

Mode: Global
Event: <Alt-braceleft>
Description: It removes a given AreaVi instance from
having output written in.

Mode: Global
Event: <Alt-bracketleft>
Description: It redirects output from sys.stdout to
a given AreaVi instance.
"""

from vyapp.app import root
from vyapp.widgets import TextWindow
import sys

class Stdout:
    """
    This class is used to wrap an AreaVi widget to be
    used as stdout channel.
    """

    def __init__(self, area):
        self.area = area
        self.area.mark_set('(CODE_MARK)', 'insert')

    def write(self, data):
        """
        This method is called to write data to the AreaVi widget.
        """

        index0 = self.area.index('(CODE_MARK)')
        self.area.insert('(CODE_MARK)', data)
        self.area.see('insert')

    def __eq__(self, other):
        return self.area == other

class Transmitter:
    """
    This class would replace sys.stdout. It is used to dispatch
    data that is written to sys.stdout, the data is dispatched to its
    child objects that have a write method.
    """

    def __init__(self, default):
        self.base    = [default]
        self.default = default

    def flush(self):
        pass

    def restore(self):
        """
        It clears all children and restores the default
        child.
        """
        self.base.clear()
        self.base.append(self.default)

    def append(self, fd):
        try:
            self.base.remove(fd)
        except ValueError:
            pass
        finally:
            self.base.append(fd)

    def discard(self, fd):
        try:
            self.base.remove(fd)
        except ValueError:
            pass

    def write(self, data):
        """
        This method is used to dispatch data to the
        children.

        If it occurs an exception when writing to some
        of the supposed fd's then it is removed from
        the list of fd's.

        Consider the situation
        that an areavi instance is added then
        destroyed. It will throw an exception
        when writing to that areavi.
        """

        for ind in self.base[:]:
            try:
                ind.write(data)
            except Exception:
                self.base.discard(ind)

class CmdOutput:
    """
    """

    def __init__(self, win):
        self.win = win

    def write(self, data):
        self.win.text.insert('end', data)
        self.win.text.see('end')

    def __eq__(self, other):
        return self.win.text == other

class OutputController:
    def __init__(self, area):
        self.area = area

        area.install('outputs', (-1, '<Alt-bracketleft>', self.add_output),
        (-1, '<Alt-braceleft>', self.rm_output),
        (-1, '<Alt-bracketright>', self.view_log),
        (-1, '<Control-Alt-bracketleft>',  self.restore_output))

    def view_log(self, event):
        code_output.display()

    def add_output(self, event):
        sys.stdout.append(Stdout(self.area))
        root.status.set_msg('Output set on: %s' % self.area.index('insert'))
        return 'break'

    def rm_output(self, event):
        sys.stdout.discard(self.area)
        root.status.set_msg('Output removed!')
        return 'break'

    def restore_output(self, event):
        sys.stdout.restore()
        root.status.set_msg('Stdout restored!')
        return 'break'

sys.stdout  = Transmitter(sys.__stdout__)
code_output = TextWindow('', title='Cmd Output')
code_output.withdraw()
sys.stdout.append(CmdOutput(code_output))
install = OutputController

<|endoftext|>


<commit_before>"""
Natural Gas Imports Script

This script calculates the imports of natural gas for a neighborhood to provide heating/cooling.
It has two loops: one for each of heating network and cooling network
This is then combined to calculate the total natural gas imports and the corresponding file is saved in the
respective folder
"""
from __future__ import division
from __future__ import print_function

import cea.config
import cea.inputlocator
import numpy as np

__author__ = "Sreepathi Bhargava Krishna"
__copyright__ = "Copyright 2018, Architecture and Building Systems - ETH Zurich"
__credits__ = ["Sreepathi Bhargava Krishna"]
__license__ = "MIT"
__version__ = "0.1"
__maintainer__ = "Daren Thomas"
__email__ = "cea@arch.ethz.ch"
__status__ = "Production"

def fuel_imports(master_to_slave_vars, heating_dispatch,
                 cooling_dispatch):

    if master_to_slave_vars.DHN_exists:
        NG_used_CHP_W = heating_dispatch["NG_CHP_req_W"]
        NG_used_BaseBoiler_W = heating_dispatch["NG_BaseBoiler_req_W"]
        NG_used_PeakBoiler_W = heating_dispatch["NG_PeakBoiler_req_W"]
        NG_used_BackupBoiler_W = heating_dispatch["NG_BackupBoiler_req_W"]
    else:
        NG_used_CHP_W = np.zeros(8760)
        NG_used_BaseBoiler_W = np.zeros(8760)
        NG_used_PeakBoiler_W = np.zeros(8760)
        NG_used_BackupBoiler_W = np.zeros(8760)

    if master_to_slave_vars.DCN_exists:
        NG_used_CCGT_W = cooling_dispatch["NG_used_CCGT_W"]
    else:
        NG_used_CCGT_W = np.zeros(8760)

    NG_total_heating_W = [a + b + c + d  for a, b, c, d in
                          zip(NG_used_CHP_W, NG_used_BaseBoiler_W, NG_used_PeakBoiler_W, NG_used_BackupBoiler_W)]

    NG_total_cooling_W = NG_used_CCGT_W

    NG_total_W = NG_total_heating_W + NG_total_cooling_W

    naturalgas_dispatch = {
        "NG_GRID_connected_W": NG_total_W,
        "NG_GRID_heating_connected_W": NG_total_heating_W,
        "NG_GRID_cooling_connected_W": NG_total_cooling_W,
        "NG_CHP_req_W": NG_used_CHP_W,
        "NG_BaseBoiler_req_W": NG_used_BaseBoiler_W,
        "NG_PeakBoiler_req_W": NG_used_PeakBoiler_W,
        "NG_BackupBoiler_req_W": NG_used_BackupBoiler_W,
        "NG_used_CCGT_W": NG_used_CCGT_W
    }
    return naturalgas_dispatch

def main(config):
    locator = cea.inputlocator.InputLocator(config.scenario)
    generation = 2
    individual = 2
    print("Calculating imports of natural gas of individual" + str(individual) + " of generation " + str(generation))
    district_heating_network = config.optimization.district_heating_network
    district_cooling_network = config.optimization.district_cooling_network

    fuel_imports(generation, individual, locator, district_heating_network, district_cooling_network)

if __name__ == '__main__':
    main(cea.config.Configuration())
<commit_msg>erasing depreciated inputs<commit_after>"""
Natural Gas Imports Script

This script calculates the imports of natural gas for a neighborhood to provide heating/cooling.
It has two loops: one for each of heating network and cooling network
This is then combined to calculate the total natural gas imports and the corresponding file is saved in the
respective folder
"""
from __future__ import division
from __future__ import print_function

import cea.config
import cea.inputlocator
import numpy as np

__author__ = "Sreepathi Bhargava Krishna"
__copyright__ = "Copyright 2018, Architecture and Building Systems - ETH Zurich"
__credits__ = ["Sreepathi Bhargava Krishna"]
__license__ = "MIT"
__version__ = "0.1"
__maintainer__ = "Daren Thomas"
__email__ = "cea@arch.ethz.ch"
__status__ = "Production"

def fuel_imports(master_to_slave_vars, heating_dispatch,
                 cooling_dispatch):

    if master_to_slave_vars.DHN_exists:
        NG_used_CHP_W = heating_dispatch["NG_CHP_req_W"]
        NG_used_BaseBoiler_W = heating_dispatch["NG_BaseBoiler_req_W"]
        NG_used_PeakBoiler_W = heating_dispatch["NG_PeakBoiler_req_W"]
        NG_used_BackupBoiler_W = heating_dispatch["NG_BackupBoiler_req_W"]
    else:
        NG_used_CHP_W = np.zeros(8760)
        NG_used_BaseBoiler_W = np.zeros(8760)
        NG_used_PeakBoiler_W = np.zeros(8760)
        NG_used_BackupBoiler_W = np.zeros(8760)

    if master_to_slave_vars.DCN_exists:
        NG_used_CCGT_W = cooling_dispatch["NG_used_CCGT_W"]
    else:
        NG_used_CCGT_W = np.zeros(8760)

    NG_total_heating_W = [a + b + c + d  for a, b, c, d in
                          zip(NG_used_CHP_W, NG_used_BaseBoiler_W, NG_used_PeakBoiler_W, NG_used_BackupBoiler_W)]

    NG_total_cooling_W = NG_used_CCGT_W

    NG_total_W = NG_total_heating_W + NG_total_cooling_W

    naturalgas_dispatch = {
        "NG_GRID_connected_W": NG_total_W,
        "NG_GRID_heating_connected_W": NG_total_heating_W,
        "NG_GRID_cooling_connected_W": NG_total_cooling_W,
        "NG_CHP_req_W": NG_used_CHP_W,
        "NG_BaseBoiler_req_W": NG_used_BaseBoiler_W,
        "NG_PeakBoiler_req_W": NG_used_PeakBoiler_W,
        "NG_BackupBoiler_req_W": NG_used_BackupBoiler_W,
        "NG_used_CCGT_W": NG_used_CCGT_W
    }
    return naturalgas_dispatch

def main(config):
    locator = cea.inputlocator.InputLocator(config.scenario)
    generation = 2
    individual = 2
    print("Calculating imports of natural gas of individual" + str(individual) + " of generation " + str(generation))
    district_heating_network = config.optimization.district_heating_network
    district_cooling_network = config.optimization.district_cooling_network

    fuel_imports(generation, individual, locator)

if __name__ == '__main__':
    main(cea.config.Configuration())
<|endoftext|>


<commit_before>from collections import namedtuple

class CommandNotFoundError(Exception):
    pass

class AmbiguousCommandError(Exception):
    pass

cmd_data = namedtuple('cmd_data','invocations prompt enabled syntax_file help flags')

# Flags
RUN_IN_OWN_CONSOLE = 0x02

HG_COMMANDS = {}

HG_COMMANDS['default'] = {
    'commit': cmd_data(
                    invocations={'commit...': 'commit -m "%(input)s"',
                                 'commit... (this file)': 'commit "%(file_name)s" -m "%(input)s"',
                                },
                    prompt='Commit message:',
                    enabled=True,
                    syntax_file='',
                    help='commit the specified files or all outstanding changes',
                    flags=0,
                    ),
    'init': cmd_data(
                    invocations={'init': 'init',
                                },
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='create a new repository in the given directory',
                    flags=RUN_IN_OWN_CONSOLE,
                    ),
    'add': cmd_data(
                    invocations={'add (this file)': 'add "%(file_name)s"',
                                 'add': 'add',
                               },
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='add the specified files on the next commit',
                    flags=0,
                    ),
    'addremove': cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='add all new files, delete all missing files',
                    flags=0,
                    ),
    'annotate': cmd_data(
                    invocations={'annotate (this file)': 'annotate "%(file_name)s"',
                                'blame (this file)': 'annotate "%(file_name)s"',
                               },
                    prompt='',
                    enabled=True,
                    syntax_file='Packages/SublimeHg/Support/Mercurial Annotate.hidden-tmLanguage',
                    help='show changeset information by line for each file',
                    flags=0,
                    ),
    'bookmark': cmd_data(
                    invocations={'bookmark (parent revision)': 'bookmark "%(input)s"',
                                },
                    prompt='Bookmark name:',
                    enabled=True,
                    syntax_file='',
                    help='track a line of development with movable markers',
                    flags=0,
                    ),
    'bookmarks': cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='track a line of development with movable markers',
                    flags=0,
                    ),
    'branch': cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='set or show the current branch name',
                    flags=0,
                    ),
    'branches': cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='list repository named branches',
                    flags=0,
                    ),
    'diff': cmd_data(
                    invocations={'diff (this file)': 'diff "%(file_name)s"',
                                 'diff': 'diff',
                        },
                    prompt='',
                    enabled=True,
                    syntax_file='Packages/Diff/Diff.tmLanguage',
                    help='diff repository (or selected files)',
                    flags=0,
                    ),
    'forget': cmd_data(
                    invocations={'forget (this file)': 'forget "%(file_name)s"',
                        },
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='forget the specified files on the next commit',
                    flags=0,
                    ),
    'grep': cmd_data(
                    invocations={'grep...': 'grep "%(input)s"',
                                },
                    prompt='Pattern (grep):',
                    enabled=True,
                    syntax_file='',
                    help='search for a pattern in specified files and revisions',
                    flags=0,
                    ),
    'heads': cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='show current repository heads or show branch heads',
                    flags=0,
                    ),
    'help': cmd_data(
                    invocations={'help...': 'help "%(input)s"',
                                 'help': 'help',
                        },
                    prompt='Help topic:',
                    enabled=True,
                    syntax_file='',
                    help='show help for a given topic or a help overview',
                    flags=0,
                    ),
    'indentify': cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='identify the working copy or specified revision',
                    flags=0,
                    ),
    'incoming': cmd_data(
                    invocations={'incoming...': 'incoming %(input)s',
                                 'incoming': 'incoming',
                        },
                    prompt='Incoming source:',
                    enabled=True,
                    syntax_file='',
                    help='show new changesets found in source',
                    flags=RUN_IN_OWN_CONSOLE,
                    ),
    'locate': cmd_data(
                    invocations={'locate': 'locate "%(input)s"'
                                },
                    prompt='Pattern:',
                    enabled=True,
                    syntax_file='',
                    help='locate files matching specific patterns',
                    flags=0,
                    ),
    'log': cmd_data(
                    invocations={'log (this file)': 'log "%(file_name)s"',
                                 'log': 'log',
                                },
                    prompt='',
                    enabled=True,
                    syntax_file='Packages/SublimeHg/Support/Mercurial Log.hidden-tmLanguage',
                    help='show revision history of entire repository or files',
                    flags=0,
                    ),
    'manifest': cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='output the current or given revision of the project manifest',
                    flags=0,
                    ),
    'merge': cmd_data(
                    invocations={'merge...': 'merge "%(input)s"',
                                 'merge': 'merge',
                                },
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='merge working directory with another revision',
                    flags=0,
                    ),
    'outgoing': cmd_data(
                    invocations={'outgoing...': 'outgoing %(input)s',
                                 'outgoing': 'outgoing',
                        },
                    prompt='Outgoing target:',
                    enabled=True,
                    syntax_file='Packages/SublimeHg/Support/Mercurial Log.hidden-tmLanguage',
                    help='show changesets not found in the destination',
                    flags=RUN_IN_OWN_CONSOLE,
                    ),
    'parents': cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='show the parents of the working directory or revision',
                    flags=0,
                    ),
    'paths': cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='show aliases for remote repositories',
                    flags=0,
                    ),
    'pull': cmd_data(
                    invocations={'pull...': 'pull %(input)s',
                                 'pull': 'pull',
                        },
                    prompt='Pull source:',
                    enabled=True,
                    syntax_file='',
                    help='pull changes from the specified source',
                    flags=RUN_IN_OWN_CONSOLE,
                    ),
    "push": cmd_data(
                    invocations={'push...': 'push %(input)s',
                                 'push': 'push',
                        },
                    prompt="Push target:",
                    enabled=True,
                    syntax_file='',
                    help='push changes to the specified destination',
                    flags=RUN_IN_OWN_CONSOLE,
                    ),
    "recover": cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='roll back an interrupted transaction',
                    flags=0,
                    ),
    "remove": cmd_data(
                    invocations={'remove (this file)': 'remove "%(file_name)s"',
                        },
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='remove the specified files on the next commit',
                    flags=0,
                    ),
    "rename": cmd_data(
                    invocations={'rename (this file)': 'rename "%(file_name)s" "%(input)s"',
                        },
                    prompt="New name:",
                    enabled=True,
                    syntax_file='',
                    help='rename files; equivalent of copy + remove',
                    flags=0,
                    ),
    "resolve": cmd_data(
                    invocations={'resolve (this file)': 'resolve "%(file_name)s"',
                        },
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='redo merges or set/view the merge status of files',
                    flags=0,
                    ),
    "revert": cmd_data(
                    invocations={'revert (this file)': 'revert "%(file_name)s"',
                        },
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='restore files to their checkout state',
                    flags=0,
                    ),
    "rollback": cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='roll back the last transaction (dangerous)',
                    flags=0,
                    ),
    "root": cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='print the root (top) of the current working directory',
                    flags=0,
                    ),
    "showconfig": cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='show combined config settings from all hgrc files',
                    flags=0,
                    ),
    "status": cmd_data(
                    invocations={'status (this file)': 'status "%(file_name)s"',
                                 'status': 'status',
                        },
                    prompt='',
                    enabled=True,
                    syntax_file='Packages/SublimeHg/Support/Mercurial Status Report.hidden-tmLanguage',
                    help='show changed files in the working directory',
                    flags=0,
                    ),
    "summary": cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='summarize working directory state',
                    flags=0,
                    ),
    "tag": cmd_data(
                    invocations={'tag...': 'tag "%(input)s"',
                        },
                    prompt="Tag name:",
                    enabled=True,
                    syntax_file='',
                    help='add one or more tags for the current or given revision',
                    flags=0,
                    ),
    "tags": cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='list repository tags',
                    flags=0,
                    ),
    "tip": cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='show the tip revision',
                    flags=0,
                    ),
    "update": cmd_data(
                    invocations={'update...': 'update "%(input)s"',
                                 'update': 'update',
                        },
                    prompt="Branch:",
                    enabled=True,
                    syntax_file='',
                    help='update working directory (or switch revisions)',
                    flags=0,
                    ),
    "verify": cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='verify the integrity of the repository',
                    flags=0,
                    ),
    "version": cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='output version and copyright information',
                    flags=0,
                    ),
    "serve": cmd_data(
                    invocations={"serve": "serve"},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='start stand-alone webserver',
                    flags=RUN_IN_OWN_CONSOLE,
                    ),
    "init": cmd_data(
                    invocations={"init (this file's directory)": "init"},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='create a new repository in the given directory',
                    flags=RUN_IN_OWN_CONSOLE,
                    ),
}

# At some point we'll let the user choose whether to load extensions.
HG_COMMANDS['mq'] = {
        "qapplied": cmd_data(
                        invocations={'qapplied': 'qapplied -s',
                                   },
                        prompt='',
                        enabled=True,
                        syntax_file='',
                        help='',
                        flags=0,
                        ),
        "qdiff": cmd_data(
                        invocations={},
                        prompt='',
                        enabled=True,
                        syntax_file='Packages/Diff/Diff.tmLanguage',
                        help='',
                        flags=0,
                        ),
        "qgoto": cmd_data(
                        invocations={'qgoto...':'qgoto "%(input)s"',
                                   },
                        prompt="Patch name:",
                        enabled=True,
                        syntax_file='',
                        help='',
                        flags=0,
                        ),
        "qheader": cmd_data(
                        invocations={'qheader...': 'qheader "%(input)s"',
                                     'qheader': 'qheader',
                                   },
                        prompt="Patch name:",
                        enabled=True,
                        syntax_file='',
                        help='',
                        flags=0,
                        ),
        "qnext": cmd_data(
                        invocations={'qnext': 'qnext -s',
                                   },
                        prompt='',
                        enabled=True,
                        syntax_file='',
                        help='',
                        flags=0,
                        ),
        "qpop": cmd_data(
                        invocations={},
                        prompt='',
                        enabled=True,
                        syntax_file='',
                        help='',
                        flags=0,
                        ),
        "qprev": cmd_data(
                        invocations={'qprev': 'qprev -s',
                                   },
                        prompt='',
                        enabled=True,
                        syntax_file='',
                        help='',
                        flags=0,
                        ),
        "qpush": cmd_data(
                        invocations={},
                        prompt='',
                        enabled=True,
                        syntax_file='',
                        help='',
                        flags=0,
                        ),
        "qrefresh": cmd_data(
                        invocations={'qrefresh... (EDIT commit message': 'qrefresh -e',
                                     'qrefresh... (NEW commit message)': 'qrefresh -m "%(input)s"',
                                     'qrefresh': 'qrefresh',
                                   },
                        prompt='Commit message:',
                        enabled=True,
                        syntax_file='',
                        help='',
                        flags=0,
                        ),
        "qfold": cmd_data(
                        invocations={'qfold...': 'qfold "%(input)s"'},
                        prompt='Patch name:',
                        enabled=True,
                        syntax_file='',
                        help='',
                        flags=0,
                        ),
        "qseries": cmd_data(
                        invocations={'qseries': 'qseries -s',
                                   },
                        prompt='',
                        enabled=True,
                        syntax_file='',
                        help='',
                        flags=0,
                        ),
        "qfinish": cmd_data(
                        invocations={'qfinish': 'qfinish "%(input)s"',
                                   },
                        prompt='Patch name:',
                        enabled=True,
                        syntax_file='',
                        help='',
                        flags=0,
                        ),
        "qnew": cmd_data(
                        invocations={'qnew': 'qnew "%(input)s"',
                                       },
                        prompt='Patch name:',
                        enabled=True,
                        syntax_file='',
                        help='',
                        flags=0,
                        ),
        "qdelete": cmd_data(
                        invocations={'qdelete': 'qdelete "%(input)s"',
                                       },
                        prompt='Patch name:',
                        enabled=True,
                        syntax_file='',
                        help='',
                        flags=0,
                        ),
        "qtop": cmd_data(
                        invocations={'qtop': 'qtop -s',
                                   },
                        prompt='',
                        enabled=True,
                        syntax_file='',
                        help='',
                        flags=0,
                        ),
        "qunapplied": cmd_data(
                        invocations={},
                        prompt='',
                        enabled=True,
                        syntax_file='',
                        help='',
                        flags=0,
                        ),
    }

def format_for_display(extension):
    all_cmds = []
    for name, cmd_data in HG_COMMANDS[extension].iteritems():
        if cmd_data.invocations:
            for display_name, invocation in cmd_data.invocations.iteritems():
                all_cmds.append([display_name, cmd_data.help])
        else:
            all_cmds.append([name, cmd_data.help])

    return sorted(all_cmds, key=lambda x: x[0])

def find_cmd(extensions, search_term):
    candidates = []
    extensions.insert(0, 'default')
    for ext in extensions:
        cmds = HG_COMMANDS[ext]
        for name, cmd_data in cmds.iteritems():
            if search_term in cmd_data.invocations:
                return cmd_data.invocations[search_term], cmd_data
                break
            elif search_term == name:
                return name, cmd_data
                break
            elif name.startswith(search_term):
                candidates.append((name, cmd_data))

    if len(candidates) == 1:
        return candidates[0]

    if len(candidates) > 1:
        raise AmbiguousCommandError
    else:
        raise CommandNotFoundError

def get_commands_by_ext(extensions):
    cmds = []
    for ext in extensions:
        if not ext.lower() == 'default':
            cmds.extend(format_for_display(ext))
    # Make sure we return at least 'default' commands.
    cmds = format_for_display('default') + cmds
    return cmds

HG_COMMANDS_AND_SHORT_HELP = format_for_display("default")
HG_COMMANDS_LIST = [x.replace('.', '') for x in HG_COMMANDS if ' ' not in x]
HG_COMMANDS_LIST = list(sorted(set(HG_COMMANDS_LIST)))
<commit_msg>add help lines<commit_after>from collections import namedtuple

class CommandNotFoundError(Exception):
    pass

class AmbiguousCommandError(Exception):
    pass

cmd_data = namedtuple('cmd_data','invocations prompt enabled syntax_file help flags')

# Flags
RUN_IN_OWN_CONSOLE = 0x02

HG_COMMANDS = {}

HG_COMMANDS['default'] = {
    'commit': cmd_data(
                    invocations={'commit...': 'commit -m "%(input)s"',
                                 'commit... (this file)': 'commit "%(file_name)s" -m "%(input)s"',
                                },
                    prompt='Commit message:',
                    enabled=True,
                    syntax_file='',
                    help='commit the specified files or all outstanding changes',
                    flags=0,
                    ),
    'init': cmd_data(
                    invocations={'init': 'init',
                                },
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='create a new repository in the given directory',
                    flags=RUN_IN_OWN_CONSOLE,
                    ),
    'add': cmd_data(
                    invocations={'add (this file)': 'add "%(file_name)s"',
                                 'add': 'add',
                               },
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='add the specified files on the next commit',
                    flags=0,
                    ),
    'addremove': cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='add all new files, delete all missing files',
                    flags=0,
                    ),
    'annotate': cmd_data(
                    invocations={'annotate (this file)': 'annotate "%(file_name)s"',
                                'blame (this file)': 'annotate "%(file_name)s"',
                               },
                    prompt='',
                    enabled=True,
                    syntax_file='Packages/SublimeHg/Support/Mercurial Annotate.hidden-tmLanguage',
                    help='show changeset information by line for each file',
                    flags=0,
                    ),
    'bookmark': cmd_data(
                    invocations={'bookmark (parent revision)': 'bookmark "%(input)s"',
                                },
                    prompt='Bookmark name:',
                    enabled=True,
                    syntax_file='',
                    help='track a line of development with movable markers',
                    flags=0,
                    ),
    'bookmarks': cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='track a line of development with movable markers',
                    flags=0,
                    ),
    'branch': cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='set or show the current branch name',
                    flags=0,
                    ),
    'branches': cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='list repository named branches',
                    flags=0,
                    ),
    'diff': cmd_data(
                    invocations={'diff (this file)': 'diff "%(file_name)s"',
                                 'diff': 'diff',
                        },
                    prompt='',
                    enabled=True,
                    syntax_file='Packages/Diff/Diff.tmLanguage',
                    help='diff repository (or selected files)',
                    flags=0,
                    ),
    'forget': cmd_data(
                    invocations={'forget (this file)': 'forget "%(file_name)s"',
                        },
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='forget the specified files on the next commit',
                    flags=0,
                    ),
    'grep': cmd_data(
                    invocations={'grep...': 'grep "%(input)s"',
                                },
                    prompt='Pattern (grep):',
                    enabled=True,
                    syntax_file='',
                    help='search for a pattern in specified files and revisions',
                    flags=0,
                    ),
    'heads': cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='show current repository heads or show branch heads',
                    flags=0,
                    ),
    'help': cmd_data(
                    invocations={'help...': 'help "%(input)s"',
                                 'help': 'help',
                        },
                    prompt='Help topic:',
                    enabled=True,
                    syntax_file='',
                    help='show help for a given topic or a help overview',
                    flags=0,
                    ),
    'indentify': cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='identify the working copy or specified revision',
                    flags=0,
                    ),
    'incoming': cmd_data(
                    invocations={'incoming...': 'incoming %(input)s',
                                 'incoming': 'incoming',
                        },
                    prompt='Incoming source:',
                    enabled=True,
                    syntax_file='',
                    help='show new changesets found in source',
                    flags=RUN_IN_OWN_CONSOLE,
                    ),
    'locate': cmd_data(
                    invocations={'locate': 'locate "%(input)s"'
                                },
                    prompt='Pattern:',
                    enabled=True,
                    syntax_file='',
                    help='locate files matching specific patterns',
                    flags=0,
                    ),
    'log': cmd_data(
                    invocations={'log (this file)': 'log "%(file_name)s"',
                                 'log': 'log',
                                },
                    prompt='',
                    enabled=True,
                    syntax_file='Packages/SublimeHg/Support/Mercurial Log.hidden-tmLanguage',
                    help='show revision history of entire repository or files',
                    flags=0,
                    ),
    'manifest': cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='output the current or given revision of the project manifest',
                    flags=0,
                    ),
    'merge': cmd_data(
                    invocations={'merge...': 'merge "%(input)s"',
                                 'merge': 'merge',
                                },
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='merge working directory with another revision',
                    flags=0,
                    ),
    'outgoing': cmd_data(
                    invocations={'outgoing...': 'outgoing %(input)s',
                                 'outgoing': 'outgoing',
                        },
                    prompt='Outgoing target:',
                    enabled=True,
                    syntax_file='Packages/SublimeHg/Support/Mercurial Log.hidden-tmLanguage',
                    help='show changesets not found in the destination',
                    flags=RUN_IN_OWN_CONSOLE,
                    ),
    'parents': cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='show the parents of the working directory or revision',
                    flags=0,
                    ),
    'paths': cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='show aliases for remote repositories',
                    flags=0,
                    ),
    'pull': cmd_data(
                    invocations={'pull...': 'pull %(input)s',
                                 'pull': 'pull',
                        },
                    prompt='Pull source:',
                    enabled=True,
                    syntax_file='',
                    help='pull changes from the specified source',
                    flags=RUN_IN_OWN_CONSOLE,
                    ),
    "push": cmd_data(
                    invocations={'push...': 'push %(input)s',
                                 'push': 'push',
                        },
                    prompt="Push target:",
                    enabled=True,
                    syntax_file='',
                    help='push changes to the specified destination',
                    flags=RUN_IN_OWN_CONSOLE,
                    ),
    "recover": cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='roll back an interrupted transaction',
                    flags=0,
                    ),
    "remove": cmd_data(
                    invocations={'remove (this file)': 'remove "%(file_name)s"',
                        },
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='remove the specified files on the next commit',
                    flags=0,
                    ),
    "rename": cmd_data(
                    invocations={'rename (this file)': 'rename "%(file_name)s" "%(input)s"',
                        },
                    prompt="New name:",
                    enabled=True,
                    syntax_file='',
                    help='rename files; equivalent of copy + remove',
                    flags=0,
                    ),
    "resolve": cmd_data(
                    invocations={'resolve (this file)': 'resolve "%(file_name)s"',
                        },
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='redo merges or set/view the merge status of files',
                    flags=0,
                    ),
    "revert": cmd_data(
                    invocations={'revert (this file)': 'revert "%(file_name)s"',
                        },
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='restore files to their checkout state',
                    flags=0,
                    ),
    "rollback": cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='roll back the last transaction (dangerous)',
                    flags=0,
                    ),
    "root": cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='print the root (top) of the current working directory',
                    flags=0,
                    ),
    "showconfig": cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='show combined config settings from all hgrc files',
                    flags=0,
                    ),
    "status": cmd_data(
                    invocations={'status (this file)': 'status "%(file_name)s"',
                                 'status': 'status',
                        },
                    prompt='',
                    enabled=True,
                    syntax_file='Packages/SublimeHg/Support/Mercurial Status Report.hidden-tmLanguage',
                    help='show changed files in the working directory',
                    flags=0,
                    ),
    "summary": cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='summarize working directory state',
                    flags=0,
                    ),
    "tag": cmd_data(
                    invocations={'tag...': 'tag "%(input)s"',
                        },
                    prompt="Tag name:",
                    enabled=True,
                    syntax_file='',
                    help='add one or more tags for the current or given revision',
                    flags=0,
                    ),
    "tags": cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='list repository tags',
                    flags=0,
                    ),
    "tip": cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='show the tip revision',
                    flags=0,
                    ),
    "update": cmd_data(
                    invocations={'update...': 'update "%(input)s"',
                                 'update': 'update',
                        },
                    prompt="Branch:",
                    enabled=True,
                    syntax_file='',
                    help='update working directory (or switch revisions)',
                    flags=0,
                    ),
    "verify": cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='verify the integrity of the repository',
                    flags=0,
                    ),
    "version": cmd_data(
                    invocations={},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='output version and copyright information',
                    flags=0,
                    ),
    "serve": cmd_data(
                    invocations={"serve": "serve"},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='start stand-alone webserver',
                    flags=RUN_IN_OWN_CONSOLE,
                    ),
    "init": cmd_data(
                    invocations={"init (this file's directory)": "init"},
                    prompt='',
                    enabled=True,
                    syntax_file='',
                    help='create a new repository in the given directory',
                    flags=RUN_IN_OWN_CONSOLE,
                    ),
}

# At some point we'll let the user choose whether to load extensions.
HG_COMMANDS['mq'] = {
        "qapplied": cmd_data(
                        invocations={'qapplied': 'qapplied -s',
                                   },
                        prompt='',
                        enabled=True,
                        syntax_file='',
                        help='print the patches already applied',
                        flags=0,
                        ),
        "qdiff": cmd_data(
                        invocations={},
                        prompt='',
                        enabled=True,
                        syntax_file='Packages/Diff/Diff.tmLanguage',
                        help='diff of the current patch and subsequent modifications',
                        flags=0,
                        ),
        "qgoto": cmd_data(
                        invocations={'qgoto...':'qgoto "%(input)s"',
                                   },
                        prompt="Patch name:",
                        enabled=True,
                        syntax_file='',
                        help='push or pop patches until named patch is at top of stack',
                        flags=0,
                        ),
        "qheader": cmd_data(
                        invocations={'qheader...': 'qheader "%(input)s"',
                                     'qheader': 'qheader',
                                   },
                        prompt="Patch name:",
                        enabled=True,
                        syntax_file='',
                        help='print the header of the topmost or specified patch',
                        flags=0,
                        ),
        "qnext": cmd_data(
                        invocations={'qnext': 'qnext -s',
                                   },
                        prompt='',
                        enabled=True,
                        syntax_file='',
                        help='print the name of the next pushable patch',
                        flags=0,
                        ),
        "qpop": cmd_data(
                        invocations={},
                        prompt='',
                        enabled=True,
                        syntax_file='',
                        help='pop the current patch off the stack',
                        flags=0,
                        ),
        "qprev": cmd_data(
                        invocations={'qprev': 'qprev -s',
                                   },
                        prompt='',
                        enabled=True,
                        syntax_file='',
                        help='print the name of the preceding applied patch',
                        flags=0,
                        ),
        "qpush": cmd_data(
                        invocations={},
                        prompt='',
                        enabled=True,
                        syntax_file='',
                        help='push the next patch onto the stack',
                        flags=0,
                        ),
        "qrefresh": cmd_data(
                        invocations={'qrefresh... (EDIT commit message': 'qrefresh -e',
                                     'qrefresh... (NEW commit message)': 'qrefresh -m "%(input)s"',
                                     'qrefresh': 'qrefresh',
                                   },
                        prompt='Commit message:',
                        enabled=True,
                        syntax_file='',
                        help='update the current patch',
                        flags=0,
                        ),
        "qfold": cmd_data(
                        invocations={'qfold...': 'qfold "%(input)s"'},
                        prompt='Patch name:',
                        enabled=True,
                        syntax_file='',
                        help='fold the named patches into the current patch',
                        flags=0,
                        ),
        "qseries": cmd_data(
                        invocations={'qseries': 'qseries -s',
                                   },
                        prompt='',
                        enabled=True,
                        syntax_file='',
                        help='print the entire series file',
                        flags=0,
                        ),
        "qfinish": cmd_data(
                        invocations={'qfinish': 'qfinish "%(input)s"',
                                   },
                        prompt='Patch name:',
                        enabled=True,
                        syntax_file='',
                        help='move applied patches into repository history',
                        flags=0,
                        ),
        "qnew": cmd_data(
                        invocations={'qnew': 'qnew "%(input)s"',
                                       },
                        prompt='Patch name:',
                        enabled=True,
                        syntax_file='',
                        help='create a new patch',
                        flags=0,
                        ),
        "qdelete": cmd_data(
                        invocations={'qdelete...': 'qdelete "%(input)s"',
                                       },
                        prompt='Patch name:',
                        enabled=True,
                        syntax_file='',
                        help='remove patches from queue',
                        flags=0,
                        ),
        "qtop": cmd_data(
                        invocations={'qtop': 'qtop -s',
                                   },
                        prompt='',
                        enabled=True,
                        syntax_file='',
                        help='print the name of the current patch',
                        flags=0,
                        ),
        "qunapplied": cmd_data(
                        invocations={},
                        prompt='',
                        enabled=True,
                        syntax_file='',
                        help='print the patches not yet applied',
                        flags=0,
                        ),
    }

def format_for_display(extension):
    all_cmds = []
    for name, cmd_data in HG_COMMANDS[extension].iteritems():
        if cmd_data.invocations:
            for display_name, invocation in cmd_data.invocations.iteritems():
                all_cmds.append([display_name, cmd_data.help])
        else:
            all_cmds.append([name, cmd_data.help])

    return sorted(all_cmds, key=lambda x: x[0])

def find_cmd(extensions, search_term):
    candidates = []
    extensions.insert(0, 'default')
    for ext in extensions:
        cmds = HG_COMMANDS[ext]
        for name, cmd_data in cmds.iteritems():
            if search_term in cmd_data.invocations:
                return cmd_data.invocations[search_term], cmd_data
                break
            elif search_term == name:
                return name, cmd_data
                break
            elif name.startswith(search_term):
                candidates.append((name, cmd_data))

    if len(candidates) == 1:
        return candidates[0]

    if len(candidates) > 1:
        raise AmbiguousCommandError
    else:
        raise CommandNotFoundError

def get_commands_by_ext(extensions):
    cmds = []
    for ext in extensions:
        if not ext.lower() == 'default':
            cmds.extend(format_for_display(ext))
    # Make sure we return at least 'default' commands.
    cmds = format_for_display('default') + cmds
    return cmds

HG_COMMANDS_AND_SHORT_HELP = format_for_display("default")
HG_COMMANDS_LIST = [x.replace('.', '') for x in HG_COMMANDS if ' ' not in x]
HG_COMMANDS_LIST = list(sorted(set(HG_COMMANDS_LIST)))
<|endoftext|>


<commit_before>
import numpy as np

def make_cell_division_times(n_divisions, n_replicates=8, std=.1, seed=None, drop_p=.05, maxtime=10):
    """
    Simulate Cell division times for n_divisions. The division times are drawn between 0 and maxtime.
    """
    seed = seed or np.random.randint(1000,10000)
    np.random.seed(seed)

    # Make random blobs of data with varying varainces, dependend on blob size.
    # This reflects cells of similar expression, and therefore, expresses cell stages/differentiation.
    class_sizes = n_replicates*(2**np.arange(0, n_divisions+1))#np.diff(np.r_[0, class_centers, n_data])
    n_data = class_sizes.sum()

    class_centers = (np.linspace(0,n_data,n_divisions+2)).astype(int)
    class_centers = np.int64((class_centers[:-1] + class_centers[1:])/2)

    class_vars = np.ones(class_sizes.size)*std#(max_variance/class_sizes.max())*class_sizes
    #class_vars = (std/class_sizes.max())*class_sizes
    # Each cell has their own center, such that the higher
    # cell stages have a plateue distribution (multimodal normal distribution)
    # such that biological variation is captured better
    t = []
    for mu, st, size in zip(np.exp(np.linspace(0,np.log(maxtime),n_data)[class_centers]), class_vars, class_sizes):
        stage = size/n_replicates
        offset = (2*st)#/float(stage)
        # make cell centers for each cell stage:
        # cell_centers = np.random.normal(mu, var/2., stage)
        cell_centers = np.random.uniform((1-.1)*mu, (1+.1)*mu, stage)
        # Then make replicates around the centers
        cell_individuals = np.random.uniform(cell_centers-offset, cell_centers+offset, (n_replicates,stage)).flatten()
        # cell_individuals = np.random.normal(cell_centers, var, (n_replicates,stage)).flatten()
        #cell_individuals = np.random.uniform((1-offset)*mu, (1+offset)*mu, n_replicates*stage)
        t = np.r_[t, cell_individuals]
    t = t[:,None]
    t.sort(0)

    labels = []
    cell_stage = 1
    for size in class_sizes:
        labels.extend([cell_stage]*size)
        cell_stage *= 2
    labels = np.r_[labels]

    # Now we want to simulate some techincal drop out events,
    # where cells where destroyed or fell out due to technical variation,
    # Thus, we go hyper geometric, and the first cell stage cannot
    # fall out:
    #dropout_p = np.zeros(n_replicates)
    #for i in range(1,n_divisions+1):
    #    stage_size = n_replicates*(2**i)
    #    dropout_p = np.r_[dropout_p, np.repeat(2.0**i, stage_size)]
    #dropout_p /= dropout_p.max()
    #dropout_p *= drop_p

    dropout_p = np.ones(n_data) * drop_p
    dropout_p[:n_replicates] = 0

    dropouts = np.random.binomial(1, 1.-dropout_p).astype(bool)

    return t[dropouts], labels[dropouts], seed

import GPy
from GPy.util import diag
from scipy.stats import norm

def simulate_latent_space(t, labels, seed=None, var=.2, split_prob=.1):
    # t needs to be sorted!
    seed = seed or np.random.randint(1000,10000)
    np.random.seed(seed)

    n_data = t.shape[0]

    ulabs = [labels[0]]

    newlabs = []

    for x in range(n_data):
        if labels[x] != ulabs[-1]:
            ulabs.append(labels[x])

    Xsim = np.zeros((n_data, 2))
    split_ends = [Xsim[0]]
    prev_ms = [[.1,.1]]
    split_end_times = [t[labels==ulabs[0].max()]]
    tmax = t.max()

    for lab in ulabs:
        fil = (lab==labels).nonzero()[0]

        # zero out, for simulating linear relation within cluster:
        new_se = []
        new_m = []
        new_set = []

        splits = np.array_split(fil, len(split_ends))

        i = 1
        for s in range(len(split_ends)):
            # for all previously done splits:
            prev_m = prev_ms[s]
            split = splits[s]
            split_end = split_ends[s]
            split_end_time = split_end_times[s]

            pre_theta = None
            prev_split_time = None
            for split in np.array_split(split, np.random.binomial(1, split_prob)+1):
                newlabs.extend(["{} {}".format(_c, i) for _c in labels[split]])
                i += 1
                # If we split a collection into two, we want the two times to match up now:
                if prev_split_time is None:
                    prev_split_time = t[split].ptp()
                else:
                    t[split.min():] -= prev_split_time
                t[split] -= (t[split.min()]-split_end_time)
                x = t[split]
                x -= x.min()
                x /= x.max()
                x *= np.sqrt(lab)

                # rotate m away a little from the previous direction:
                if pre_theta is None:
                    pre_theta = theta = np.random.uniform(-45, 45)
                else:
                    theta = ((pre_theta+90)%90)-90
                theta *= (np.pi/180.) # radians for rotation matrix
                rot_m = np.array([[np.cos(theta), -np.sin(theta)], [np.sin(theta), np.cos(theta)]])
                m = np.dot(rot_m, prev_m)

                #print (m*x).shape, m, x.shape, Xsim[split].shape
                v = (x.mean(0) - np.abs((-x+x.mean(0))))
                v -= v.min(0)-1e-6
                v /= v.max(0)
                v *= var*t[split]/tmax

                Xsim[split] = np.random.normal(split_end + m*x, v)
                #new_se.append(np.percentile(Xsim[split], 100, 0))
                p = (m*x[-1])
                p /= 4*np.sqrt(GPy.util.linalg.tdot(p))
                new_se.append(split_end + (m*x[-1]) + p)
                new_m.append(m)
                new_set.append(t[split.max()])

        split_ends = new_se
        prev_ms = new_m
        split_end_times = new_set
        # The split probability goes up every time the cell stage changes:

        split_prob = min(1., split_prob*2)

    Xsim -= Xsim.mean(0)
    Xsim /= Xsim.std(0)
    #Xsim += np.random.normal(0,var,Xsim.shape)

    return Xsim, seed, np.asarray(newlabs)

def simulate_new_Y(Xsim, t, p_dims, num_classes=10,noise_var=.2):
    n_data = Xsim.shape[0]
    Y = np.empty((n_data, p_dims))

    splits = np.random.choice(p_dims, replace=False, size=num_classes)
    splits.sort()

    for sub in np.array_split(range(p_dims), splits):
        ky_sim = (GPy.kern.RBF(1, variance=1., lengthscale=5, active_dims=[2])
                  + GPy.kern.RBF(2, variance=1., lengthscale=.7)
                  + GPy.kern.Linear(2, variances=.01)
                  + GPy.kern.RBF(2, ARD=True, variance=1, lengthscale=[np.random.uniform(5,7),np.random.uniform(3,5)])
                  #+ GPy.kern.LogisticBasisFuncKernel(2, np.random.uniform(0,10), variance=1, slope=1, active_dims=[1,2])
                  + GPy.kern.White(3,variance=(noise_var)**2)
                  )
        Ky_sim = ky_sim.K(np.c_[Xsim, t])
        Y[:, sub] = np.random.multivariate_normal(np.zeros(n_data), Ky_sim, 2).T.dot(np.random.normal(0,1,(2, sub.size)))
    Y -= Y.mean(0)
    Y /= Y.std(0)
    return Y

def guo_simulation(p_dims=48, n_divisions=6, seed=None):
    t, labels, seed = make_cell_division_times(n_divisions, n_replicates=9, seed=seed, std=.03, drop_p=.6)
    c = np.log2(labels) / n_divisions
    #c = t
    xvar = .6
    Xsim, seed, labels = simulate_latent_space(t, labels, var=xvar, seed=seed, split_prob=.01)
    def simulate_new():
        return simulate_new_Y(Xsim, t, p_dims, num_classes=48, noise_var=.7)
    return Xsim, simulate_new, t, c, labels, seed<commit_msg>[simulations] redoing guo simulation<commit_after>
import numpy as np

def make_cell_division_times(n_divisions, n_replicates=8, std=.1, seed=None, drop_p=.05, maxtime=10):
    """
    Simulate Cell division times for n_divisions. The division times are drawn between 0 and maxtime.
    """
    seed = seed or np.random.randint(1000,10000)
    np.random.seed(seed)

    # Make random blobs of data with varying varainces, dependend on blob size.
    # This reflects cells of similar expression, and therefore, expresses cell stages/differentiation.
    class_sizes = n_replicates*(2**np.arange(0, n_divisions+1))#np.diff(np.r_[0, class_centers, n_data])
    n_data = class_sizes.sum()

    class_centers = (np.linspace(0,n_data,n_divisions+2)).astype(int)
    class_centers = np.int64((class_centers[:-1] + class_centers[1:])/2)

    class_vars = np.ones(class_sizes.size)*std#(max_variance/class_sizes.max())*class_sizes
    #class_vars = (std/class_sizes.max())*class_sizes
    # Each cell has their own center, such that the higher
    # cell stages have a plateue distribution (multimodal normal distribution)
    # such that biological variation is captured better
    t = []
    for mu, st, size in zip(np.exp(np.linspace(0,np.log(maxtime),n_data)[class_centers]), class_vars, class_sizes):
        stage = size/n_replicates
        offset = (2*st)#/float(stage)
        # make cell centers for each cell stage:
        # cell_centers = np.random.normal(mu, var/2., stage)
        cell_centers = np.random.uniform((1-.1)*mu, (1+.1)*mu, stage)
        # Then make replicates around the centers
        cell_individuals = np.random.uniform(cell_centers-offset, cell_centers+offset, (n_replicates,stage)).flatten()
        # cell_individuals = np.random.normal(cell_centers, var, (n_replicates,stage)).flatten()
        #cell_individuals = np.random.uniform((1-offset)*mu, (1+offset)*mu, n_replicates*stage)
        t = np.r_[t, cell_individuals]
    t = t[:,None]
    t.sort(0)

    labels = []
    cell_stage = 1
    for size in class_sizes:
        labels.extend([cell_stage]*size)
        cell_stage *= 2
    labels = np.r_[labels]

    # Now we want to simulate some techincal drop out events,
    # where cells where destroyed or fell out due to technical variation,
    # Thus, we go hyper geometric, and the first cell stage cannot
    # fall out:
    #dropout_p = np.zeros(n_replicates)
    #for i in range(1,n_divisions+1):
    #    stage_size = n_replicates*(2**i)
    #    dropout_p = np.r_[dropout_p, np.repeat(2.0**i, stage_size)]
    #dropout_p /= dropout_p.max()
    #dropout_p *= drop_p

    dropout_p = np.ones(n_data) * drop_p
    dropout_p[:n_replicates] = 0

    dropouts = np.random.binomial(1, 1.-dropout_p).astype(bool)

    return t[dropouts], labels[dropouts], seed

import GPy
from GPy.util import diag
from scipy.stats import norm

def simulate_latent_space(t, labels, seed=None, var=.2, split_prob=.1):
    # t needs to be sorted!
    seed = seed or np.random.randint(1000,10000)
    np.random.seed(seed)

    n_data = t.shape[0]

    ulabs = [labels[0]]

    newlabs = []

    for x in range(n_data):
        if labels[x] != ulabs[-1]:
            ulabs.append(labels[x])

    Xsim = np.zeros((n_data, 2))
    split_ends = [Xsim[0]]
    prev_ms = [[.1,.1]]
    split_end_times = [t[labels==ulabs[0].max()]]
    tmax = t.max()

    for lab in ulabs:
        fil = (lab==labels).nonzero()[0]

        # zero out, for simulating linear relation within cluster:
        new_se = []
        new_m = []
        new_set = []

        splits = np.array_split(fil, len(split_ends))

        i = 1
        for s in range(len(split_ends)):
            # for all previously done splits:
            prev_m = prev_ms[s]
            split = splits[s]
            split_end = split_ends[s]
            split_end_time = split_end_times[s]

            pre_theta = None
            prev_split_time = None
            for split in np.array_split(split, np.random.binomial(1, split_prob)+1):
                newlabs.extend(["{} {}".format(_c, i) for _c in labels[split]])
                i += 1
                # If we split a collection into two, we want the two times to match up now:
                if prev_split_time is None:
                    prev_split_time = t[split].ptp()
                else:
                    t[split.min():] -= prev_split_time
                t[split] -= (t[split.min()]-split_end_time)
                x = t[split]
                x -= x.min()
                x /= x.max()
                x *= np.sqrt(lab)

                # rotate m away a little from the previous direction:
                if pre_theta is None:
                    pre_theta = theta = np.random.uniform(-45, 45)
                else:
                    theta = ((pre_theta+90)%90)-90
                theta *= (np.pi/180.) # radians for rotation matrix
                rot_m = np.array([[np.cos(theta), -np.sin(theta)], [np.sin(theta), np.cos(theta)]])
                m = np.dot(rot_m, prev_m)

                #print (m*x).shape, m, x.shape, Xsim[split].shape
                v = (x.mean(0) - np.abs((-x+x.mean(0))))
                v -= v.min(0)-1e-6
                v /= v.max(0)
                v *= var*t[split]/tmax

                Xsim[split] = np.random.normal(split_end + m*x, v)
                #new_se.append(np.percentile(Xsim[split], 100, 0))
                p = (m*x[-1])
                p /= 4*np.sqrt(GPy.util.linalg.tdot(p))
                new_se.append(split_end + (m*x[-1]) + p)
                new_m.append(m)
                new_set.append(t[split.max()])

        split_ends = new_se
        prev_ms = new_m
        split_end_times = new_set
        # The split probability goes up every time the cell stage changes:

        split_prob = min(1., split_prob*2)

    Xsim -= Xsim.mean(0)
    Xsim /= Xsim.std(0)
    #Xsim += np.random.normal(0,var,Xsim.shape)

    return Xsim, seed, np.asarray(newlabs)

def simulate_new_Y(Xsim, t, p_dims, num_classes=10,noise_var=.2):
    n_data = Xsim.shape[0]
    Y = np.empty((n_data, p_dims))

    splits = np.random.choice(p_dims, replace=False, size=num_classes)
    splits.sort()

    #for sub in np.array_split(range(p_dims), splits):
    for sub in np.array_split(range(p_dims), splits):
        ky_sim = (GPy.kern.RBF(1, variance=1e-6, lengthscale=5, active_dims=[2])
                  + GPy.kern.RBF(2, variance=.4, lengthscale=.7)
                  + GPy.kern.Linear(2, variances=.1)
                  + GPy.kern.RBF(2, ARD=True, variance=1, lengthscale=[9,5])
                  #+ GPy.kern.LogisticBasisFuncKernel(2, np.random.uniform(0,10), variance=1, slope=1, active_dims=[1,2])
                  + GPy.kern.White(3,variance=(noise_var)**2)
                  )
        Ky_sim = ky_sim.K(np.c_[Xsim, t])
        Y[:, sub] = np.random.multivariate_normal(np.zeros(n_data), Ky_sim, 2).T.dot(np.random.normal(0,1,(2, sub.size)))
    #Ky_sim = ky_sim.K(np.c_[Xsim, t])
    #Y = np.random.multivariate_normal(np.zeros(n_data), Ky_sim, p_dims).T
    Y -= Y.mean(0)
    Y /= Y.std(0)
    return Y

def simulate_new_Y_old(Xsim, t, p_dims, num_classes=10,noise_var=.2):
    n_data = Xsim.shape[0]
    Y = np.empty((n_data, p_dims))

    splits = np.random.choice(p_dims, replace=False, size=num_classes)
    splits.sort()

    for sub in np.array_split(range(p_dims), splits):
        ky_sim = (GPy.kern.RBF(1, variance=1., lengthscale=5, active_dims=[2])
                  + GPy.kern.RBF(2, variance=1., lengthscale=.7)
                  + GPy.kern.Linear(2, variances=.01)
                  + GPy.kern.RBF(2, ARD=True, variance=1, lengthscale=[np.random.uniform(5,7),np.random.uniform(3,5)])
                  #+ GPy.kern.LogisticBasisFuncKernel(2, np.random.uniform(0,10), variance=1, slope=1, active_dims=[1,2])
                  + GPy.kern.White(3,variance=(noise_var)**2)
                  )
        Ky_sim = ky_sim.K(np.c_[Xsim, t])
        Y[:, sub] = np.random.multivariate_normal(np.zeros(n_data), Ky_sim, 2).T.dot(np.random.normal(0,1,(2, sub.size)))
    Y -= Y.mean(0)
    Y /= Y.std(0)
    return Y

def guo_simulation(p_dims=48, n_divisions=6, seed=None):
    t, labels, seed = make_cell_division_times(n_divisions, n_replicates=9, seed=seed, std=.01, drop_p=.6)
    c = np.log2(labels) / n_divisions
    #c = t
    xvar = 1
    Xsim, seed, labels = simulate_latent_space(t, labels, var=xvar, seed=seed, split_prob=.01)
    def simulate_new():
        return simulate_new_Y(Xsim, t, 48, num_classes=48, noise_var=.37)
    return Xsim, simulate_new, t, c, labels, seed

def guo_simulation_old(p_dims=48, n_divisions=6, seed=None):
    t, labels, seed = make_cell_division_times(n_divisions, n_replicates=9, seed=seed, std=.03, drop_p=.6)
    c = np.log2(labels) / n_divisions
    #c = t
    xvar = .6
    Xsim, seed, labels = simulate_latent_space(t, labels, var=xvar, seed=seed, split_prob=.01)
    def simulate_new():
        return simulate_new_Y(Xsim, t, p_dims, num_classes=48, noise_var=.7)
    return Xsim, simulate_new, t, c, labels, seed<|endoftext|>


<commit_before>from django.utils.translation import ugettext_lazy as _
from django.core.urlresolvers import reverse
from django.db import models

from wafer.snippets.markdown_field import MarkdownTextField

from wafer.talks.models import Talk
from wafer.pages.models import Page

class Venue(models.Model):
    """Information about a venue for conference events"""
    order = models.IntegerField(default=1)

    name = models.CharField(max_length=1024)

    notes = MarkdownTextField(
        help_text=_("Notes or directions that will be useful to"
                    " conference attendees"))

    class Meta:
        ordering = ['order', 'name']

    def __unicode__(self):
        return u'%s' % self.name

    def get_absolute_url(self):
        return reverse('wafer_venue', args=(self.pk,))

class Slot(models.Model):

    # XXX: We're trading flexibility for ease of implementation here.
    #      Revisit this when we have a better idea of the actual
    #      requirements.

    start_time = models.DateTimeField(null=True, blank=True)

    end_time = models.DateTimeField(null=True, blank=True)

    def __unicode__(self):
        return u'Slot: %s - %s' % (self.start_time.isoformat(),
                                   self.end_time.isoformat())

class ScheduleItem(models.Model):

    venue = models.ForeignKey(Venue)

    # Items can span multiple slots (tutorials, etc).
    slots = models.ManyToManyField(Slot)

    talk = models.ForeignKey(Talk, null=True)
    page = models.ForeignKey(Page, null=True)
    details = MarkdownTextField(
        null=False, blank=True,
        help_text=_("Additional details (if required)"))

    notes = models.TextField(
        null=False, blank=True,
        help_text=_("Notes for the conference organisers"))

    def get_desc(self):
        if self.details:
            if self.talk:
                return '%s - %s' % (self.talk.title, self.details)
            elif self.page:
                return '%s - %s' % (self.page.name, self.details)
            return self.details
        elif self.talk:
            return self.talk.title
        elif self.page:
            return self.page.name
        return ''

    def get_url(self):
        if self.talk:
            return self.talk.get_absolute_url()
        elif self.page:
            return self.page.get_absolute_url()
        return None

    def get_details(self):
        return self.get_desc()

    def __unicode__(self):
        return u'%s in %s' % (self.get_desc(), self.venue)
<commit_msg>Tweak required fields on models<commit_after>from django.utils.translation import ugettext_lazy as _
from django.core.urlresolvers import reverse
from django.db import models

from wafer.snippets.markdown_field import MarkdownTextField

from wafer.talks.models import Talk
from wafer.pages.models import Page

class Venue(models.Model):
    """Information about a venue for conference events"""
    order = models.IntegerField(default=1)

    name = models.CharField(max_length=1024)

    notes = MarkdownTextField(
        null=False, blank=True,
        help_text=_("Notes or directions that will be useful to"
                    " conference attendees"))

    class Meta:
        ordering = ['order', 'name']

    def __unicode__(self):
        return u'%s' % self.name

    def get_absolute_url(self):
        return reverse('wafer_venue', args=(self.pk,))

class Slot(models.Model):

    # XXX: We're trading flexibility for ease of implementation here.
    #      Revisit this when we have a better idea of the actual
    #      requirements.

    start_time = models.DateTimeField(null=True, blank=True)

    end_time = models.DateTimeField(null=True, blank=True)

    def __unicode__(self):
        return u'Slot: %s - %s' % (self.start_time.isoformat(),
                                   self.end_time.isoformat())

class ScheduleItem(models.Model):

    venue = models.ForeignKey(Venue)

    # Items can span multiple slots (tutorials, etc).
    slots = models.ManyToManyField(Slot)

    talk = models.ForeignKey(Talk, null=True, blank=True)
    page = models.ForeignKey(Page, null=True, blank=True)
    details = MarkdownTextField(
        null=False, blank=True,
        help_text=_("Additional details (if required)"))

    notes = models.TextField(
        null=False, blank=True,
        help_text=_("Notes for the conference organisers"))

    def get_desc(self):
        if self.details:
            if self.talk:
                return '%s - %s' % (self.talk.title, self.details)
            elif self.page:
                return '%s - %s' % (self.page.name, self.details)
            return self.details
        elif self.talk:
            return self.talk.title
        elif self.page:
            return self.page.name
        return ''

    def get_url(self):
        if self.talk:
            return self.talk.get_absolute_url()
        elif self.page:
            return self.page.get_absolute_url()
        return None

    def get_details(self):
        return self.get_desc()

    def __unicode__(self):
        return u'%s in %s' % (self.get_desc(), self.venue)
<|endoftext|>


<commit_before>from twisted.web.error import Error
from twisted.web import http

from vulcan.utils import safe_format

class RateLimitReached(Exception):
    def __init__(self, retry_seconds):
        self.retry_seconds = retry_seconds
        description = safe_format(
            "Rate limit reached. Retry in {} second{}",
            str(retry_seconds),
            's' if retry_seconds > 1 else '')

        Exception.__init__(self, description)

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return self.__dict__ == other.__dict__

class AuthorizationFailed(Error):
    pass

class TimeoutError(Exception):
    """Raised when time expires in timeout decorator"""

# http://tools.ietf.org/html/rfc6585
TOO_MANY_REQUESTS = 429

RESPONSES = {TOO_MANY_REQUESTS: "Too Many Requests"}
RESPONSES.update(http.RESPONSES)
<commit_msg>provide default code, message for AuthorizationFailed<commit_after>from twisted.web.error import Error
from twisted.web import http
from twisted.web.http import UNAUTHORIZED

from vulcan.utils import safe_format

# http://tools.ietf.org/html/rfc6585
TOO_MANY_REQUESTS = 429

RESPONSES = {TOO_MANY_REQUESTS: "Too Many Requests"}
RESPONSES.update(http.RESPONSES)

class RateLimitReached(Exception):
    def __init__(self, retry_seconds):
        self.retry_seconds = retry_seconds
        description = safe_format(
            "Rate limit reached. Retry in {} second{}",
            str(retry_seconds),
            's' if retry_seconds > 1 else '')

        Exception.__init__(self, description)

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return self.__dict__ == other.__dict__

class AuthorizationFailed(Error):
    def __init__(self,
                 code=http.UNAUTHORIZED,
                 message=RESPONSES[UNAUTHORIZED],
                 response=None):
        Error.__init__(self, code, message, response)

class TimeoutError(Exception):
    """Raised when time expires in timeout decorator"""
<|endoftext|>


<commit_before>import click
from shub.deploy import list_targets

from shub import exceptions as shub_exceptions
from shub_image import utils

SHORT_HELP = "Test a built image with Scrapy Cloud contract"
HELP = """ TODO """

SH_EP_SCRAPY_WARNING = \
    'You should add scrapinghub-entrypoint-scrapy dependency to your' \
    ' requirements.txt or to Dockerfile to run the image with Scrapy Cloud'

@click.command(help=HELP, short_help=SHORT_HELP)
@click.argument("target", required=False, default="default")
@click.option("-l", "--list-targets", help="list available targets",
              is_flag=True, is_eager=True, expose_value=False,
              callback=list_targets)
@click.option("-d", "--debug", help="debug mode", is_flag=True)
@click.option("--version", help="release version")
@click.pass_context
def cli(ctx, target, debug, version):
    ctx.obj = {'debug': debug}
    config = utils.load_release_config()
    image = config.get_image(target)
    version = version or config.get_version()
    image_name = utils.format_image_name(image, version)
    docker_client = utils.get_docker_client()
    for check in [_check_image_exists,
                  _check_start_crawl_entry,
                  _check_list_spiders_entry,
                  _check_sh_entrypoint]:
        check(image_name, docker_client)

def _check_image_exists(image_name, docker_client):
    """Check that the image exists on local machine."""
    # if there's no docker lib, the command will fail earlier
    # with an exception when getting a client in get_docker_client()
    from docker.errors import NotFound
    try:
        docker_client.inspect_image(image_name)
    except NotFound as exc:
        utils.debug_log("{}".format(exc))
        raise shub_exceptions.NotFoundException(
            "The image doesn't exist yet, please use build command at first.")

def _check_sh_entrypoint(image_name, docker_client):
    """Check that the image has scrapinghub-entrypoint-scrapy pkg"""
    status, logs = _run_docker_command(
        docker_client, image_name,
        ['pip', 'show', 'scrapinghub-entrypoint-scrapy'])
    if status != 0 or not logs:
        raise shub_exceptions.NotFoundException(SH_EP_SCRAPY_WARNING)

def _check_list_spiders_entry(image_name, docker_client):
    """Check that the image has list-spiders entrypoint"""
    status, logs = _run_docker_command(
        docker_client, image_name, ['which', 'list-spiders'])
    if status != 0 or not logs:
        raise shub_exceptions.NotFoundException(
            "list-spiders command is not found in the image.\n"
            "Please upgrade your scrapinghub-entrypoint-scrapy.")

def _check_start_crawl_entry(image_name, docker_client):
    """Check that the image has start-crawl entrypoint"""
    status, logs = _run_docker_command(
        docker_client, image_name, ['which', 'start-crawl'])
    if status != 0 or not logs:
        raise shub_exceptions.NotFoundException(
            "start-crawl command is not found in the image.\n"
            + SH_EP_SCRAPY_WARNING)

def _run_docker_command(client, image_name, command):
    """A helper to execute an arbitrary cmd with given docker image"""
    container = client.create_container(image=image_name, command=command)
    client.start(container)
    statuscode = client.wait(container=container['Id'])
    logs = client.logs(container=container['Id'], stdout=True,
                       stderr=True if statuscode else False,
                       stream=False, timestamps=False)
    utils.debug_log("{} results:\n{}".format(command, logs))
    return statuscode, logs
<commit_msg>Specify sh-entrypoint-scrapy version<commit_after>import click
from shub.deploy import list_targets

from shub import exceptions as shub_exceptions
from shub_image import utils

SHORT_HELP = "Test a built image with Scrapy Cloud contract"
HELP = """ TODO """

SH_EP_SCRAPY_WARNING = \
    'You should add scrapinghub-entrypoint-scrapy(>=0.7.0) dependency' \
    ' to your requirements.txt or to Dockerfile to run the image with' \
    ' Scrapy Cloud.'

@click.command(help=HELP, short_help=SHORT_HELP)
@click.argument("target", required=False, default="default")
@click.option("-l", "--list-targets", help="list available targets",
              is_flag=True, is_eager=True, expose_value=False,
              callback=list_targets)
@click.option("-d", "--debug", help="debug mode", is_flag=True)
@click.option("--version", help="release version")
@click.pass_context
def cli(ctx, target, debug, version):
    ctx.obj = {'debug': debug}
    config = utils.load_release_config()
    image = config.get_image(target)
    version = version or config.get_version()
    image_name = utils.format_image_name(image, version)
    docker_client = utils.get_docker_client()
    for check in [_check_image_exists,
                  _check_start_crawl_entry,
                  _check_list_spiders_entry,
                  _check_sh_entrypoint]:
        check(image_name, docker_client)

def _check_image_exists(image_name, docker_client):
    """Check that the image exists on local machine."""
    # if there's no docker lib, the command will fail earlier
    # with an exception when getting a client in get_docker_client()
    from docker.errors import NotFound
    try:
        docker_client.inspect_image(image_name)
    except NotFound as exc:
        utils.debug_log("{}".format(exc))
        raise shub_exceptions.NotFoundException(
            "The image doesn't exist yet, please use build command at first.")

def _check_sh_entrypoint(image_name, docker_client):
    """Check that the image has scrapinghub-entrypoint-scrapy pkg"""
    status, logs = _run_docker_command(
        docker_client, image_name,
        ['pip', 'show', 'scrapinghub-entrypoint-scrapy'])
    if status != 0 or not logs:
        raise shub_exceptions.NotFoundException(SH_EP_SCRAPY_WARNING)

def _check_list_spiders_entry(image_name, docker_client):
    """Check that the image has list-spiders entrypoint"""
    status, logs = _run_docker_command(
        docker_client, image_name, ['which', 'list-spiders'])
    if status != 0 or not logs:
        raise shub_exceptions.NotFoundException(
            "list-spiders command is not found in the image.\n"
            "Please upgrade your scrapinghub-entrypoint-scrapy(>=0.7.0)")

def _check_start_crawl_entry(image_name, docker_client):
    """Check that the image has start-crawl entrypoint"""
    status, logs = _run_docker_command(
        docker_client, image_name, ['which', 'start-crawl'])
    if status != 0 or not logs:
        raise shub_exceptions.NotFoundException(
            "start-crawl command is not found in the image.\n"
            + SH_EP_SCRAPY_WARNING)

def _run_docker_command(client, image_name, command):
    """A helper to execute an arbitrary cmd with given docker image"""
    container = client.create_container(image=image_name, command=command)
    client.start(container)
    statuscode = client.wait(container=container['Id'])
    logs = client.logs(container=container['Id'], stdout=True,
                       stderr=True if statuscode else False,
                       stream=False, timestamps=False)
    utils.debug_log("{} results:\n{}".format(command, logs))
    return statuscode, logs
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import os, threading, tinify
from urlparse import urlparse
from django.db.models.signals import post_save
from django.templatetags.static import StaticNode
from django.conf import settings

aws_key_id = os.getenv('AWS_ACCESS_KEY_ID')
aws_secret = os.getenv('AWS_SECRET_ACCESS_KEY')
aws_bucket = os.getenv('AWS_STORAGE_BUCKET_NAME')
aws_region = os.getenv('AWS_REGION')
cf_key = os.getenv('CF_API_KEY')
tinify.key = os.getenv('TINYPNG_API_KEY')

# If cloudflare key is not set, then we don't need to import Cloudflare
if cf_key:
    import CloudFlare

class TinyPngOptimizeThread(threading.Thread):
    def __init__(self, instance, **kwargs):
        self.instance = instance
        super(TinyPngOptimizeThread, self).__init__(**kwargs)

    def run(self):
        if tinify.key != None:
            # If aws keys are available, use them to fetch the image and write back
            if aws_key_id and aws_secret and aws_bucket and aws_region:
                source_url_http = urlparse(StaticNode.handle_simple(self.instance.file.name), scheme='http').geturl()
                source_url_https = urlparse(StaticNode.handle_simple(self.instance.file.name), scheme='https').geturl()
                source = tinify.from_url(source_url_https)
                path = "%s/%s" % (aws_bucket, self.instance.file.name)
                source.store(service='s3',aws_access_key_id=aws_key_id,aws_secret_access_key=aws_secret,region=aws_region,path=path)
                if cf_key:
                    cf = CloudFlare.CloudFlare()
                    cf.zones.purge_cache.delete('cf_key', data={'files':[source_url_http, source_url_https]})
            # Else we grab the local image, optimize it and override the local file
            else:
                path = os.getcwd()+self.instance.url
                source = tinify.from_file(path)
                source.to_file(path)
        else:
            print "No tinify key"

<commit_msg>Update models.py<commit_after>from __future__ import unicode_literals

import os, threading, tinify
from urlparse import urlparse
from django.db.models.signals import post_save
from django.templatetags.static import StaticNode
from django.conf import settings

aws_key_id = os.getenv('AWS_ACCESS_KEY_ID')
aws_secret = os.getenv('AWS_SECRET_ACCESS_KEY')
aws_bucket = os.getenv('AWS_STORAGE_BUCKET_NAME')
aws_region = os.getenv('AWS_REGION')
cf_key = os.getenv('CF_API_KEY')
tinify.key = os.getenv('TINYPNG_API_KEY')

# If cloudflare key is not set, then we don't need to import Cloudflare
if cf_key:
    import CloudFlare

class TinyPngOptimizeThread(threading.Thread):
    def __init__(self, instance, **kwargs):
        self.instance = instance
        super(TinyPngOptimizeThread, self).__init__(**kwargs)

    def run(self):
        if tinify.key != None:
            # If aws keys are available, use them to fetch the image and write back
            if aws_key_id and aws_secret and aws_bucket and aws_region:
                source_url_http = urlparse(StaticNode.handle_simple(self.instance.file.name), scheme='http').geturl()
                source_url_https = urlparse(StaticNode.handle_simple(self.instance.file.name), scheme='https').geturl()
                source = tinify.from_url(source_url_https)
                path = "%s/%s" % (aws_bucket, self.instance.file.name)
                source.store(service='s3',aws_access_key_id=aws_key_id,aws_secret_access_key=aws_secret,region=aws_region,path=path)
                if cf_key:
                    cf = CloudFlare.CloudFlare()
                    cf.zones.purge_cache.delete(cf_key, data={'files':[source_url_http, source_url_https]})
            # Else we grab the local image, optimize it and override the local file
            else:
                path = os.getcwd()+self.instance.url
                source = tinify.from_file(path)
                source.to_file(path)
        else:
            print "No tinify key"

<|endoftext|>