<commit_before>import asyncio
import logging
import uuid
import random
from functools import partial
from aiogear.packet import Type
from aiogear.mixin import GearmanProtocolMixin

logger = logging.getLogger(__name__)

class Client(GearmanProtocolMixin, asyncio.Protocol):
    def __init__(self, loop=None):
        super(Client, self).__init__(loop=loop)
        self.transport = None

        self.submit_job = partial(self._submit_job, Type.SUBMIT_JOB)
        self.submit_job_bg = partial(self._submit_job, Type.SUBMIT_JOB_BG)
        self.submit_job_high = partial(self._submit_job, Type.SUBMIT_JOB_HIGH)
        self.submit_job_high_bg = partial(self._submit_job, Type.SUBMIT_JOB_HIGH_BG)
        self.submit_job_low = partial(self._submit_job, Type.SUBMIT_JOB_LOW)
        self.submit_job_low_bg = partial(self._submit_job, Type.SUBMIT_JOB_LOW_BG)

        self.handles = {}
        self._closing = None

    def __del__(self):
        try:
            self.disconnect()
        except AttributeError:
            pass

    @staticmethod
    def uuid():
        # 0x00 is used as delimiter in gearman protocol
        # replace it with random printable character.
        replacement = chr(random.randint(32, 126)).encode('ascii')
        return uuid.uuid4().bytes.replace(b'\0', replacement)

    def connection_made(self, transport):
        self.transport = transport

    def connection_lost(self, exc):
        self.transport = None
        if self._closing:
            self._closing.set_result(exc)

    async def _submit_job(self, packet, name, *args, **kwargs):
        uuid = kwargs.pop('uuid', None)
        if uuid is None:
            uuid = self.uuid()
        self.send(packet, name, uuid, *args)
        job_created = await self.wait_for(Type.JOB_CREATED)
        f = self.wait_for(Type.WORK_COMPLETE, Type.WORK_FAIL, Type.WORK_EXCEPTION)
        handle = job_created.handle
        self.handles[handle] = f
        f.add_done_callback(lambda _: self.handles.pop(handle))
        return job_created

    def submit_job_sched(self, name, dt, *args, **kwargs):
        sched_args = [str(int(x)) for x in dt.strftime('%M %H %d %m %w').split()]
        return self._submit_job(Type.SUBMIT_JOB_SCHED, name, *(sched_args + list(args)), **kwargs)

    def get_status(self, handle):
        self.send(Type.GET_STATUS, handle)
        return self.wait_for(Type.STATUS_RES)

    def get_status_unique(self, uuid):
        self.send(Type.STATUS_RES_UNIQUE, uuid)
        return self.wait_for(Type.STATUS_RES_UNIQUE)

    def option_req(self, option):
        self.send(Type.OPTION_REQ, option)
        return self.wait_for(Type.OPTION_RES, Type.ERROR)

    def wait_job(self, handle):
        f = self.handles.get(handle)
        if not f:
            raise RuntimeError('Unable to find handle {} in handles {}'.format(handle, self.handles))
        return f

    def disconnect(self):
        self.transport.close()
        f = self._closing = self.loop.create_future()
        return f

    close = disconnect
<commit_msg>client: fixed possible race condition on submit_job.<commit_after>import asyncio
import logging
import uuid
import random
from functools import partial
from aiogear.packet import Type
from aiogear.mixin import GearmanProtocolMixin

logger = logging.getLogger(__name__)

class Client(GearmanProtocolMixin, asyncio.Protocol):
    def __init__(self, loop=None):
        super(Client, self).__init__(loop=loop)
        self.transport = None

        self.submit_job = partial(self._submit_job, Type.SUBMIT_JOB)
        self.submit_job_bg = partial(self._submit_job, Type.SUBMIT_JOB_BG)
        self.submit_job_high = partial(self._submit_job, Type.SUBMIT_JOB_HIGH)
        self.submit_job_high_bg = partial(self._submit_job, Type.SUBMIT_JOB_HIGH_BG)
        self.submit_job_low = partial(self._submit_job, Type.SUBMIT_JOB_LOW)
        self.submit_job_low_bg = partial(self._submit_job, Type.SUBMIT_JOB_LOW_BG)

        self.handles = {}
        self._closing = None

    def __del__(self):
        try:
            self.disconnect()
        except AttributeError:
            pass

    @staticmethod
    def uuid():
        # 0x00 is used as delimiter in gearman protocol
        # replace it with random printable character.
        replacement = chr(random.randint(32, 126)).encode('ascii')
        return uuid.uuid4().bytes.replace(b'\0', replacement)

    def connection_made(self, transport):
        self.transport = transport

    def connection_lost(self, exc):
        self.transport = None
        if self._closing:
            self._closing.set_result(exc)

    async def _submit_job(self, packet, name, *args, **kwargs):
        uuid = kwargs.pop('uuid', None)
        if uuid is None:
            uuid = self.uuid()
        jc_f = self.wait_for(Type.JOB_CREATED)
        w_f = self.wait_for(Type.WORK_COMPLETE, Type.WORK_FAIL, Type.WORK_EXCEPTION)
        self.send(packet, name, uuid, *args)
        job_created = await jc_f
        handle = job_created.handle
        self.handles[handle] = w_f
        w_f.add_done_callback(lambda _: self.handles.pop(handle))
        return job_created

    def submit_job_sched(self, name, dt, *args, **kwargs):
        sched_args = [str(int(x)) for x in dt.strftime('%M %H %d %m %w').split()]
        return self._submit_job(Type.SUBMIT_JOB_SCHED, name, *(sched_args + list(args)), **kwargs)

    def get_status(self, handle):
        self.send(Type.GET_STATUS, handle)
        return self.wait_for(Type.STATUS_RES)

    def get_status_unique(self, uuid):
        self.send(Type.STATUS_RES_UNIQUE, uuid)
        return self.wait_for(Type.STATUS_RES_UNIQUE)

    def option_req(self, option):
        self.send(Type.OPTION_REQ, option)
        return self.wait_for(Type.OPTION_RES, Type.ERROR)

    def wait_job(self, handle):
        f = self.handles.get(handle)
        if not f:
            raise RuntimeError('Unable to find handle {} in handles {}'.format(handle, self.handles))
        return f

    def disconnect(self):
        self.transport.close()
        f = self._closing = self.loop.create_future()
        return f

    close = disconnect
<|endoftext|>


<commit_before>import re
import string
import logging

from collections import MutableMapping

from .param import Param
from .uri import Uri
from .utils import gen_str

CONTACT_PATTERNS = [
    # unquoted name
    re.compile('^(?P<name>[a-zA-Z0-9\-\.!%\*_\+`\'~]*)'
               '[ \t]*'
               '<(?P<uri>[^>]+)>'
               '[ \t]*'
               '(?:;(?P<params>[^\?]*))?'),
    # quoted name
    re.compile('^(?:"(?P<name>[^"]+)")'
               '[ \t]*'
               '<(?P<uri>[^>]+)>'
               '[ \t]*'
               '(?:;(?P<params>[^\?]*))?'),
    # no name
    re.compile('(?P<name>)'
               '[ \t]*'
               '(?P<uri>[^ ;]+)'
               '[ \t]*'
               '(?:;(?P<params>[^\?]*))?'),
]

LOG = logging.getLogger(__name__)

class Contact(MutableMapping):
    def __init__(self, *args, **kwargs):
        self._contact = dict(*args, **kwargs)

        params = self._contact.get('params')
        if not params:
            self._contact['params'] = Param()
        if not isinstance(params, Param):
            self._contact['params'] = Param(self._contact['params'])

        uri = self._contact.get('uri')
        if not isinstance(uri, Uri):
            self._contact['uri'] = Uri(self._contact['uri'])

    def add_tag(self):
        if 'tag' not in self._contact['params']:
            self._contact['params']['tag'] = gen_str(16, string.digits + 'abcdef')

    @classmethod
    def from_header(cls, contact):
        for s in CONTACT_PATTERNS:
            m = s.match(contact)
            if m:
                return cls(m.groupdict())
        else:
            raise ValueError('Not valid contact address')

    def __str__(self):
        r = ''
        if self._contact.get('name'):  # Check if name exist and is not empty
            r += '"%s" ' % self._contact['name']
        r += self._contact['uri'].contact_repr()
        params = self._contact['params']
        if params:
            r += ';%s' % str(params)
        return r

    # MutableMapping API
    def __eq__(self, other):
        return self is other

    def __getitem__(self, key):
        return self._contact[key]

    def __setitem__(self, key, value):
        self._contact[key] = value

    def __delitem__(self, key):
        del self._contact[key]

    def __len__(self):
        return len(self._contact)

    def __iter__(self):
        return iter(self._contact)
<commit_msg>Add useful properties to get useful contact information<commit_after>import re
import string
import logging

from collections import MutableMapping

from .param import Param
from .uri import Uri
from .utils import gen_str

CONTACT_PATTERNS = [
    # unquoted name
    re.compile('^(?P<name>[a-zA-Z0-9\-\.!%\*_\+`\'~]*)'
               '[ \t]*'
               '<(?P<uri>[^>]+)>'
               '[ \t]*'
               '(?:;(?P<params>[^\?]*))?'),
    # quoted name
    re.compile('^(?:"(?P<name>[^"]+)")'
               '[ \t]*'
               '<(?P<uri>[^>]+)>'
               '[ \t]*'
               '(?:;(?P<params>[^\?]*))?'),
    # no name
    re.compile('(?P<name>)'
               '[ \t]*'
               '(?P<uri>[^ ;]+)'
               '[ \t]*'
               '(?:;(?P<params>[^\?]*))?'),
]

LOG = logging.getLogger(__name__)

class Contact(MutableMapping):
    def __init__(self, *args, **kwargs):
        self._contact = dict(*args, **kwargs)

        params = self._contact.get('params')
        if not params:
            self._contact['params'] = Param()
        if not isinstance(params, Param):
            self._contact['params'] = Param(self._contact['params'])

        uri = self._contact.get('uri')
        if not isinstance(uri, Uri):
            self._contact['uri'] = Uri(self._contact['uri'])

    def add_tag(self):
        if 'tag' not in self._contact['params']:
            self._contact['params']['tag'] = gen_str(16, string.digits + 'abcdef')

    @classmethod
    def from_header(cls, contact):
        for s in CONTACT_PATTERNS:
            m = s.match(contact)
            if m:
                return cls(m.groupdict())
        else:
            raise ValueError('Not valid contact address')

    def __str__(self):
        r = ''
        if self._contact.get('name'):  # Check if name exist and is not empty
            r += '"%s" ' % self._contact['name']
        r += self._contact['uri'].contact_repr()
        params = self._contact['params']
        if params:
            r += ';%s' % str(params)
        return r

    @property
    def scheme(self):
        return self._contact['uri']['scheme']

    @property
    def transport(self):
        transport = self._contact['params'].get('transport')
        if not transport:
            return 'tcp' if self.scheme == 'sips' else 'udp'
        return transport

    @property
    def host(self):
        return self._contact['uri']['host']

    @property
    def port(self):
        port = self._contact['uri'].get('port')
        if not port:
            if self.scheme == 'sips':
                return 5061
            elif self.transport == 'udp':
                return 5060
            else:
                return 5080
        return port

    @property
    def details(self):
        return self.scheme, self.transport, self.host, self.port

    # MutableMapping API
    def __eq__(self, other):
        return self is other

    def __getitem__(self, key):
        return self._contact[key]

    def __setitem__(self, key, value):
        self._contact[key] = value

    def __delitem__(self, key):
        del self._contact[key]

    def __len__(self):
        return len(self._contact)

    def __iter__(self):
        return iter(self._contact)
<|endoftext|>


<commit_before>import argparse
import asyncio
import logging.config
import multiprocessing
import operator
import os
import signal
import sys
import time
from functools import partial, reduce
from pathlib import Path
from urllib.parse import splittype  # type: ignore
from urllib.request import urlopen

from . import utils
from .core import command
from .core.config import Config
from .core.context import Context, GroupResolver
from .core.plugin import Plugin, search_plugins

parser = argparse.ArgumentParser(prefix_chars='-+')

group = parser.add_mutually_exclusive_group(required=False)
group.add_argument('+g', '++groups', nargs='+', action='append',
                   metavar='GROUP', help='Run groups')
group.add_argument('-g', '--groups', nargs='*', action='append',
                   dest='exclude_groups',
                   metavar='GROUP', help='Run all exclude groups')

parser.add_argument('--multiprocessing', action='store_true')
parser.add_argument('-i', '--interact', action='store_true')
parser.add_argument('-I', '--interact-kernel', action='store_true')
parser.add_argument('-l', '--logging', help='logging level')

PROMPT = "======== Running aioworkers ========\n" \
    "(Press CTRL+C to quit)"

class PidFileType(argparse.FileType):
    def __call__(self, string):
        f = super().__call__(string)
        with f:
            f.write(str(os.getpid()))
        return f

parser.add_argument(
    '--pid-file', help='Process ID file',
    type=PidFileType('w'))

try:
    import uvloop
except ImportError:
    pass
else:
    asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())

context = Context(Config())

def main(*config_files, args=None, config_dirs=(),
         commands=(), config_dict=None):
    cwd = os.getcwd()
    if cwd not in sys.path:
        sys.path.insert(0, cwd)
    context.config.search_dirs.extend(config_dirs)

    if not commands:
        p = Path(sys.argv[0])
        if __package__ in (p.parent.name, p.name):
            commands += __name__,
        elif p.name.startswith('__'):
            commands += p.parent.name,
        else:
            commands += p.name,

    plugins = search_plugins()
    plugins.extend(search_plugins(*commands, force=True))
    for i in plugins:
        i.add_arguments(parser)

    if args is None:
        args, argv = parser.parse_known_args()
        cmds = list(commands)
        while argv and not argv[0].startswith('-'):
            cmds.append(argv.pop(0))
        if getattr(args, 'config', None):
            config_files += tuple(args.config)
        if getattr(args, 'config_stdin', None):
            assert not args.interact, 'Can not be used --config-stdin' \
                                      ' with --interact'
            config_dict = utils.load_from_fd(sys.stdin.buffer)
        if args.logging:
            logging.basicConfig(level=args.logging.upper())
    else:
        cmds, argv = list(commands), []

    config = context.config
    plugins.extend(search_plugins(*cmds))
    for p in plugins:
        args, argv = p.parse_known_args(args=argv, namespace=args)
        config.load(*p.configs)
        config.update(p.get_config())
    cmds = [cmd for cmd in cmds if cmd not in sys.modules]

    config.load(*config_files)
    config_dict and config.update(config_dict)

    def sum_g(list_groups):
        if list_groups:
            return set(reduce(operator.add, list_groups))

    run = partial(
        loop_run,
        group_resolver=GroupResolver(
            include=sum_g(args.groups),
            exclude=sum_g(args.exclude_groups),
            all_groups=args.exclude_groups is not None,
            default=True,
        ), cmds=cmds, argv=argv, ns=args, prompt=PROMPT)

    try:
        if args.multiprocessing:
            with context.processes():
                print(PROMPT)
                logger = multiprocessing.get_logger()
                processes = process_iter(config.get('processes', {}))
                for p in processes:
                    logger.info('Create process %s', p['name'])
                    p['process'] = create_process(p)
                while True:
                    multiprocessing.connection.wait(
                        map(lambda x: x['process'].sentinel, processes),
                    )
                    for p in processes:
                        proc = p['process']  # type: multiprocessing.Process
                        if not proc.is_alive():
                            logger.critical('Recreate process %s', p['name'])
                            p['process'] = create_process(p)
                    time.sleep(1)

        elif args.interact:
            from .core.interact import shell
            args.print = lambda *args: None
            shell(run)
        elif args.interact_kernel:
            from .core.interact import kernel
            kernel(run)
        else:
            run()
    except KeyboardInterrupt:
        pass
    finally:
        sig = signal.SIGTERM
        msg = ''
        while multiprocessing.active_children():
            msg and print(msg)
            for p in multiprocessing.active_children():
                os.kill(p.pid, sig)
            time.sleep(0.3)
            msg = 'killall children'
            sig = signal.SIGKILL

def process_iter(cfg):
    result = []
    for k, v in cfg.items():
        if 'count' in v:
            for i in range(v.get_int('count')):
                result.append({
                    'name': '{}-{}'.format(k, i),
                    'groups': v.get('groups', ()),
                })
        else:
            result.append({
                'name': k,
                'groups': v.get('groups', ()),
            })
    return result

def create_process(cfg):
    p = multiprocessing.Process(
        target=loop_run,
        kwargs=dict(
            group_resolver=GroupResolver(
                include=set(cfg['groups']),
                exclude=set(),
                all_groups=False,
                default=True,
            ),
            process_name=cfg['name'],
        ),
        name=cfg['name'],
        daemon=True,
    )
    p.start()
    return p

def loop_run(
    conf=None, future=None,
    group_resolver=None,
    ns=None, cmds=None,
    argv=None, loop=None,
    prompt=None,
    process_name=None,
):
    if process_name:
        utils.setproctitle(process_name)
    if loop is None:
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)
    if conf:
        context.set_config(conf)
    if loop is not None:
        context.set_loop(loop)
    if group_resolver is not None:
        context.set_group_resolver(group_resolver)

    if not cmds:
        cmds = ['run_forever']
        prompt and print(prompt)
    argv = argv or []
    ns = ns or argparse.Namespace()
    with utils.monkey_close(loop), context:
        if future is not None:
            future.set_result(context)
        for cmd in cmds:
            try:
                result = command.run(cmd, context, argv=argv, ns=ns)
            except command.CommandNotFound:
                print('Command {} not found'.format(cmd))
                continue
            if result is not None:
                print('{} => {}'.format(cmd, result))
        if hasattr(loop, 'shutdown_asyncgens'):
            loop.run_until_complete(loop.shutdown_asyncgens())

class UriType(argparse.FileType):
    def __call__(self, string):
        t, path = splittype(string)
        if not t:
            return super().__call__(string)
        return urlopen(string)

class ExtendAction(argparse.Action):
    def __call__(self, parser, namespace, values, option_string=None):
        getattr(namespace, self.dest).extend(values)

class plugin(Plugin):
    def add_arguments(self, parser):
        default = []
        parser.set_defaults(config=default)
        parser.add_argument(
            '-c', '--config', nargs='+', action=ExtendAction,
            type=UriType('r', encoding='utf-8'),
        )
        parser.add_argument('--config-stdin', action='store_true')

def main_with_conf(*args, **kwargs):
    import warnings
    warnings.warn(
        'Deprecated main_with_conf, use main',
        DeprecationWarning, stacklevel=2,
    )
    main(*args, **kwargs)

if __name__ == '__main__':
    main()
<commit_msg>fix import multiprocessing.connection<commit_after>import argparse
import asyncio
import logging.config
import multiprocessing.connection
import operator
import os
import signal
import sys
import time
from functools import partial, reduce
from pathlib import Path
from urllib.parse import splittype  # type: ignore
from urllib.request import urlopen

from . import utils
from .core import command
from .core.config import Config
from .core.context import Context, GroupResolver
from .core.plugin import Plugin, search_plugins

parser = argparse.ArgumentParser(prefix_chars='-+')

group = parser.add_mutually_exclusive_group(required=False)
group.add_argument('+g', '++groups', nargs='+', action='append',
                   metavar='GROUP', help='Run groups')
group.add_argument('-g', '--groups', nargs='*', action='append',
                   dest='exclude_groups',
                   metavar='GROUP', help='Run all exclude groups')

parser.add_argument('--multiprocessing', action='store_true')
parser.add_argument('-i', '--interact', action='store_true')
parser.add_argument('-I', '--interact-kernel', action='store_true')
parser.add_argument('-l', '--logging', help='logging level')

PROMPT = "======== Running aioworkers ========\n" \
    "(Press CTRL+C to quit)"

class PidFileType(argparse.FileType):
    def __call__(self, string):
        f = super().__call__(string)
        with f:
            f.write(str(os.getpid()))
        return f

parser.add_argument(
    '--pid-file', help='Process ID file',
    type=PidFileType('w'))

try:
    import uvloop
except ImportError:
    pass
else:
    asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())

context = Context(Config())

def main(*config_files, args=None, config_dirs=(),
         commands=(), config_dict=None):
    cwd = os.getcwd()
    if cwd not in sys.path:
        sys.path.insert(0, cwd)
    context.config.search_dirs.extend(config_dirs)

    if not commands:
        p = Path(sys.argv[0])
        if __package__ in (p.parent.name, p.name):
            commands += __name__,
        elif p.name.startswith('__'):
            commands += p.parent.name,
        else:
            commands += p.name,

    plugins = search_plugins()
    plugins.extend(search_plugins(*commands, force=True))
    for i in plugins:
        i.add_arguments(parser)

    if args is None:
        args, argv = parser.parse_known_args()
        cmds = list(commands)
        while argv and not argv[0].startswith('-'):
            cmds.append(argv.pop(0))
        if getattr(args, 'config', None):
            config_files += tuple(args.config)
        if getattr(args, 'config_stdin', None):
            assert not args.interact, 'Can not be used --config-stdin' \
                                      ' with --interact'
            config_dict = utils.load_from_fd(sys.stdin.buffer)
        if args.logging:
            logging.basicConfig(level=args.logging.upper())
    else:
        cmds, argv = list(commands), []

    config = context.config
    plugins.extend(search_plugins(*cmds))
    for p in plugins:
        args, argv = p.parse_known_args(args=argv, namespace=args)
        config.load(*p.configs)
        config.update(p.get_config())
    cmds = [cmd for cmd in cmds if cmd not in sys.modules]

    config.load(*config_files)
    config_dict and config.update(config_dict)

    def sum_g(list_groups):
        if list_groups:
            return set(reduce(operator.add, list_groups))

    run = partial(
        loop_run,
        group_resolver=GroupResolver(
            include=sum_g(args.groups),
            exclude=sum_g(args.exclude_groups),
            all_groups=args.exclude_groups is not None,
            default=True,
        ), cmds=cmds, argv=argv, ns=args, prompt=PROMPT)

    try:
        if args.multiprocessing:
            with context.processes():
                print(PROMPT)
                logger = multiprocessing.get_logger()
                processes = process_iter(config.get('processes', {}))
                for p in processes:
                    logger.info('Create process %s', p['name'])
                    p['process'] = create_process(p)
                while True:
                    multiprocessing.connection.wait(
                        map(lambda x: x['process'].sentinel, processes),
                    )
                    for p in processes:
                        proc = p['process']  # type: multiprocessing.Process
                        if not proc.is_alive():
                            logger.critical('Recreate process %s', p['name'])
                            p['process'] = create_process(p)
                    time.sleep(1)

        elif args.interact:
            from .core.interact import shell
            args.print = lambda *args: None
            shell(run)
        elif args.interact_kernel:
            from .core.interact import kernel
            kernel(run)
        else:
            run()
    except KeyboardInterrupt:
        pass
    finally:
        sig = signal.SIGTERM
        msg = ''
        while multiprocessing.active_children():
            msg and print(msg)
            for p in multiprocessing.active_children():
                os.kill(p.pid, sig)
            time.sleep(0.3)
            msg = 'killall children'
            sig = signal.SIGKILL

def process_iter(cfg):
    result = []
    for k, v in cfg.items():
        if 'count' in v:
            for i in range(v.get_int('count')):
                result.append({
                    'name': '{}-{}'.format(k, i),
                    'groups': v.get('groups', ()),
                })
        else:
            result.append({
                'name': k,
                'groups': v.get('groups', ()),
            })
    return result

def create_process(cfg):
    p = multiprocessing.Process(
        target=loop_run,
        kwargs=dict(
            group_resolver=GroupResolver(
                include=set(cfg['groups']),
                exclude=set(),
                all_groups=False,
                default=True,
            ),
            process_name=cfg['name'],
        ),
        name=cfg['name'],
        daemon=True,
    )
    p.start()
    return p

def loop_run(
    conf=None, future=None,
    group_resolver=None,
    ns=None, cmds=None,
    argv=None, loop=None,
    prompt=None,
    process_name=None,
):
    if process_name:
        utils.setproctitle(process_name)
    if loop is None:
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)
    if conf:
        context.set_config(conf)
    if loop is not None:
        context.set_loop(loop)
    if group_resolver is not None:
        context.set_group_resolver(group_resolver)

    if not cmds:
        cmds = ['run_forever']
        prompt and print(prompt)
    argv = argv or []
    ns = ns or argparse.Namespace()
    with utils.monkey_close(loop), context:
        if future is not None:
            future.set_result(context)
        for cmd in cmds:
            try:
                result = command.run(cmd, context, argv=argv, ns=ns)
            except command.CommandNotFound:
                print('Command {} not found'.format(cmd))
                continue
            if result is not None:
                print('{} => {}'.format(cmd, result))
        if hasattr(loop, 'shutdown_asyncgens'):
            loop.run_until_complete(loop.shutdown_asyncgens())

class UriType(argparse.FileType):
    def __call__(self, string):
        t, path = splittype(string)
        if not t:
            return super().__call__(string)
        return urlopen(string)

class ExtendAction(argparse.Action):
    def __call__(self, parser, namespace, values, option_string=None):
        getattr(namespace, self.dest).extend(values)

class plugin(Plugin):
    def add_arguments(self, parser):
        default = []
        parser.set_defaults(config=default)
        parser.add_argument(
            '-c', '--config', nargs='+', action=ExtendAction,
            type=UriType('r', encoding='utf-8'),
        )
        parser.add_argument('--config-stdin', action='store_true')

def main_with_conf(*args, **kwargs):
    import warnings
    warnings.warn(
        'Deprecated main_with_conf, use main',
        DeprecationWarning, stacklevel=2,
    )
    main(*args, **kwargs)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>__version__ = '0.11.0'
version_info = __version__.split('.')

from .client import ZKClient  # noqa
from .protocol import WatchEvent  # noqa
from .protocol.acl import ACL  # noqa
from .retry import RetryPolicy  # noqa
<commit_msg>Roll 0.12.0<commit_after>__version__ = '0.12.0'
version_info = __version__.split('.')

from .client import ZKClient  # noqa
from .protocol import WatchEvent  # noqa
from .protocol.acl import ACL  # noqa
from .retry import RetryPolicy  # noqa
<|endoftext|>


<commit_before>from gui.ui_mainwindow import Ui_MainWindow
from PySide.QtGui import QApplication, QMainWindow
from PySide import QtCore
from lyrical_picker import LyricalPicker

class LyricalControl(QMainWindow, Ui_MainWindow):
    def __init__(self, projector_window, db):
        QMainWindow.__init__(self)
        self.projector = projector_window
        self.setupUi(self)
        self.songs = []
        self.db = db

        self.song_list.clicked.connect(self.click_song)
        self.lyrics.clicked.connect(self.update_screen)
        self.add_button.clicked.connect(self.show_picker)

    def add_song(self, song):
        self.songs.append(song)
        self.song_list.addItem(song.title)

    def remove_song(self):
        #controller.song_list.removeCurrentRow(0)
        pass

    def edit_song(self):
        #controller.song_list.getCurrentRow()
        pass

    def show_lyrics_list(self, lyrics):
        self.lyrics.clear()
        self.lyrics.insertItems(0, lyrics)
        self.lyrics.setCurrentRow(0)

    def keyPressEvent(self, e):
        if e.key() == QtCore.Qt.Key_Escape:
            sys.exit()

    @QtCore.Slot()
    def click_song(self):
        '''When clicked, show correct lyrics
        '''
        if self.song_list.count() > 0 and any(self.songs):
            i = self.song_list.currentRow()
            self.show_lyrics_list(self.songs[i].lyrics_list())

    @QtCore.Slot()
    def update_screen(self):
        title = self.song_list.currentItem().text()
        content = self.lyrics.currentItem().text()
        self.projector.update(Slide(title, content))

    @QtCore.Slot()
    def show_picker(self):
        self.picker = LyricalPicker(self.db, self)
        self.picker.show()

<commit_msg>Import Slide<commit_after>from gui.ui_mainwindow import Ui_MainWindow
from PySide.QtGui import QApplication, QMainWindow
from PySide import QtCore
from lyrical_picker import LyricalPicker
from slide import Slide

class LyricalControl(QMainWindow, Ui_MainWindow):
    def __init__(self, projector_window, db):
        QMainWindow.__init__(self)
        self.projector = projector_window
        self.setupUi(self)
        self.songs = []
        self.db = db

        self.song_list.clicked.connect(self.click_song)
        self.lyrics.clicked.connect(self.update_screen)
        self.add_button.clicked.connect(self.show_picker)

    def add_song(self, song):
        self.songs.append(song)
        self.song_list.addItem(song.title)

    def remove_song(self):
        #controller.song_list.removeCurrentRow(0)
        pass

    def edit_song(self):
        #controller.song_list.getCurrentRow()
        pass

    def show_lyrics_list(self, lyrics):
        self.lyrics.clear()
        self.lyrics.insertItems(0, lyrics)
        self.lyrics.setCurrentRow(0)

    def keyPressEvent(self, e):
        if e.key() == QtCore.Qt.Key_Escape:
            sys.exit()

    @QtCore.Slot()
    def click_song(self):
        '''When clicked, show correct lyrics
        '''
        if self.song_list.count() > 0 and any(self.songs):
            i = self.song_list.currentRow()
            self.show_lyrics_list(self.songs[i].lyrics_list())

    @QtCore.Slot()
    def update_screen(self):
        title = self.song_list.currentItem().text()
        content = self.lyrics.currentItem().text()
        self.projector.update(Slide(title, content))

    @QtCore.Slot()
    def show_picker(self):
        self.picker = LyricalPicker(self.db, self)
        self.picker.show()

<|endoftext|>


<commit_before>from django.conf.urls import url

from api.nodes import views
from api.identifiers import views as identifier_views
from website import settings

urlpatterns = [
    # Examples:
    # url(r'^$', 'api.views.home', name='home'),
    # url(r'^blog/', include('blog.urls')),
    url(r'^$', views.NodeList.as_view(), name=views.NodeList.view_name),
    url(r'^(?P<node_id>\w+)/$', views.NodeDetail.as_view(), name=views.NodeDetail.view_name),
    url(r'^(?P<node_id>\w+)/contributors/$', views.NodeContributorsList.as_view(), name=views.NodeContributorsList.view_name),
    url(r'^(?P<node_id>\w+)/contributors/(?P<user_id>\w+)/$', views.NodeContributorDetail.as_view(), name=views.NodeContributorDetail.view_name),
    url(r'^(?P<node_id>\w+)/children/$', views.NodeChildrenList.as_view(), name=views.NodeChildrenList.view_name),
    url(r'^(?P<node_id>\w+)/forks/$', views.NodeForksList.as_view(), name=views.NodeForksList.view_name),
    url(r'^(?P<node_id>\w+)/files/$', views.NodeProvidersList.as_view(), name=views.NodeProvidersList.view_name),
    url(r'^(?P<node_id>\w+)/files/providers/(?P<provider>\w+)/?$', views.NodeProviderDetail.as_view(), name=views.NodeProviderDetail.view_name),
    url(r'^(?P<node_id>\w+)/files/(?P<provider>\w+)(?P<path>/(?:.*/)?)$', views.NodeFilesList.as_view(), name=views.NodeFilesList.view_name),
    url(r'^(?P<node_id>\w+)/files/(?P<provider>\w+)(?P<path>/.+[^/])$', views.NodeFileDetail.as_view(), name=views.NodeFileDetail.view_name),
    url(r'^(?P<node_id>\w+)/comments/$', views.NodeCommentsList.as_view(), name=views.NodeCommentsList.view_name),
    url(r'^(?P<node_id>\w+)/logs/$', views.NodeLogList.as_view(), name=views.NodeLogList.view_name),
    url(r'^(?P<node_id>\w+)/institution/$', views.NodeInstitutionDetail.as_view(), name=views.NodeInstitutionDetail.view_name),
    url(r'^(?P<node_id>\w+)/relationships/institution/$', views.NodeInstitutionRelationship.as_view(), name=views.NodeInstitutionRelationship.view_name),
    url(r'^(?P<node_id>\w+)/node_links/$', views.NodeLinksList.as_view(), name=views.NodeLinksList.view_name),
    url(r'^(?P<node_id>\w+)/node_links/(?P<node_link_id>\w+)/', views.NodeLinksDetail.as_view(), name=views.NodeLinksDetail.view_name),
    url(r'^(?P<node_id>\w+)/wikis/$', views.NodeWikiList.as_view(), name=views.NodeWikiList.view_name),
]

# Routes only active in local/staging environments
if settings.DEV_MODE:
    urlpatterns.extend([
        url(r'^(?P<node_id>\w+)/registrations/$', views.NodeRegistrationsList.as_view(), name=views.NodeRegistrationsList.view_name),

        # Custom citations
        url(r'^(?P<node_id>\w+)/citations/$', views.NodeAlternativeCitationsList.as_view(), name=views.NodeAlternativeCitationsList.view_name),
        url(r'^(?P<node_id>\w+)/citations/(?P<citation_id>\w+)/$', views.NodeAlternativeCitationDetail.as_view(), name=views.NodeAlternativeCitationDetail.view_name),
    ])
<commit_msg>Remove unused import<commit_after>from django.conf.urls import url

from api.nodes import views
from website import settings

urlpatterns = [
    # Examples:
    # url(r'^$', 'api.views.home', name='home'),
    # url(r'^blog/', include('blog.urls')),
    url(r'^$', views.NodeList.as_view(), name=views.NodeList.view_name),
    url(r'^(?P<node_id>\w+)/$', views.NodeDetail.as_view(), name=views.NodeDetail.view_name),
    url(r'^(?P<node_id>\w+)/contributors/$', views.NodeContributorsList.as_view(), name=views.NodeContributorsList.view_name),
    url(r'^(?P<node_id>\w+)/contributors/(?P<user_id>\w+)/$', views.NodeContributorDetail.as_view(), name=views.NodeContributorDetail.view_name),
    url(r'^(?P<node_id>\w+)/children/$', views.NodeChildrenList.as_view(), name=views.NodeChildrenList.view_name),
    url(r'^(?P<node_id>\w+)/forks/$', views.NodeForksList.as_view(), name=views.NodeForksList.view_name),
    url(r'^(?P<node_id>\w+)/files/$', views.NodeProvidersList.as_view(), name=views.NodeProvidersList.view_name),
    url(r'^(?P<node_id>\w+)/files/providers/(?P<provider>\w+)/?$', views.NodeProviderDetail.as_view(), name=views.NodeProviderDetail.view_name),
    url(r'^(?P<node_id>\w+)/files/(?P<provider>\w+)(?P<path>/(?:.*/)?)$', views.NodeFilesList.as_view(), name=views.NodeFilesList.view_name),
    url(r'^(?P<node_id>\w+)/files/(?P<provider>\w+)(?P<path>/.+[^/])$', views.NodeFileDetail.as_view(), name=views.NodeFileDetail.view_name),
    url(r'^(?P<node_id>\w+)/comments/$', views.NodeCommentsList.as_view(), name=views.NodeCommentsList.view_name),
    url(r'^(?P<node_id>\w+)/logs/$', views.NodeLogList.as_view(), name=views.NodeLogList.view_name),
    url(r'^(?P<node_id>\w+)/institution/$', views.NodeInstitutionDetail.as_view(), name=views.NodeInstitutionDetail.view_name),
    url(r'^(?P<node_id>\w+)/relationships/institution/$', views.NodeInstitutionRelationship.as_view(), name=views.NodeInstitutionRelationship.view_name),
    url(r'^(?P<node_id>\w+)/node_links/$', views.NodeLinksList.as_view(), name=views.NodeLinksList.view_name),
    url(r'^(?P<node_id>\w+)/node_links/(?P<node_link_id>\w+)/', views.NodeLinksDetail.as_view(), name=views.NodeLinksDetail.view_name),
    url(r'^(?P<node_id>\w+)/wikis/$', views.NodeWikiList.as_view(), name=views.NodeWikiList.view_name),
]

# Routes only active in local/staging environments
if settings.DEV_MODE:
    urlpatterns.extend([
        url(r'^(?P<node_id>\w+)/registrations/$', views.NodeRegistrationsList.as_view(), name=views.NodeRegistrationsList.view_name),

        # Custom citations
        url(r'^(?P<node_id>\w+)/citations/$', views.NodeAlternativeCitationsList.as_view(), name=views.NodeAlternativeCitationsList.view_name),
        url(r'^(?P<node_id>\w+)/citations/(?P<citation_id>\w+)/$', views.NodeAlternativeCitationDetail.as_view(), name=views.NodeAlternativeCitationDetail.view_name),
    ])
<|endoftext|>


<commit_before>"""gelpaths.py: solution paths for group elastic net.

This module implements a 2-stage process where group elastic net is used to find
the support, and ridge regression is performed on it.
"""

import math
import sys

import torch

from gel.ridgepaths import ridge_paths

def _find_support(B, ns, supp_thresh):
    """Find features with non-zero coefficients."""
    support = (B.norm(p=2, dim=1, keepdim=True) >= supp_thresh).expand_as(B)
    support = torch.cat([s_j[:n_j] for s_j, n_j in zip(support, ns)])
    support = torch.nonzero(support)[:, 0]
    if not support.numel():
        support = None
    return support

def compute_ls_grid(As, y, sns_vec, m, ks, n_ls, l_eps, dtype):
    """Compute l values for each given k and return a dictionary mapping
    k to a list (in decreasing order) of lambda values.

    Arguments have the same meaning as in gel_paths2. sns_vec is a vector of
    sns_j values as opposed to the matrix computed in gel_paths2.
    """
    ls_grid = {}
    # The bound is given by max{||A_j'@(y - b_0)||/(m*sqrt{n_j}*k)} where b_0 =
    # 1'@y/m. So most things can be precomputed.
    l_max_b_0 = y.mean()
    l_max_unscaled = max(
        (A_j.t() @ (y - l_max_b_0)).norm(p=2) / (m * sns_j)
        for A_j, sns_j in zip(As, sns_vec)
    )
    for k in ks:
        l_max = l_max_unscaled / k
        if n_ls == 1:
            ls_grid[k] = [l_max]
        else:
            l_min = l_max * l_eps
            ls = torch.logspace(
                math.log10(l_min), math.log10(l_max), steps=n_ls, dtype=dtype
            )
            ls = sorted([l.item() for l in ls], reverse=True)
            ls_grid[k] = ls
    return ls_grid

def gel_paths(
    gel_solve,
    gel_solve_kwargs,
    make_A,
    As,
    y,
    l_1s,
    l_2s,
    l_rs,
    summ_fun,
    supp_thresh=1e-6,
    device=torch.device("cpu"),
    verbose=False,
    aux_rel_tol=1e-3,
    dtype=torch.float32,
):
    """Solve group elastic net to find support and perform ridge on it.

    The problem is solved for multiple values of l_1, l_2, and l_r (the ridge
    regularization), and the result for each is summarized using the given
    summary function.

    Arguments:
        gel_solve, make_A: functions from a gel implementation to be used
            internally.
        gel_solve_kwargs: dictionary of keyword arguments to be passed to
            gel_solve.
        As: list of feature matrices (same as in make_A). All features should be
            centered.
        y: vector of predictions.
        l_1s, l_2s, l_rs: list of values for l_1, l_2, and l_r respectively.
        summ_fun: function to summarize results (same as in ridge_paths).
        supp_thresh: for computing support, 2-norms below this value are
            considered 0.
        device: torch device.
        verbose: enable/disable verbosity.
        aux_rel_tol: relative tolerance for solving auxiliary problems.
        dtype: torch dtype.

    The function returns a dictionary mapping (l_1, l_2, l_r) values to their
    summaries.
    """
    # We have to loop over l_1s, l_2s, and l_rs. Since multiple ridge
    # regressions can be performed efficiently with ridge_paths, l_rs should be
    # the inner most loop. Further, with l_1^1 > l_1^2 > ..., group elastic net
    # for (l_1^j, l_2) can be solved efficiently by warm starting at the
    # solution for (l_1^{j-1}, l_2). So, l_2s should be the outer most loop.

    # First compute various required variables.
    l_1s = sorted(l_1s, reverse=True)
    p = len(As)
    m = As[0].shape[0]
    As = [A_j.to(device, dtype) for A_j in As]
    ns = torch.tensor([A_j.shape[1] for A_j in As])
    B_zeros = torch.zeros(p, ns.max().item(), device=device, dtype=dtype)
    sns = ns.to(device, dtype).sqrt().unsqueeze(1).expand_as(B_zeros)

    # Form the A matrix as needed by gel_solve.
    A = make_A(As, ns, device, dtype)

    # Form X which combines all the As (for ridge_paths).
    X = torch.cat(As, dim=1)
    X = X.t()  # ridge_paths expects a pxm matrix

    y = y.to(device, dtype)
    if "Cs" in gel_solve_kwargs:
        gel_solve_kwargs["Cs"] = [
            C_j.to(device, dtype) for C_j in gel_solve_kwargs["Cs"]
        ]
    if "Is" in gel_solve_kwargs:
        gel_solve_kwargs["Is"] = [
            I_j.to(device, dtype) for I_j in gel_solve_kwargs["Is"]
        ]

    gel_solve_kwargs_aux = gel_solve_kwargs.copy()
    gel_solve_kwargs_aux["rel_tol"] = aux_rel_tol
    summaries = {}

    for l_2 in l_2s:
        b_init = 0.0, B_zeros  # reset b_init for each l_2

        # Find a good initialization to solve for the first (l_1, l_2) pair.
        if l_2 != 0 and l_1s[0] != 0:
            # Find k corresponding to the l_1, l_2.
            # l_1 = k*l, and l_2 = (1 - k)*l.
            k = 1.0 / (1.0 + l_2 / l_1s[0])

            # Compute l_max corresponding to k.
            l_aux = compute_ls_grid(As, y, sns[:, 0], m, [k], 1, None, dtype)[k]
            l_aux = l_aux[0]

            # Solve with k, l_aux.
            if verbose:
                print(
                    "Solving auxiliary problem to get good initialization",
                    file=sys.stderr,
                )
            b_init = gel_solve(
                A,
                y,
                k * l_aux,
                (1.0 - k) * l_aux,
                ns,
                b_init,
                verbose=verbose,
                **gel_solve_kwargs,
            )
            if verbose:
                print("Done solving auxiliary problem", file=sys.stderr)

        for l_1 in l_1s:
            # Solve group elastic net initializing at the previous solution.
            b_0, B = gel_solve(
                A, y, l_1, l_2, ns, b_init, verbose=verbose, **gel_solve_kwargs
            )
            b_init = b_0, B

            # Find support.
            support = _find_support(B, ns, supp_thresh)
            if verbose:
                support_size = 0 if support is None else len(support)
                print("Support size: {}".format(support_size), file=sys.stderr)

            # Solve ridge on support and store summaries.
            if support is not None:
                X_supp = X if support_size == X.shape[0] else X[support]
            else:
                X_supp = None
            ridge_summaries = ridge_paths(
                X_supp, y, support, l_rs, summ_fun, verbose
            )
            del X_supp
            for l_r, summary in ridge_summaries.items():
                summaries[(l_1, l_2, l_r)] = summary
            if verbose:
                print(file=sys.stderr)

    return summaries

def gel_paths2(
    gel_solve,
    gel_solve_kwargs,
    make_A,
    As,
    y,
    ks,
    n_ls,
    l_eps,
    l_rs,
    summ_fun,
    supp_thresh=1e-6,
    device=torch.device("cpu"),
    verbose=False,
    ls_grid=None,
    aux_rel_tol=1e-3,
    dtype=torch.float32,
):
    """Solve for paths with a reparametrized group elastic net.

    Arguments:
        gel_solve, make_A: functions from a gel implementation to be used
            internally.
        gel_solve_kwargs: dictionary of keyword arguments to be passed to
            gel_solve.
        As: list of feature matrices (same as in make_A). All features should be
            centered.
        y: vector of predictions.
        ks: list of values for trading-off l1 and l2 regularizations.
        n_ls: number of lambda values.
        l_eps: ratio of minimum to maximum lambda value.
        l_rs: list of ridge regularization values.
        summ_fun: function to summarize results (same as in ridge_paths).
        supp_thresh: for computing support, 2-norms below this value are
            considered 0.
        device: torch device.
        verbose: enable/disable verbosity.
        ls_grid: pre-computed dictionary mapping each k in ks to a list of l
            values (in decreasing order). If this argument is not None, n_ls and
            l_eps are ignored.
        aux_rel_tol: relative tolerance for solving auxiliary problems.
        dtype: torch dtype.

    The regularization terms can be rewritten as

        l*(k*||b_j|| + (1 - k)*||b_j||^2)

    where k controls the tradeoff between the two norms, and l controls the
    overall strength. This function takes a list of tradeoff values through ks.
    For each k, an upper bound can be found on l (which will lead to an empty
    support). Using this upper bound, n_ls l values are computed on a log scale
    such that l_min / l_max = l_eps. Other arguments are same as in gel_paths.

    The function returns a dictionary mapping (l_1, l_2, l_r) values to their
    summaries.
    """
    # Setup is mostly identical to gel_paths.
    p = len(As)
    m = As[0].shape[0]
    As = [A_j.to(device, dtype) for A_j in As]
    ns = torch.tensor([A_j.shape[1] for A_j in As])
    B_zeros = torch.zeros(p, ns.max(), device=device, dtype=dtype)
    sns = ns.to(device, dtype).sqrt().unsqueeze(1).expand_as(B_zeros)
    A = make_A(As, ns, device, dtype)
    X = torch.cat(As, dim=1)
    X = X.t()

    y = y.to(device, dtype)
    if "Cs" in gel_solve_kwargs:
        gel_solve_kwargs["Cs"] = [
            C_j.to(device) for C_j in gel_solve_kwargs["Cs"]
        ]
    if "Is" in gel_solve_kwargs:
        gel_solve_kwargs["Is"] = [
            I_j.to(device) for I_j in gel_solve_kwargs["Is"]
        ]

    if ls_grid is None:
        ls_grid = compute_ls_grid(As, y, sns[:, 0], m, ks, n_ls, l_eps, dtype)
        ls_grid_self = None
    else:
        # Compute l values with self data to get good initializations.
        ls_grid_self = compute_ls_grid(
            As, y, sns[:, 0], m, ks, n_ls, l_eps, dtype
        )
        gel_solve_kwargs_aux = gel_solve_kwargs.copy()
        gel_solve_kwargs_aux["rel_tol"] = aux_rel_tol

    summaries = {}
    for k in ks:
        b_init = 0.0, B_zeros  # reset the initial value for each k
        ls = ls_grid[k]
        full_support = False

        if ls_grid_self is not None:
            # Solve with self l values to get a better initialization.
            # This _greatly_ speeds up the optimization.
            if verbose:
                print(
                    "Solving auxiliary problems to get good initialization",
                    file=sys.stderr,
                )
            ls_self = ls_grid_self[k]
            for l_self in ls_self:
                if l_self < ls[0]:
                    break
                b_init = gel_solve(
                    A,
                    y,
                    k * l_self,
                    (1.0 - k) * l_self,
                    ns,
                    b_init,
                    verbose=verbose,
                    **gel_solve_kwargs_aux,
                )
            if verbose:
                print("Done solving auxiliary problems", file=sys.stderr)

        ridge_summaries = None
        for l in ls:
            # Convert k, l into l_1, l_2.
            l_1, l_2 = k * l, (1.0 - k) * l

            if full_support:
                # Just copy the previous summaries.
                for l_r, summary in ridge_summaries.items():
                    summaries[(l_1, l_2, l_r)] = summary
                continue

            # Rest is similar to gel_paths.
            b_0, B = gel_solve(
                A, y, l_1, l_2, ns, b_init, verbose=verbose, **gel_solve_kwargs
            )
            b_init = b_0, B

            # Find support.
            support = _find_support(B, ns, supp_thresh)
            support_size = 0 if support is None else len(support)
            full_support = support_size == X.shape[0]
            if verbose:
                print("Support size: {}".format(support_size), file=sys.stderr)

            # Solve ridge on support and store summaries.
            if support is not None:
                X_supp = X if full_support else X[support]
            else:
                X_supp = None
            ridge_summaries = ridge_paths(
                X_supp, y, support, l_rs, summ_fun, verbose
            )
            del X_supp
            for l_r, summary in ridge_summaries.items():
                summaries[(l_1, l_2, l_r)] = summary
            if verbose:
                print(file=sys.stderr)

    return summaries
<commit_msg>Fix support_size error<commit_after>"""gelpaths.py: solution paths for group elastic net.

This module implements a 2-stage process where group elastic net is used to find
the support, and ridge regression is performed on it.
"""

import math
import sys

import torch

from gel.ridgepaths import ridge_paths

def _find_support(B, ns, supp_thresh):
    """Find features with non-zero coefficients."""
    support = (B.norm(p=2, dim=1, keepdim=True) >= supp_thresh).expand_as(B)
    support = torch.cat([s_j[:n_j] for s_j, n_j in zip(support, ns)])
    support = torch.nonzero(support)[:, 0]
    if not support.numel():
        support = None
    return support

def compute_ls_grid(As, y, sns_vec, m, ks, n_ls, l_eps, dtype):
    """Compute l values for each given k and return a dictionary mapping
    k to a list (in decreasing order) of lambda values.

    Arguments have the same meaning as in gel_paths2. sns_vec is a vector of
    sns_j values as opposed to the matrix computed in gel_paths2.
    """
    ls_grid = {}
    # The bound is given by max{||A_j'@(y - b_0)||/(m*sqrt{n_j}*k)} where b_0 =
    # 1'@y/m. So most things can be precomputed.
    l_max_b_0 = y.mean()
    l_max_unscaled = max(
        (A_j.t() @ (y - l_max_b_0)).norm(p=2) / (m * sns_j)
        for A_j, sns_j in zip(As, sns_vec)
    )
    for k in ks:
        l_max = l_max_unscaled / k
        if n_ls == 1:
            ls_grid[k] = [l_max]
        else:
            l_min = l_max * l_eps
            ls = torch.logspace(
                math.log10(l_min), math.log10(l_max), steps=n_ls, dtype=dtype
            )
            ls = sorted([l.item() for l in ls], reverse=True)
            ls_grid[k] = ls
    return ls_grid

def gel_paths(
    gel_solve,
    gel_solve_kwargs,
    make_A,
    As,
    y,
    l_1s,
    l_2s,
    l_rs,
    summ_fun,
    supp_thresh=1e-6,
    device=torch.device("cpu"),
    verbose=False,
    aux_rel_tol=1e-3,
    dtype=torch.float32,
):
    """Solve group elastic net to find support and perform ridge on it.

    The problem is solved for multiple values of l_1, l_2, and l_r (the ridge
    regularization), and the result for each is summarized using the given
    summary function.

    Arguments:
        gel_solve, make_A: functions from a gel implementation to be used
            internally.
        gel_solve_kwargs: dictionary of keyword arguments to be passed to
            gel_solve.
        As: list of feature matrices (same as in make_A). All features should be
            centered.
        y: vector of predictions.
        l_1s, l_2s, l_rs: list of values for l_1, l_2, and l_r respectively.
        summ_fun: function to summarize results (same as in ridge_paths).
        supp_thresh: for computing support, 2-norms below this value are
            considered 0.
        device: torch device.
        verbose: enable/disable verbosity.
        aux_rel_tol: relative tolerance for solving auxiliary problems.
        dtype: torch dtype.

    The function returns a dictionary mapping (l_1, l_2, l_r) values to their
    summaries.
    """
    # We have to loop over l_1s, l_2s, and l_rs. Since multiple ridge
    # regressions can be performed efficiently with ridge_paths, l_rs should be
    # the inner most loop. Further, with l_1^1 > l_1^2 > ..., group elastic net
    # for (l_1^j, l_2) can be solved efficiently by warm starting at the
    # solution for (l_1^{j-1}, l_2). So, l_2s should be the outer most loop.

    # First compute various required variables.
    l_1s = sorted(l_1s, reverse=True)
    p = len(As)
    m = As[0].shape[0]
    As = [A_j.to(device, dtype) for A_j in As]
    ns = torch.tensor([A_j.shape[1] for A_j in As])
    B_zeros = torch.zeros(p, ns.max().item(), device=device, dtype=dtype)
    sns = ns.to(device, dtype).sqrt().unsqueeze(1).expand_as(B_zeros)

    # Form the A matrix as needed by gel_solve.
    A = make_A(As, ns, device, dtype)

    # Form X which combines all the As (for ridge_paths).
    X = torch.cat(As, dim=1)
    X = X.t()  # ridge_paths expects a pxm matrix

    y = y.to(device, dtype)
    if "Cs" in gel_solve_kwargs:
        gel_solve_kwargs["Cs"] = [
            C_j.to(device, dtype) for C_j in gel_solve_kwargs["Cs"]
        ]
    if "Is" in gel_solve_kwargs:
        gel_solve_kwargs["Is"] = [
            I_j.to(device, dtype) for I_j in gel_solve_kwargs["Is"]
        ]

    gel_solve_kwargs_aux = gel_solve_kwargs.copy()
    gel_solve_kwargs_aux["rel_tol"] = aux_rel_tol
    summaries = {}

    for l_2 in l_2s:
        b_init = 0.0, B_zeros  # reset b_init for each l_2

        # Find a good initialization to solve for the first (l_1, l_2) pair.
        if l_2 != 0 and l_1s[0] != 0:
            # Find k corresponding to the l_1, l_2.
            # l_1 = k*l, and l_2 = (1 - k)*l.
            k = 1.0 / (1.0 + l_2 / l_1s[0])

            # Compute l_max corresponding to k.
            l_aux = compute_ls_grid(As, y, sns[:, 0], m, [k], 1, None, dtype)[k]
            l_aux = l_aux[0]

            # Solve with k, l_aux.
            if verbose:
                print(
                    "Solving auxiliary problem to get good initialization",
                    file=sys.stderr,
                )
            b_init = gel_solve(
                A,
                y,
                k * l_aux,
                (1.0 - k) * l_aux,
                ns,
                b_init,
                verbose=verbose,
                **gel_solve_kwargs,
            )
            if verbose:
                print("Done solving auxiliary problem", file=sys.stderr)

        for l_1 in l_1s:
            # Solve group elastic net initializing at the previous solution.
            b_0, B = gel_solve(
                A, y, l_1, l_2, ns, b_init, verbose=verbose, **gel_solve_kwargs
            )
            b_init = b_0, B

            # Find support.
            support = _find_support(B, ns, supp_thresh)
            support_size = 0 if support is None else len(support)
            if verbose:
                print("Support size: {}".format(support_size), file=sys.stderr)

            # Solve ridge on support and store summaries.
            if support is not None:
                X_supp = X if support_size == X.shape[0] else X[support]
            else:
                X_supp = None
            ridge_summaries = ridge_paths(
                X_supp, y, support, l_rs, summ_fun, verbose
            )
            del X_supp
            for l_r, summary in ridge_summaries.items():
                summaries[(l_1, l_2, l_r)] = summary
            if verbose:
                print(file=sys.stderr)

    return summaries

def gel_paths2(
    gel_solve,
    gel_solve_kwargs,
    make_A,
    As,
    y,
    ks,
    n_ls,
    l_eps,
    l_rs,
    summ_fun,
    supp_thresh=1e-6,
    device=torch.device("cpu"),
    verbose=False,
    ls_grid=None,
    aux_rel_tol=1e-3,
    dtype=torch.float32,
):
    """Solve for paths with a reparametrized group elastic net.

    Arguments:
        gel_solve, make_A: functions from a gel implementation to be used
            internally.
        gel_solve_kwargs: dictionary of keyword arguments to be passed to
            gel_solve.
        As: list of feature matrices (same as in make_A). All features should be
            centered.
        y: vector of predictions.
        ks: list of values for trading-off l1 and l2 regularizations.
        n_ls: number of lambda values.
        l_eps: ratio of minimum to maximum lambda value.
        l_rs: list of ridge regularization values.
        summ_fun: function to summarize results (same as in ridge_paths).
        supp_thresh: for computing support, 2-norms below this value are
            considered 0.
        device: torch device.
        verbose: enable/disable verbosity.
        ls_grid: pre-computed dictionary mapping each k in ks to a list of l
            values (in decreasing order). If this argument is not None, n_ls and
            l_eps are ignored.
        aux_rel_tol: relative tolerance for solving auxiliary problems.
        dtype: torch dtype.

    The regularization terms can be rewritten as

        l*(k*||b_j|| + (1 - k)*||b_j||^2)

    where k controls the tradeoff between the two norms, and l controls the
    overall strength. This function takes a list of tradeoff values through ks.
    For each k, an upper bound can be found on l (which will lead to an empty
    support). Using this upper bound, n_ls l values are computed on a log scale
    such that l_min / l_max = l_eps. Other arguments are same as in gel_paths.

    The function returns a dictionary mapping (l_1, l_2, l_r) values to their
    summaries.
    """
    # Setup is mostly identical to gel_paths.
    p = len(As)
    m = As[0].shape[0]
    As = [A_j.to(device, dtype) for A_j in As]
    ns = torch.tensor([A_j.shape[1] for A_j in As])
    B_zeros = torch.zeros(p, ns.max(), device=device, dtype=dtype)
    sns = ns.to(device, dtype).sqrt().unsqueeze(1).expand_as(B_zeros)
    A = make_A(As, ns, device, dtype)
    X = torch.cat(As, dim=1)
    X = X.t()

    y = y.to(device, dtype)
    if "Cs" in gel_solve_kwargs:
        gel_solve_kwargs["Cs"] = [
            C_j.to(device) for C_j in gel_solve_kwargs["Cs"]
        ]
    if "Is" in gel_solve_kwargs:
        gel_solve_kwargs["Is"] = [
            I_j.to(device) for I_j in gel_solve_kwargs["Is"]
        ]

    if ls_grid is None:
        ls_grid = compute_ls_grid(As, y, sns[:, 0], m, ks, n_ls, l_eps, dtype)
        ls_grid_self = None
    else:
        # Compute l values with self data to get good initializations.
        ls_grid_self = compute_ls_grid(
            As, y, sns[:, 0], m, ks, n_ls, l_eps, dtype
        )
        gel_solve_kwargs_aux = gel_solve_kwargs.copy()
        gel_solve_kwargs_aux["rel_tol"] = aux_rel_tol

    summaries = {}
    for k in ks:
        b_init = 0.0, B_zeros  # reset the initial value for each k
        ls = ls_grid[k]
        full_support = False

        if ls_grid_self is not None:
            # Solve with self l values to get a better initialization.
            # This _greatly_ speeds up the optimization.
            if verbose:
                print(
                    "Solving auxiliary problems to get good initialization",
                    file=sys.stderr,
                )
            ls_self = ls_grid_self[k]
            for l_self in ls_self:
                if l_self < ls[0]:
                    break
                b_init = gel_solve(
                    A,
                    y,
                    k * l_self,
                    (1.0 - k) * l_self,
                    ns,
                    b_init,
                    verbose=verbose,
                    **gel_solve_kwargs_aux,
                )
            if verbose:
                print("Done solving auxiliary problems", file=sys.stderr)

        ridge_summaries = None
        for l in ls:
            # Convert k, l into l_1, l_2.
            l_1, l_2 = k * l, (1.0 - k) * l

            if full_support:
                # Just copy the previous summaries.
                for l_r, summary in ridge_summaries.items():
                    summaries[(l_1, l_2, l_r)] = summary
                continue

            # Rest is similar to gel_paths.
            b_0, B = gel_solve(
                A, y, l_1, l_2, ns, b_init, verbose=verbose, **gel_solve_kwargs
            )
            b_init = b_0, B

            # Find support.
            support = _find_support(B, ns, supp_thresh)
            support_size = 0 if support is None else len(support)
            full_support = support_size == X.shape[0]
            if verbose:
                print("Support size: {}".format(support_size), file=sys.stderr)

            # Solve ridge on support and store summaries.
            if support is not None:
                X_supp = X if full_support else X[support]
            else:
                X_supp = None
            ridge_summaries = ridge_paths(
                X_supp, y, support, l_rs, summ_fun, verbose
            )
            del X_supp
            for l_r, summary in ridge_summaries.items():
                summaries[(l_1, l_2, l_r)] = summary
            if verbose:
                print(file=sys.stderr)

    return summaries
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
'''
Stabilization techniques for PDEs with dominating convection.
The classical article about SUPG is :cite:`brooks`; for an overview
of methods, see :cite:`sold1`, :cite:`sold2`, and :cite:`bgs2004`.
'''
from dolfin import (
    Expression
    )

def supg(convection, diffusion, element_degree):
    cppcode = '''#include <dolfin/mesh/Vertex.h>

class SupgStab : public Expression {
public:
double sigma;
int p;
boost::shared_ptr<GenericFunction> convection;
boost::shared_ptr<Mesh> mesh;

SupgStab(): Expression(2)
{}

void eval(Array<double>& b_tau,
          const Array<double>& x,
          const ufc::cell& c
          ) const
{
  assert(b_tau.size() == x.size());
  convection->eval(b_tau, x, c);
  double conv_norm = 0.0;
  for (uint i = 0; i < b_tau.size(); ++i)
    conv_norm += b_tau[i]*b_tau[i];
  conv_norm = sqrt(conv_norm);

  if (conv_norm > DOLFIN_EPS)
  {
    Cell cell(*mesh, c.index);

    // Compute the directed diameter of the cell, cf.
    //
    //    On spurious oscillations at layers diminishing (SOLD) methods
    //    for convection-diffusion equations: Part II - Analysis for P1 and Q1
    //    finite elements;
    //    Volker John, Petr Knobloch;
    //    Comput. Methods Appl. Mech. Engrg. 197 (2008) 1997-2014.
    //
    //    diam(cell, s) = 2*||s|| / sum_{nodes n_i} |s.\\grad\\psi|
    //
    // where \\psi is the P_1 basis function of n_i.
    //
    const double area = cell.volume();
    const unsigned int* vertices = cell.entities(0);
    assert(vertices);

    double sum = 0.0;
    for (int i=0; i<3; i++)
    {
      for (int j=i+1; j<3; j++)
      {
        // Get edge coords.
        const dolfin::Vertex v0(*mesh, vertices[i]);
        const dolfin::Vertex v1(*mesh, vertices[j]);
        const Point p0 = v0.point();
        const Point p1 = v1.point();
        const double e0 = p0[0] - p1[0];
        const double e1 = p0[1] - p1[1];

        // Note that
        //
        //     \\grad\\psi = ortho_edge / edgelength / height
        //               = ortho_edge / (2*area)
        //
        // so
        //
        //   (v.\\grad\\psi) = (v.ortho_edge) / (2*area).
        //
        // Move the constant factors out of the summation.
        //
        // It would be really nice if we could just do
        //    edge.dot((-v[1], v[0]))
        // but unfortunately, edges just dot with other edges.
        sum += fabs(e1*b_tau[0] - e0*b_tau[1]);
      }
    }
    const double h = 4 * conv_norm * area / sum;

    //// The alternative for the lazy:
    //const double h = cell.diameter();

    // Just a little sanity check here.
    assert(h <= cell.diameter());

    const double Pe = 0.5*conv_norm * h/(p*sigma);
    assert(Pe > 0.0);
    double xi = 0.0;
    //if (Pe > DOLFIN_EPS)
    //  xi = 1.0/tanh(Pe) - 1.0/Pe;
    // lim_{a->0} (1.0/tanh(a) - 1.0/a)/a = 1/3.
    //xi = min(1.0, Pe/3.0);
    if (Pe > 1.0)
      xi = 1.0 - 1.0/Pe;

    const double tau = 0.5*h*xi / (p*conv_norm);

    if (tau > 1.0e3)
    {
      std::cout << "tau   = " << tau << std::endl;
      std::cout << "||b|| = " << conv_norm << std::endl;
      std::cout << "Pe    = " << Pe << std::endl;
      std::cout << "h     = " << h << std::endl;
      std::cout << "xi    = " << xi << std::endl;
      throw 1;
    }

    // tau*b:
    for (uint i = 0; i < b_tau.size(); ++i)
      b_tau[i] *= 0.5*h*xi / (p*conv_norm);
  }
  else
  {
    for (uint i = 0; i < b_tau.size(); ++i)
      b_tau[i] = 0.0;
  }
  // We don't need no else-clause here since the (negibly small) convection
  // is already stored in v.

  return;
}
};
'''
    # TODO set degree
    b_tau = Expression(cppcode, degree=5)
    b_tau.convection = convection
    b_tau.mesh = convection.function_space().mesh()
    b_tau.sigma = diffusion
    b_tau.p = element_degree
    return b_tau

def supg2(mesh, convection, diffusion_coefficient, element_degree):
    cppcode = '''#include <dolfin/mesh/Vertex.h>

class SupgStab : public Expression {
public:
double sigma;
int p;
std::shared_ptr<GenericFunction> convection;
std::shared_ptr<Mesh> mesh;

SupgStab(): Expression()
{}

void eval(Array<double>& tau,
          const Array<double>& x,
          const ufc::cell& c
          ) const
{
  Array<double> v(x.size());
  convection->eval(v, x, c);
  double conv_norm = 0.0;
  for (uint i = 0; i < v.size(); ++i)
    conv_norm += v[i]*v[i];
  conv_norm = sqrt(conv_norm);

  if (conv_norm < DOLFIN_EPS) {
    tau[0] = 0.0;
  }

  Cell cell(*mesh, c.index);

  // Compute the directed diameter of the cell, cf.
  //
  // On spurious oscillations at layers diminishing (SOLD) methods for
  // convection-diffusion equations: Part II - Analysis for P1 and Q1 finite
  // elements
  // Volker John, Petr Knobloch;
  // Comput. Methods Appl. Mech. Engrg. 197 (2008) 1997-2014.
  //
  // The diameter in a direction s is defined as
  //
  //    diam(cell, s) = 2*||s|| / sum_{nodes n_i} |s.\\grad\\psi|
  //
  // where \\psi is the P_1 basis function of n_i.
  //
  const double area = cell.volume();
  const unsigned int* vertices = cell.entities(0);
  assert(vertices);

  double sum = 0.0;
  for (int i=0; i<3; i++) {
    for (int j=i+1; j<3; j++) {
      // Get edge coords.
      const dolfin::Vertex v0(*mesh, vertices[i]);
      const dolfin::Vertex v1(*mesh, vertices[j]);
      const Point p0 = v0.point();
      const Point p1 = v1.point();
      const double e0 = p0[0] - p1[0];
      const double e1 = p0[1] - p1[1];

      // Note that
      //
      //   \\grad\\psi = ortho_edge / edgelength / height
      //               = ortho_edge / (2*area)
      //
      // so
      //
      //   (v.\\grad\\psi) = (v.ortho_edge) / (2*area).
      //
      // Move the constant factors out of the summation.
      //
      // It would be really nice if we could just do
      //    edge.dot((-v[1], v[0]))
      // but unfortunately, edges just dot with other edges.
      sum += fabs(e1*v[0] - e0*v[1]);
    }
  }
  const double h = 4 * conv_norm * area / sum;

  // // The alternative for the lazy:
  // const double h = cell.diameter();

  // // Just a little sanity check here.
  // const double eps = 1.0e-12;
  // if (h > cell.diameter() + eps) {
  //     std::cout << "The directed diameter h (" << h << ") "
  //               << "should not be larger than the actual cell diameter "
  //               << "(" << cell.diameter() << ")."
  //               << std::endl;
  // }
  // assert(h < cell.diameter() + eps);

  const double Pe = 0.5*conv_norm * h/(p*sigma);
  assert(Pe > 0.0);
  // We'd like to compute `xi = 1.0/tanh(Pe) - 1.0/Pe`. This expression can
  // hardly be evaluated for small Pe, see
  // <https://stackoverflow.com/a/43279491/353337>. Hence, use its Taylor
  // expansion around 0.
  const double xi = Pe > 1.0e-5 ?
      1.0/tanh(Pe) - 1.0/Pe :
      Pe/3.0 - Pe*Pe*Pe / 45.0 + 2.0/945.0 * Pe*Pe*Pe*Pe*Pe;

  tau[0] = 0.5*h*xi / (p*conv_norm);

  //if (tau[0] > 1.0e3)
  //{
  //  std::cout << "tau   = " << tau[0] << std::endl;
  //  std::cout << "||b|| = " << conv_norm << std::endl;
  //  std::cout << "Pe    = " << Pe << std::endl;
  //  std::cout << "h     = " << h << std::endl;
  //  std::cout << "xi    = " << xi << std::endl;
  //  //throw 1;
  //}

  return;
}
};
'''
    tau = Expression(cppcode, degree=1)
    tau.convection = convection
    tau.mesh = mesh  # convection.function_space().mesh()
    tau.sigma = diffusion_coefficient
    tau.p = element_degree

    return tau
<commit_msg>fix for supg stabilization<commit_after># -*- coding: utf-8 -*-
#
'''
Stabilization techniques for PDEs with dominating convection.
The classical article about SUPG is :cite:`brooks`; for an overview
of methods, see :cite:`sold1`, :cite:`sold2`, and :cite:`bgs2004`.
'''
from dolfin import (
    Expression
    )

def supg(convection, diffusion, element_degree):
    '''For each cell :math:`K`, this function computes

    ..math::
        \\tau = \\frac{h}{2\\|b\\|} \\left(\\frac{1}{\\tanh Pe} - \\frac{1}{Pe}

    with the element diameter in the direction of the convection vector
    :math:`b` and the Péclet number :math:`Pe = \\frac{\\|b\\|
    h}{2\\varepsilon}`; see (3) in :cite:`sold2`.
    '''
    cppcode = '''#include <dolfin/mesh/Vertex.h>

class SupgStab : public Expression {
public:
double sigma;
int p;
boost::shared_ptr<GenericFunction> convection;
boost::shared_ptr<Mesh> mesh;

SupgStab(): Expression(2)
{}

void eval(
  Array<double>& b_tau,
  const Array<double>& x,
  const ufc::cell& c
  ) const
{
  assert(b_tau.size() == x.size());
  convection->eval(b_tau, x, c);
  double conv_norm = 0.0;
  for (uint i = 0; i < b_tau.size(); ++i)
    conv_norm += b_tau[i]*b_tau[i];
  conv_norm = sqrt(conv_norm);

  if (conv_norm < DOLFIN_EPS) {
    for (uint i = 0; i < b_tau.size(); ++i)
      b_tau[i] = 0.0;
    return;
  }

  Cell cell(*mesh, c.index);

  //// The alternative for the lazy:
  //const double h = cell.diameter();

  // Compute the directed diameter of the cell, cf. :cite:`sold2`.
  //
  //    diam(cell, s) = 2*||s|| / sum_{nodes n_i} |s.\\grad\\psi|
  //
  // where \\psi is the P_1 basis function of n_i.
  //
  const double area = cell.volume();
  const unsigned int* vertices = cell.entities(0);
  assert(vertices);
  double sum = 0.0;
  for (int i=0; i<3; i++) {
    for (int j=i+1; j<3; j++) {
      // Get edge coords.
      const dolfin::Vertex v0(*mesh, vertices[i]);
      const dolfin::Vertex v1(*mesh, vertices[j]);
      const Point p0 = v0.point();
      const Point p1 = v1.point();
      const double e0 = p0[0] - p1[0];
      const double e1 = p0[1] - p1[1];

      // Note that
      //
      //     \\grad\\psi = ortho_edge / edgelength / height
      //               = ortho_edge / (2*area)
      //
      // so
      //
      //   (v.\\grad\\psi) = (v.ortho_edge) / (2*area).
      //
      // Move the constant factors out of the summation.
      //
      // It would be really nice if we could just do
      //    edge.dot((-v[1], v[0]))
      // but unfortunately, edges just dot with other edges.
      sum += fabs(e1*b_tau[0] - e0*b_tau[1]);
    }
  }
  const double h = 4 * conv_norm * area / sum;

  // Just a little sanity check here.
  assert(h <= cell.diameter());

  const double Pe = 0.5*conv_norm * h/(p*sigma);
  assert(Pe > 0.0);

  // We'd like to compute `xi = 1.0/tanh(Pe) - 1.0/Pe`. This expression can
  // hardly be evaluated for small Pe, see
  // <https://stackoverflow.com/a/43279491/353337>. Hence, use its Taylor
  // expansion around 0.
  const double xi = Pe > 1.0e-5 ?
      1.0/tanh(Pe) - 1.0/Pe :
      Pe/3.0 - Pe*Pe*Pe / 45.0 + 2.0/945.0 * Pe*Pe*Pe*Pe*Pe;
  // const double xi =  Pe > 1.0 ? 1.0 - 1.0/Pe : 0.0;

  const double tau = 0.5*h*xi / (p*conv_norm);

  if (tau > 1.0e3)
  {
    std::cout << "tau   = " << tau << std::endl;
    std::cout << "||b|| = " << conv_norm << std::endl;
    std::cout << "Pe    = " << Pe << std::endl;
    std::cout << "h     = " << h << std::endl;
    std::cout << "xi    = " << xi << std::endl;
    throw 1;
  }

  // tau*b:
  for (uint i = 0; i < b_tau.size(); ++i)
    b_tau[i] *= 0.5*h*xi / (p*conv_norm);

  return;
}
};
'''
    # TODO set degree
    b_tau = Expression(cppcode, degree=5)
    b_tau.convection = convection
    b_tau.mesh = convection.function_space().mesh()
    b_tau.sigma = diffusion
    b_tau.p = element_degree
    return b_tau

def supg2(mesh, convection, diffusion_coefficient, element_degree):
    cppcode = '''#include <dolfin/mesh/Vertex.h>

class SupgStab : public Expression {
public:
double sigma;
int p;
std::shared_ptr<GenericFunction> convection;
std::shared_ptr<Mesh> mesh;

SupgStab(): Expression()
{}

void eval(Array<double>& tau,
          const Array<double>& x,
          const ufc::cell& c
          ) const
{
  Array<double> v(x.size());
  convection->eval(v, x, c);
  double conv_norm = 0.0;
  for (uint i = 0; i < v.size(); ++i)
    conv_norm += v[i]*v[i];
  conv_norm = sqrt(conv_norm);

  if (conv_norm < DOLFIN_EPS) {
    tau[0] = 0.0;
  }

  Cell cell(*mesh, c.index);

  // Compute the directed diameter of the cell, cf.
  //
  // On spurious oscillations at layers diminishing (SOLD) methods for
  // convection-diffusion equations: Part II - Analysis for P1 and Q1 finite
  // elements
  // Volker John, Petr Knobloch;
  // Comput. Methods Appl. Mech. Engrg. 197 (2008) 1997-2014.
  //
  // The diameter in a direction s is defined as
  //
  //    diam(cell, s) = 2*||s|| / sum_{nodes n_i} |s.\\grad\\psi|
  //
  // where \\psi is the P_1 basis function of n_i.
  //
  const double area = cell.volume();
  const unsigned int* vertices = cell.entities(0);
  assert(vertices);

  double sum = 0.0;
  for (int i=0; i<3; i++) {
    for (int j=i+1; j<3; j++) {
      // Get edge coords.
      const dolfin::Vertex v0(*mesh, vertices[i]);
      const dolfin::Vertex v1(*mesh, vertices[j]);
      const Point p0 = v0.point();
      const Point p1 = v1.point();
      const double e0 = p0[0] - p1[0];
      const double e1 = p0[1] - p1[1];

      // Note that
      //
      //   \\grad\\psi = ortho_edge / edgelength / height
      //               = ortho_edge / (2*area)
      //
      // so
      //
      //   (v.\\grad\\psi) = (v.ortho_edge) / (2*area).
      //
      // Move the constant factors out of the summation.
      //
      // It would be really nice if we could just do
      //    edge.dot((-v[1], v[0]))
      // but unfortunately, edges just dot with other edges.
      sum += fabs(e1*v[0] - e0*v[1]);
    }
  }
  const double h = 4 * conv_norm * area / sum;

  // // The alternative for the lazy:
  // const double h = cell.diameter();

  // // Just a little sanity check here.
  // const double eps = 1.0e-12;
  // if (h > cell.diameter() + eps) {
  //     std::cout << "The directed diameter h (" << h << ") "
  //               << "should not be larger than the actual cell diameter "
  //               << "(" << cell.diameter() << ")."
  //               << std::endl;
  // }
  // assert(h < cell.diameter() + eps);

  const double Pe = 0.5*conv_norm * h/(p*sigma);
  assert(Pe > 0.0);
  // We'd like to compute `xi = 1.0/tanh(Pe) - 1.0/Pe`. This expression can
  // hardly be evaluated for small Pe, see
  // <https://stackoverflow.com/a/43279491/353337>. Hence, use its Taylor
  // expansion around 0.
  const double xi = Pe > 1.0e-5 ?
      1.0/tanh(Pe) - 1.0/Pe :
      Pe/3.0 - Pe*Pe*Pe / 45.0 + 2.0/945.0 * Pe*Pe*Pe*Pe*Pe;

  tau[0] = 0.5*h*xi / (p*conv_norm);

  //if (tau[0] > 1.0e3)
  //{
  //  std::cout << "tau   = " << tau[0] << std::endl;
  //  std::cout << "||b|| = " << conv_norm << std::endl;
  //  std::cout << "Pe    = " << Pe << std::endl;
  //  std::cout << "h     = " << h << std::endl;
  //  std::cout << "xi    = " << xi << std::endl;
  //  //throw 1;
  //}

  return;
}
};
'''
    tau = Expression(cppcode, degree=1)
    tau.convection = convection
    tau.mesh = mesh  # convection.function_space().mesh()
    tau.sigma = diffusion_coefficient
    tau.p = element_degree

    return tau
<|endoftext|>


<commit_before>__author__ = 'Aaron Anderson'

import re

from boto.s3.connection import S3Connection

class NexradQuery(object):
    def __init__(self):
        self.year_re = re.compile('^(\d{4})/')
        self.month_re = re.compile('^\d{4}/(\d{2})')
        self.day_re = re.compile('^\d{4}/\d{2}/(\d{2})')
        self.radar_re = re.compile('^\d{4}/\d{2}/\d{2}/(....)/')
        self.scan_re = re.compile('^\d{4}/\d{2}/\d{2}/..../(.*.gz)')
        self.s3conn = S3Connection(anon=True)
        self.bucket = self.s3conn.get_bucket('noaa-nexrad-level2')

    def get_available_years(self):
        """
        This method allows you to get the years that are currently available.
        :rtype : list
        :return: A list of strings representing the years available
        """
        years = []
        resp = list(self.bucket.list("", "/"))
        for each in resp:
            match = self.year_re.match(each.name)
            if match is not None:
                years.append(match.group(1))
        return years

    def get_available_months(self, year):
        """
        This method allows you to get the available months in a given year.
        :rtype : list
        :param year: string - the year we are requesting available months for (i.e. 2010)
        :return: A list of strings representing the months available for that year
        """
        months = []
        resp = list(self.bucket.list("%s/" % year, "/"))
        for each in resp:
            match = self.month_re.match(each.name)
            if match is not None:
                months.append(match.group(1))
        return months

    def get_available_days(self, year, month):
        """
        This method allows you to get the available days in a given month and year.
        :rtype : list
        :param year: string - the year we are requesting available days for (i.e 2010)
        :param month: string - the month we are requesting available days for (i.e. 05)
        :return: A list of strings representing the days available in the given month and year
        """
        days = []
        resp = list(self.bucket.list("%s/%s/" % (year, month), "/"))
        for each in resp:
            match = self.day_re.match(each.name)
            if match is not None:
                days.append(match.group(1))
        return days

    def get_available_radars(self, year, month, day):
        """
        This method allows you to get the available radars in a given day, month, and year.
        :rtype : list
        :param year: string - the year we are requesting available radars for (i.e 2010)
        :param month: string - the month we are requesting available radars for (i.e. 05)
        :param day: string - the day we are requesting available radars for (i.e. 01)
        :return: A list of string representing the radar sites available in the given
        day, month, and year
        """
        radars = []
        resp = list(self.bucket.list("%s/%s/%s/" % (year, month, day), "/"))
        for each in resp:
            match = self.radar_re.match(each.name)
            if match is not None:
                radars.append(match.group(1))
        return radars

    def get_available_scans(self, year, month, day, radar):
        """
        This method allows you to get the available radar scans for a given radar, day, month, and year.
        :rtype : list
        :param year: string - the year we are requesting available scans for (i.e 2010)
        :param month: string - the month we are requesting available scans for (i.e. 05)
        :param day: string - the day we are requesting available scans for (i.e. 01)
        :param radar: string - the radar id we are requesting available scans for (i.e. KTLX)
        :return: A list of string representing the radar scans available for a given radar,
        day, month, and year
        """
        scans = []
        resp = list(self.bucket.list("%s/%s/%s/%s/" % (year, month, day, radar)))
        for each in resp:
            match = self.scan_re.match(each.name)
            if match is not None:
                scans.append(match.group(1))
        return scans

if __name__ == '__main__':
    query = NexradQuery()
    print query.get_available_years()
    print query.get_available_months("2010")
    print query.get_available_days("2010", "02")
    print query.get_available_radars("2010", "02", "02")
    print query.get_available_scans("2010", "02", "02", "KTLX")
<commit_msg>set to private variables<commit_after>__author__ = 'Aaron Anderson'

import re

from boto.s3.connection import S3Connection

class NexradQuery(object):
    def __init__(self):
        self._year_re = re.compile('^(\d{4})/')
        self._month_re = re.compile('^\d{4}/(\d{2})')
        self._day_re = re.compile('^\d{4}/\d{2}/(\d{2})')
        self._radar_re = re.compile('^\d{4}/\d{2}/\d{2}/(....)/')
        self._scan_re = re.compile('^\d{4}/\d{2}/\d{2}/..../(.*.gz)')
        self._s3conn = S3Connection(anon=True)
        self._bucket = self._s3conn.get_bucket('noaa-nexrad-level2')

    def get_available_years(self):
        """
        This method allows you to get the years that are currently available.
        :rtype : list
        :return: A list of strings representing the years available
        """
        years = []
        resp = list(self._bucket.list("", "/"))
        for each in resp:
            match = self._year_re.match(each.name)
            if match is not None:
                years.append(match.group(1))
        return years

    def get_available_months(self, year):
        """
        This method allows you to get the available months in a given year.
        :rtype : list
        :param year: string - the year we are requesting available months for (i.e. 2010)
        :return: A list of strings representing the months available for that year
        """
        months = []
        resp = list(self._bucket.list("%s/" % year, "/"))
        for each in resp:
            match = self._month_re.match(each.name)
            if match is not None:
                months.append(match.group(1))
        return months

    def get_available_days(self, year, month):
        """
        This method allows you to get the available days in a given month and year.
        :rtype : list
        :param year: string - the year we are requesting available days for (i.e 2010)
        :param month: string - the month we are requesting available days for (i.e. 05)
        :return: A list of strings representing the days available in the given month and year
        """
        days = []
        resp = list(self._bucket.list("%s/%s/" % (year, month), "/"))
        for each in resp:
            match = self._day_re.match(each.name)
            if match is not None:
                days.append(match.group(1))
        return days

    def get_available_radars(self, year, month, day):
        """
        This method allows you to get the available radars in a given day, month, and year.
        :rtype : list
        :param year: string - the year we are requesting available radars for (i.e 2010)
        :param month: string - the month we are requesting available radars for (i.e. 05)
        :param day: string - the day we are requesting available radars for (i.e. 01)
        :return: A list of string representing the radar sites available in the given
        day, month, and year
        """
        radars = []
        resp = list(self._bucket.list("%s/%s/%s/" % (year, month, day), "/"))
        for each in resp:
            match = self._radar_re.match(each.name)
            if match is not None:
                radars.append(match.group(1))
        return radars

    def get_available_scans(self, year, month, day, radar):
        """
        This method allows you to get the available radar scans for a given radar, day, month, and year.
        :rtype : list
        :param year: string - the year we are requesting available scans for (i.e 2010)
        :param month: string - the month we are requesting available scans for (i.e. 05)
        :param day: string - the day we are requesting available scans for (i.e. 01)
        :param radar: string - the radar id we are requesting available scans for (i.e. KTLX)
        :return: A list of string representing the radar scans available for a given radar,
        day, month, and year
        """
        scans = []
        resp = list(self._bucket.list("%s/%s/%s/%s/" % (year, month, day, radar)))
        for each in resp:
            match = self._scan_re.match(each.name)
            if match is not None:
                scans.append(match.group(1))
        return scans

if __name__ == '__main__':
    query = NexradQuery()
    print query.get_available_years()
    print query.get_available_months("2010")
    print query.get_available_days("2010", "02")
    print query.get_available_radars("2010", "02", "02")
    print query.get_available_scans("2010", "02", "02", "KTLX")
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os
import sys
import imp
import contextlib

class ExampleCollector(object):

    def __init__(self, paths):
        self.paths = paths

    def modules(self):
        for path in self._collect_files_containing_examples():
            with self._load_module_from(path) as module:
                if self._has_examples(module):
                    yield module

    def _collect_files_containing_examples(self):
        collected = []
        for path in self.paths:
            if not os.path.exists(path):
                continue

            if os.path.isdir(path):
                collected.extend(self._collect_files_in_directory(path))
            else:
                collected.append(path)
        return collected

    def _collect_files_in_directory(self, directory):
        collected = []
        for root, dirs, files in os.walk(directory):
            collected.extend([os.path.join(root, file_) for file_ in files if file_.endswith('_spec.py')])

        collected.sort()
        return collected

    @contextlib.contextmanager
    def _load_module_from(self, path):
        name = path.replace('.py', '')
        try:
            yield imp.load_source(name, path)
        finally:
            if name in sys.modules:
                del sys.modules[name]

    def _has_examples(self, module):
        return hasattr(module, 'examples')

<commit_msg>Avoid RuntimeWarning.<commit_after># -*- coding: utf-8 -*-

import os
import sys
import imp
import contextlib

class ExampleCollector(object):

    def __init__(self, paths):
        self.paths = paths

    def modules(self):
        for path in self._collect_files_containing_examples():
            with self._load_module_from(path) as module:
                if self._has_examples(module):
                    yield module

    def _collect_files_containing_examples(self):
        collected = []
        for path in self.paths:
            if not os.path.exists(path):
                continue

            if os.path.isdir(path):
                collected.extend(self._collect_files_in_directory(path))
            else:
                collected.append(path)
        return collected

    def _collect_files_in_directory(self, directory):
        collected = []
        for root, dirs, files in os.walk(directory):
            collected.extend([os.path.join(self._normalize_path(root), file_)
                    for file_ in files if file_.endswith('_spec.py')])
        collected.sort()
        return collected

    def _normalize_path(self, path):
        return os.path.normpath(path)

    @contextlib.contextmanager
    def _load_module_from(self, path):
        name = path.replace('.py', '')
        try:
            yield imp.load_source(name, path)
        finally:
            if name in sys.modules:
                del sys.modules[name]

    def _has_examples(self, module):
        return hasattr(module, 'examples')

<|endoftext|>


<commit_before>from manage_users.exceptions import (DuplicateUserError, NoSuchUserError,
                                     SSHKeyNotFoundError)

class DuplicateUserError(Exception):
    """
        Raised when an attempt is made to create a user which already exists.
    """

    def __init__(self, user):
        """
            Create the duplicate user exception.

            Keyword arguments:
            user -- The name of the user that the collision occurred for.
        """
        super(DuplicateUserError, self).__init__(
            'Cannot create {user}, as this user already exists.'.format(
                user=user
            ),
        )

class NoSuchUserError(Exception):
    """
        Raised when an attempt is made to operate on a nonexistent user.
    """

    def __init__(self, user):
        """
            Create the missing user exception.

            Keyword arguments:
            user -- The name of the user that could not be found.
        """
        super(DuplicateUserError, self).__init__(
            'Could not modify {user} as they do not exist.'.format(
                user=user
            ),
        )

class SSHKeyNotFoundError(Exception):
    """
        Raised when an attempt is made to act on an SSH key for a user where
        the user does not have that many keys, or the user does not exist.
    """

    def __init__(self, user, key_id):
        """
            Create missing SSH key exception.

            Keyword arguments:
            user -- The name of the user that the key could not be found for.
            key_id -- The ID of the key that could not be found.
        """
        message =  'Could not find {key_id} for {user}.'.format(
            user=user
        )
        message += (' Either the user does not have this many keys, or'
                    ' they do not exist.')

        super(DuplicateUserError, self).__init__(message)
<commit_msg>Remove scratchpad import statement<commit_after>class DuplicateUserError(Exception):
    """
        Raised when an attempt is made to create a user which already exists.
    """

    def __init__(self, user):
        """
            Create the duplicate user exception.

            Keyword arguments:
            user -- The name of the user that the collision occurred for.
        """
        super(DuplicateUserError, self).__init__(
            'Cannot create {user}, as this user already exists.'.format(
                user=user
            ),
        )

class NoSuchUserError(Exception):
    """
        Raised when an attempt is made to operate on a nonexistent user.
    """

    def __init__(self, user):
        """
            Create the missing user exception.

            Keyword arguments:
            user -- The name of the user that could not be found.
        """
        super(DuplicateUserError, self).__init__(
            'Could not modify {user} as they do not exist.'.format(
                user=user
            ),
        )

class SSHKeyNotFoundError(Exception):
    """
        Raised when an attempt is made to act on an SSH key for a user where
        the user does not have that many keys, or the user does not exist.
    """

    def __init__(self, user, key_id):
        """
            Create missing SSH key exception.

            Keyword arguments:
            user -- The name of the user that the key could not be found for.
            key_id -- The ID of the key that could not be found.
        """
        message =  'Could not find {key_id} for {user}.'.format(
            user=user
        )
        message += (' Either the user does not have this many keys, or'
                    ' they do not exist.')

        super(DuplicateUserError, self).__init__(message)
<|endoftext|>


<commit_before>#!/usr/bin/python

from azuremodules import *

import argparse
import sys
import time
        #for error checking
parser = argparse.ArgumentParser()

parser.add_argument('-e', '--expected', help='specify expected DiskSize in KB', required=True)
args = parser.parse_args()
                #if no value specified then stop
expectedDiskSize = args.expected

def RunTest(expectedSize):
    UpdateState("TestRunning")
    RunLog.info("Checking DiskSize...")
    output = Run("df / | awk 'NR==2' | awk '{print $2}'")
    ActualSize = float(output)

    if (ActualSize < expectedSize*1.1 and ActualSize > expectedSize*0.9) :
        RunLog.info('Root file disk size is: %s', output)
        ResultLog.info('PASS')
        UpdateState("TestCompleted")
    else :
        RunLog.error('Getting the Disk SizeError over 10 percent different from Original OSImage: %s', output)
        ResultLog.error('FAIL')
        UpdateState("TestCompleted")

RunTest(float(expectedDiskSize))<commit_msg>Fix : BVT-VERIFY-DISKSIZE<commit_after>#!/usr/bin/python

from azuremodules import *

import argparse
import sys
import time
        #for error checking
parser = argparse.ArgumentParser()

parser.add_argument('-e', '--expected', help='specify expected DiskSize in KB', required=True)
args = parser.parse_args()
                #if no value specified then stop
expectedDiskSize = args.expected

def RunTest(expectedSize):
    UpdateState("TestRunning")
    RunLog.info("Checking DiskSize...")
    output = Run("fdisk -l | awk '/sda/ {print $5;}' | awk 'NR==1'")
    ActualSize = float(output)

    if (ActualSize < expectedSize*1.1 and ActualSize > expectedSize*0.9) :
        RunLog.info('/dev/sda disk size is: %s', output)
        ResultLog.info('PASS')
        UpdateState("TestCompleted")
    else :
        RunLog.error('Getting the Disk SizeError over 10 percent different from Original OSImage: %s', output)
        ResultLog.error('FAIL')
        UpdateState("TestCompleted")

RunTest(float(expectedDiskSize))<|endoftext|>