<commit_before># -*- coding: utf-8 -*-
"""
  monotonic
  ~~~~~~~~~

  This module provides a ``monotonic()`` function which returns the
  value (in fractional seconds) of a clock which never goes backwards.

  On Python 3.3 or newer, ``monotonic`` will be an alias of
  ``time.monotonic`` from the standard library. On older versions,
  it will fall back to an equivalent implementation:

  +-------------+----------------------------------------+
  | Linux, BSD  | ``clock_gettime(3)``                   |
  +-------------+----------------------------------------+
  | Windows     | ``GetTickCount`` or ``GetTickCount64`` |
  +-------------+----------------------------------------+
  | OS X        | ``mach_absolute_time``                 |
  +-------------+----------------------------------------+

  If no suitable implementation exists for the current platform,
  attempting to import this module (or to import from it) will
  cause a ``RuntimeError`` exception to be raised.

  Copyright 2014, 2015, 2016 Ori Livneh <ori@wikimedia.org>

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.

"""
import ctypes
import ctypes.util
import os
import sys
import threading
import time

__all__ = ('monotonic',)

try:
    monotonic = time.monotonic
except AttributeError:
    try:
        if sys.platform == 'darwin':  # OS X, iOS
            # See Technical Q&A QA1398 of the Mac Developer Library:
            #  <https://developer.apple.com/library/mac/qa/qa1398/>
            libc = ctypes.CDLL('/usr/lib/libc.dylib', use_errno=True)

            class mach_timebase_info_data_t(ctypes.Structure):
                """System timebase info. Defined in <mach/mach_time.h>."""
                _fields_ = (('numer', ctypes.c_uint32),
                            ('denom', ctypes.c_uint32))

            mach_absolute_time = libc.mach_absolute_time
            mach_absolute_time.restype = ctypes.c_uint64

            timebase = mach_timebase_info_data_t()
            libc.mach_timebase_info(ctypes.byref(timebase))
            ticks_per_second = timebase.numer / timebase.denom * 1.0e9

            def monotonic():
                """Monotonic clock, cannot go backward."""
                return mach_absolute_time() / ticks_per_second

        elif sys.platform.startswith('win32') or sys.platform.startswith('cygwin'):
            if sys.platform.startswith('cygwin'):
                # Note: cygwin implements clock_gettime (CLOCK_MONOTONIC = 4) since
                # version 1.7.6. Using raw WinAPI for maximum version compatibility.

                # Ugly hack using the wrong calling convention (in 32-bit mode)
                # because ctypes has no windll under cygwin (and it also seems that
                # the code letting you select stdcall in _ctypes doesn't exist under
                # the preprocessor definitions relevant to cygwin).
                # This is 'safe' because:
                # 1. The ABI of GetTickCount and GetTickCount64 is identical for
                #    both calling conventions because they both have no parameters.
                # 2. libffi masks the problem because after making the call it doesn't
                #    touch anything through esp and epilogue code restores a correct
                #    esp from ebp afterwards.
                try:
                    kernel32 = ctypes.cdll.kernel32
                except OSError:  # 'No such file or directory'
                    kernel32 = ctypes.cdll.LoadLibrary('kernel32.dll')
            else:
                kernel32 = ctypes.windll.kernel32

            GetTickCount64 = getattr(kernel32, 'GetTickCount64', None)
            if GetTickCount64:
                # Windows Vista / Windows Server 2008 or newer.
                GetTickCount64.restype = ctypes.c_ulonglong

                def monotonic():
                    """Monotonic clock, cannot go backward."""
                    return GetTickCount64() / 1000.0

            else:
                # Before Windows Vista.
                GetTickCount = kernel32.GetTickCount
                GetTickCount.restype = ctypes.c_uint32

                get_tick_count_lock = threading.Lock()
                get_tick_count_last_sample = 0
                get_tick_count_wraparounds = 0

                def monotonic():
                    """Monotonic clock, cannot go backward."""
                    global get_tick_count_last_sample
                    global get_tick_count_wraparounds

                    with get_tick_count_lock:
                        current_sample = GetTickCount()
                        if current_sample < get_tick_count_last_sample:
                            get_tick_count_wraparounds += 1
                        get_tick_count_last_sample = current_sample

                        final_milliseconds = get_tick_count_wraparounds << 32
                        final_milliseconds += get_tick_count_last_sample
                        return final_milliseconds / 1000.0

        else:
            try:
                clock_gettime = ctypes.CDLL(ctypes.util.find_library('c'),
                                            use_errno=True).clock_gettime
            except Exception:
                clock_gettime = ctypes.CDLL(ctypes.util.find_library('rt'),
                                            use_errno=True).clock_gettime

            class timespec(ctypes.Structure):
                """Time specification, as described in clock_gettime(3)."""
                _fields_ = (('tv_sec', ctypes.c_long),
                            ('tv_nsec', ctypes.c_long))

            if sys.platform.startswith('linux'):
                CLOCK_MONOTONIC = 1
            elif sys.platform.startswith('freebsd'):
                CLOCK_MONOTONIC = 4
            elif sys.platform.startswith('sunos5'):
                CLOCK_MONOTONIC = 4
            elif 'bsd' in sys.platform:
                CLOCK_MONOTONIC = 3
            elif sys.platform.startswith('aix'):
                CLOCK_MONOTONIC = ctypes.c_longlong(10)

            def monotonic():
                """Monotonic clock, cannot go backward."""
                ts = timespec()
                if clock_gettime(CLOCK_MONOTONIC, ctypes.pointer(ts)):
                    errno = ctypes.get_errno()
                    raise OSError(errno, os.strerror(errno))
                return ts.tv_sec + ts.tv_nsec / 1.0e9

        # Perform a sanity-check.
        if monotonic() - monotonic() > 0:
            raise ValueError('monotonic() is not monotonic!')

    except Exception as e:
        raise RuntimeError('no suitable implementation for this system: ' + repr(e))
<commit_msg>[catapult] Fix a bug in third party monotonic<commit_after># -*- coding: utf-8 -*-
"""
  monotonic
  ~~~~~~~~~

  This module provides a ``monotonic()`` function which returns the
  value (in fractional seconds) of a clock which never goes backwards.

  On Python 3.3 or newer, ``monotonic`` will be an alias of
  ``time.monotonic`` from the standard library. On older versions,
  it will fall back to an equivalent implementation:

  +-------------+----------------------------------------+
  | Linux, BSD  | ``clock_gettime(3)``                   |
  +-------------+----------------------------------------+
  | Windows     | ``GetTickCount`` or ``GetTickCount64`` |
  +-------------+----------------------------------------+
  | OS X        | ``mach_absolute_time``                 |
  +-------------+----------------------------------------+

  If no suitable implementation exists for the current platform,
  attempting to import this module (or to import from it) will
  cause a ``RuntimeError`` exception to be raised.

  Copyright 2014, 2015, 2016 Ori Livneh <ori@wikimedia.org>

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.

"""
import ctypes
import ctypes.util
import os
import sys
import threading
import time

__all__ = ('monotonic',)

try:
    monotonic = time.monotonic
except AttributeError:
    try:
        if sys.platform == 'darwin':  # OS X, iOS
            # See Technical Q&A QA1398 of the Mac Developer Library:
            #  <https://developer.apple.com/library/mac/qa/qa1398/>
            libc = ctypes.CDLL('/usr/lib/libc.dylib', use_errno=True)

            class mach_timebase_info_data_t(ctypes.Structure):
                """System timebase info. Defined in <mach/mach_time.h>."""
                _fields_ = (('numer', ctypes.c_uint32),
                            ('denom', ctypes.c_uint32))

            mach_absolute_time = libc.mach_absolute_time
            mach_absolute_time.restype = ctypes.c_uint64

            timebase = mach_timebase_info_data_t()
            libc.mach_timebase_info(ctypes.byref(timebase))
            nanoseconds_in_second = 1.0e9

            def monotonic():
                """Monotonic clock, cannot go backward."""
                nanoseconds = mach_absolute_time() * timebase.numer / timebase.denom
                return nanoseconds / nanoseconds_in_second

        elif sys.platform.startswith('win32') or sys.platform.startswith('cygwin'):
            if sys.platform.startswith('cygwin'):
                # Note: cygwin implements clock_gettime (CLOCK_MONOTONIC = 4) since
                # version 1.7.6. Using raw WinAPI for maximum version compatibility.

                # Ugly hack using the wrong calling convention (in 32-bit mode)
                # because ctypes has no windll under cygwin (and it also seems that
                # the code letting you select stdcall in _ctypes doesn't exist under
                # the preprocessor definitions relevant to cygwin).
                # This is 'safe' because:
                # 1. The ABI of GetTickCount and GetTickCount64 is identical for
                #    both calling conventions because they both have no parameters.
                # 2. libffi masks the problem because after making the call it doesn't
                #    touch anything through esp and epilogue code restores a correct
                #    esp from ebp afterwards.
                try:
                    kernel32 = ctypes.cdll.kernel32
                except OSError:  # 'No such file or directory'
                    kernel32 = ctypes.cdll.LoadLibrary('kernel32.dll')
            else:
                kernel32 = ctypes.windll.kernel32

            GetTickCount64 = getattr(kernel32, 'GetTickCount64', None)
            if GetTickCount64:
                # Windows Vista / Windows Server 2008 or newer.
                GetTickCount64.restype = ctypes.c_ulonglong

                def monotonic():
                    """Monotonic clock, cannot go backward."""
                    return GetTickCount64() / 1000.0

            else:
                # Before Windows Vista.
                GetTickCount = kernel32.GetTickCount
                GetTickCount.restype = ctypes.c_uint32

                get_tick_count_lock = threading.Lock()
                get_tick_count_last_sample = 0
                get_tick_count_wraparounds = 0

                def monotonic():
                    """Monotonic clock, cannot go backward."""
                    global get_tick_count_last_sample
                    global get_tick_count_wraparounds

                    with get_tick_count_lock:
                        current_sample = GetTickCount()
                        if current_sample < get_tick_count_last_sample:
                            get_tick_count_wraparounds += 1
                        get_tick_count_last_sample = current_sample

                        final_milliseconds = get_tick_count_wraparounds << 32
                        final_milliseconds += get_tick_count_last_sample
                        return final_milliseconds / 1000.0

        else:
            try:
                clock_gettime = ctypes.CDLL(ctypes.util.find_library('c'),
                                            use_errno=True).clock_gettime
            except Exception:
                clock_gettime = ctypes.CDLL(ctypes.util.find_library('rt'),
                                            use_errno=True).clock_gettime

            class timespec(ctypes.Structure):
                """Time specification, as described in clock_gettime(3)."""
                _fields_ = (('tv_sec', ctypes.c_long),
                            ('tv_nsec', ctypes.c_long))

            if sys.platform.startswith('linux'):
                CLOCK_MONOTONIC = 1
            elif sys.platform.startswith('freebsd'):
                CLOCK_MONOTONIC = 4
            elif sys.platform.startswith('sunos5'):
                CLOCK_MONOTONIC = 4
            elif 'bsd' in sys.platform:
                CLOCK_MONOTONIC = 3
            elif sys.platform.startswith('aix'):
                CLOCK_MONOTONIC = ctypes.c_longlong(10)

            def monotonic():
                """Monotonic clock, cannot go backward."""
                ts = timespec()
                if clock_gettime(CLOCK_MONOTONIC, ctypes.pointer(ts)):
                    errno = ctypes.get_errno()
                    raise OSError(errno, os.strerror(errno))
                return ts.tv_sec + ts.tv_nsec / 1.0e9

        # Perform a sanity-check.
        if monotonic() - monotonic() > 0:
            raise ValueError('monotonic() is not monotonic!')

    except Exception as e:
        raise RuntimeError('no suitable implementation for this system: ' + repr(e))
<|endoftext|>


<commit_before>#!/usr/bin/env python3.5

import sys
from time import ctime
from tweepy import API
from tweepy import Stream
from tweepy import OAuthHandler
from tweepy.streaming import StreamListener
from credentials import *
from tweepy.utils import import_simplejson
import markovify
import random
import argparse

json = import_simplejson()

class Listener(StreamListener):
    def __init__(self, api, followed_user_id, followed_user_handle, mock_mode):
        super().__init__(api)
        self.tweet_data = []
        self.followed_user_id = followed_user_id
        self.followed_user_handle = followed_user_handle

        self.mock_mode = mock_mode
        self.reply_list = []
        self.next_reply = ''
        self.load_next_reply(mock_mode)

    def on_error(self, error):
        print("Returned error code %s" % error)
        return False

    def on_status(self, status):
        if status.user.id == self.followed_user_id:
            tweet_text = '@%s %s' % (self.followed_user_handle, self.next_reply)
            self.api.update_status(tweet_text, in_reply_to_status_id=status.id)

            print('%s: Tweeted: %s' % (ctime(), tweet_text))

            if self.mock_mode:
                self.update_mock_text(status.text)

            self.load_next_reply(self.mock_mode)

    def load_next_reply(self, mock=False):
        if not mock:
            with open('reply_list.txt', 'r') as reply_list_file:
                self.reply_list = reply_list_file.readlines()

            self.next_reply = random.choice(self.reply_list)

        else:
            with open('user_tweet_history.txt') as user_tweet_history_file:
                text = user_tweet_history_file.read()

            text_model = markovify.Text(text)
            self.next_reply = text_model.make_short_sentence(140)

    @staticmethod
    def update_mock_text(text):
        with open('user_tweet_history.txt', 'wa') as user_tweet_history_fd:
            user_tweet_history_fd.write(text)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--handle',
                        required=True,
                        type=str,
                        dest='followed_handle',
                        action='store',
                        help='Twitter handle (without @)')
    parser.add_argument('--mock',
                        dest='mock_mode',
                        default=False,
                        action='store_true',
                        help='enable mock mode')
    args = parser.parse_args()

    auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
    auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
    api = API(auth)

    found_users = api.lookup_users(screen_names=[str(args.followed_handle)])

    if len(found_users) != 1:
        print('Lookup for twitter handle %s failed' % args.followed_handle)
        sys.exit()

    followed_user_id = found_users[0].id

    twitterStream = Stream(auth, Listener(api, followed_user_id, args.followed_handle, args.mock_mode))
    twitterStream.filter(follow=[str(followed_user_id)], async=True)
<commit_msg>Write write to user_tweet_history<commit_after>#!/usr/bin/env python3.5

import sys
from time import ctime
from tweepy import API
from tweepy import Stream
from tweepy import OAuthHandler
from tweepy.streaming import StreamListener
from credentials import *
from tweepy.utils import import_simplejson
import markovify
import random
import argparse

json = import_simplejson()

class Listener(StreamListener):
    def __init__(self, api, followed_user_id, followed_user_handle, mock_mode):
        super().__init__(api)
        self.tweet_data = []
        self.followed_user_id = followed_user_id
        self.followed_user_handle = followed_user_handle

        self.mock_mode = mock_mode
        self.reply_list = []
        self.next_reply = ''
        self.load_next_reply(mock_mode)

    def on_error(self, error):
        print("Returned error code %s" % error)
        return False

    def on_status(self, status):
        if status.user.id == self.followed_user_id:
            tweet_text = '@%s %s' % (self.followed_user_handle, self.next_reply)
            self.api.update_status(tweet_text, in_reply_to_status_id=status.id)

            print('%s: Tweeted: %s' % (ctime(), tweet_text))

            if self.mock_mode:
                self.update_mock_text(status.text)

            self.load_next_reply(self.mock_mode)

    def load_next_reply(self, mock=False):
        if not mock:
            with open('reply_list.txt', 'r') as reply_list_file:
                self.reply_list = reply_list_file.readlines()

            self.next_reply = random.choice(self.reply_list)

        else:
            with open('user_tweet_history.txt') as user_tweet_history_file:
                text = user_tweet_history_file.read()

            text_model = markovify.Text(text)
            self.next_reply = text_model.make_short_sentence(140)

    @staticmethod
    def update_mock_text(text):
        with open('user_tweet_history.txt', 'a') as user_tweet_history_fd:
            user_tweet_history_fd.write("\n")
            user_tweet_history_fd.write(text)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--handle',
                        required=True,
                        type=str,
                        dest='followed_handle',
                        action='store',
                        help='Twitter handle (without @)')
    parser.add_argument('--mock',
                        dest='mock_mode',
                        default=False,
                        action='store_true',
                        help='enable mock mode')
    args = parser.parse_args()

    auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
    auth.set_access_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
    api = API(auth)

    found_users = api.lookup_users(screen_names=[str(args.followed_handle)])

    if len(found_users) != 1:
        print('Lookup for twitter handle %s failed' % args.followed_handle)
        sys.exit()

    followed_user_id = found_users[0].id

    twitterStream = Stream(auth, Listener(api, followed_user_id, args.followed_handle, args.mock_mode))
    twitterStream.filter(follow=[str(followed_user_id)], async=True)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright (c) 2015 Angel Terrones (<angelterrones@gmail.com>)
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

from myhdl import Signal
from myhdl import modbv
from myhdl import always_comb
from myhdl import concat
from myhdl import instances
from Core.consts import Consts

class IMMGen:
    """
    Generate the immediate values.
    """
    def __init__(self,
                 sel:         Signal,
                 instruction: Signal,
                 imm:         Signal):
        """
        Initializes the IO ports.

        :param sel:         Select the type of instruction/immediate
        :param instruction: Current instruction
        :param imm:         A 32-bit immediate value
        """
        self.sel         = sel
        self.instruction = instruction
        self.imm         = imm

    def GetRTL(self):
        """
        Defines module behavior
        """
        sign   = Signal(False)
        b30_20 = Signal(modbv(0)[11:])
        b19_12 = Signal(modbv(0)[8:])
        b11    = Signal(False)
        b10_5  = Signal(modbv(0)[6:])
        b4_1   = Signal(modbv(0)[4:])
        b0     = Signal(False)

        @always_comb
        def _sign():
            sign.next     = False if self.sel == Consts.IMM_Z else self.instruction[31]

        @always_comb
        def rtl():
            b30_20.next   = self.instruction[31:20] if self.sel == Consts.IMM_U else concat(*[sign for _ in range(11)])
            b19_12.next   = self.instruction[20:12] if (self.sel == Consts.IMM_U or self.sel == Consts.IMM_UJ) else concat(*[sign for _ in range(8)])
            b11.next      = (False if (self.sel == Consts.IMM_U or self.sel == Consts.IMM_Z) else
                             (self.instruction[20] if self.sel == Consts.IMM_UJ else
                              (self.instruction[7] if self.sel == Consts.IMM_SB else sign)))
            b10_5.next    = 0 if (self.sel == Consts.IMM_U or self.sel == Consts.IMM_Z) else self.instruction[31:25]
            b4_1.next     = (0 if self.sel == Consts.IMM_U else
                             (self.instruction[12:8] if (self.sel == Consts.IMM_S or self.sel == Consts.IMM_SB) else
                              (self.instruction[20:16] if self.sel == Consts.IMM_Z else self.instruction[25:21])))
            b0.next       = (self.instruction[7] if self.sel == Consts.IMM_S else
                             (self.instruction[20] if self.sel == Consts.IMM_I else
                              (self.instruction[15] if self.sel == Consts.IMM_Z else False)))

        @always_comb
        def imm_concat():
            self.imm.next = concat(sign, b30_20, b19_12, b11, b10_5, b4_1, b0)

        return instances()

# Local Variables:
# flycheck-flake8-maximum-line-length: 200
# flycheck-flake8rc: ".flake8rc"
# End:
<commit_msg>Fix for translation.<commit_after>#!/usr/bin/env python
# Copyright (c) 2015 Angel Terrones (<angelterrones@gmail.com>)
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

from myhdl import Signal
from myhdl import modbv
from myhdl import always_comb
from myhdl import concat
from myhdl import instances
from Core.consts import Consts

class IMMGen:
    """
    Generate the immediate values.
    """
    def __init__(self,
                 sel:         Signal,
                 instruction: Signal,
                 imm:         Signal):
        """
        Initializes the IO ports.

        :param sel:         Select the type of instruction/immediate
        :param instruction: Current instruction
        :param imm:         A 32-bit immediate value
        """
        self.sel         = sel
        self.instruction = instruction
        self.imm         = imm

    def GetRTL(self):
        """
        Defines module behavior
        """
        sign   = Signal(False)
        b30_20 = Signal(modbv(0)[11:])
        b19_12 = Signal(modbv(0)[8:])
        b11    = Signal(False)
        b10_5  = Signal(modbv(0)[6:])
        b4_1   = Signal(modbv(0)[4:])
        b0     = Signal(False)

        @always_comb
        def _sign():
            sign.next     = False if self.sel == Consts.IMM_Z else self.instruction[31]

        @always_comb
        def rtl():
            b30_20.next   = self.instruction[31:20] if self.sel == Consts.IMM_U else concat(sign, sign, sign, sign, sign, sign, sign, sign, sign, sign, sign)
            b19_12.next   = self.instruction[20:12] if (self.sel == Consts.IMM_U or self.sel == Consts.IMM_UJ) else concat(sign, sign, sign, sign, sign, sign, sign, sign)
            b11.next      = (False if (self.sel == Consts.IMM_U or self.sel == Consts.IMM_Z) else
                             (self.instruction[20] if self.sel == Consts.IMM_UJ else
                              (self.instruction[7] if self.sel == Consts.IMM_SB else sign)))
            b10_5.next    = 0 if (self.sel == Consts.IMM_U or self.sel == Consts.IMM_Z) else self.instruction[31:25]
            b4_1.next     = (0 if self.sel == Consts.IMM_U else
                             (self.instruction[12:8] if (self.sel == Consts.IMM_S or self.sel == Consts.IMM_SB) else
                              (self.instruction[20:16] if self.sel == Consts.IMM_Z else self.instruction[25:21])))
            b0.next       = (self.instruction[7] if self.sel == Consts.IMM_S else
                             (self.instruction[20] if self.sel == Consts.IMM_I else
                              (self.instruction[15] if self.sel == Consts.IMM_Z else False)))

        @always_comb
        def imm_concat():
            self.imm.next = concat(sign, b30_20, b19_12, b11, b10_5, b4_1, b0)

        return instances()

# Local Variables:
# flycheck-flake8-maximum-line-length: 200
# flycheck-flake8rc: ".flake8rc"
# End:
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Convolutional network example.

Run the training for 50 epochs with
```
python __init__.py --num-epochs 50
```
It is going to reach around 0.8% error rate on the test set.

"""
from __future__ import print_function

import sys
import logging
import numpy
import os
import subprocess
from argparse import ArgumentParser

import theano
from theano import tensor

from blocks.algorithms import (
    GradientDescent, RMSProp, Adam,
    Restrict, CompositeRule, VariableClipping)
from blocks.bricks.base import application
from blocks.bricks import (MLP, Rectifier, Initializable, FeedforwardSequence,
                           Softmax, Activation)
from blocks.bricks.conv import (Convolutional, ConvolutionalSequence,
                                Flattener, MaxPooling)
from blocks.bricks.cost import CategoricalCrossEntropy, MisclassificationRate
from blocks.bricks.simple import Linear
from blocks.extensions import FinishAfter, Timing, Printing
from blocks.extensions.monitoring import (DataStreamMonitoring,
                                          TrainingDataMonitoring)
from blocks.extensions.saveload import Checkpoint
from blocks.extensions.training import TrackTheBest
from blocks.extensions.predicates import OnLogRecord
from blocks.serialization import load_parameters
from blocks.initialization import Constant, Uniform
from blocks.main_loop import MainLoop
from blocks.model import Model
from blocks.monitoring import aggregation
from blocks.filter import VariableFilter
from blocks.graph import ComputationGraph, apply_dropout
from blocks.roles import OUTPUT, WEIGHT
from fuel.datasets import DogsVsCats
from fuel.schemes import ShuffledScheme, SequentialExampleScheme
from fuel.streams import DataStream, ServerDataStream
from toolz.itertoolz import interleave

from server import add_transformers

logger = logging.getLogger(__name__)

class LeNet(FeedforwardSequence, Initializable):
    """LeNet-like convolutional network.

    The class implements LeNet, which is a convolutional sequence with
    an MLP on top (several fully-connected layers). For details see
    [LeCun95]_.

    .. [LeCun95] LeCun, Yann, et al.
       *Comparison of learning algorithms for handwritten digit
       recognition.*,
       International conference on artificial neural networks. Vol. 60.

    Parameters
    ----------
    conv_activations : list of :class:`.Brick`
        Activations for convolutional network.
    num_channels : int
        Number of channels in the input image.
    image_shape : tuple
        Input image shape.
    filter_sizes : list of tuples
        Filter sizes of :class:`.blocks.conv.ConvolutionalLayer`.
    feature_maps : list
        Number of filters for each of convolutions.
    pooling_sizes : list of tuples
        Sizes of max pooling for each convolutional layer.
    top_mlp_activations : list of :class:`.blocks.bricks.Activation`
        List of activations for the top MLP.
    top_mlp_dims : list
        Numbers of hidden units and the output dimension of the top MLP.
    conv_step : tuples
        Step of convolution (similar for all layers).
    border_mode : str
        Border mode of convolution (similar for all layers).

    """
    def __init__(self, conv_activations, num_channels, image_shape,
                 filter_sizes, feature_maps, pooling_sizes,
                 top_mlp_activations, top_mlp_dims,
                 conv_step=None, border_mode='valid', **kwargs):
        if conv_step is None:
            self.conv_step = (1, 1)
        else:
            self.conv_step = conv_step
        self.num_channels = num_channels
        self.image_shape = image_shape
        self.top_mlp_activations = top_mlp_activations
        self.top_mlp_dims = top_mlp_dims
        self.border_mode = border_mode

        conv_parameters = zip(filter_sizes, feature_maps)

        # Construct convolutional layers with corresponding parameters
        self.layers = list(interleave([
            (Convolutional(filter_size=filter_size,
                           num_filters=num_filter,
                           step=self.conv_step,
                           border_mode=self.border_mode,
                           name='conv_{}'.format(i))
             for i, (filter_size, num_filter)
             in enumerate(conv_parameters)),
            conv_activations,
            (MaxPooling(size, name='pool_{}'.format(i))
             for i, size in enumerate(pooling_sizes))]))

        self.conv_sequence = ConvolutionalSequence(self.layers, num_channels,
                                                   image_size=image_shape)

        # Construct a top MLP
        self.top_mlp = MLP(top_mlp_activations, top_mlp_dims)

        # We need to flatten the output of the last convolutional layer.
        # This brick accepts a tensor of dimension (batch_size, ...) and
        # returns a matrix (batch_size, features)
        self.flattener = Flattener()
        application_methods = [self.conv_sequence.apply, self.flattener.apply,
                               self.top_mlp.apply]
        super(LeNet, self).__init__(application_methods, **kwargs)

    @property
    def output_dim(self):
        return self.top_mlp_dims[-1]

    @output_dim.setter
    def output_dim(self, value):
        self.top_mlp_dims[-1] = value

    def _push_allocation_config(self):
        self.conv_sequence._push_allocation_config()
        conv_out_dim = self.conv_sequence.get_dim('output')

        self.top_mlp.activations = self.top_mlp_activations
        self.top_mlp.dims = [numpy.prod(conv_out_dim)] + self.top_mlp_dims

    @application(inputs=['image'])
    def apply_5windows(self, image):
        width, height = self.image_shape
        # the dimension 0 stands for the
        hor_offset = (image.shape[1] - width) / 2
        ver_offset = (image.shape[2] - height) / 2
        x_views = tensor.concatenate(
            [image[None, :, :width, :height],
             image[None, :, :width, -height:],
             image[None, :, -width:, :height],
             image[None, :, -width:, -height:],
             image[None, :,
                   hor_offset:hor_offset + width, ver_offset:ver_offset + height]],
             axis=0)
        return self.apply(x_views).mean(axis=0)[None, :]

def main(mode, save_to, num_epochs, load_params=None, feature_maps=None, mlp_hiddens=None,
         conv_sizes=None, pool_sizes=None,
         batch_size=None, num_batches=None, algo=None,
         test_set=None, valid_examples=None,
         dropout=None, max_norm=None):
    if feature_maps is None:
        feature_maps = [20, 50, 50]
    if mlp_hiddens is None:
        mlp_hiddens = [500]
    if conv_sizes is None:
        conv_sizes = [5, 5, 5]
    if pool_sizes is None:
        pool_sizes = [2, 2, 2]
    if batch_size is None:
        batch_size = 500
    if valid_examples is None:
        valid_examples = 2500
    if test_set is None:
        test_set = 'test'
    if algo is None:
        algo = 'rmsprop'

    image_size = (128, 128)
    output_size = 2

    if (len(feature_maps) != len(conv_sizes) or
        len(feature_maps) != len(pool_sizes)):
        raise ValueError("OMG, inconsistent arguments")

    # Use ReLUs everywhere and softmax for the final prediction
    conv_activations = [Rectifier() for _ in feature_maps]
    mlp_activations = [Rectifier() for _ in mlp_hiddens] + [Softmax()]
    convnet = LeNet(conv_activations, 3, image_size,
                    filter_sizes=zip(conv_sizes, conv_sizes),
                    feature_maps=feature_maps,
                    pooling_sizes=zip(pool_sizes, pool_sizes),
                    top_mlp_activations=mlp_activations,
                    top_mlp_dims=mlp_hiddens + [output_size],
                    border_mode='full',
                    weights_init=Uniform(width=.1),
                    biases_init=Constant(0))
    # We push initialization config to set different initialization schemes
    # for convolutional layers.
    convnet.initialize()
    logging.info("Input dim: {} {} {}".format(
        *convnet.children[0].get_dim('input_')))
    for i, layer in enumerate(convnet.layers):
        if isinstance(layer, Activation):
            logging.info("Layer {} ({})".format(
                i, layer.__class__.__name__))
        else:
            logging.info("Layer {} ({}) dim: {} {} {}".format(
                i, layer.__class__.__name__, *layer.get_dim('output')))

    single_x = tensor.tensor3('image_features')
    x = tensor.tensor4('image_features')
    single_y = tensor.lvector('targets')
    y = tensor.lmatrix('targets')

    # Training
    probs = convnet.apply(x)
    cost = (CategoricalCrossEntropy().apply(y.flatten(), probs)
            .copy(name='cost'))
    error_rate = (MisclassificationRate().apply(y.flatten(), probs)
                  .copy(name='error_rate'))

    cg = ComputationGraph([cost, error_rate])
    if dropout:
        relu_outputs = VariableFilter(bricks=[Rectifier], roles=[OUTPUT])(cg)
        cg = apply_dropout(cg, relu_outputs, dropout)
        cost, error_rate = cg.outputs

    # Validation
    valid_probs = convnet.apply_5windows(single_x)
    valid_cost = (CategoricalCrossEntropy().apply(single_y, valid_probs)
            .copy(name='cost'))
    valid_error_rate = (MisclassificationRate().apply(
        single_y, valid_probs).copy(name='error_rate'))

    model = Model([cost, error_rate])
    if load_params:
        logger.info("Loaded params from {}".format(load_params))
        with open(load_params, 'r') as src:
            model.set_parameter_values(load_parameters(src))

    # Training stream with random cropping
    train = DogsVsCats(("train",), subset=slice(None, 25000 - valid_examples, None))
    train_str =  DataStream(
        train, iteration_scheme=ShuffledScheme(train.num_examples, batch_size))
    train_str = add_transformers(train_str, random_crop=True)

    # Validation stream without cropping
    valid = DogsVsCats(("train",), subset=slice(25000 - valid_examples, None, None))
    valid_str = DataStream(
        valid, iteration_scheme=SequentialExampleScheme(valid.num_examples))
    valid_str = add_transformers(valid_str)

    if mode == 'train':
        directory, _ = os.path.split(sys.argv[0])
        env = dict(os.environ)
        env['THEANO_FLAGS'] = 'floatX=float32'
        port = numpy.random.randint(1025, 10000)
        server = subprocess.Popen(
            [directory + '/server.py',
             str(25000 - valid_examples), str(batch_size), str(port)],
            env=env, stderr=subprocess.STDOUT)
        train_str = ServerDataStream(
            ('image_features', 'targets'), produces_examples=False,
            port=port)

        save_to_base, save_to_extension = os.path.splitext(save_to)

        # Train with simple SGD
        if algo == 'rmsprop':
            step_rule = RMSProp(decay_rate=0.999, learning_rate=0.0003)
        elif algo == 'adam':
            step_rule = Adam()
        else:
            assert False
        if max_norm:
            conv_params = VariableFilter(bricks=[Convolutional], roles=[WEIGHT])(cg)
            linear_params = VariableFilter(bricks=[Linear], roles=[WEIGHT])(cg)
            step_rule = CompositeRule(
                [step_rule,
                 Restrict(VariableClipping(max_norm, axis=0), linear_params),
                 Restrict(VariableClipping(max_norm, axis=(1, 2, 3)), conv_params)])

        algorithm = GradientDescent(
            cost=cost, parameters=model.parameters,
            step_rule=step_rule)
        # `Timing` extension reports time for reading data, aggregating a batch
        # and monitoring;
        # `ProgressBar` displays a nice progress bar during training.
        extensions = [Timing(every_n_batches=100),
                    FinishAfter(after_n_epochs=num_epochs,
                                after_n_batches=num_batches),
                    DataStreamMonitoring(
                        [valid_cost, valid_error_rate],
                        valid_str,
                        prefix="valid"),
                    TrainingDataMonitoring(
                        [cost, error_rate,
                        aggregation.mean(algorithm.total_gradient_norm)],
                        prefix="train",
                        after_epoch=True),
                    TrackTheBest("valid_error_rate"),
                    Checkpoint(save_to, save_separately=['log'],
                                before_training=True, after_epoch=True)
                        .add_condition(
                            ['after_epoch'],
                            OnLogRecord("valid_error_rate_best_so_far"),
                            (save_to_base + '_best' + save_to_extension,)),
                    Printing(every_n_batches=100)]

        model = Model(cost)

        main_loop = MainLoop(
            algorithm,
            train_str,
            model=model,
            extensions=extensions)
        try:
            main_loop.run()
        finally:
            server.terminate()
    elif mode == 'test':
        classify = theano.function([single_x], valid_probs.argmax())
        test = DogsVsCats((test_set,))
        test_str = DataStream(
            test, iteration_scheme=SequentialExampleScheme(test.num_examples))
        test_str = add_transformers(test_str)
        correct = 0
        with open(save_to, 'w') as dst:
            print("id", "label", sep=',', file=dst)
            for index, example in enumerate(test_str.get_epoch_iterator()):
                image = example[0]
                prediction = classify(image)
                print(index + 1, classify(image), sep=',', file=dst)
                if len(example) > 1 and prediction == example[1]:
                    correct += 1
        print(correct / float(test.num_examples))
    else:
        assert False

if __name__ == "__main__":
    logging.basicConfig(
        level=logging.DEBUG,
        format="%(asctime)s: %(name)s: %(levelname)s: %(message)s")
    parser = ArgumentParser("An example of training a convolutional network "
                            "on the MNIST dataset.")
    parser.add_argument("mode",
                        help="What to do", choices=['train', 'test'])
    parser.add_argument("save_to",
                        help="Destination to save the state of the training "
                             "process.")

    parser.add_argument("--num-epochs", type=int, default=2,
                        help="Number of training epochs to do.")
    parser.add_argument("--load-params", help="Path to load parameters from")
    parser.add_argument("--batch-size", type=int,
                        help="Batch size.")
    parser.add_argument("--algo", choices=['rmsprop', 'adam'],
                        help="The algorithm to use.")
    parser.add_argument("--dropout", type=float,
                        help="Dropout coefficient")
    parser.add_argument("--max-norm", type=float,
                        help="Dropout coefficient")

    parser.add_argument("--test-set", type=str)
    parser.add_argument("--valid-examples", type=int)

    parser.add_argument("--feature-maps", type=int, nargs='+',
                        help="List of feature maps numbers.")
    parser.add_argument("--mlp-hiddens", type=int, nargs='+',
                        help="List of numbers of hidden units for the MLP.")
    parser.add_argument("--conv-sizes", type=int, nargs='+',
                        help="Convolutional kernels sizes. The kernels are "
                        "always square.")
    parser.add_argument("--pool-sizes", type=int, nargs='+',
                        help="Pooling sizes. The pooling windows are always "
                             "square. Should be the same length as "
                             "--conv-sizes.")

    args = parser.parse_args()
    main(**vars(args))
<commit_msg>Configurable stride for the first layer<commit_after>#!/usr/bin/env python
"""Convolutional network example.

Run the training for 50 epochs with
```
python __init__.py --num-epochs 50
```
It is going to reach around 0.8% error rate on the test set.

"""
from __future__ import print_function

import sys
import logging
import numpy
import os
import subprocess
from argparse import ArgumentParser

import theano
from theano import tensor

from blocks.algorithms import (
    GradientDescent, RMSProp, Adam,
    Restrict, CompositeRule, VariableClipping)
from blocks.bricks.base import application
from blocks.bricks import (MLP, Rectifier, Initializable, FeedforwardSequence,
                           Softmax, Activation)
from blocks.bricks.conv import (Convolutional, ConvolutionalSequence,
                                Flattener, MaxPooling)
from blocks.bricks.cost import CategoricalCrossEntropy, MisclassificationRate
from blocks.bricks.simple import Linear
from blocks.extensions import FinishAfter, Timing, Printing
from blocks.extensions.monitoring import (DataStreamMonitoring,
                                          TrainingDataMonitoring)
from blocks.extensions.saveload import Checkpoint
from blocks.extensions.training import TrackTheBest
from blocks.extensions.predicates import OnLogRecord
from blocks.serialization import load_parameters
from blocks.initialization import Constant, Uniform
from blocks.main_loop import MainLoop
from blocks.model import Model
from blocks.monitoring import aggregation
from blocks.filter import VariableFilter
from blocks.graph import ComputationGraph, apply_dropout
from blocks.roles import OUTPUT, WEIGHT
from fuel.datasets import DogsVsCats
from fuel.schemes import ShuffledScheme, SequentialExampleScheme
from fuel.streams import DataStream, ServerDataStream
from toolz.itertoolz import interleave

from server import add_transformers

logger = logging.getLogger(__name__)

class LeNet(FeedforwardSequence, Initializable):
    """LeNet-like convolutional network.

    The class implements LeNet, which is a convolutional sequence with
    an MLP on top (several fully-connected layers). For details see
    [LeCun95]_.

    .. [LeCun95] LeCun, Yann, et al.
       *Comparison of learning algorithms for handwritten digit
       recognition.*,
       International conference on artificial neural networks. Vol. 60.

    Parameters
    ----------
    conv_activations : list of :class:`.Brick`
        Activations for convolutional network.
    num_channels : int
        Number of channels in the input image.
    image_shape : tuple
        Input image shape.
    filter_sizes : list of tuples
        Filter sizes of :class:`.blocks.conv.ConvolutionalLayer`.
    feature_maps : list
        Number of filters for each of convolutions.
    pooling_sizes : list of tuples
        Sizes of max pooling for each convolutional layer.
    top_mlp_activations : list of :class:`.blocks.bricks.Activation`
        List of activations for the top MLP.
    top_mlp_dims : list
        Numbers of hidden units and the output dimension of the top MLP.
    stride : int
        Step of convolution for the first layer, 1 will be used
        for all other layers.
    border_mode : str
        Border mode of convolution (similar for all layers).

    """
    def __init__(self, conv_activations, num_channels, image_shape,
                 filter_sizes, feature_maps, pooling_sizes,
                 top_mlp_activations, top_mlp_dims,
                 stride, border_mode='valid', **kwargs):
        self.stride = stride
        self.num_channels = num_channels
        self.image_shape = image_shape
        self.top_mlp_activations = top_mlp_activations
        self.top_mlp_dims = top_mlp_dims
        self.border_mode = border_mode

        conv_parameters = zip(filter_sizes, feature_maps)

        # Construct convolutional layers with corresponding parameters
        self.layers = list(interleave([
            (Convolutional(filter_size=filter_size,
                           num_filters=num_filter,
                           step=(1, 1) if i > 0 else (self.stride, self.stride),
                           border_mode=self.border_mode,
                           name='conv_{}'.format(i))
             for i, (filter_size, num_filter)
             in enumerate(conv_parameters)),
            conv_activations,
            (MaxPooling(size, name='pool_{}'.format(i))
             for i, size in enumerate(pooling_sizes))]))

        self.conv_sequence = ConvolutionalSequence(self.layers, num_channels,
                                                   image_size=image_shape)

        # Construct a top MLP
        self.top_mlp = MLP(top_mlp_activations, top_mlp_dims)

        # We need to flatten the output of the last convolutional layer.
        # This brick accepts a tensor of dimension (batch_size, ...) and
        # returns a matrix (batch_size, features)
        self.flattener = Flattener()
        application_methods = [self.conv_sequence.apply, self.flattener.apply,
                               self.top_mlp.apply]
        super(LeNet, self).__init__(application_methods, **kwargs)

    @property
    def output_dim(self):
        return self.top_mlp_dims[-1]

    @output_dim.setter
    def output_dim(self, value):
        self.top_mlp_dims[-1] = value

    def _push_allocation_config(self):
        self.conv_sequence._push_allocation_config()
        conv_out_dim = self.conv_sequence.get_dim('output')

        self.top_mlp.activations = self.top_mlp_activations
        self.top_mlp.dims = [numpy.prod(conv_out_dim)] + self.top_mlp_dims

    @application(inputs=['image'])
    def apply_5windows(self, image):
        width, height = self.image_shape
        # the dimension 0 stands for the
        hor_offset = (image.shape[1] - width) / 2
        ver_offset = (image.shape[2] - height) / 2
        x_views = tensor.concatenate(
            [image[None, :, :width, :height],
             image[None, :, :width, -height:],
             image[None, :, -width:, :height],
             image[None, :, -width:, -height:],
             image[None, :,
                   hor_offset:hor_offset + width, ver_offset:ver_offset + height]],
             axis=0)
        return self.apply(x_views).mean(axis=0)[None, :]

def main(mode, save_to, num_epochs, load_params=None,
         feature_maps=None, mlp_hiddens=None, conv_sizes=None, pool_sizes=None, stride=None,
         batch_size=None, num_batches=None, algo=None,
         test_set=None, valid_examples=None,
         dropout=None, max_norm=None):
    if feature_maps is None:
        feature_maps = [20, 50, 50]
    if mlp_hiddens is None:
        mlp_hiddens = [500]
    if conv_sizes is None:
        conv_sizes = [5, 5, 5]
    if pool_sizes is None:
        pool_sizes = [2, 2, 2]
    if batch_size is None:
        batch_size = 500
    if valid_examples is None:
        valid_examples = 2500
    if stride is None:
        stride = 1
    if test_set is None:
        test_set = 'test'
    if algo is None:
        algo = 'rmsprop'

    image_size = (128, 128)
    output_size = 2

    if (len(feature_maps) != len(conv_sizes) or
        len(feature_maps) != len(pool_sizes)):
        raise ValueError("OMG, inconsistent arguments")

    # Use ReLUs everywhere and softmax for the final prediction
    conv_activations = [Rectifier() for _ in feature_maps]
    mlp_activations = [Rectifier() for _ in mlp_hiddens] + [Softmax()]
    convnet = LeNet(conv_activations, 3, image_size,
                    stride=stride,
                    filter_sizes=zip(conv_sizes, conv_sizes),
                    feature_maps=feature_maps,
                    pooling_sizes=zip(pool_sizes, pool_sizes),
                    top_mlp_activations=mlp_activations,
                    top_mlp_dims=mlp_hiddens + [output_size],
                    border_mode='full',
                    weights_init=Uniform(width=.1),
                    biases_init=Constant(0))
    # We push initialization config to set different initialization schemes
    # for convolutional layers.
    convnet.initialize()
    logging.info("Input dim: {} {} {}".format(
        *convnet.children[0].get_dim('input_')))
    for i, layer in enumerate(convnet.layers):
        if isinstance(layer, Activation):
            logging.info("Layer {} ({})".format(
                i, layer.__class__.__name__))
        else:
            logging.info("Layer {} ({}) dim: {} {} {}".format(
                i, layer.__class__.__name__, *layer.get_dim('output')))

    single_x = tensor.tensor3('image_features')
    x = tensor.tensor4('image_features')
    single_y = tensor.lvector('targets')
    y = tensor.lmatrix('targets')

    # Training
    probs = convnet.apply(x)
    cost = (CategoricalCrossEntropy().apply(y.flatten(), probs)
            .copy(name='cost'))
    error_rate = (MisclassificationRate().apply(y.flatten(), probs)
                  .copy(name='error_rate'))

    cg = ComputationGraph([cost, error_rate])
    if dropout:
        relu_outputs = VariableFilter(bricks=[Rectifier], roles=[OUTPUT])(cg)
        cg = apply_dropout(cg, relu_outputs, dropout)
        cost, error_rate = cg.outputs

    # Validation
    valid_probs = convnet.apply_5windows(single_x)
    valid_cost = (CategoricalCrossEntropy().apply(single_y, valid_probs)
            .copy(name='cost'))
    valid_error_rate = (MisclassificationRate().apply(
        single_y, valid_probs).copy(name='error_rate'))

    model = Model([cost, error_rate])
    if load_params:
        logger.info("Loaded params from {}".format(load_params))
        with open(load_params, 'r') as src:
            model.set_parameter_values(load_parameters(src))

    # Training stream with random cropping
    train = DogsVsCats(("train",), subset=slice(None, 25000 - valid_examples, None))
    train_str =  DataStream(
        train, iteration_scheme=ShuffledScheme(train.num_examples, batch_size))
    train_str = add_transformers(train_str, random_crop=True)

    # Validation stream without cropping
    valid = DogsVsCats(("train",), subset=slice(25000 - valid_examples, None, None))
    valid_str = DataStream(
        valid, iteration_scheme=SequentialExampleScheme(valid.num_examples))
    valid_str = add_transformers(valid_str)

    if mode == 'train':
        directory, _ = os.path.split(sys.argv[0])
        env = dict(os.environ)
        env['THEANO_FLAGS'] = 'floatX=float32'
        port = numpy.random.randint(1025, 10000)
        server = subprocess.Popen(
            [directory + '/server.py',
             str(25000 - valid_examples), str(batch_size), str(port)],
            env=env, stderr=subprocess.STDOUT)
        train_str = ServerDataStream(
            ('image_features', 'targets'), produces_examples=False,
            port=port)

        save_to_base, save_to_extension = os.path.splitext(save_to)

        # Train with simple SGD
        if algo == 'rmsprop':
            step_rule = RMSProp(decay_rate=0.999, learning_rate=0.0003)
        elif algo == 'adam':
            step_rule = Adam()
        else:
            assert False
        if max_norm:
            conv_params = VariableFilter(bricks=[Convolutional], roles=[WEIGHT])(cg)
            linear_params = VariableFilter(bricks=[Linear], roles=[WEIGHT])(cg)
            step_rule = CompositeRule(
                [step_rule,
                 Restrict(VariableClipping(max_norm, axis=0), linear_params),
                 Restrict(VariableClipping(max_norm, axis=(1, 2, 3)), conv_params)])

        algorithm = GradientDescent(
            cost=cost, parameters=model.parameters,
            step_rule=step_rule)
        # `Timing` extension reports time for reading data, aggregating a batch
        # and monitoring;
        # `ProgressBar` displays a nice progress bar during training.
        extensions = [Timing(every_n_batches=100),
                    FinishAfter(after_n_epochs=num_epochs,
                                after_n_batches=num_batches),
                    DataStreamMonitoring(
                        [valid_cost, valid_error_rate],
                        valid_str,
                        prefix="valid"),
                    TrainingDataMonitoring(
                        [cost, error_rate,
                        aggregation.mean(algorithm.total_gradient_norm)],
                        prefix="train",
                        after_epoch=True),
                    TrackTheBest("valid_error_rate"),
                    Checkpoint(save_to, save_separately=['log'],
                                before_training=True, after_epoch=True)
                        .add_condition(
                            ['after_epoch'],
                            OnLogRecord("valid_error_rate_best_so_far"),
                            (save_to_base + '_best' + save_to_extension,)),
                    Printing(every_n_batches=100)]

        model = Model(cost)

        main_loop = MainLoop(
            algorithm,
            train_str,
            model=model,
            extensions=extensions)
        try:
            main_loop.run()
        finally:
            server.terminate()
    elif mode == 'test':
        classify = theano.function([single_x], valid_probs.argmax())
        test = DogsVsCats((test_set,))
        test_str = DataStream(
            test, iteration_scheme=SequentialExampleScheme(test.num_examples))
        test_str = add_transformers(test_str)
        correct = 0
        with open(save_to, 'w') as dst:
            print("id", "label", sep=',', file=dst)
            for index, example in enumerate(test_str.get_epoch_iterator()):
                image = example[0]
                prediction = classify(image)
                print(index + 1, classify(image), sep=',', file=dst)
                if len(example) > 1 and prediction == example[1]:
                    correct += 1
        print(correct / float(test.num_examples))
    else:
        assert False

if __name__ == "__main__":
    logging.basicConfig(
        level=logging.DEBUG,
        format="%(asctime)s: %(name)s: %(levelname)s: %(message)s")
    parser = ArgumentParser("An example of training a convolutional network "
                            "on the MNIST dataset.")
    parser.add_argument("mode",
                        help="What to do", choices=['train', 'test'])
    parser.add_argument("save_to",
                        help="Destination to save the state of the training "
                             "process.")

    parser.add_argument("--num-epochs", type=int, default=2,
                        help="Number of training epochs to do.")
    parser.add_argument("--load-params", help="Path to load parameters from")
    parser.add_argument("--batch-size", type=int,
                        help="Batch size.")
    parser.add_argument("--algo", choices=['rmsprop', 'adam'],
                        help="The algorithm to use.")
    parser.add_argument("--dropout", type=float,
                        help="Dropout coefficient")
    parser.add_argument("--max-norm", type=float,
                        help="Dropout coefficient")

    parser.add_argument("--test-set", type=str)
    parser.add_argument("--valid-examples", type=int)

    parser.add_argument("--stride", type=int,
                        help="Stride for the first layer")
    parser.add_argument("--feature-maps", type=int, nargs='+',
                        help="List of feature maps numbers.")
    parser.add_argument("--mlp-hiddens", type=int, nargs='+',
                        help="List of numbers of hidden units for the MLP.")
    parser.add_argument("--conv-sizes", type=int, nargs='+',
                        help="Convolutional kernels sizes. The kernels are "
                        "always square.")
    parser.add_argument("--pool-sizes", type=int, nargs='+',
                        help="Pooling sizes. The pooling windows are always "
                             "square. Should be the same length as "
                             "--conv-sizes.")

    args = parser.parse_args()
    main(**vars(args))
<|endoftext|>


<commit_before>import urllib

from django import forms
from django.contrib.localflavor.us import forms as lfus

from etcetera.structure import models as structure
from etcetera.service import models as service

SERVICE_CHOICES = (
	('repair','Problem'),
	('install','Install'),
)

class ServiceForm(forms.Form):
	name = forms.CharField(
		max_length=200,
		help_text='Your first and last name'
	)
	department = forms.CharField(
		max_length=100,
		help_text='Your university department'
	)
	phone = lfus.USPhoneNumberField(help_text='Format: ###-###-####')
	email = forms.EmailField(help_text='Your university email address')
	coordinator_check = forms.BooleanField(
		label='Have you consulted the classroom coordinator yet?',
		required=False,
		help_text='Check this only if you have already consulted \
			a classroom coordinator'
	)
	#building = forms.CharField(max_length=25)
	building = forms.ModelChoiceField(
		queryset=structure.Building.objects.all(),
		required=True,
		help_text='The university location of the request.'
	)
	room = forms.CharField(label="Room #", max_length=15)
	work_type = forms.ChoiceField(
		label='Request Type',
		choices=SERVICE_CHOICES,
		help_text='If new equipment is being installed, choose Install. \
			Otherwise, choose Problem'
	)
	equipment_text = forms.CharField(
		label='Equipment',
		max_length=75,
		required=False,
		help_text='The equipment in question, either a description or \
			make/model'
	)
	barcode = forms.CharField(
		label="Barcode?",
		max_length=6,
		required=False,
		help_text='If present, the number on the ETC barcode sticker'
	)
	description = forms.CharField(
		widget=forms.Textarea,
		help_text="Give a description of the request. If it's a repair, \
			what's broken, what are the symptoms? If it's an install, what \
			sort of equipment needs to be installed, and how? Be as specific \
			as possible so we can serve you best."
	)

class WorkOrderModelForm(forms.ModelForm):
	uncomplete = forms.BooleanField(required=False)
	barcode = forms.CharField(
		label="ETC Barcode",
		max_length=6,
		required=False,
	)

	class Meta:
		model = service.WorkOrder
		exclude = ('creation_date',)

class SearchForm(forms.Form):
	q = forms.CharField(max_length=50)

	def as_url_args(self):
		return urllib.urlencode(self.cleaned_data)<commit_msg>Added pretty full name selection in service forms.<commit_after>import urllib

from django import forms
from django.contrib.localflavor.us import forms as lfus
from django.contrib.auth import models as auth

from etcetera.structure import models as structure
from etcetera.service import models as service
from etcetera.extras import forms as ef

SERVICE_CHOICES = (
	('repair','Problem'),
	('install','Install'),
)

class ServiceForm(forms.Form):
	name = forms.CharField(
		max_length=200,
		help_text='Your first and last name'
	)
	department = forms.CharField(
		max_length=100,
		help_text='Your university department'
	)
	phone = lfus.USPhoneNumberField(help_text='Format: ###-###-####')
	email = forms.EmailField(help_text='Your university email address')
	coordinator_check = forms.BooleanField(
		label='Have you consulted the classroom coordinator yet?',
		required=False,
		help_text='Check this only if you have already consulted \
			a classroom coordinator'
	)
	#building = forms.CharField(max_length=25)
	building = forms.ModelChoiceField(
		queryset=structure.Building.objects.all(),
		required=True,
		help_text='The university location of the request.'
	)
	room = forms.CharField(label="Room #", max_length=15)
	work_type = forms.ChoiceField(
		label='Request Type',
		choices=SERVICE_CHOICES,
		help_text='If new equipment is being installed, choose Install. \
			Otherwise, choose Problem'
	)
	equipment_text = forms.CharField(
		label='Equipment',
		max_length=75,
		required=False,
		help_text='The equipment in question, either a description or \
			make/model'
	)
	barcode = forms.CharField(
		label="Barcode?",
		max_length=6,
		required=False,
		help_text='If present, the number on the ETC barcode sticker'
	)
	description = forms.CharField(
		widget=forms.Textarea,
		help_text="Give a description of the request. If it's a repair, \
			what's broken, what are the symptoms? If it's an install, what \
			sort of equipment needs to be installed, and how? Be as specific \
			as possible so we can serve you best."
	)

class WorkOrderModelForm(forms.ModelForm):
	uncomplete = forms.BooleanField(required=False)
	barcode = forms.CharField(
		label="ETC Barcode",
		max_length=6,
		required=False,
	)
	technician = ef.UserModelChoiceField(auth.User.objects.all().order_by('last_name'))

	class Meta:
		model = service.WorkOrder
		exclude = ('creation_date',)

class SearchForm(forms.Form):
	q = forms.CharField(max_length=50)

	def as_url_args(self):
		return urllib.urlencode(self.cleaned_data)<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright Hybrid Logic Ltd.  See LICENSE file for details.

"""
Script executed by Buildbot on the Mac mini build slave, this instantiates the
OS X Virtual Machine to a snapshot with a clean homebrew install, then executes
via SSH the brew update, install and test commands to verify a successful
installation of a homebrew recipe.

The host machine must have:
        * VMWare Fusion installed,
        * A VMWare OS X VM available at a particular location

The guest virtual machine must have:
        * A snapshot with a particular name, homebrew-clean
        * Homebrew installed and available
"""

import os
import sys
import urllib2

from subprocess import check_output, CalledProcessError

from flocker.common._ssh import run_with_fabric, Run

YOSEMITE_VMX_PATH = (
    '"'
    "{home}/Documents/Virtual Machines.localized/"
    "OS X 10.10.vmwarevm/OS X 10.10.vmx"
    '"'
).format(home=os.getenv("HOME"))

YOSEMITE_SNAPSHOT = "homebrew-clean"

VM_USERNAME = "ClusterHQVM"

VM_HOST = "10.0.126.88"

def main():
    try:
        if len(sys.argv) < 2:
            raise Exception("URL of homebrew recipe not specified")
        recipe_url = sys.argv[1]
        # Open the recipe URL to validate and verify.
        recipe_resource = urllib2.urlopen(recipe_url)
        revert_result = check_output([
            "vmrun", "revertToSnapshot", YOSEMITE_VMX_PATH, YOSEMITE_SNAPSHOT,
        ])
        start_result = check_output([
            "vmrun", "start", YOSEMITE_VMX_PATH, "nogui",
        ])
        run_with_fabric(VM_USERNAME, VM_HOST, commands=[
            Run(command="brew update"),
            Run(command="brew install {url}".format(url=recipe_url)),
            Run(command="brew test {url}".format(url=recipe_url)),
        ])
        stop_result = check_output([
            "vmrun", "stop", YOSEMITE_VMX_PATH, "hard",
        ])
        print "Done."
        sys.exit(0)
    except CalledProcessError as e:
        print (
            "Error: Command {cmd} terminated with exit status {code}.").format(
            cmd=" ".join(e.cmd), code=e.returncode
        )
        sys.exit(1)
    except Exception as e:
        print "Error: {error}.".format(error=str(e))
        sys.exit(1)

if __name__ == "__main__":
    main()
<commit_msg>Comment fix<commit_after>#!/usr/bin/env python
# Copyright Hybrid Logic Ltd.  See LICENSE file for details.

"""
Script executed by Buildbot on the Mac mini build slave, this instantiates the
OS X Virtual Machine to a snapshot with a clean homebrew install, then executes
via SSH the brew update, install and test commands to verify a successful
installation of a homebrew recipe.

The host machine must have:
        * VMWare Fusion installed,
        * A VMWare OS X VM available at a particular location

The guest virtual machine must have:
        * A snapshot with a particular name, homebrew-clean
        * Homebrew installed and available
"""

import os
import sys
import urllib2

from subprocess import check_output, CalledProcessError

from flocker.common._ssh import run_with_fabric, Run

YOSEMITE_VMX_PATH = (
    '"'
    "{home}/Documents/Virtual Machines.localized/"
    "OS X 10.10.vmwarevm/OS X 10.10.vmx"
    '"'
).format(home=os.getenv("HOME"))

YOSEMITE_SNAPSHOT = "homebrew-clean"

VM_USERNAME = "ClusterHQVM"

VM_HOST = "10.0.126.88"

def main():
    try:
        if len(sys.argv) < 2:
            raise Exception("URL of homebrew recipe not specified")
        recipe_url = sys.argv[1]
        # Open the recipe URL just to validate and verify that it exists.
        # We do not need to read its content.
        recipe_resource = urllib2.urlopen(recipe_url)
        revert_result = check_output([
            "vmrun", "revertToSnapshot", YOSEMITE_VMX_PATH, YOSEMITE_SNAPSHOT,
        ])
        start_result = check_output([
            "vmrun", "start", YOSEMITE_VMX_PATH, "nogui",
        ])
        run_with_fabric(VM_USERNAME, VM_HOST, commands=[
            Run(command="brew update"),
            Run(command="brew install {url}".format(url=recipe_url)),
            Run(command="brew test {url}".format(url=recipe_url)),
        ])
        stop_result = check_output([
            "vmrun", "stop", YOSEMITE_VMX_PATH, "hard",
        ])
        print "Done."
        sys.exit(0)
    except CalledProcessError as e:
        print (
            "Error: Command {cmd} terminated with exit status {code}.").format(
            cmd=" ".join(e.cmd), code=e.returncode
        )
        sys.exit(1)
    except Exception as e:
        print "Error: {error}.".format(error=str(e))
        sys.exit(1)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from django.shortcuts import get_object_or_404, render_to_response
from django.template import RequestContext
from django.conf import settings
from django.contrib.auth.decorators import login_required

from django.utils.translation import ugettext_lazy as _

from friends.models import *
from friends.forms import JoinRequestForm
from account.forms import SignupForm

# @@@ if made more generic these could be moved to django-friends proper

@login_required
def friends(request):
    if request.method == "POST":
        if request.POST["action"] == "accept":
            invitation_id = request.POST["invitation"]
            try:
                invitation = FriendshipInvitation.objects.get(id=invitation_id)
                if invitation.to_user == request.user:
                    invitation.accept()
                    request.user.message_set.create(message=_("Accepted friendship request from %(from_user)s") % {'from_user': invitation.from_user})
            except FriendshipInvitation.DoesNotExist:
                pass
        elif request.POST["action"] == "invite": # invite to join
            join_request_form = JoinRequestForm(request.POST)
            if join_request_form.is_valid():
                join_request_form.save(request.user)
                join_request_form = JoinRequestForm() # @@@
    else:
        join_request_form = JoinRequestForm()

    invites_received = request.user.invitations_to.all().order_by("-sent")
    invites_sent = request.user.invitations_from.all().order_by("-sent")
    joins_sent = request.user.join_from.all().order_by("-sent")

    return render_to_response("friends_app/invitations.html", {
        "join_request_form": join_request_form,
        "invites_received": invites_received,
        "invites_sent": invites_sent,
        "joins_sent": joins_sent,
    }, context_instance=RequestContext(request))

def accept_join(request, confirmation_key):
    join_invitation = get_object_or_404(JoinInvitation, confirmation_key = confirmation_key.lower())
    if request.user.is_authenticated():
        return render_to_response("account/signup.html", {
            "contact_email": settings.CONTACT_EMAIL,
        }, context_instance=RequestContext(request))
    else:
        form = SignupForm(initial={"email": join_invitation.contact.email, "confirmation_key": join_invitation.confirmation_key })
        return render_to_response("account/signup.html", {
            "form": form,
            "contact_email": settings.CONTACT_EMAIL,
        }, context_instance=RequestContext(request))
<commit_msg>fixed problem with accepting from invitation page<commit_after>from django.shortcuts import get_object_or_404, render_to_response
from django.template import RequestContext
from django.conf import settings
from django.contrib.auth.decorators import login_required

from django.utils.translation import ugettext_lazy as _

from friends.models import *
from friends.forms import JoinRequestForm
from account.forms import SignupForm

# @@@ if made more generic these could be moved to django-friends proper

@login_required
def friends(request):
    if request.method == "POST":
        if request.POST["action"] == "accept":
            invitation_id = request.POST["invitation"]
            try:
                invitation = FriendshipInvitation.objects.get(id=invitation_id)
                if invitation.to_user == request.user:
                    invitation.accept()
                    request.user.message_set.create(message=_("Accepted friendship request from %(from_user)s") % {'from_user': invitation.from_user})
            except FriendshipInvitation.DoesNotExist:
                pass
            join_request_form = JoinRequestForm()
        elif request.POST["action"] == "invite": # invite to join
            join_request_form = JoinRequestForm(request.POST)
            if join_request_form.is_valid():
                join_request_form.save(request.user)
                join_request_form = JoinRequestForm() # @@@
    else:
        join_request_form = JoinRequestForm()

    invites_received = request.user.invitations_to.all().order_by("-sent")
    invites_sent = request.user.invitations_from.all().order_by("-sent")
    joins_sent = request.user.join_from.all().order_by("-sent")

    return render_to_response("friends_app/invitations.html", {
        "join_request_form": join_request_form,
        "invites_received": invites_received,
        "invites_sent": invites_sent,
        "joins_sent": joins_sent,
    }, context_instance=RequestContext(request))

def accept_join(request, confirmation_key):
    join_invitation = get_object_or_404(JoinInvitation, confirmation_key = confirmation_key.lower())
    if request.user.is_authenticated():
        return render_to_response("account/signup.html", {
            "contact_email": settings.CONTACT_EMAIL,
        }, context_instance=RequestContext(request))
    else:
        form = SignupForm(initial={"email": join_invitation.contact.email, "confirmation_key": join_invitation.confirmation_key })
        return render_to_response("account/signup.html", {
            "form": form,
            "contact_email": settings.CONTACT_EMAIL,
        }, context_instance=RequestContext(request))
<|endoftext|>


<commit_before>"""
Tests for the MDK low-level protocol code.
"""

from collections import deque

import hypothesis.strategies as st
from hypothesis import given, assume

from mdk_protocol import SendWithAcks

class SendToServer(object):
    """
    Implement the SendAckableEvent interface.
    """
    def __init__(self, simulator):
        self.simulator = simulator

    def send(self, event):
        self.simulator._send_to_server(event)

class Payload(object):
    """
    Implement the AckablePayload interface.
    """
    def __init__(self, message):
        self.message = message

    def getTimestamp(self):
        return 1

class NetworkSimulator(object):
    """
    Simulate connection between SendWithAcks and a server.
    """
    def __init__(self):
        self.server_received = set()
        self.client = SendWithAcks()

    def _send_to_server(self, event):
        """
        Send a AckableEvent to the server.
        """
        self.connection_to_server.appendleft(event)

    def reconnect(self):
        """
        Disconnect client from server and then reconnect.
        """
        # Wipe existing connection, if any:
        self.connection_to_server = deque()
        self.connection_to_client = deque()
        # Notify client:
        self.client.onConnected(SendToServer(self))

    def tick(self):
        """
        One pseudo-second has passed.
        """
        # Connection delivers acks to client, if there is any.
        if self.connection_to_client:
            ack = self.connection_to_client.pop()
            self.client.onAck(ack)

        # Connection delivers message to server, if there is any.  The server
        # then acks it. Do this after ack delivery so it takes another tick for
        # acks to be delivered.
        if self.connection_to_server:
            message = self.connection_to_server.pop()
            self.server_received.add(message.payload)
            self.connection_to_client.appendleft(message.sequence)

        # Tell client time has passed:
        self.client.onPump(SendToServer(self))

def test_sendWithAcks_delivery():
    """
    Demonstrate that sendWithAcks algorithm delivers messages even though
    connections are dropped.

    Basic test setup:

    1. SendWithAcks is told to send 10 messages.
    2. Messages get added to a Connection.
    3. Once a (fake) second a message is moved from Connection to Server.
    4. Once a second the Server sends an Ack to Connection.
    5. Once a second the Acks from Connection are delivered to SendWithAcks.

    Every once in a while the Connection disconnects, and everything in its
    buffers is wiped and never delivered. It then reconnects.

    At the end of large-number-of-seconds all messages should have been
    delivered to Server and SendWithAcks should have no buffered messages.
    """
    simulator = NetworkSimulator()
    messages = set(Payload("message{}".format(i)) for i in range(10))
    for m in messages:
        simulator.client.send("my_type", m)

    simulator.reconnect()
    for _ in range(10000):
        simulator.tick()

    assert simulator.server_received == messages
    assert len(simulator.client._buffered) == 0
    assert len(simulator.client._inFlight) == 0
<commit_msg>Unreliable connections demonstrating bugs in SendWithAcks.<commit_after>"""
Tests for the MDK low-level protocol code.
"""

from collections import deque

import hypothesis.strategies as st
from hypothesis import given, assume

from mdk_protocol import SendWithAcks

class SendToServer(object):
    """
    Implement the SendAckableEvent interface.
    """
    def __init__(self, simulator):
        self.simulator = simulator

    def send(self, event):
        self.simulator._send_to_server(event)

class Payload(object):
    """
    Implement the AckablePayload interface.
    """
    def __init__(self, message):
        self.message = message

    def __repr__(self):
        return "<Payload: {}>".format(self.message)

    def getTimestamp(self):
        return 1342342

class NetworkSimulator(object):
    """
    Simulate connection between SendWithAcks and a server.
    """
    def __init__(self):
        self.server_received = set()
        self.client = SendWithAcks()

    def _send_to_server(self, event):
        """
        Send a AckableEvent to the server.
        """
        self.connection_to_server.appendleft(event)

    def reconnect(self):
        """
        Disconnect client from server and then reconnect.
        """
        # Wipe existing connection, if any:
        self.connection_to_server = deque()
        self.connection_to_client = deque()
        # Notify client:
        self.client.onConnected(SendToServer(self))

    def tick(self):
        """
        One pseudo-second has passed.
        """
        # Connection delivers acks to client, if there is any.
        if self.connection_to_client:
            ack = self.connection_to_client.pop()
            self.client.onAck(ack)

        # Connection delivers message to server, if there is any.  The server
        # then acks it. Do this after ack delivery so it takes another tick for
        # acks to be delivered.
        if self.connection_to_server:
            message = self.connection_to_server.pop()
            self.server_received.add(message.payload)
            if message.sync:
                # Send ack since sync flag was set:
                self.connection_to_client.appendleft(message.sequence)

        # Tell client time has passed:
        self.client.onPump(SendToServer(self))

@given(st.streaming(st.integers(min_value=1, max_value=5)))
def test_sendWithAcks_delivery(disconnect_intervals):
    """
    Demonstrate that sendWithAcks algorithm delivers messages even though
    connections are dropped.

    Basic test setup:

    1. SendWithAcks is told to send 10 messages.
    2. Messages get added to a Connection.
    3. Once a (fake) second a message is moved from Connection to Server.
    4. Once a second the Server sends an Ack to Connection.
    5. Once a second the Acks from Connection are delivered to SendWithAcks.

    Every once in a while the Connection disconnects, and everything in its
    buffers is wiped and never delivered. It then reconnects.

    At the end of large-number-of-seconds all messages should have been
    delivered to Server and SendWithAcks should have no buffered messages.
    """
    simulator = NetworkSimulator()
    messages = set(Payload("message{}".format(i)) for i in range(10))
    for m in messages:
        simulator.client.send("my_type", m)

    # Do some delivery interrupted by disconnects:
    disconnect_intervals = iter(disconnect_intervals)
    ticks_until_reconnect = 1
    for _ in range(100):
        ticks_until_reconnect -= 1
        if ticks_until_reconnect == 0:
            simulator.reconnect()
            ticks_until_reconnect = next(disconnect_intervals)
        simulator.tick()

    # Now reliably deliver anything that's left:
    for _ in range(100):
        simulator.tick()

    assert simulator.server_received == messages
    assert len(simulator.client._buffered) == 0
    #assert len(simulator.client._inFlight) == 0
<|endoftext|>


<commit_before>import speech_recognition as sr #Import the speech recognition library
from sympy import * #Necessary for commented out code, but comment this import if not using them
#Setup speech
r = sr.Recognizer()
#Code to get speech

with sr.Microphone() as source: #Need this block every time you want to input speech
	#print("Do you want to solve an expression or solve for a variable?") #To be implemented lated with sympy
	print("Say your equation:")
	#audio = input("Input your equation")
	audio = r.listen(source) #Sets a variable called audio to the audio input, to be later interpreted
######################################
#EQUATION CODE
#equation = input("Input your equation: \n") #Uncomment to test this code
#x = Symbol('x')
#print(solve(equation, x))
#equation.replace("y", "")
#equation.replace("=", "")
################SAMPLE CODE######################
#limit(sin(x)/x, x, 0) #LIMIT
#integrate(exp(x)*sin(x) + exp(x)*cos(x), x) #INTEGRAL
#diff(sin(x)*exp(x), x) #DERIVATIVE

keys = [line.rstrip('\n') for line in open('keys.txt')] #Puts every line in keys.txt in a list
results = [line.rstrip('\n') for line in open('results.txt')] #Same thing, but for results.txt
#Below are the try and except blocks for google speech recognition

try:
	equation = r.recognize_google(audio).lower() #Takes whatever the speech interpretor took the input as and makes it lower case to fit the dictionary
	#equation = raw_input("Input equation: ") # For typing in equation
	print(equation) #Prints the equation, check to see if it recognized your speech correctly
	for i in range(len(keys)): #Goes through the equation replacing any phrases with their mathematical equivalents
		equation = equation.replace(keys[i], results[i])

	try:
		ev = eval(equation) #eval is a standard evaluating function in python. Later it should be replaced with a manual evaluator
		print(equation)
		print(ev)
	except: #In case the input doesn't explicitly state an end parentheses, adds one
		try:
			equation = equation + ")" #Add a paren to the end
			ev = eval(equation) #Attemps to evaluate again
			print(equation)
			print(ev) #Prints the evaluation
		except:
			print("Unable to evaulate equation") #If all fails, just give up
except sr.UnknownValueError: #This is the most common error, try audio again, making certain the program can clearly hear you
    print("Google Speech Recognition could not understand audio") #The two excepts below are the standard for google speech recognition
except sr.RequestError as e:
    print("Could not request results from Google Speech Recognition service; {0}".format(e))
<commit_msg>Added more commented code<commit_after>import speech_recognition as sr #Import the speech recognition library
from sympy import * #Necessary for commented out code, but comment this import if not using them
#Setup speech
r = sr.Recognizer()
#Code to get speech

with sr.Microphone() as source: #Need this block every time you want to input speech
	#print("Do you want to solve an expression or solve for a variable?") #To be implemented lated with sympy
	print("Say your equation:")
	#audio = input("Input your equation")
	#x = Symbol('x')
	#diff(audio, x)						#For direct derivative typed input
	audio = r.listen(source) #Sets a variable called audio to the audio input, to be later interpreted
######################################
#EQUATION CODE
#equation = input("Input your equation: \n") #Uncomment to test this code
#x = Symbol('x')
#print(solve(equation, x))
#equation.replace("y", "")
#equation.replace("=", "")
################SAMPLE CODE######################
#limit(sin(x)/x, x, 0) #LIMIT
#integrate(exp(x)*sin(x) + exp(x)*cos(x), x) #INTEGRAL
#diff(sin(x)*exp(x), x) #DERIVATIVE

keys = [line.rstrip('\n') for line in open('keys.txt')] #Puts every line in keys.txt in a list
results = [line.rstrip('\n') for line in open('results.txt')] #Same thing, but for results.txt
#Below are the try and except blocks for google speech recognition

try:
	equation = r.recognize_google(audio).lower() #Takes whatever the speech interpretor took the input as and makes it lower case to fit the dictionary
	#equation = raw_input("Input equation: ") # For typing in equation
	print(equation) #Prints the equation, check to see if it recognized your speech correctly
	for i in range(len(keys)): #Goes through the equation replacing any phrases with their mathematical equivalents
		equation = equation.replace(keys[i], results[i])

	try:
		ev = eval(equation) #eval is a standard evaluating function in python. Later it should be replaced with a manual evaluator
		print(equation)
		print(ev)
	except: #In case the input doesn't explicitly state an end parentheses, adds one
		try:
			equation = equation + ")" #Add a paren to the end
			ev = eval(equation) #Attemps to evaluate again
			print(equation)
			print(ev) #Prints the evaluation
		except:
			print("Unable to evaulate equation") #If all fails, just give up
except sr.UnknownValueError: #This is the most common error, try audio again, making certain the program can clearly hear you
    print("Google Speech Recognition could not understand audio") #The two excepts below are the standard for google speech recognition
except sr.RequestError as e:
    print("Could not request results from Google Speech Recognition service; {0}".format(e))
<|endoftext|>


<commit_before>#! /usr/bin/env python
"""
Investigate the reliability of age fitting.

Generates a varied set of synthetic astrometry datasets.
Run traceback and fit, compare fitted age with ground truth age.
"""
from chronostar import synthesiser as syn
import numpy as np
from subprocess import call
from chronostar.fit_groups import fit_groups
import pdb
import matplotlib.pyplot as plt

group_pars = np.array([
    # X, Y, Z,U,V,W,dX,dY,dZ,dV,Cxy,Cxz,Cyz,age,N
    [ 0, 0, 0,0,0,0,10,10,10, 3,  0,  0,  0,20,30],
    [ 0,10, 0,0,0,0, 5, 5, 5, 3,0.6,0.5,0.0,20,40],
    [10, 0, 0,0,0,0,10,10,10, 3,0.4,0.2,0.0,20,50],
    [10,10, 0,0,0,0, 5, 5, 5, 3,  0,0.2,0.0,10,60],
    [10,10,10,0,0,0,10,10,10, 3,  0,0.2,0.0,10,20],
    ])
# error = 0.01
ntests = group_pars.shape[0]
true_ages = []
fitted_ages = []

base_group = [ 0,10, 0,0,0,0, 5, 5, 5, 3,0.6,0.5,0.0]
for age in [5,7.5,10,12.5,15,17.5,20,22.5,25,27.5,30]:
    for nstars in [15, 30, 60]:
        group_pars = base_group + [age] + [nstars]
        syn.synthesise_data(1, group_pars, 0.01)
        astr_filename =\
            "data/synth_data_1groups_{}stars.pkl".format(nstars)
        tb_filename =\
            "data/tb_synth_data_1groups_{}stars.pkl".format(nstars)
        call([
            "python", "generate_default_tb.py",
            astr_filename,
            ])
        best_fit = fit_groups(
            1000,1000,1,0,tb_filename,tstamp=nstars,noplots=True)

        true_ages.append(age)
        fitted_ages.append(best_fit[-2])

plt.plot(true_ages, fitted_ages, 'o')
pdb.set_trace()

# synthesise a bunch of solo groups with varying error
# errors = [0.01, 0.02, 0.05, 0.1]

# for error in errors:

# # synthesise a bunch of solo groups
# for i in range(5):
#     syn.synthesise_data(1, group_pars[i], error)
#
# # synthesise a bunch of paired groups
# for i in range(4):
#     syn.synthesise_data(2, group_pars[i:i+2], error)
#
# # synthesise a couple of three groups
# syn.synthesise_data(3, group_pars[0:3], error)
# syn.synthesise_data(3, group_pars[2:5], error)

<commit_msg>Better plot labelling for age diog<commit_after>#! /usr/bin/env python
"""
Investigate the reliability of age fitting.

Generates a varied set of synthetic astrometry datasets.
Run traceback and fit, compare fitted age with ground truth age.
"""
from chronostar import synthesiser as syn
import numpy as np
from subprocess import call
from chronostar.fit_groups import fit_groups
import pdb
import matplotlib.pyplot as plt

group_pars = np.array([
    # X, Y, Z,U,V,W,dX,dY,dZ,dV,Cxy,Cxz,Cyz,age,N
    [ 0, 0, 0,0,0,0,10,10,10, 3,  0,  0,  0,20,30],
    [ 0,10, 0,0,0,0, 5, 5, 5, 3,0.6,0.5,0.0,20,40],
    [10, 0, 0,0,0,0,10,10,10, 3,0.4,0.2,0.0,20,50],
    [10,10, 0,0,0,0, 5, 5, 5, 3,  0,0.2,0.0,10,60],
    [10,10,10,0,0,0,10,10,10, 3,  0,0.2,0.0,10,20],
    ])
# error = 0.01
ntests = group_pars.shape[0]
true_ages = []
fitted_ages = []

base_group = [ 0,10, 0,0,0,0, 5, 5, 5, 3,0.6,0.5,0.0]
for age in [5,7.5,10,12.5,15,17.5,20,22.5,25,27.5,30]:
    for nstars in [15, 30, 60]:
        group_pars = base_group + [age] + [nstars]
        syn.synthesise_data(1, group_pars, 0.01)
        astr_filename =\
            "data/synth_data_1groups_{}stars.pkl".format(nstars)
        tb_filename =\
            "data/tb_synth_data_1groups_{}stars.pkl".format(nstars)
        call([
            "python", "generate_default_tb.py",
            astr_filename,
            ])
        best_fit = fit_groups(
            1000,1000,1,0,tb_filename,tstamp=nstars,noplots=True)

        true_ages.append(age)
        fitted_ages.append(best_fit[-2])

plt.plot(true_ages[0::3], fitted_ages[0::3], "bo", label="15 stars")
plt.plot(true_ages[1::3], fitted_ages[1::3], "g+", label="30 stars")
plt.plot(true_ages[2::3], fitted_ages[2::3], "yx", label="60 stars")
plt.legend(loc='best'); plt.xlabel("True age"); plt.ylabel("Fitted age")

pdb.set_trace()

# synthesise a bunch of solo groups with varying error
# errors = [0.01, 0.02, 0.05, 0.1]

# for error in errors:

# # synthesise a bunch of solo groups
# for i in range(5):
#     syn.synthesise_data(1, group_pars[i], error)
#
# # synthesise a bunch of paired groups
# for i in range(4):
#     syn.synthesise_data(2, group_pars[i:i+2], error)
#
# # synthesise a couple of three groups
# syn.synthesise_data(3, group_pars[0:3], error)
# syn.synthesise_data(3, group_pars[2:5], error)

<|endoftext|>