<commit_before># Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import asyncio

from asyncio.streams import FlowControlMixin

try:
    from http_parser.parser import HttpParser
except ImportError:
    from http_parser.pyparser import HttpParser

class HTTPProtocol(FlowControlMixin, asyncio.Protocol):

    def __init__(self, stream_reader, callback, loop=None):
        super().__init__(loop=loop)
        self._stream_reader = stream_reader
        self._stream_writer = None

        self._callback = callback
        self._task = None

    def connection_made(self, transport):
        self._parser = HttpParser()

        self._stream_reader.set_transport(transport)
        self._stream_writer = asyncio.StreamWriter(
            transport,
            self,
            self._stream_reader,
            self._loop,
        )

    def connection_lost(self, exc):
        if exc is None:
            self._stream_reader.feed_eof()
        else:
            self._stream_reader.set_exception(exc)

        super().connection_lost(exc)

    def data_received(self, data):
        # Parse our incoming data with our HTTP parser
        self._parser.execute(data, len(data))

        # If we have not already handled the headers and we've gotten all of
        # them, then invoke the callback with the headers in them.
        if self._task is None and self._parser.is_headers_complete():
            coro = self.dispatch(
                self._parser.get_headers(),
                self._stream_reader,
                self._stream_writer,
            )
            self._task = asyncio.Task(coro, loop=self._loop)

        # Determine if we have any data in the body buffer and if so feed it
        # to our StreamReader
        if self._parser.is_partial_body():
            self._stream_reader.feed_data(self._parser.recv_body())

        # Determine if we've completed the end of the HTTP request, if we have
        # then we should close our stream reader because there is nothing more
        # to read.
        if self._parser.is_message_complete():
            self._stream_reader.feed_eof()

    def eof_received(self):
        # We've gotten an EOF from the client, so we'll propagate this to our
        # StreamReader
        self._stream_reader.feed_eof()

    @asyncio.coroutine
    def dispatch(self, headers, reader, writer):
        # Get the status, headers, and body from the callback. The body must
        # be iterable, and each item can either be a bytes object, or an
        # asyncio coroutine, in which case we'll ``yield from`` on it to wait
        # for it's value.
        status, resp_headers, body = yield from self._callback(headers, reader)

        # Write out the status line to the client for this request
        # TODO: We probably don't want to hard code HTTP/1.1 here
        writer.write(b"HTTP/1.1 " + status + b"\r\n")

        # Write out the headers, taking special care to ensure that any
        # mandatory headers are added.
        # TODO: We need to handle some required headers
        for key, values in resp_headers.items():
            # In order to handle headers which need to have multiple values
            # like Set-Cookie, we allow the value of the header to be an
            # iterable instead of a bytes object, in which case we'll write
            # multiple header lines for this header.
            if isinstance(values, (bytes, bytearray)):
                values = [values]

            for value in values:
                writer.write(key + b": " + value + b"\r\n")

        # Before we get to the body, we need to write a blank line to separate
        # the headers and the response body
        writer.write(b"\r\n")

        for chunk in body:
            # If the chunk is a coroutine, then we want to wait for the result
            # before we write it.
            if asyncio.iscoroutine(chunk):
                chunk = yield from chunk

            # Write our chunk out to the connect client
            writer.write(chunk)

        # We've written everything in our iterator, so we want to close the
        # connection.
        writer.close()

class HTTPServer:

    protocol_class = HTTPProtocol

    def __init__(self, callback, loop=None):
        self.callback = callback
        self.loop = None

    def __call__(self):
        reader = asyncio.StreamReader(loop=self.loop)
        protocol = self.protocol_class(reader, self.callback, loop=self.loop)
        return protocol
<commit_msg>Clearer variable names<commit_after># Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import asyncio

from asyncio.streams import FlowControlMixin

try:
    from http_parser.parser import HttpParser
except ImportError:
    from http_parser.pyparser import HttpParser

class HTTPProtocol(FlowControlMixin, asyncio.Protocol):

    def __init__(self, stream_reader, callback, loop=None):
        super().__init__(loop=loop)
        self._stream_reader = stream_reader
        self._stream_writer = None

        self._callback = callback
        self._task = None

    def connection_made(self, transport):
        self._parser = HttpParser()

        self._stream_reader.set_transport(transport)
        self._stream_writer = asyncio.StreamWriter(
            transport,
            self,
            self._stream_reader,
            self._loop,
        )

    def connection_lost(self, exc):
        if exc is None:
            self._stream_reader.feed_eof()
        else:
            self._stream_reader.set_exception(exc)

        super().connection_lost(exc)

    def data_received(self, data):
        # Parse our incoming data with our HTTP parser
        self._parser.execute(data, len(data))

        # If we have not already handled the headers and we've gotten all of
        # them, then invoke the callback with the headers in them.
        if self._task is None and self._parser.is_headers_complete():
            coro = self.dispatch(
                self._parser.get_headers(),
                self._stream_reader,
                self._stream_writer,
            )
            self._task = asyncio.Task(coro, loop=self._loop)

        # Determine if we have any data in the body buffer and if so feed it
        # to our StreamReader
        if self._parser.is_partial_body():
            self._stream_reader.feed_data(self._parser.recv_body())

        # Determine if we've completed the end of the HTTP request, if we have
        # then we should close our stream reader because there is nothing more
        # to read.
        if self._parser.is_message_complete():
            self._stream_reader.feed_eof()

    def eof_received(self):
        # We've gotten an EOF from the client, so we'll propagate this to our
        # StreamReader
        self._stream_reader.feed_eof()

    @asyncio.coroutine
    def dispatch(self, headers, request_body, response):
        # Get the status, headers, and body from the callback. The body must
        # be iterable, and each item can either be a bytes object, or an
        # asyncio coroutine, in which case we'll ``yield from`` on it to wait
        # for it's value.
        status, resp_headers, body = yield from self._callback(
            headers,
            request_body,
        )

        # Write out the status line to the client for this request
        # TODO: We probably don't want to hard code HTTP/1.1 here
        response.write(b"HTTP/1.1 " + status + b"\r\n")

        # Write out the headers, taking special care to ensure that any
        # mandatory headers are added.
        # TODO: We need to handle some required headers
        for key, values in resp_headers.items():
            # In order to handle headers which need to have multiple values
            # like Set-Cookie, we allow the value of the header to be an
            # iterable instead of a bytes object, in which case we'll write
            # multiple header lines for this header.
            if isinstance(values, (bytes, bytearray)):
                values = [values]

            for value in values:
                response.write(key + b": " + value + b"\r\n")

        # Before we get to the body, we need to write a blank line to separate
        # the headers and the response body
        response.write(b"\r\n")

        for chunk in body:
            # If the chunk is a coroutine, then we want to wait for the result
            # before we write it.
            if asyncio.iscoroutine(chunk):
                chunk = yield from chunk

            # Write our chunk out to the connect client
            response.write(chunk)

        # We've written everything in our iterator, so we want to close the
        # connection.
        response.close()

class HTTPServer:

    protocol_class = HTTPProtocol

    def __init__(self, callback, loop=None):
        self.callback = callback
        self.loop = None

    def __call__(self):
        reader = asyncio.StreamReader(loop=self.loop)
        protocol = self.protocol_class(reader, self.callback, loop=self.loop)
        return protocol
<|endoftext|>


<commit_before>from .decorators import get, post, route
from .routes import get

def application(request):
    # TODO Does it make more sense to do route first?
    # get("/") > request > do_response()

    request > get("/") > do_response()
    request > get("/msg") > say_hello()

@route
def do_response(request):
    return "Hello!\n"

@route
def say_hello(request):
    who = request.args.get('who', 'World')
    return "Hello, %s!\n" % who

class Router(object):
    pass

router = Router()

@router.get("/")
def index(request, response):
    return "Index\n"

@router.use
def middleware(request, response, _next):
    request.msg = "Hello, %s!"
    _next()

msg = router.route("/msg")

@msg.get
def get_msg(request, response):
    return request.msg % request.args.get("who", "World")

@msg.post
def post_msg(request, response):
    response.location = "/msg?who=%s" % request.args.get("who", "Poster")

@router.param("user")
def add_user(self, request):
    mapped_names = {
        't': 'Travis',
        'p': 'Peter',
    }
    request.user = mapped_names.get(request.params['user'], 'Whoever You Are')

@router.get("/msg/<user>")
def handle(request):
    return request.msg % request.user
<commit_msg>Remove old sample -- no longer valid<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python2
# -*- coding: utf-8 -*-

import os
import json
import datetime
from time import time

from plugins.plugin import Plugin

from bytebot_config import BYTEBOT_PLUGIN_CONFIG

with open(BYTEBOT_PLUGIN_CONFIG['ccc32c3']['events_json_path']) as json_file:
    json_data = json.load(json_file)

class ccc32c3(Plugin):

    def __init__(self):
        pass

    def registerCommand(self, irc):
        irc.registerCommand("!32c3", "32c3")

    def get_plays(self, hall):
        now_playing = json_data['schedule'][
            'conference']['days'][0]['rooms'][hall][0]
        now_playing['date'] = "2099-12-31T23:59:59+01:00"

        next_playing = json_data['schedule'][
            'conference']['days'][0]['rooms'][hall][0]
        next_playing['date'] = "2099-12-31T23:59:59+01:00"

        for days in json_data['schedule']['conference']['days']:

            for event in days['rooms'][hall]:
                z = (datetime.datetime.strptime(event['date'][
                     0:-6], "%Y-%m-%dT%H:%M:%S") - datetime.datetime.now())

                if(z.days < 0):
                    now_playing = event

            for event in days['rooms'][hall]:
                z = (datetime.datetime.strptime(event['date'][
                     0:-6], "%Y-%m-%dT%H:%M:%S") - datetime.datetime.now())

                if(z.days >= 0):
                    if(datetime.datetime.strptime(
                            event['date'][0:-6],
                            "%Y-%m-%dT%H:%M:%S") <
                       datetime.datetime.strptime(
                            next_playing['date'][0:-6],
                            "%Y-%m-%dT%H:%M:%S")):

                        next_playing = event

        return now_playing, next_playing

    def conv(self, t):
        x = datetime.datetime.strptime(t[0:-6], "%Y-%m-%dT%H:%M:%S")
        return x.strftime("%H:%M")

    def getLineOfPersons(self, event):
        line = ""
        for persons in event['persons']:
            line += "@" + persons['public_name'] + " "
        return line

    def onPrivmsg(self, irc, msg, channel, user):
        if msg.find("!32c3") == -1:
            return

        self.irc = irc
        self.channel = channel

        if msg.find("now") != -1:
            now, next = self.get_plays("Hall 1")
            irc.msg(channel, ("Hall 1: " +
                              self.conv(now['date']) +
                              " " +
                              now['title'] +
                              " " +
                              self.getLineOfPersons(now)
                              ).encode("utf-8", "ignore"))

            now, next = self.get_plays("Hall 2")
            irc.msg(channel, ("Hall 2: " +
                              self.conv(now['date']) +
                              " " +
                              now['title'] +
                              " " +
                              self.getLineOfPersons(now)
                              ).encode("utf-8", "ignore"))

            now, next = self.get_plays("Hall G")
            irc.msg(channel, ("Hall G: " +
                              self.conv(now['date']) +
                              " " +
                              now['title'] +
                              " " +
                              self.getLineOfPersons(now)
                              ).encode("utf-8", "ignore"))

            now, next = self.get_plays("Hall 6")
            irc.msg(channel, ("Hall 6: " +
                              self.conv(now['date']) +
                              " " +
                              now['title'] +
                              " " +
                              self.getLineOfPersons(now)
                              ).encode("utf-8", "ignore"))

            irc.last_ccc32c3 = time()

        if msg.find("next") != -1:
            now, next = self.get_plays("Hall 1")
            irc.msg(channel, ("Hall 1: " +
                              self.conv(next['date']) +
                              " " +
                              next['title'] +
                              " " +
                              self.getLineOfPersons(next)
                              ).encode("utf-8", "ignore"))

            now, next = self.get_plays("Hall 2")
            irc.msg(channel, ("Hall 2: " +
                              self.conv(next['date']) +
                              " " +
                              next['title'] +
                              " " +
                              self.getLineOfPersons(next)
                              ).encode("utf-8", "ignore"))

            now, next = self.get_plays("Hall G")
            irc.msg(channel, ("Hall G: " +
                              self.conv(next['date']) +
                              " " +
                              next['title'] +
                              " " +
                              self.getLineOfPersons(next)
                              ).encode("utf-8", "ignore"))

            now, next = self.get_plays("Hall 6")
            irc.msg(channel, ("Hall 6: " +
                              self.conv(next['date']) +
                              " " +
                              next['title'] +
                              " " +
                              self.getLineOfPersons(next)
                              ).encode("utf-8", "ignore"))

            irc.last_ccc32c3 = time()
<commit_msg>removed unused line<commit_after>#!/usr/bin/env python2
# -*- coding: utf-8 -*-

import json
import datetime
from time import time

from plugins.plugin import Plugin

from bytebot_config import BYTEBOT_PLUGIN_CONFIG

with open(BYTEBOT_PLUGIN_CONFIG['ccc32c3']['events_json_path']) as json_file:
    json_data = json.load(json_file)

class ccc32c3(Plugin):

    def __init__(self):
        pass

    def registerCommand(self, irc):
        irc.registerCommand("!32c3", "32c3")

    def get_plays(self, hall):
        now_playing = json_data['schedule'][
            'conference']['days'][0]['rooms'][hall][0]
        now_playing['date'] = "2099-12-31T23:59:59+01:00"

        next_playing = json_data['schedule'][
            'conference']['days'][0]['rooms'][hall][0]
        next_playing['date'] = "2099-12-31T23:59:59+01:00"

        for days in json_data['schedule']['conference']['days']:

            for event in days['rooms'][hall]:
                z = (datetime.datetime.strptime(event['date'][
                     0:-6], "%Y-%m-%dT%H:%M:%S") - datetime.datetime.now())

                if(z.days < 0):
                    now_playing = event

            for event in days['rooms'][hall]:
                z = (datetime.datetime.strptime(event['date'][
                     0:-6], "%Y-%m-%dT%H:%M:%S") - datetime.datetime.now())

                if(z.days >= 0):
                    if(datetime.datetime.strptime(
                            event['date'][0:-6],
                            "%Y-%m-%dT%H:%M:%S") <
                       datetime.datetime.strptime(
                            next_playing['date'][0:-6],
                            "%Y-%m-%dT%H:%M:%S")):

                        next_playing = event

        return now_playing, next_playing

    def conv(self, t):
        x = datetime.datetime.strptime(t[0:-6], "%Y-%m-%dT%H:%M:%S")
        return x.strftime("%H:%M")

    def getLineOfPersons(self, event):
        line = ""
        for persons in event['persons']:
            line += "@" + persons['public_name'] + " "
        return line

    def onPrivmsg(self, irc, msg, channel, user):
        if msg.find("!32c3") == -1:
            return

        self.irc = irc
        self.channel = channel

        if msg.find("now") != -1:
            now, next = self.get_plays("Hall 1")
            irc.msg(channel, ("Hall 1: " +
                              self.conv(now['date']) +
                              " " +
                              now['title'] +
                              " " +
                              self.getLineOfPersons(now)
                              ).encode("utf-8", "ignore"))

            now, next = self.get_plays("Hall 2")
            irc.msg(channel, ("Hall 2: " +
                              self.conv(now['date']) +
                              " " +
                              now['title'] +
                              " " +
                              self.getLineOfPersons(now)
                              ).encode("utf-8", "ignore"))

            now, next = self.get_plays("Hall G")
            irc.msg(channel, ("Hall G: " +
                              self.conv(now['date']) +
                              " " +
                              now['title'] +
                              " " +
                              self.getLineOfPersons(now)
                              ).encode("utf-8", "ignore"))

            now, next = self.get_plays("Hall 6")
            irc.msg(channel, ("Hall 6: " +
                              self.conv(now['date']) +
                              " " +
                              now['title'] +
                              " " +
                              self.getLineOfPersons(now)
                              ).encode("utf-8", "ignore"))

            irc.last_ccc32c3 = time()

        if msg.find("next") != -1:
            now, next = self.get_plays("Hall 1")
            irc.msg(channel, ("Hall 1: " +
                              self.conv(next['date']) +
                              " " +
                              next['title'] +
                              " " +
                              self.getLineOfPersons(next)
                              ).encode("utf-8", "ignore"))

            now, next = self.get_plays("Hall 2")
            irc.msg(channel, ("Hall 2: " +
                              self.conv(next['date']) +
                              " " +
                              next['title'] +
                              " " +
                              self.getLineOfPersons(next)
                              ).encode("utf-8", "ignore"))

            now, next = self.get_plays("Hall G")
            irc.msg(channel, ("Hall G: " +
                              self.conv(next['date']) +
                              " " +
                              next['title'] +
                              " " +
                              self.getLineOfPersons(next)
                              ).encode("utf-8", "ignore"))

            now, next = self.get_plays("Hall 6")
            irc.msg(channel, ("Hall 6: " +
                              self.conv(next['date']) +
                              " " +
                              next['title'] +
                              " " +
                              self.getLineOfPersons(next)
                              ).encode("utf-8", "ignore"))

            irc.last_ccc32c3 = time()
<|endoftext|>


<commit_before>"""Stomp Protocol Connectivity

    This provides basic connectivity to a message broker supporting the 'stomp' protocol.
    At the moment ACK, SEND, SUBSCRIBE, UNSUBSCRIBE, BEGIN, ABORT, COMMIT, CONNECT and DISCONNECT operations
    are supported.

    This changes the previous version which required a listener per subscription -- now a listener object
    just calls the 'addlistener' method and will receive all messages sent in response to all/any subscriptions.
    (The reason for the change is that the handling of an 'ack' becomes problematic unless the listener mechanism
    is decoupled from subscriptions).

    Note that you must 'start' an instance of Connection to begin receiving messages.  For example:

        conn = stomp.Connection([('localhost', 62003)], 'myuser', 'mypass')
        conn.start()

    Meta-Data
    ---------
    Author: Jason R Briggs
    License: http://www.apache.org/licenses/LICENSE-2.0
    Start Date: 2005/12/01
    Last Revision Date: $Date: 2008/09/11 00:16 $

    Notes/Attribution
    -----------------
    * uuid method courtesy of Carl Free Jr:
      http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/213761
    * patch from Andreas Schobel
    * patches from Julian Scheid of Rising Sun Pictures (http://open.rsp.com.au)
    * patch from Fernando
    * patches from Eugene Strulyov

    Updates
    -------
    * 2007/03/31 : (Andreas Schobel) patch to fix newlines problem in ActiveMQ 4.1
    * 2007/09    : (JRB) updated to get stomp.py working in Jython as well as Python
    * 2007/09/05 : (Julian Scheid) patch to allow sending custom headers
    * 2007/09/18 : (JRB) changed code to use logging instead of just print. added logger for jython to work
    * 2007/09/18 : (Julian Scheid) various updates, including:
       - change incoming message handling so that callbacks are invoked on the listener not only for MESSAGE, but also for
            CONNECTED, RECEIPT and ERROR frames.
       - callbacks now get not only the payload but any headers specified by the server
       - all outgoing messages now sent via a single method
       - only one connection used
       - change to use thread instead of threading
       - sends performed on the calling thread
       - receiver loop now deals with multiple messages in one received chunk of data
       - added reconnection attempts and connection fail-over
       - changed defaults for "user" and "passcode" to None instead of empty string (fixed transmission of those values)
       - added readline support
    * 2008/03/26 : (Fernando) added cStringIO for faster performance on large messages
    * 2008/09/10 : (Eugene) remove lower() on headers to support case-sensitive header names
    * 2008/09/11 : (JRB) fix incompatibilities with RabbitMQ, add wait for socket-connect
    * 2008/10/28 : (Eugene) add jms map (from stomp1.1 ideas)
    * 2008/11/25 : (Eugene) remove superfluous (incorrect) locking code
    * 2009/02/05 : (JRB) remove code to replace underscores with dashes in header names (causes a problem in rabbit-mq)
    * 2009/03/29 : (JRB) minor change to add logging config file
                   (JRB) minor change to add socket timeout, suggested by Israel
    * 2009/04/01 : (Gavin) patch to change md5 to hashlib (for 2.6 compatibility)
    * 2009/04/02 : (Fernando Ciciliati) fix overflow bug when waiting too long to connect to the broker
    * 2009/05/28 : (Martin Pieuchot) patch to support version of Python pre 2.5

"""

__version__ = '2.0.2'
__all__ = [ 'stomp' ]

import stomp
import cli
import listener

Connection = stomp.Connection
ConnectionListener = listener.ConnectionListener
StatsListener = listener.StatsListener<commit_msg>update version<commit_after>"""Stomp Protocol Connectivity

    This provides basic connectivity to a message broker supporting the 'stomp' protocol.
    At the moment ACK, SEND, SUBSCRIBE, UNSUBSCRIBE, BEGIN, ABORT, COMMIT, CONNECT and DISCONNECT operations
    are supported.

    This changes the previous version which required a listener per subscription -- now a listener object
    just calls the 'addlistener' method and will receive all messages sent in response to all/any subscriptions.
    (The reason for the change is that the handling of an 'ack' becomes problematic unless the listener mechanism
    is decoupled from subscriptions).

    Note that you must 'start' an instance of Connection to begin receiving messages.  For example:

        conn = stomp.Connection([('localhost', 62003)], 'myuser', 'mypass')
        conn.start()

    Meta-Data
    ---------
    Author: Jason R Briggs
    License: http://www.apache.org/licenses/LICENSE-2.0
    Start Date: 2005/12/01
    Last Revision Date: $Date: 2008/09/11 00:16 $

    Notes/Attribution
    -----------------
    * uuid method courtesy of Carl Free Jr:
      http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/213761
    * patch from Andreas Schobel
    * patches from Julian Scheid of Rising Sun Pictures (http://open.rsp.com.au)
    * patch from Fernando
    * patches from Eugene Strulyov

"""

__version__ = '2.0.3'
__all__ = [ 'stomp' ]

import stomp
import cli
import listener

Connection = stomp.Connection
ConnectionListener = listener.ConnectionListener
StatsListener = listener.StatsListener<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function, unicode_literals, division
import six
from importlib import import_module
from armet import exceptions
from armet.resources import options
from armet.resources.attributes import Attribute

class ResourceBase(type):

    #! Options class to use to expand options.
    options = options.ResourceOptions

    @staticmethod
    def _is_resource(name, bases):
        if name == 'NewBase':
            # This is a six contrivance; not a real class.
            return False

        for base in bases:
            if base.__name__ == 'NewBase':
                # This is a six contrivance; move along.
                continue

            if isinstance(base, ResourceBase):
                # This is some sort of derived resource; good.
                return True

        # This is not derived at all from Resource (eg. is Resource)
        return False

    def __new__(cls, name, bases, attrs):
        if not cls._is_resource(name, bases):
            # This is not an actual resource.
            return super(ResourceBase, cls).__new__(cls, name, bases, attrs)

        # Gather the attributes of all options classes.
        metadata = {}
        for base in bases:
            meta = getattr(base, 'Meta', None)
            if meta:
                metadata.update(**meta.__dict__)

        if attrs.get('Meta'):
            metadata.update(**attrs['Meta'].__dict__)

        # Expand the options class with the gathered metadata.
        meta = attrs['meta'] = cls.options(metadata, name,
            [getattr(base, 'Meta', None) for base in bases])

        # Apply the HTTP connector.
        # Mangle the bases. Note that this does not actually change the bases
        # in posterity, it only changes the bases at class object creation.
        # The significance here is that class may derive and change their
        # connectors.
        print(meta.connectors)
        connector = import_module('{}.resources'.format(
            meta.connectors['http']))

        new_bases = []
        for base in bases:
            if base is not import_module('armet.resources').Resource:
                new_bases.append(base)

            else:
                new_bases.append(connector.Resource)

        # Gather declared attributes from ourself and base classes.
        # TODO: We'll likely need a hook here for ORMs
        attributes = {}
        for base in bases:
            if hasattr(base, 'attributes'):
                attributes.update(**base.attributes)

        for index, attribute in six.iteritems(attrs):
            if isinstance(attribute, Attribute):
                attributes[index] = attribute

        # Append include directives here
        attributes.update(**meta.include)

        # Store the gathered attributes
        attrs['attributes'] = attributes

        # Construct the class object.
        self = super(ResourceBase, cls).__new__(cls, name, tuple(new_bases),
            attrs)

        # Return the constructed object.
        return self
<commit_msg>Removed print statement.<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function, unicode_literals, division
import six
from importlib import import_module
from armet import exceptions
from armet.resources import options
from armet.resources.attributes import Attribute

class ResourceBase(type):

    #! Options class to use to expand options.
    options = options.ResourceOptions

    @staticmethod
    def _is_resource(name, bases):
        if name == 'NewBase':
            # This is a six contrivance; not a real class.
            return False

        for base in bases:
            if base.__name__ == 'NewBase':
                # This is a six contrivance; move along.
                continue

            if isinstance(base, ResourceBase):
                # This is some sort of derived resource; good.
                return True

        # This is not derived at all from Resource (eg. is Resource)
        return False

    def __new__(cls, name, bases, attrs):
        if not cls._is_resource(name, bases):
            # This is not an actual resource.
            return super(ResourceBase, cls).__new__(cls, name, bases, attrs)

        # Gather the attributes of all options classes.
        metadata = {}
        for base in bases:
            meta = getattr(base, 'Meta', None)
            if meta:
                metadata.update(**meta.__dict__)

        if attrs.get('Meta'):
            metadata.update(**attrs['Meta'].__dict__)

        # Expand the options class with the gathered metadata.
        meta = attrs['meta'] = cls.options(metadata, name,
            [getattr(base, 'Meta', None) for base in bases])

        # Apply the HTTP connector.
        # Mangle the bases. Note that this does not actually change the bases
        # in posterity, it only changes the bases at class object creation.
        # The significance here is that class may derive and change their
        # connectors.
        connector = import_module('{}.resources'.format(
            meta.connectors['http']))

        new_bases = []
        for base in bases:
            if base is not import_module('armet.resources').Resource:
                new_bases.append(base)

            else:
                new_bases.append(connector.Resource)

        # Gather declared attributes from ourself and base classes.
        # TODO: We'll likely need a hook here for ORMs
        attributes = {}
        for base in bases:
            if hasattr(base, 'attributes'):
                attributes.update(**base.attributes)

        for index, attribute in six.iteritems(attrs):
            if isinstance(attribute, Attribute):
                attributes[index] = attribute

        # Append include directives here
        attributes.update(**meta.include)

        # Store the gathered attributes
        attrs['attributes'] = attributes

        # Construct the class object.
        self = super(ResourceBase, cls).__new__(cls, name, tuple(new_bases),
            attrs)

        # Return the constructed object.
        return self
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function
import datetime
import json
import random
import multiprocessing
import sys
import threading
import time
import imp
import re
import os
import shutil
from logging import getLogger

import numpy as np
from PIL import Image
import six
import cPickle as pickle
from six.moves import queue

import chainer
from chainer import computational_graph
from chainer import cuda
from chainer import optimizers
from chainer import serializers

VALIDATION_TIMING = 500 # ORIGINAL 50000

logger = getLogger(__name__)

def load_image_list(path):
    tuples = []
    for line in open(path):
        pair = line.strip().split()
        tuples.append((pair[0], np.int32(pair[1])))
    return tuples

def read_image(path, model_insize, mean_image, center=False, flip=False):
    cropwidth = 256 - model_insize
    image = np.asarray(Image.open(path))
    if len(image.shape) == 3:
        image = image.transpose(2, 0, 1)
    else:
        zeros = np.zeros((256,256))
        image = np.array([image, zeros, zeros])
    if center:
        top = left = cropwidth / 2
    else:
        top = random.randint(0, cropwidth - 1)
        left = random.randint(0, cropwidth - 1)
    bottom = model_insize + top
    right = model_insize + left
    image = image[:, top:bottom, left:right].astype(np.float32)
    image -= mean_image[:, top:bottom, left:right]
    image /= 255
    if flip and random.randint(0, 1) == 0:
        return np.fliplr(image)
    else:
        return image

def feed_data(train_list, val_list, mean_image, batchsize, val_batchsize, model, loaderjob, epoch, optimizer, data_q, avoid_flipping):
    i = 0
    count = 0
    x_batch = np.ndarray((batchsize, 3, model.insize, model.insize), dtype=np.float32)
    y_batch = np.ndarray((batchsize,), dtype=np.int32)
    val_x_batch = np.ndarray((val_batchsize, 3, model.insize, model.insize), dtype=np.float32)
    val_y_batch = np.ndarray((val_batchsize,), dtype=np.int32)

    batch_pool = [None] * batchsize
    val_batch_pool = [None] * val_batchsize
    pool = multiprocessing.Pool(loaderjob)
    data_q.put('train')
    use_flip = True

    if avoid_flipping == 1:
        use_flip = False

    for epoch in six.moves.range(1, 1 + epoch):
        logger.info('epoch: {0}'.format(epoch))
        logger.info('learning rate: {0}'.format(optimizer.lr))
        perm = np.random.permutation(len(train_list))
        for idx in perm:
            path, label = train_list[idx]
            batch_pool[i] = pool.apply_async(read_image, (path, model.insize, mean_image, False, use_flip))
            y_batch[i] = label
            i += 1

            if i == batchsize:
                for j, x in enumerate(batch_pool):
                    x_batch[j] = x.get()
                data_q.put((x_batch.copy(), y_batch.copy(), epoch))
                i= 0

            count += 1
            if count % 1000 == 0:
                data_q.put('val')
                j = 0
                for path, label in val_list:
                    val_batch_pool[j] = pool.apply_async(read_image, (path, model.insize, mean_image, True, False))
                    val_y_batch[j] = label
                    j += 1

                    if j == val_batchsize:
                        for k, x in enumerate(val_batch_pool):
                            val_x_batch[k] = x.get()
                        data_q.put((val_x_batch.copy(), val_y_batch.copy(), epoch))
                        j = 0
                data_q.put('train')

        optimizer.lr *= 0.97
    pool.close()
    pool.join()
    data_q.put('end')
    return

def log_result(batchsize, val_batchsize, log_file,log_html, res_q):
    fH = open(log_html, 'w')
    fH.flush()

    f = open(log_file, 'w')
    f.write("count\tepoch\taccuracy\tloss\taccuracy(val)\tloss(val)\n")
    f.flush()
    count = 0
    train_count = 0
    train_cur_loss = 0
    train_cur_accuracy = 0
    begin_at = time.time()
    val_begin_at = None
    while True:
        result = res_q.get()
        if result == 'end':
            break
        elif result == 'train':
            train = True
            if val_begin_at is not None:
                begin_at += time.time() - val_begin_at
                val_begin_at = None
            continue
        elif result == 'val':
            train = False
            val_count = val_loss = val_accuracy = 0
            val_begin_at = time.time()
            continue
        loss, accuracy, epoch = result
        if train:
            train_count += 1
            duration = time.time() - begin_at
            throughput = train_count * batchsize / duration
            fH.write(
                '\rtrain {} updates ({} samples) time: {} ({} images/sec)
'
                .format(train_count, train_count * batchsize, datetime.timedelta(seconds=duration), throughput))
            fH.flush()
            f.write(str(count) + "\t" + str(epoch) + "\t" + str(accuracy) + "\t" + str(loss) + "\t\t\n")
            f.flush()
            count += 1
            train_cur_loss += loss
            train_cur_accuracy += accuracy
            if train_count % 1000 == 0:
                mean_loss = train_cur_loss / 1000
                mean_error = 1 - train_cur_accuracy / 10000
                fH.write(""+json.dumps({'type': 'train', 'iteration': train_count, 'error': mean_error, 'loss': mean_loss})+"
")
                fH.flush()
                sys.stdout.flush()
                train_cur_loss = 0
                train_cur_accuracy = 0
        else:
            val_count += val_batchsize
            duration = time.time() - val_begin_at
            throughput = val_count / duration
            fH.write(
                '\rval {} batches ({} samples) time: {} ({} images/sec)'
                .format(val_count / val_batchsize, val_count, datetime.timedelta(seconds=duration), throughput)
            )
            fH.flush()
            val_loss += loss
            val_accuracy += accuracy
            if val_count == VALIDATION_TIMING:
                mean_loss = val_loss * val_batchsize / VALIDATION_TIMING
                mean_accuracy = val_accuracy * val_batchsize / VALIDATION_TIMING
                fH.write(""+json.dumps({'type': 'val', 'iteration': train_count, 'error': (1 - mean_accuracy), 'loss': mean_loss})+"
")
                fH.flush()
                f.write(str(count) + "\t" + str(epoch) + "\t\t\t" + str(mean_accuracy) + "\t" + str(mean_loss) + "\n")
                count += 1
                f.flush()
                sys.stdout.flush()
    f.close()
    fH.close()

def train_loop(model, output_dir, xp, optimizer, res_q, data_q):
    graph_generated = False
    while True:
        while data_q.empty():
            time.sleep(0.1)
        inp = data_q.get()
        if inp == 'end':
            res_q.put('end')
            break
        elif inp == 'train':
            res_q.put('train')
            model.train = True
            continue
        elif inp == 'val':
            res_q.put('val')
            model.train = False
            continue
        volatile = 'off' if model.train else 'on'
        x = chainer.Variable(xp.asarray(inp[0]), volatile=volatile)
        t = chainer.Variable(xp.asarray(inp[1]), volatile=volatile)
        if model.train:
            optimizer.update(model, x, t)
            if not graph_generated:
                with open(output_dir + os.sep + 'graph.dot', 'w') as o:
                    o.write(computational_graph.build_computational_graph((model.loss,)).dump())
                graph_generated = True
        else:
            model(x, t)

        serializers.save_hdf5(output_dir + os.sep + 'model%04d'%inp[2], model)
        #serializers.save_hdf5(output_dir + os.sep + 'optimizer%04d'%inp[2], optimizer)
        res_q.put((float(model.loss.data), float(model.accuracy.data), inp[2]))
        del x, t

def load_module(dir_name, symbol):
    (file, path, description) = imp.find_module(symbol,[dir_name])
    return imp.load_module(symbol, file, path, description)

def do_train(
        db_model,
        root_output_dir,
        batchsize=32,
        val_batchsize=250,
        gpu=-1,
        loaderjob=20,
        pretrained_model=""
    ):
    logger.info('Start imagenet train. model_id: {0} gpu: {1}, pretrained_model: {2}'.format(db_model.id, gpu, pretrained_model))
    # start initialization
    if gpu >= 0:
        cuda.check_cuda_available()
    xp = cuda.cupy if gpu >= 0 else np

    train_list = load_image_list(os.path.join(db_model.prepared_file_path, 'train.txt'))
    val_list   = load_image_list(os.path.join(db_model.prepared_file_path, 'test.txt'))
    mean_image = pickle.load(open(os.path.join(db_model.prepared_file_path, 'mean.npy'), 'rb'))

    # @see http://qiita.com/progrommer/items/abd2276f314792c359da
    (model_dir, model_name) = os.path.split(db_model.network_path)
    model_name = re.sub(r"\.py$", "", model_name)
    model_module = load_module(model_dir, model_name)
    model = model_module.Network()

    # create directory for saving trained models
    if db_model.trained_model_path is None:
        db_model.trained_model_path = os.path.join(root_output_dir, model_name)
    if not os.path.exists(db_model.trained_model_path):
        os.mkdir(db_model.trained_model_path)

    # Load pretrained model
    if pretrained_model is not None and pretrained_model.find("model") > -1:
        logger.info("load pretrained model : "+db_model.trained_model_path + os.sep +pretrained_model)
        serializers.load_hdf5(os.path.join(db_model.trained_model_path,pretrained_model), model)
        # delete old models
        try:
            shutil.copyfile(os.path.join(db_model.trained_model_path, pretrained_model), os.path.join(db_model.trained_model_path, 'previous_' + pretrained_model))
        except Exception as e:
            logger.exception('Could not copy {0} to {1}. {2}'.format(os.path.join(db_model.trained_model_path, pretrained_model), os.path.join(db_model.trained_model_path, 'previous_' + pretrained_model), e))
            raise e
        pretrained_models = sorted(os.listdir(db_model.trained_model_path), reverse=True)
        for m in pretrained_models:
            if m.startswith('model') and pretrained_model != m:
                try:
                    os.remove(os.path.join(db_model.trained_model_path, m))
                except Exception as e:
                    logger.exception('Could not remove old models: {0} {1}'.format(os.path.join(db_model.trained_model_path, m), e))
                    raise e

    # delete layer visualization cache
    for f in os.listdir(db_model.trained_model_path):
        if os.path.isdir(os.path.join(db_model.trained_model_path, f)):
            try:
                shutil.rmtree(os.path.join(db_model.trained_model_path, f))
            except Exception as e:
                logger.exception('Could not remove visualization cache. {0}'.format(e))
                raise e

    if gpu >= 0:
        cuda.get_device(gpu).use()
        model.to_gpu()

    optimizer = optimizers.MomentumSGD(lr=0.01, momentum=0.9)
    optimizer.setup(model)

    data_q = queue.Queue(maxsize=1)
    res_q = queue.Queue()

    db_model.is_trained = 1
    db_model.update_and_commit()

    # Invoke threads
    feeder = threading.Thread(
        target=feed_data,
        args=(
            train_list,
            val_list,
            mean_image,
            batchsize,
            val_batchsize,
            model,
            loaderjob,
            db_model.epoch,
            optimizer,
            data_q,
            db_model.avoid_flipping
        )
    )
    feeder.daemon = True
    feeder.start()
    train_logger = threading.Thread(
        target=log_result,
        args=(
            batchsize,
            val_batchsize,
            os.path.join(db_model.trained_model_path, 'line_graph.tsv'),
            os.path.join(db_model.trained_model_path, 'log.html'),
            res_q
        )
    )
    train_logger.daemon = True
    train_logger.start()
    train_loop(
        model,
        db_model.trained_model_path,
        xp,
        optimizer,
        res_q,
        data_q
    )
    feeder.join()
    train_logger.join()

    # post-processing
    db_model.is_trained = 2
    db_model.pid = None
    db_model.update_and_commit()
    if os.path.exists(os.path.join(db_model.trained_model_path,  'previous_' + pretrained_model)):
        #delete backup file
        try:
            os.remove(os.path.join(db_model.trained_model_path, 'previous_' + pretrained_model))
        except Exception as e:
            logger.exception('Could not delete backuped model: {0} {1}'.format(os.path.join(db_model.trained_model_path, 'previous_' + pretrained_model), e))
            raise e
    # delete prepared images
    for f in os.listdir(db_model.prepared_file_path):
        (head, ext) = os.path.splitext(f)
        ext = ext.lower()
        if ext in ['.jpg', '.jpeg', '.gif', '.png']:
            try:
                os.remove(os.path.join(db_model.prepared_file_path, f))
            except Exception as e:
                logger.exception('Could not remove prepared file: {0} {1}'.format(os.path.join(db_model.prepared_file_path, f), e))
                raise e
    logger.info('Finish imagenet train. model_id: {0}'.format(db_model.id))
<commit_msg>do validation regardless of number of train image<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function
import datetime
import json
import random
import multiprocessing
import sys
import threading
import time
import imp
import re
import os
import shutil
from logging import getLogger

import numpy as np
from PIL import Image
import six
import cPickle as pickle
from six.moves import queue

import chainer
from chainer import computational_graph
from chainer import cuda
from chainer import optimizers
from chainer import serializers

VALIDATION_TIMING = 500 # ORIGINAL 50000

logger = getLogger(__name__)

def load_image_list(path):
    tuples = []
    for line in open(path):
        pair = line.strip().split()
        tuples.append((pair[0], np.int32(pair[1])))
    return tuples

def read_image(path, model_insize, mean_image, center=False, flip=False):
    cropwidth = 256 - model_insize
    image = np.asarray(Image.open(path))
    if len(image.shape) == 3:
        image = image.transpose(2, 0, 1)
    else:
        zeros = np.zeros((256,256))
        image = np.array([image, zeros, zeros])
    if center:
        top = left = cropwidth / 2
    else:
        top = random.randint(0, cropwidth - 1)
        left = random.randint(0, cropwidth - 1)
    bottom = model_insize + top
    right = model_insize + left
    image = image[:, top:bottom, left:right].astype(np.float32)
    image -= mean_image[:, top:bottom, left:right]
    image /= 255
    if flip and random.randint(0, 1) == 0:
        return np.fliplr(image)
    else:
        return image

def feed_data(train_list, val_list, mean_image, batchsize, val_batchsize, model, loaderjob, epoch, optimizer, data_q, avoid_flipping):
    denominator = 1000 if len(train_list) > 1000 else len(train_list)
    i = 0
    count = 0
    x_batch = np.ndarray((batchsize, 3, model.insize, model.insize), dtype=np.float32)
    y_batch = np.ndarray((batchsize,), dtype=np.int32)
    val_x_batch = np.ndarray((val_batchsize, 3, model.insize, model.insize), dtype=np.float32)
    val_y_batch = np.ndarray((val_batchsize,), dtype=np.int32)

    batch_pool = [None] * batchsize
    val_batch_pool = [None] * val_batchsize
    pool = multiprocessing.Pool(loaderjob)
    data_q.put('train')
    use_flip = True

    if avoid_flipping == 1:
        use_flip = False

    for epoch in six.moves.range(1, 1 + epoch):
        logger.info('epoch: {0}'.format(epoch))
        logger.info('learning rate: {0}'.format(optimizer.lr))
        perm = np.random.permutation(len(train_list))
        for idx in perm:
            path, label = train_list[idx]
            batch_pool[i] = pool.apply_async(read_image, (path, model.insize, mean_image, False, use_flip))
            y_batch[i] = label
            i += 1

            if i == batchsize:
                for j, x in enumerate(batch_pool):
                    x_batch[j] = x.get()
                data_q.put((x_batch.copy(), y_batch.copy(), epoch))
                i= 0

            count += 1
            if count % denominator == 0:
                data_q.put('val')
                j = 0
                for path, label in val_list:
                    val_batch_pool[j] = pool.apply_async(read_image, (path, model.insize, mean_image, True, False))
                    val_y_batch[j] = label
                    j += 1

                    if j == val_batchsize:
                        for k, x in enumerate(val_batch_pool):
                            val_x_batch[k] = x.get()
                        data_q.put((val_x_batch.copy(), val_y_batch.copy(), epoch))
                        j = 0
                data_q.put('train')

        optimizer.lr *= 0.97
    pool.close()
    pool.join()
    data_q.put('end')
    return

def log_result(batchsize, val_batchsize, log_file,log_html, res_q):
    fH = open(log_html, 'w')
    fH.flush()

    f = open(log_file, 'w')
    f.write("count\tepoch\taccuracy\tloss\taccuracy(val)\tloss(val)\n")
    f.flush()
    count = 0
    train_count = 0
    train_cur_loss = 0
    train_cur_accuracy = 0
    begin_at = time.time()
    val_begin_at = None
    while True:
        result = res_q.get()
        if result == 'end':
            break
        elif result == 'train':
            train = True
            if val_begin_at is not None:
                begin_at += time.time() - val_begin_at
                val_begin_at = None
            continue
        elif result == 'val':
            train = False
            val_count = val_loss = val_accuracy = 0
            val_begin_at = time.time()
            continue
        loss, accuracy, epoch = result
        if train:
            train_count += 1
            duration = time.time() - begin_at
            throughput = train_count * batchsize / duration
            fH.write(
                '\rtrain {} updates ({} samples) time: {} ({} images/sec)
'
                .format(train_count, train_count * batchsize, datetime.timedelta(seconds=duration), throughput))
            fH.flush()
            f.write(str(count) + "\t" + str(epoch) + "\t" + str(accuracy) + "\t" + str(loss) + "\t\t\n")
            f.flush()
            count += 1
            train_cur_loss += loss
            train_cur_accuracy += accuracy
            if train_count % 1000 == 0:
                mean_loss = train_cur_loss / 1000
                mean_error = 1 - train_cur_accuracy / 10000
                fH.write(""+json.dumps({'type': 'train', 'iteration': train_count, 'error': mean_error, 'loss': mean_loss})+"
")
                fH.flush()
                sys.stdout.flush()
                train_cur_loss = 0
                train_cur_accuracy = 0
        else:
            val_count += val_batchsize
            duration = time.time() - val_begin_at
            throughput = val_count / duration
            fH.write(
                '\rval {} batches ({} samples) time: {} ({} images/sec)'
                .format(val_count / val_batchsize, val_count, datetime.timedelta(seconds=duration), throughput)
            )
            fH.flush()
            val_loss += loss
            val_accuracy += accuracy
            if val_count == VALIDATION_TIMING:
                mean_loss = val_loss * val_batchsize / VALIDATION_TIMING
                mean_accuracy = val_accuracy * val_batchsize / VALIDATION_TIMING
                fH.write(""+json.dumps({'type': 'val', 'iteration': train_count, 'error': (1 - mean_accuracy), 'loss': mean_loss})+"
")
                fH.flush()
                f.write(str(count) + "\t" + str(epoch) + "\t\t\t" + str(mean_accuracy) + "\t" + str(mean_loss) + "\n")
                count += 1
                f.flush()
                sys.stdout.flush()
    f.close()
    fH.close()

def train_loop(model, output_dir, xp, optimizer, res_q, data_q):
    graph_generated = False
    while True:
        while data_q.empty():
            time.sleep(0.1)
        inp = data_q.get()
        if inp == 'end':
            res_q.put('end')
            break
        elif inp == 'train':
            res_q.put('train')
            model.train = True
            continue
        elif inp == 'val':
            res_q.put('val')
            model.train = False
            continue
        volatile = 'off' if model.train else 'on'
        x = chainer.Variable(xp.asarray(inp[0]), volatile=volatile)
        t = chainer.Variable(xp.asarray(inp[1]), volatile=volatile)
        if model.train:
            optimizer.update(model, x, t)
            if not graph_generated:
                with open(output_dir + os.sep + 'graph.dot', 'w') as o:
                    o.write(computational_graph.build_computational_graph((model.loss,)).dump())
                graph_generated = True
        else:
            model(x, t)

        serializers.save_hdf5(output_dir + os.sep + 'model%04d'%inp[2], model)
        #serializers.save_hdf5(output_dir + os.sep + 'optimizer%04d'%inp[2], optimizer)
        res_q.put((float(model.loss.data), float(model.accuracy.data), inp[2]))
        del x, t

def load_module(dir_name, symbol):
    (file, path, description) = imp.find_module(symbol,[dir_name])
    return imp.load_module(symbol, file, path, description)

def do_train(
        db_model,
        root_output_dir,
        batchsize=32,
        val_batchsize=250,
        gpu=-1,
        loaderjob=20,
        pretrained_model=""
    ):
    logger.info('Start imagenet train. model_id: {0} gpu: {1}, pretrained_model: {2}'.format(db_model.id, gpu, pretrained_model))
    # start initialization
    if gpu >= 0:
        cuda.check_cuda_available()
    xp = cuda.cupy if gpu >= 0 else np

    train_list = load_image_list(os.path.join(db_model.prepared_file_path, 'train.txt'))
    val_list   = load_image_list(os.path.join(db_model.prepared_file_path, 'test.txt'))
    mean_image = pickle.load(open(os.path.join(db_model.prepared_file_path, 'mean.npy'), 'rb'))

    # @see http://qiita.com/progrommer/items/abd2276f314792c359da
    (model_dir, model_name) = os.path.split(db_model.network_path)
    model_name = re.sub(r"\.py$", "", model_name)
    model_module = load_module(model_dir, model_name)
    model = model_module.Network()

    # create directory for saving trained models
    if db_model.trained_model_path is None:
        db_model.trained_model_path = os.path.join(root_output_dir, model_name)
    if not os.path.exists(db_model.trained_model_path):
        os.mkdir(db_model.trained_model_path)

    # Load pretrained model
    if pretrained_model is not None and pretrained_model.find("model") > -1:
        logger.info("load pretrained model : "+db_model.trained_model_path + os.sep +pretrained_model)
        serializers.load_hdf5(os.path.join(db_model.trained_model_path,pretrained_model), model)
        # delete old models
        try:
            shutil.copyfile(os.path.join(db_model.trained_model_path, pretrained_model), os.path.join(db_model.trained_model_path, 'previous_' + pretrained_model))
        except Exception as e:
            logger.exception('Could not copy {0} to {1}. {2}'.format(os.path.join(db_model.trained_model_path, pretrained_model), os.path.join(db_model.trained_model_path, 'previous_' + pretrained_model), e))
            raise e
        pretrained_models = sorted(os.listdir(db_model.trained_model_path), reverse=True)
        for m in pretrained_models:
            if m.startswith('model') and pretrained_model != m:
                try:
                    os.remove(os.path.join(db_model.trained_model_path, m))
                except Exception as e:
                    logger.exception('Could not remove old models: {0} {1}'.format(os.path.join(db_model.trained_model_path, m), e))
                    raise e

    # delete layer visualization cache
    for f in os.listdir(db_model.trained_model_path):
        if os.path.isdir(os.path.join(db_model.trained_model_path, f)):
            try:
                shutil.rmtree(os.path.join(db_model.trained_model_path, f))
            except Exception as e:
                logger.exception('Could not remove visualization cache. {0}'.format(e))
                raise e

    if gpu >= 0:
        cuda.get_device(gpu).use()
        model.to_gpu()

    optimizer = optimizers.MomentumSGD(lr=0.01, momentum=0.9)
    optimizer.setup(model)

    data_q = queue.Queue(maxsize=1)
    res_q = queue.Queue()

    db_model.is_trained = 1
    db_model.update_and_commit()

    # Invoke threads
    feeder = threading.Thread(
        target=feed_data,
        args=(
            train_list,
            val_list,
            mean_image,
            batchsize,
            val_batchsize,
            model,
            loaderjob,
            db_model.epoch,
            optimizer,
            data_q,
            db_model.avoid_flipping
        )
    )
    feeder.daemon = True
    feeder.start()
    train_logger = threading.Thread(
        target=log_result,
        args=(
            batchsize,
            val_batchsize,
            os.path.join(db_model.trained_model_path, 'line_graph.tsv'),
            os.path.join(db_model.trained_model_path, 'log.html'),
            res_q
        )
    )
    train_logger.daemon = True
    train_logger.start()
    train_loop(
        model,
        db_model.trained_model_path,
        xp,
        optimizer,
        res_q,
        data_q
    )
    feeder.join()
    train_logger.join()

    # post-processing
    db_model.is_trained = 2
    db_model.pid = None
    db_model.update_and_commit()
    if os.path.exists(os.path.join(db_model.trained_model_path,  'previous_' + pretrained_model)):
        #delete backup file
        try:
            os.remove(os.path.join(db_model.trained_model_path, 'previous_' + pretrained_model))
        except Exception as e:
            logger.exception('Could not delete backuped model: {0} {1}'.format(os.path.join(db_model.trained_model_path, 'previous_' + pretrained_model), e))
            raise e
    # delete prepared images
    for f in os.listdir(db_model.prepared_file_path):
        (head, ext) = os.path.splitext(f)
        ext = ext.lower()
        if ext in ['.jpg', '.jpeg', '.gif', '.png']:
            try:
                os.remove(os.path.join(db_model.prepared_file_path, f))
            except Exception as e:
                logger.exception('Could not remove prepared file: {0} {1}'.format(os.path.join(db_model.prepared_file_path, f), e))
                raise e
    logger.info('Finish imagenet train. model_id: {0}'.format(db_model.id))
<|endoftext|>


<commit_before>import os
import unittest
import xml.dom.minidom

from click.testing import CliRunner
from junit_conversor import _parse, _convert
from junit_conversor.cli import conversion

current_dir = os.path.dirname(os.path.realpath(__file__))
project_root = os.path.join(current_dir, os.pardir)
output_dir = os.path.join(current_dir, 'output')
example_files_dir = os.path.join(current_dir, 'flake8_example_results')
failed_flake8 = os.path.join(example_files_dir, 'failed_flake8.txt')
failed_flake8_with_invalid_lines = os.path.join(example_files_dir, 'failed_flake8_with_invalid_lines.txt')
valid_flake8 = os.path.join(example_files_dir, 'valid_flake8.txt')

junit_conversor_cli = os.path.join(current_dir, os.pardir, 'bin', 'junit_conversor')

class ParseTest(unittest.TestCase):
    def test_should_parse_a_flake8_file_with_errors(self):
        parsed = _parse(failed_flake8)

        self.assertEqual(parsed, {
            "tests/subject/__init__.py": [
                {"file": "tests/subject/__init__.py", "line": "1", "col": "1", "detail": "F401 'os' imported but unused", "code": "F401"},
                {"file": "tests/subject/__init__.py", "line": "3", "col": "1", "detail": "E302 expected 2 blank lines, found 1", "code": "E302"},
            ],
            "tests/subject/example.py": [
                {"file": "tests/subject/example.py", "line": "4", "col": "1", "detail": "E302 expected 2 blank lines, found 1", "code": "E302"},
            ]
        })

    def test_should_return_an_empty_dict_when_parsing_a_flake8_success_file(self):
        self.assertEqual({}, _parse(valid_flake8))

    def test_should_skip_invalid_lines(self):
        parsed = _parse(failed_flake8_with_invalid_lines)

        self.assertEqual(parsed, {
            "tests/subject/__init__.py": [
                {"file": "tests/subject/__init__.py", "line": "1", "col": "1", "detail": "F401 'os' imported but unused", "code": "F401"},
                {"file": "tests/subject/__init__.py", "line": "3", "col": "1", "detail": "E302 expected 2 blank lines, found 1", "code": "E302"},
            ],
            "tests/subject/example.py": [
                {"file": "tests/subject/example.py", "line": "4", "col": "1", "detail": "E302 expected 2 blank lines, found 1", "code": "E302"},
            ]
        })

class ConvertTest(unittest.TestCase):
    def setUp(self):
        self.destination = os.path.join(output_dir, 'junit.xml')

        try:
            os.remove(self.destination)
        except OSError:
            pass

    def assertXmlIsValid(self, xml_file):
        try:
            with open(xml_file) as f:
                content = f.read()

            xml.dom.minidom.parseString(content)
        except xml.parsers.expat.ExpatError:
            raise Exception('The specified file is not a valid XML (%s)'
                            % content[0:30])

    def test_should_convert_a_file_with_flake8_errors_to_junit_xml(self):
        _convert(failed_flake8, self.destination)

        self.assertTrue(os.path.exists(self.destination), 'The xml file should exist')
        self.assertXmlIsValid(self.destination)

    def test_should_not_create_a_file_if_there_are_no_errors(self):
        _convert(valid_flake8, self.destination)
        self.assertFalse(os.path.exists(self.destination), 'The xml file should not exist')

class JunitConversorCliTest(unittest.TestCase):
    runner = CliRunner()

    def test_should_fail_if_source_file_is_not_given(self):
        result = self.runner.invoke(conversion, [])

        self.assertEqual(2, result.exit_code)
        self.assertIn('Missing argument "source"', result.output)

    def test_should_fail_if_destination_file_is_not_given(self):
        result = self.runner.invoke(conversion, [failed_flake8])

        self.assertEqual(2, result.exit_code)
        self.assertIn('Missing argument "destination', result.output)

    def test_should_make_a_simple_conversion(self):
        with self.runner.isolated_filesystem():
            result = self.runner.invoke(conversion, [failed_flake8, 'result.xml'])

            self.assertEqual(0, result.exit_code)
            self.assertEqual('Conversion done\n', result.output)
<commit_msg>Refactor tests<commit_after>import os
import unittest
import xml.dom.minidom

from click.testing import CliRunner
from junit_conversor import _parse, _convert
from junit_conversor.cli import conversion

current_dir = os.path.dirname(os.path.realpath(__file__))
project_root = os.path.join(current_dir, os.pardir)
output_dir = os.path.join(current_dir, 'output')
example_files_dir = os.path.join(current_dir, 'flake8_example_results')
failed_flake8 = os.path.join(example_files_dir, 'failed_flake8.txt')
failed_flake8_with_invalid_lines = os.path.join(example_files_dir, 'failed_flake8_with_invalid_lines.txt')
valid_flake8 = os.path.join(example_files_dir, 'valid_flake8.txt')

junit_conversor_cli = os.path.join(current_dir, os.pardir, 'bin', 'junit_conversor')

class TestCase(unittest.TestCase):
    def assertXmlIsValid(self, xml_file):
        try:
            with open(xml_file) as f:
                content = f.read()

            xml.dom.minidom.parseString(content)
        except xml.parsers.expat.ExpatError:
            raise Exception('The specified file is not a valid XML (%s)'
                            % content[0:30])

    def assertFileExist(self, file_name):
        self.assertTrue(os.path.exists(file_name), 'File %s does not exist' % file_name)

    def assertFileDoesNotExist(self, file_name):
        self.assertFalse(os.path.exists(file_name), 'File %s exist' % file_name)

class ParseTest(TestCase):
    def test_should_parse_a_flake8_file_with_errors(self):
        parsed = _parse(failed_flake8)

        self.assertEqual(parsed, {
            "tests/subject/__init__.py": [
                {"file": "tests/subject/__init__.py", "line": "1", "col": "1", "detail": "F401 'os' imported but unused", "code": "F401"},
                {"file": "tests/subject/__init__.py", "line": "3", "col": "1", "detail": "E302 expected 2 blank lines, found 1", "code": "E302"},
            ],
            "tests/subject/example.py": [
                {"file": "tests/subject/example.py", "line": "4", "col": "1", "detail": "E302 expected 2 blank lines, found 1", "code": "E302"},
            ]
        })

    def test_should_return_an_empty_dict_when_parsing_a_flake8_success_file(self):
        self.assertEqual({}, _parse(valid_flake8))

    def test_should_skip_invalid_lines(self):
        parsed = _parse(failed_flake8_with_invalid_lines)

        self.assertEqual(parsed, {
            "tests/subject/__init__.py": [
                {"file": "tests/subject/__init__.py", "line": "1", "col": "1", "detail": "F401 'os' imported but unused", "code": "F401"},
                {"file": "tests/subject/__init__.py", "line": "3", "col": "1", "detail": "E302 expected 2 blank lines, found 1", "code": "E302"},
            ],
            "tests/subject/example.py": [
                {"file": "tests/subject/example.py", "line": "4", "col": "1", "detail": "E302 expected 2 blank lines, found 1", "code": "E302"},
            ]
        })

class ConvertTest(TestCase):
    def setUp(self):
        self.destination = os.path.join(output_dir, 'junit.xml')

        try:
            os.remove(self.destination)
        except OSError:
            pass

    def test_should_convert_a_file_with_flake8_errors_to_junit_xml(self):
        _convert(failed_flake8, self.destination)

        self.assertFileExist(self.destination)
        self.assertXmlIsValid(self.destination)

    def test_should_not_create_a_file_if_there_are_no_errors(self):
        _convert(valid_flake8, self.destination)
        self.assertFileDoesNotExist(self.destination)

class JunitConversorCliTest(TestCase):
    runner = CliRunner()

    def test_should_fail_if_source_file_is_not_given(self):
        result = self.runner.invoke(conversion, [])

        self.assertEqual(2, result.exit_code)
        self.assertIn('Missing argument "source"', result.output)

    def test_should_fail_if_destination_file_is_not_given(self):
        result = self.runner.invoke(conversion, [failed_flake8])

        self.assertEqual(2, result.exit_code)
        self.assertIn('Missing argument "destination', result.output)

    def test_should_make_a_simple_conversion(self):
        with self.runner.isolated_filesystem():
            result = self.runner.invoke(conversion, [failed_flake8, 'result.xml'])

            self.assertEqual(0, result.exit_code)
            self.assertEqual('Conversion done\n', result.output)
            self.assertFileExist('result.xml')
            self.assertXmlIsValid('result.xml')
<|endoftext|>


<commit_before>import unittest
from edtf_validate import valid_edtf

class isValidBadInput(unittest.TestCase):

    def testIntervalMalformed(self):
        # is_valid_interval should fail if not 8601 extended interval
        self.assertFalse(valid_edtf.is_valid_interval('2012/2011'))

    def testEDTFDateTime(self):
        # is_valid should fail if match doesn't exist
        self.assertFalse(valid_edtf.is_valid("2012-10-10T10:10:1"))
        self.assertFalse(valid_edtf.is_valid("2012-10-10T1:10:10"))
        self.assertFalse(valid_edtf.is_valid("2012-10-10T10:1:10"))
        self.assertFalse(valid_edtf.is_valid("2012-10-10t10:10:10"))
        self.assertFalse(valid_edtf.is_valid("2012-10-10T10:10:10Z10"))
        self.assertTrue(valid_edtf.is_valid("2012-10-10T10:10:10Z"))

    def testEDTFInterval(self):
        self.assertTrue(valid_edtf.is_valid("-1000/-0999"))
        self.assertTrue(valid_edtf.is_valid("-1000/-0090-10"))
        self.assertTrue(valid_edtf.is_valid("-1000/2000"))
        self.assertTrue(valid_edtf.is_valid("1000/2000"))
        self.assertTrue(valid_edtf.is_valid("unknown/2000"))
        self.assertTrue(valid_edtf.is_valid("unknown/open"))
        self.assertTrue(valid_edtf.is_valid("0000/0000"))
        self.assertTrue(valid_edtf.is_valid("0000-02/1111"))
        self.assertTrue(valid_edtf.is_valid("0000-01/0000-01-03"))
        self.assertTrue(valid_edtf.is_valid("0000-01-13/0000-01-23"))
        self.assertTrue(valid_edtf.is_valid("1111-01-01/1111"))
        self.assertTrue(valid_edtf.is_valid("0000-01/0000"))
        self.assertFalse(valid_edtf.is_valid("2012/2013/1234/55BULBASAUR"))
        self.assertFalse(valid_edtf.is_valid("NONE/unknown"))
        self.assertFalse(valid_edtf.is_valid("2012///4444"))
        self.assertFalse(valid_edtf.is_valid("2012\\2013"))
        self.assertFalse(valid_edtf.is_valid("2000/12-12"))
        self.assertTrue(valid_edtf.is_valid("2000-uu/2012"))
        self.assertTrue(valid_edtf.is_valid("2000-12-uu/2012"))
        self.assertTrue(valid_edtf.is_valid("2000-uu-10/2012"))
        self.assertTrue(valid_edtf.is_valid("2000-uu-uu/2012"))
        self.assertTrue(valid_edtf.is_valid("2000/2000-uu-uu"))
        self.assertFalse(valid_edtf.is_valid("0800/-0999"))
        self.assertFalse(valid_edtf.is_valid("-1000/-2000"))
        self.assertFalse(valid_edtf.is_valid("1000/-2000"))
        self.assertFalse(valid_edtf.is_valid("y-61000/-2000"))
        self.assertFalse(valid_edtf.is_valid("0001/0000"))
        self.assertFalse(valid_edtf.is_valid("0000-01-03/0000-01"))
        self.assertFalse(valid_edtf.is_valid("0000/-0001"))
        self.assertFalse(valid_edtf.is_valid("0000-02/0000"))
        self.assertTrue(valid_edtf.is_valid("198u/199u"))
        self.assertTrue(valid_edtf.is_valid("198u/1999"))
        self.assertTrue(valid_edtf.is_valid("1987/199u"))
        self.assertTrue(valid_edtf.is_valid("1984-11-2u/1999-01-01"))
        self.assertTrue(valid_edtf.is_valid("1984-11-12/1984-11-uu"))
        self.assertTrue(valid_edtf.is_valid("198u-11-uu/198u-11-30"))
        # Tests with negative years and unspecified parts do not pass.
        # self.assertTrue(valid_edtf.is_valid("-194u-11-uu/194u-11-30"))
        self.assertTrue(valid_edtf.is_valid("-1980-11-01/1989-11-30"))
        self.assertTrue(valid_edtf.is_valid("1919-uu-02/1919-uu-01"))
        self.assertTrue(valid_edtf.is_valid("-1981-11-10/1980-11-30"))

    def testLevel0(self):
        self.assertTrue(valid_edtf.isLevel0('2001-02-03'))
        self.assertTrue(valid_edtf.isLevel0('2008-12'))
        self.assertTrue(valid_edtf.isLevel0('2008'))
        self.assertTrue(valid_edtf.isLevel0('-0999'))
        self.assertTrue(valid_edtf.isLevel0('-9999'))
        self.assertTrue(valid_edtf.isLevel0('0000'))
        self.assertTrue(valid_edtf.isLevel0('2001-02-03T09:30:01'))
        self.assertTrue(valid_edtf.isLevel0('2004-01-01T10:10:10Z'))
        self.assertTrue(valid_edtf.isLevel0('2012-10-10T10:10:10Z'))
        self.assertTrue(valid_edtf.isLevel0('2004-01-01T10:10:10+05:00'))
        self.assertTrue(valid_edtf.isLevel0('1964/2008'))
        self.assertTrue(valid_edtf.isLevel0('2004-06/2006-08'))
        self.assertTrue(valid_edtf.isLevel0('2004-02-01/2005-02-08'))
        self.assertTrue(valid_edtf.isLevel0('2004-02-01/2005-02'))
        self.assertTrue(valid_edtf.isLevel0('2004-02-01/2005'))
        self.assertTrue(valid_edtf.isLevel0('2005/2006-02'))
        self.assertFalse(valid_edtf.isLevel0("1863- 03-29"))
        self.assertFalse(valid_edtf.isLevel0(" 1863-03-29"))
        self.assertFalse(valid_edtf.isLevel0("1863-03 -29"))
        self.assertFalse(valid_edtf.isLevel0("1863-03- 29"))
        self.assertFalse(valid_edtf.isLevel0("1863-03-29 "))
        self.assertFalse(valid_edtf.isLevel0("18 63-03-29"))
        self.assertFalse(valid_edtf.isLevel0("1863-0 3-29"))

    def testLevel1(self):
        self.assertTrue(valid_edtf.isLevel1('1984?'))
        self.assertTrue(valid_edtf.isLevel1('2004-06?'))
        self.assertTrue(valid_edtf.isLevel1('2004-06-11?'))
        self.assertTrue(valid_edtf.isLevel1('1984~'))
        self.assertTrue(valid_edtf.isLevel1('1984?~'))
        self.assertTrue(valid_edtf.isLevel1('199u'))
        self.assertTrue(valid_edtf.isLevel1('19uu'))
        self.assertTrue(valid_edtf.isLevel1('1999-uu'))
        self.assertTrue(valid_edtf.isLevel1('1999-01-uu'))
        self.assertTrue(valid_edtf.isLevel1('1999-uu-uu'))
        self.assertTrue(valid_edtf.isLevel1('unknown/2006'))
        self.assertTrue(valid_edtf.isLevel1('2004-06-01/unknown'))
        self.assertTrue(valid_edtf.isLevel1('2004-01-01/open'))
        self.assertTrue(valid_edtf.isLevel1('1984~/2004-06'))
        self.assertTrue(valid_edtf.isLevel1('1984/2004-06~'))
        self.assertTrue(valid_edtf.isLevel1('1984~/2004~'))
        self.assertTrue(valid_edtf.isLevel1('1984?/2004?~'))
        self.assertTrue(valid_edtf.isLevel1('1984-06?/2004-08?'))
        self.assertTrue(valid_edtf.isLevel1('1984-06-02?/2004-08-08~'))
        self.assertTrue(valid_edtf.isLevel1('1984-06-02?/unknown'))
        self.assertTrue(valid_edtf.isLevel1('y170000002'))
        self.assertTrue(valid_edtf.isLevel1('y-170000002'))
        self.assertTrue(valid_edtf.isLevel1('2001-21'))
        self.assertTrue(valid_edtf.isLevel1('2003-22'))
        self.assertTrue(valid_edtf.isLevel1('2000-23'))
        self.assertTrue(valid_edtf.isLevel1('2010-24'))

    def testLevel2(self):
        self.assertTrue(valid_edtf.isLevel2('2004?-06-11'))
        self.assertTrue(valid_edtf.isLevel2('2004-06~-11'))
        self.assertTrue(valid_edtf.isLevel2('2004-(06)?-11'))
        self.assertTrue(valid_edtf.isLevel2('2004-06-(11)~'))
        self.assertTrue(valid_edtf.isLevel2('2004-(06)?~'))
        self.assertTrue(valid_edtf.isLevel2('2004-(06-11)?'))
        self.assertTrue(valid_edtf.isLevel2('2004?-06-(11)~'))
        self.assertTrue(valid_edtf.isLevel2('(2004-(06)~)?'))
        self.assertTrue(valid_edtf.isLevel2('2004?-(06)?~'))
        self.assertTrue(valid_edtf.isLevel2('(2004)?-06-04~'))
        self.assertTrue(valid_edtf.isLevel2('(2011)-06-04~'))
        self.assertTrue(valid_edtf.isLevel2('2011-(06-04)~'))
        self.assertTrue(valid_edtf.isLevel2('2011-23~'))
        self.assertTrue(valid_edtf.isLevel2('156u-12-25'))
        self.assertTrue(valid_edtf.isLevel2('15uu-12-25'))
        self.assertTrue(valid_edtf.isLevel2('15uu-12-uu'))
        self.assertTrue(valid_edtf.isLevel2('1560-uu-25'))
        self.assertTrue(valid_edtf.isLevel2('[1667,1668, 1670..1672]'))
        self.assertTrue(valid_edtf.isLevel2('[..1760-12-03]'))
        self.assertTrue(valid_edtf.isLevel2('[1760-12..]'))
        self.assertTrue(valid_edtf.isLevel2('[1760-01, 1760-02, 1760-12..]'))
        self.assertTrue(valid_edtf.isLevel2('[1667, 1760-12]'))
        self.assertTrue(valid_edtf.isLevel2('{1667,1668, 1670..1672}'))
        self.assertTrue(valid_edtf.isLevel2('{1960, 1961-12}'))
        self.assertTrue(valid_edtf.isLevel2('196x'))
        self.assertTrue(valid_edtf.isLevel2('19xx'))
        self.assertTrue(valid_edtf.isLevel2('2004-06-(01)~/2004-06-(20)~'))
        self.assertTrue(valid_edtf.isLevel2('2004-06-uu/2004-07-03'))
        self.assertTrue(valid_edtf.isLevel2('y17e7'))
        self.assertTrue(valid_edtf.isLevel2('y-17e7'))
        self.assertTrue(valid_edtf.isLevel2('y17101e4p3'))
        self.assertTrue(valid_edtf.isLevel2('2001-21^southernHemisphere'))
        self.assertFalse(valid_edtf.is_valid("1960-06-31"))
        self.assertFalse(valid_edtf.is_valid("[1 760-01, 1760-02, 1760-12..]"))

    def testEDTFDateMatch(self):
        # is_valid should fail if match doesn't exist
        self.assertFalse(valid_edtf.is_valid("+20067890?~"))
        self.assertFalse(valid_edtf.is_valid("y2006"))
        self.assertFalse(valid_edtf.is_valid("-0000"))
        self.assertFalse(valid_edtf.is_valid("+y20067890-14-10?~"))
        self.assertFalse(valid_edtf.is_valid("+20067890?~"))
        self.assertFalse(valid_edtf.is_valid("+2006?~"))

if __name__ == "__main__":
    unittest.main()
    print 'done'
<commit_msg>Only level 0 specifies negative dates and that doesn't allow unspecified chars<commit_after>import unittest
from edtf_validate import valid_edtf

class isValidBadInput(unittest.TestCase):

    def testIntervalMalformed(self):
        # is_valid_interval should fail if not 8601 extended interval
        self.assertFalse(valid_edtf.is_valid_interval('2012/2011'))

    def testEDTFDateTime(self):
        # is_valid should fail if match doesn't exist
        self.assertFalse(valid_edtf.is_valid("2012-10-10T10:10:1"))
        self.assertFalse(valid_edtf.is_valid("2012-10-10T1:10:10"))
        self.assertFalse(valid_edtf.is_valid("2012-10-10T10:1:10"))
        self.assertFalse(valid_edtf.is_valid("2012-10-10t10:10:10"))
        self.assertFalse(valid_edtf.is_valid("2012-10-10T10:10:10Z10"))
        self.assertTrue(valid_edtf.is_valid("2012-10-10T10:10:10Z"))

    def testEDTFInterval(self):
        self.assertTrue(valid_edtf.is_valid("-1000/-0999"))
        self.assertTrue(valid_edtf.is_valid("-1000/-0090-10"))
        self.assertTrue(valid_edtf.is_valid("-1000/2000"))
        self.assertTrue(valid_edtf.is_valid("1000/2000"))
        self.assertTrue(valid_edtf.is_valid("unknown/2000"))
        self.assertTrue(valid_edtf.is_valid("unknown/open"))
        self.assertTrue(valid_edtf.is_valid("0000/0000"))
        self.assertTrue(valid_edtf.is_valid("0000-02/1111"))
        self.assertTrue(valid_edtf.is_valid("0000-01/0000-01-03"))
        self.assertTrue(valid_edtf.is_valid("0000-01-13/0000-01-23"))
        self.assertTrue(valid_edtf.is_valid("1111-01-01/1111"))
        self.assertTrue(valid_edtf.is_valid("0000-01/0000"))
        self.assertFalse(valid_edtf.is_valid("2012/2013/1234/55BULBASAUR"))
        self.assertFalse(valid_edtf.is_valid("NONE/unknown"))
        self.assertFalse(valid_edtf.is_valid("2012///4444"))
        self.assertFalse(valid_edtf.is_valid("2012\\2013"))
        self.assertFalse(valid_edtf.is_valid("2000/12-12"))
        self.assertTrue(valid_edtf.is_valid("2000-uu/2012"))
        self.assertTrue(valid_edtf.is_valid("2000-12-uu/2012"))
        self.assertTrue(valid_edtf.is_valid("2000-uu-10/2012"))
        self.assertTrue(valid_edtf.is_valid("2000-uu-uu/2012"))
        self.assertTrue(valid_edtf.is_valid("2000/2000-uu-uu"))
        self.assertFalse(valid_edtf.is_valid("0800/-0999"))
        self.assertFalse(valid_edtf.is_valid("-1000/-2000"))
        self.assertFalse(valid_edtf.is_valid("1000/-2000"))
        self.assertFalse(valid_edtf.is_valid("y-61000/-2000"))
        self.assertFalse(valid_edtf.is_valid("0001/0000"))
        self.assertFalse(valid_edtf.is_valid("0000-01-03/0000-01"))
        self.assertFalse(valid_edtf.is_valid("0000/-0001"))
        self.assertFalse(valid_edtf.is_valid("0000-02/0000"))
        self.assertTrue(valid_edtf.is_valid("198u/199u"))
        self.assertTrue(valid_edtf.is_valid("198u/1999"))
        self.assertTrue(valid_edtf.is_valid("1987/199u"))
        self.assertTrue(valid_edtf.is_valid("1984-11-2u/1999-01-01"))
        self.assertTrue(valid_edtf.is_valid("1984-11-12/1984-11-uu"))
        self.assertTrue(valid_edtf.is_valid("198u-11-uu/198u-11-30"))
        self.assertTrue(valid_edtf.is_valid("-1980-11-01/1989-11-30"))
        self.assertTrue(valid_edtf.is_valid("1919-uu-02/1919-uu-01"))
        self.assertTrue(valid_edtf.is_valid("-1981-11-10/1980-11-30"))

    def testLevel0(self):
        self.assertTrue(valid_edtf.isLevel0('2001-02-03'))
        self.assertTrue(valid_edtf.isLevel0('2008-12'))
        self.assertTrue(valid_edtf.isLevel0('2008'))
        self.assertTrue(valid_edtf.isLevel0('-0999'))
        self.assertTrue(valid_edtf.isLevel0('-9999'))
        self.assertTrue(valid_edtf.isLevel0('0000'))
        self.assertTrue(valid_edtf.isLevel0('2001-02-03T09:30:01'))
        self.assertTrue(valid_edtf.isLevel0('2004-01-01T10:10:10Z'))
        self.assertTrue(valid_edtf.isLevel0('2012-10-10T10:10:10Z'))
        self.assertTrue(valid_edtf.isLevel0('2004-01-01T10:10:10+05:00'))
        self.assertTrue(valid_edtf.isLevel0('1964/2008'))
        self.assertTrue(valid_edtf.isLevel0('2004-06/2006-08'))
        self.assertTrue(valid_edtf.isLevel0('2004-02-01/2005-02-08'))
        self.assertTrue(valid_edtf.isLevel0('2004-02-01/2005-02'))
        self.assertTrue(valid_edtf.isLevel0('2004-02-01/2005'))
        self.assertTrue(valid_edtf.isLevel0('2005/2006-02'))
        self.assertFalse(valid_edtf.isLevel0("1863- 03-29"))
        self.assertFalse(valid_edtf.isLevel0(" 1863-03-29"))
        self.assertFalse(valid_edtf.isLevel0("1863-03 -29"))
        self.assertFalse(valid_edtf.isLevel0("1863-03- 29"))
        self.assertFalse(valid_edtf.isLevel0("1863-03-29 "))
        self.assertFalse(valid_edtf.isLevel0("18 63-03-29"))
        self.assertFalse(valid_edtf.isLevel0("1863-0 3-29"))

    def testLevel1(self):
        self.assertTrue(valid_edtf.isLevel1('1984?'))
        self.assertTrue(valid_edtf.isLevel1('2004-06?'))
        self.assertTrue(valid_edtf.isLevel1('2004-06-11?'))
        self.assertTrue(valid_edtf.isLevel1('1984~'))
        self.assertTrue(valid_edtf.isLevel1('1984?~'))
        self.assertTrue(valid_edtf.isLevel1('199u'))
        self.assertTrue(valid_edtf.isLevel1('19uu'))
        self.assertTrue(valid_edtf.isLevel1('1999-uu'))
        self.assertTrue(valid_edtf.isLevel1('1999-01-uu'))
        self.assertTrue(valid_edtf.isLevel1('1999-uu-uu'))
        self.assertTrue(valid_edtf.isLevel1('unknown/2006'))
        self.assertTrue(valid_edtf.isLevel1('2004-06-01/unknown'))
        self.assertTrue(valid_edtf.isLevel1('2004-01-01/open'))
        self.assertTrue(valid_edtf.isLevel1('1984~/2004-06'))
        self.assertTrue(valid_edtf.isLevel1('1984/2004-06~'))
        self.assertTrue(valid_edtf.isLevel1('1984~/2004~'))
        self.assertTrue(valid_edtf.isLevel1('1984?/2004?~'))
        self.assertTrue(valid_edtf.isLevel1('1984-06?/2004-08?'))
        self.assertTrue(valid_edtf.isLevel1('1984-06-02?/2004-08-08~'))
        self.assertTrue(valid_edtf.isLevel1('1984-06-02?/unknown'))
        self.assertTrue(valid_edtf.isLevel1('y170000002'))
        self.assertTrue(valid_edtf.isLevel1('y-170000002'))
        self.assertTrue(valid_edtf.isLevel1('2001-21'))
        self.assertTrue(valid_edtf.isLevel1('2003-22'))
        self.assertTrue(valid_edtf.isLevel1('2000-23'))
        self.assertTrue(valid_edtf.isLevel1('2010-24'))

    def testLevel2(self):
        self.assertTrue(valid_edtf.isLevel2('2004?-06-11'))
        self.assertTrue(valid_edtf.isLevel2('2004-06~-11'))
        self.assertTrue(valid_edtf.isLevel2('2004-(06)?-11'))
        self.assertTrue(valid_edtf.isLevel2('2004-06-(11)~'))
        self.assertTrue(valid_edtf.isLevel2('2004-(06)?~'))
        self.assertTrue(valid_edtf.isLevel2('2004-(06-11)?'))
        self.assertTrue(valid_edtf.isLevel2('2004?-06-(11)~'))
        self.assertTrue(valid_edtf.isLevel2('(2004-(06)~)?'))
        self.assertTrue(valid_edtf.isLevel2('2004?-(06)?~'))
        self.assertTrue(valid_edtf.isLevel2('(2004)?-06-04~'))
        self.assertTrue(valid_edtf.isLevel2('(2011)-06-04~'))
        self.assertTrue(valid_edtf.isLevel2('2011-(06-04)~'))
        self.assertTrue(valid_edtf.isLevel2('2011-23~'))
        self.assertTrue(valid_edtf.isLevel2('156u-12-25'))
        self.assertTrue(valid_edtf.isLevel2('15uu-12-25'))
        self.assertTrue(valid_edtf.isLevel2('15uu-12-uu'))
        self.assertTrue(valid_edtf.isLevel2('1560-uu-25'))
        self.assertTrue(valid_edtf.isLevel2('[1667,1668, 1670..1672]'))
        self.assertTrue(valid_edtf.isLevel2('[..1760-12-03]'))
        self.assertTrue(valid_edtf.isLevel2('[1760-12..]'))
        self.assertTrue(valid_edtf.isLevel2('[1760-01, 1760-02, 1760-12..]'))
        self.assertTrue(valid_edtf.isLevel2('[1667, 1760-12]'))
        self.assertTrue(valid_edtf.isLevel2('{1667,1668, 1670..1672}'))
        self.assertTrue(valid_edtf.isLevel2('{1960, 1961-12}'))
        self.assertTrue(valid_edtf.isLevel2('196x'))
        self.assertTrue(valid_edtf.isLevel2('19xx'))
        self.assertTrue(valid_edtf.isLevel2('2004-06-(01)~/2004-06-(20)~'))
        self.assertTrue(valid_edtf.isLevel2('2004-06-uu/2004-07-03'))
        self.assertTrue(valid_edtf.isLevel2('y17e7'))
        self.assertTrue(valid_edtf.isLevel2('y-17e7'))
        self.assertTrue(valid_edtf.isLevel2('y17101e4p3'))
        self.assertTrue(valid_edtf.isLevel2('2001-21^southernHemisphere'))
        self.assertFalse(valid_edtf.is_valid("1960-06-31"))
        self.assertFalse(valid_edtf.is_valid("[1 760-01, 1760-02, 1760-12..]"))

    def testEDTFDateMatch(self):
        # is_valid should fail if match doesn't exist
        self.assertFalse(valid_edtf.is_valid("+20067890?~"))
        self.assertFalse(valid_edtf.is_valid("y2006"))
        self.assertFalse(valid_edtf.is_valid("-0000"))
        self.assertFalse(valid_edtf.is_valid("+y20067890-14-10?~"))
        self.assertFalse(valid_edtf.is_valid("+20067890?~"))
        self.assertFalse(valid_edtf.is_valid("+2006?~"))

if __name__ == "__main__":
    unittest.main()
    print 'done'
<|endoftext|>