<commit_before>#!/usr/bin/env python3

# Copyright 2014 Jussi Pakkanen

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This is a helper script for IDE developers. It allows you to
extract information such as list of targets, files, compiler flags,
tests and so on. All output is in JSON for simple parsing.

Currently only works for the Ninja backend. Others use generated
project files and don't need this info."""

import json, pickle
import coredata, build, optinterpreter
from optparse import OptionParser
import sys, os

parser = OptionParser()
parser.add_option('--list-targets', action='store_true', dest='list_targets', default=False)
parser.add_option('--target-files', action='store', dest='target_files', default=None)
parser.add_option('--buildsystem-files', action='store_true', dest='buildsystem_files', default=False)
parser.add_option('--buildoptions', action='store_true', dest='buildoptions', default=False)

def list_targets(coredata, builddata):
    tlist = []
    for target in builddata.get_targets().values():
        t = {}
        t['name'] = target.get_basename()
        t['filename'] = os.path.join(target.subdir, target.get_filename())
        if isinstance(target, build.Executable):
            typename = 'executable'
        elif isinstance(target, build.SharedLibrary):
            typename = 'shared library'
        elif isinstance(target, build.StaticLibrary):
            typename = 'static library'
        else:
            typename = 'unknown'
        t['type'] = typename
        if target.should_install():
            t['installed'] = True
        else:
            t['installed'] = False
        tlist.append(t)
    print(json.dumps(tlist))

def list_target_files(target_name, coredata, builddata):
    try:
        t = builddata.targets[target_name]
        sources = t.sources + t.extra_files
        subdir = t.subdir
    except KeyError:
        print("Unknown target %s." % target_name)
        sys.exit(1)
    sources = [os.path.join(subdir, i) for i in sources]
    print(json.dumps(sources))

def list_buildoptions(coredata, builddata):
    buildtype= {'choices': ['plain', 'debug', 'debugoptimized', 'release'],
                'type' : 'combo',
                'value' : coredata.buildtype,
                'description' : 'Build type',
                'name' : 'type'}
    strip = {'value' : coredata.strip,
             'type' : 'boolean',
             'description' : 'Strip on install',
             'name' : 'strip'}
    coverage = {'value': coredata.coverage,
                'type' : 'boolean',
                'description' : 'Enable coverage',
                'name' : 'coverage'}
    pch = {'value' : coredata.use_pch,
           'type' : 'boolean',
           'description' : 'Use precompiled headers',
           'name' : 'pch'}
    unity = {'value' : coredata.unity,
             'type' : 'boolean',
             'description' : 'Unity build',
             'name' : 'unity'}
    optlist = [buildtype, strip, coverage, pch, unity]
    options = coredata.user_options
    keys = list(options.keys())
    keys.sort()
    for key in keys:
        opt = options[key]
        optdict = {}
        optdict['name'] = key
        optdict['value'] = opt.value
        if isinstance(opt, optinterpreter.UserStringOption):
            typestr = 'string'
        elif isinstance(opt, optinterpreter.UserBooleanOption):
            typestr = 'boolean'
        elif isinstance(opt, optinterpreter.UserComboOption):
            optdict['choices'] = opt.choices
            typestr = 'combo'
        else:
            raise RuntimeError("Unknown option type")
        optdict['type'] = typestr
        optdict['description'] = opt.description
        optlist.append(optdict)
    print(json.dumps(optlist))

def list_buildsystem_files(coredata, builddata):
    src_dir = builddata.environment.get_source_dir()
    # I feel dirty about this. But only slightly.
    filelist = []
    for root, dirs, files in os.walk(src_dir):
        for f in files:
            if f == 'meson.build' or f == 'meson_options.txt':
                filelist.append(os.path.relpath(os.path.join(root, f), src_dir))
    print(json.dumps(filelist))

if __name__ == '__main__':
    (options, args) = parser.parse_args()
    if len(args) > 1:
        print('Too many arguments')
        sys.exit(1)
    elif len(args) == 1:
        bdir = args[0]
    else:
        bdir = ''
    corefile = os.path.join(bdir, 'meson-private/coredata.dat')
    buildfile = os.path.join(bdir, 'meson-private/build.dat')
    coredata = pickle.load(open(corefile, 'rb'))
    builddata = pickle.load(open(buildfile, 'rb'))
    if options.list_targets:
        list_targets(coredata, builddata)
    elif options.target_files is not None:
        list_target_files(options.target_files, coredata, builddata)
    elif options.buildsystem_files:
        list_buildsystem_files(coredata, builddata)
    elif options.buildoptions:
        list_buildoptions(coredata, builddata)
    else:
        print('No command specified')
        sys.exit(1)
<commit_msg>Introspector can print out test info.<commit_after>#!/usr/bin/env python3

# Copyright 2014 Jussi Pakkanen

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This is a helper script for IDE developers. It allows you to
extract information such as list of targets, files, compiler flags,
tests and so on. All output is in JSON for simple parsing.

Currently only works for the Ninja backend. Others use generated
project files and don't need this info."""

import json, pickle
import coredata, build, optinterpreter
from optparse import OptionParser
import sys, os

parser = OptionParser()
parser.add_option('--targets', action='store_true', dest='list_targets', default=False,
                  help='List top level targets.')
parser.add_option('--target-files', action='store', dest='target_files', default=None,
                  help='List source files for a given target.')
parser.add_option('--buildsystem-files', action='store_true', dest='buildsystem_files', default=False,
                  help='List files that make up the build system.')
parser.add_option('--buildoptions', action='store_true', dest='buildoptions', default=False,
                  help='List all build options.')
parser.add_option('--tests', action='store_true', dest='tests', default=False,
                  help='List all unit tests.')

def list_targets(coredata, builddata):
    tlist = []
    for target in builddata.get_targets().values():
        t = {}
        t['name'] = target.get_basename()
        t['filename'] = os.path.join(target.subdir, target.get_filename())
        if isinstance(target, build.Executable):
            typename = 'executable'
        elif isinstance(target, build.SharedLibrary):
            typename = 'shared library'
        elif isinstance(target, build.StaticLibrary):
            typename = 'static library'
        else:
            typename = 'unknown'
        t['type'] = typename
        if target.should_install():
            t['installed'] = True
        else:
            t['installed'] = False
        tlist.append(t)
    print(json.dumps(tlist))

def list_target_files(target_name, coredata, builddata):
    try:
        t = builddata.targets[target_name]
        sources = t.sources + t.extra_files
        subdir = t.subdir
    except KeyError:
        print("Unknown target %s." % target_name)
        sys.exit(1)
    sources = [os.path.join(subdir, i) for i in sources]
    print(json.dumps(sources))

def list_buildoptions(coredata, builddata):
    buildtype= {'choices': ['plain', 'debug', 'debugoptimized', 'release'],
                'type' : 'combo',
                'value' : coredata.buildtype,
                'description' : 'Build type',
                'name' : 'type'}
    strip = {'value' : coredata.strip,
             'type' : 'boolean',
             'description' : 'Strip on install',
             'name' : 'strip'}
    coverage = {'value': coredata.coverage,
                'type' : 'boolean',
                'description' : 'Enable coverage',
                'name' : 'coverage'}
    pch = {'value' : coredata.use_pch,
           'type' : 'boolean',
           'description' : 'Use precompiled headers',
           'name' : 'pch'}
    unity = {'value' : coredata.unity,
             'type' : 'boolean',
             'description' : 'Unity build',
             'name' : 'unity'}
    optlist = [buildtype, strip, coverage, pch, unity]
    options = coredata.user_options
    keys = list(options.keys())
    keys.sort()
    for key in keys:
        opt = options[key]
        optdict = {}
        optdict['name'] = key
        optdict['value'] = opt.value
        if isinstance(opt, optinterpreter.UserStringOption):
            typestr = 'string'
        elif isinstance(opt, optinterpreter.UserBooleanOption):
            typestr = 'boolean'
        elif isinstance(opt, optinterpreter.UserComboOption):
            optdict['choices'] = opt.choices
            typestr = 'combo'
        else:
            raise RuntimeError("Unknown option type")
        optdict['type'] = typestr
        optdict['description'] = opt.description
        optlist.append(optdict)
    print(json.dumps(optlist))

def list_buildsystem_files(coredata, builddata):
    src_dir = builddata.environment.get_source_dir()
    # I feel dirty about this. But only slightly.
    filelist = []
    for root, _, files in os.walk(src_dir):
        for f in files:
            if f == 'meson.build' or f == 'meson_options.txt':
                filelist.append(os.path.relpath(os.path.join(root, f), src_dir))
    print(json.dumps(filelist))

def list_tests(testdata):
    result = []
    for t in testdata:
        to = {}
        to['cmd'] = [t.fname] + t.cmd_args
        to['env'] = t.env
        to['name'] = t.name
        result.append(to)
    print(json.dumps(result))

if __name__ == '__main__':
    (options, args) = parser.parse_args()
    if len(args) > 1:
        print('Too many arguments')
        sys.exit(1)
    elif len(args) == 1:
        bdir = args[0]
    else:
        bdir = ''
    corefile = os.path.join(bdir, 'meson-private/coredata.dat')
    buildfile = os.path.join(bdir, 'meson-private/build.dat')
    testfile = os.path.join(bdir, 'meson-private/meson_test_setup.dat')
    coredata = pickle.load(open(corefile, 'rb'))
    builddata = pickle.load(open(buildfile, 'rb'))
    testdata = pickle.load(open(testfile, 'rb'))
    if options.list_targets:
        list_targets(coredata, builddata)
    elif options.target_files is not None:
        list_target_files(options.target_files, coredata, builddata)
    elif options.buildsystem_files:
        list_buildsystem_files(coredata, builddata)
    elif options.buildoptions:
        list_buildoptions(coredata, builddata)
    elif options.tests:
        list_tests(testdata)
    else:
        print('No command specified')
        sys.exit(1)
<|endoftext|>


<commit_before>from django.contrib import messages
from django.db import transaction
from django.shortcuts import render, redirect, get_object_or_404
from django.urls.base import reverse
from django.views.generic.base import View

from decorators import inchar_required
from messaging.models import MessageRecipient, MessageRelationship, CharacterMessage, MessageRecipientGroup
from messaging.shortcuts import create_message, add_character_recipient, \
    add_recipients_for_reply, add_organization_recipient
from organization.models import Organization
from character.models import Character

def recipient_required_decorator(func):
    @inchar_required
    def inner(*args, **kwargs):
        recipient_id = kwargs.get('recipient_id')
        args[0].recipient = get_object_or_404(
            MessageRecipient, id=recipient_id, character=args[0].hero)
        return func(*args, **kwargs)
    return inner

@inchar_required
def home(request):
    context = {
        'tab': 'new',
        'recipient_list': request.hero.messagerecipient_set.filter(read=False)
    }
    return render(request, 'messaging/message_list.html', context)

class ComposeView(View):
    def get(self,
            request, character_id=None, prefilled_text='', reply_to=None):
        if character_id:
            target_character = get_object_or_404(
                Character, id=character_id, world=request.hero.world)
        else:
            target_character = None

        context = {
            'reply_to': reply_to,
            'prefilled_text': prefilled_text,
            'tab': 'compose',
            'target_character': target_character,
        }
        return render(request, 'messaging/compose.html', context)

    class RecipientBuildingException(Exception):
        pass

    @transaction.atomic
    def post(self, request):
        message_body = request.POST.get('message_body')
        reply_to = (
            get_object_or_404(
                MessageRecipient,
                id=request.POST.get('reply_to'),
                character=request.hero
            )
            if request.POST.get('reply_to') else None
        )

        if not message_body:
            messages.error(request, "Please write some message.", "danger")
            return redirect(
                request.META.get('HTTP_REFERER', reverse('messaging:compose')))

        if len(message_body) > 10000:
            return self.fail_post_gracefully(
                request,
                reply_to,
                message_body,
                error_message="This message is too long."
            )

        if not reply_to and not request.POST.getlist('recipient'):
            return self.fail_post_gracefully(
                request,
                reply_to,
                message_body,
                error_message="You must choose at least one recipient."
            )

        message = create_message(
            template='messaging/messages/character_written_message.html',
            world=request.hero.world,
            category=None,
            template_context={'content': message_body},
            sender=request.hero
        )

        if reply_to:
            add_recipients_for_reply(message, reply_to)
        else:
            try:
                self.create_recipients_from_post_data(request, message)
            except ComposeView.RecipientBuildingException as e:
                return self.fail_post_gracefully(
                    request, reply_to, message_body, error_message=str(e)
                )

        messages.success(request, "Message sent.", "success")
        return redirect('messaging:sent')

    def fail_post_gracefully(
            self, request, reply_to, prefilled_body, error_message):
        messages.error(request, error_message, "danger")
        return self.get(request, None, prefilled_body, reply_to)

    def create_recipients_from_post_data(self, request, message):
        raw_recipients = request.POST.getlist('recipient')
        organization_count = 0
        character_count = 0

        for raw_recipient in raw_recipients:
            split = raw_recipient.split('_')

            if split[0] == 'settlement':
                for character in request.hero.location.character_set.filter(
                        paused=False):
                    character_count += 1
                    add_character_recipient(message, character)
            elif split[0] == 'region':
                for character in Character.objects.filter(
                        location__tile=request.hero.location.tile,
                        paused=False):
                    character_count += 1
                    add_character_recipient(message, character)
            elif split[0] == 'organization':
                organization_count += 1
                organization = get_object_or_404(Organization, id=split[1])
                add_organization_recipient(message, organization)
            elif split[0] == 'character':
                character_count += 1
                character = get_object_or_404(Character, id=split[1])
                add_character_recipient(message, character)
            else:
                message.delete()
                raise ComposeView.RecipientBuildingException(
                    "Invalid recipient.")

        if organization_count > 4 or character_count > 40:
            message.delete()
            raise ComposeView.RecipientBuildingException(
                "Too many recipients.")

@inchar_required
def add_contact(request, character_id):
    target_character = get_object_or_404(
        Character, id=character_id, world=request.hero.world)
    created = MessageRelationship.objects.get_or_create(
        from_character=request.hero, to_character=target_character)[1]
    if created:
        messages.success(request, "Character added to contacts", "success")
    return redirect(
        request.META.get('HTTP_REFERER', reverse('character:character_home')))

@inchar_required
def remove_contact(request, character_id):
    target_character = get_object_or_404(
        Character, id=character_id, world=request.hero.world)
    try:
        target_relationship = MessageRelationship.objects.get(
            from_character=request.hero, to_character=target_character)
        target_relationship.delete()
        messages.success(request, "Character removed contacts", "success")
    except MessageRelationship.DoesNotExist:
        pass
    return redirect(
        request.META.get('HTTP_REFERER', reverse('character:character_home')))

@inchar_required
def mark_all_read(request):
    MessageRecipient.objects.filter(
        character=request.hero, read=False).update(read=True)
    return redirect(
        request.META.get('HTTP_REFERER', reverse('character:character_home')))

@recipient_required_decorator
def mark_read(request, recipient_id):
    request.recipient.read = not request.recipient.read
    request.recipient.save()
    return redirect(
        request.META.get('HTTP_REFERER', reverse('messaging:home')))

@recipient_required_decorator
def mark_favourite(request, recipient_id):
    request.recipient.favourite = not request.recipient.favourite
    request.recipient.save()
    return redirect(
        request.META.get('HTTP_REFERER', reverse('messaging:home')))

@inchar_required
def messages_list(request):
    context = {
        'tab': 'all',
        'recipient_list': request.hero.messagerecipient_set.all()
    }
    return render(request, 'messaging/message_list.html', context)

@inchar_required
def favourites_list(request):
    context = {
        'tab': 'favourites',
        'recipient_list': request.hero.messagerecipient_set.filter(
            favourite=True
        )
    }
    return render(request, 'messaging/message_list.html', context)

@inchar_required
def sent_list(request):
    context = {
        'tab': 'sent',
        'message_list': request.hero.messages_sent
    }
    return render(request, 'messaging/sent_list.html', context)

@recipient_required_decorator
def reply(request, recipient_id, prefilled_text=''):
    view = ComposeView()
    return view.get(
        request, reply_to=request.recipient, prefilled_text=prefilled_text)
<commit_msg>Reduce duplication in messaging views<commit_after>from django.contrib import messages
from django.db import transaction
from django.shortcuts import render, redirect, get_object_or_404
from django.urls.base import reverse
from django.views.generic.base import View

from decorators import inchar_required
from messaging.models import MessageRecipient, MessageRelationship
from messaging.shortcuts import create_message, add_character_recipient, \
    add_recipients_for_reply, add_organization_recipient
from organization.models import Organization
from character.models import Character

def recipient_required_decorator(func):
    @inchar_required
    def inner(*args, **kwargs):
        recipient_id = kwargs.get('recipient_id')
        args[0].recipient = get_object_or_404(
            MessageRecipient, id=recipient_id, character=args[0].hero)
        return func(*args, **kwargs)
    return inner

@inchar_required
def generic_message_list(request, tab, recipient_list,
                         template='messaging/message_list.html'):
    context = {
        'tab': tab,
        'recipient_list': recipient_list
    }
    return render(request, template, context)

@inchar_required
def home(request):
    return generic_message_list(
        request,
        tab='new',
        recipient_list=request.hero.messagerecipient_set.filter(read=False)
    )

@inchar_required
def messages_list(request):
    return generic_message_list(
        request,
        tab='all',
        recipient_list=request.hero.messagerecipient_set.all()
    )

@inchar_required
def favourites_list(request):
    return generic_message_list(
        request,
        tab='favourites',
        recipient_list=request.hero.messagerecipient_set.filter(
            favourite=True
        )
    )

@inchar_required
def sent_list(request):
    context = {
        'tab': 'sent',
        'message_list': request.hero.messages_sent
    }
    return render(request, 'messaging/sent_list.html', context)

class ComposeView(View):
    def get(self,
            request, character_id=None, prefilled_text='', reply_to=None):
        if character_id:
            target_character = get_object_or_404(
                Character, id=character_id, world=request.hero.world)
        else:
            target_character = None

        context = {
            'reply_to': reply_to,
            'prefilled_text': prefilled_text,
            'tab': 'compose',
            'target_character': target_character,
        }
        return render(request, 'messaging/compose.html', context)

    class RecipientBuildingException(Exception):
        pass

    @transaction.atomic
    def post(self, request):
        message_body = request.POST.get('message_body')
        reply_to = (
            get_object_or_404(
                MessageRecipient,
                id=request.POST.get('reply_to'),
                character=request.hero
            )
            if request.POST.get('reply_to') else None
        )

        if not message_body:
            messages.error(request, "Please write some message.", "danger")
            return redirect(
                request.META.get('HTTP_REFERER', reverse('messaging:compose')))

        if len(message_body) > 10000:
            return self.fail_post_gracefully(
                request,
                reply_to,
                message_body,
                error_message="This message is too long."
            )

        if not reply_to and not request.POST.getlist('recipient'):
            return self.fail_post_gracefully(
                request,
                reply_to,
                message_body,
                error_message="You must choose at least one recipient."
            )

        message = create_message(
            template='messaging/messages/character_written_message.html',
            world=request.hero.world,
            category=None,
            template_context={'content': message_body},
            sender=request.hero
        )

        if reply_to:
            add_recipients_for_reply(message, reply_to)
        else:
            try:
                self.create_recipients_from_post_data(request, message)
            except ComposeView.RecipientBuildingException as e:
                return self.fail_post_gracefully(
                    request, reply_to, message_body, error_message=str(e)
                )

        messages.success(request, "Message sent.", "success")
        return redirect('messaging:sent')

    def fail_post_gracefully(
            self, request, reply_to, prefilled_body, error_message):
        messages.error(request, error_message, "danger")
        return self.get(request, None, prefilled_body, reply_to)

    def create_recipients_from_post_data(self, request, message):
        raw_recipients = request.POST.getlist('recipient')
        organization_count = 0
        character_count = 0

        for raw_recipient in raw_recipients:
            split = raw_recipient.split('_')

            if split[0] == 'settlement':
                for character in request.hero.location.character_set.filter(
                        paused=False):
                    character_count += 1
                    add_character_recipient(message, character)
            elif split[0] == 'region':
                for character in Character.objects.filter(
                        location__tile=request.hero.location.tile,
                        paused=False):
                    character_count += 1
                    add_character_recipient(message, character)
            elif split[0] == 'organization':
                organization_count += 1
                organization = get_object_or_404(Organization, id=split[1])
                add_organization_recipient(message, organization)
            elif split[0] == 'character':
                character_count += 1
                character = get_object_or_404(Character, id=split[1])
                add_character_recipient(message, character)
            else:
                message.delete()
                raise ComposeView.RecipientBuildingException(
                    "Invalid recipient.")

        if organization_count > 4 or character_count > 40:
            message.delete()
            raise ComposeView.RecipientBuildingException(
                "Too many recipients.")

@inchar_required
def add_contact(request, character_id):
    target_character = get_object_or_404(
        Character, id=character_id, world=request.hero.world)
    created = MessageRelationship.objects.get_or_create(
        from_character=request.hero, to_character=target_character)[1]
    if created:
        messages.success(request, "Character added to contacts", "success")
    return redirect(
        request.META.get('HTTP_REFERER', reverse('character:character_home')))

@inchar_required
def remove_contact(request, character_id):
    target_character = get_object_or_404(
        Character, id=character_id, world=request.hero.world)
    try:
        target_relationship = MessageRelationship.objects.get(
            from_character=request.hero, to_character=target_character)
        target_relationship.delete()
        messages.success(request, "Character removed contacts", "success")
    except MessageRelationship.DoesNotExist:
        pass
    return redirect(
        request.META.get('HTTP_REFERER', reverse('character:character_home')))

@inchar_required
def mark_all_read(request):
    MessageRecipient.objects.filter(
        character=request.hero, read=False).update(read=True)
    return redirect(
        request.META.get('HTTP_REFERER', reverse('character:character_home')))

@recipient_required_decorator
def mark_read(request, recipient_id):
    request.recipient.read = not request.recipient.read
    request.recipient.save()
    return redirect(
        request.META.get('HTTP_REFERER', reverse('messaging:home')))

@recipient_required_decorator
def mark_favourite(request, recipient_id):
    request.recipient.favourite = not request.recipient.favourite
    request.recipient.save()
    return redirect(
        request.META.get('HTTP_REFERER', reverse('messaging:home')))

@recipient_required_decorator
def reply(request, recipient_id, prefilled_text=''):
    view = ComposeView()
    return view.get(
        request, reply_to=request.recipient, prefilled_text=prefilled_text)
<|endoftext|>


<commit_before># For using Velvet de-novo assembler

from helps import *

def velvet(**ops):
    ''' Performs a de novo assembly using Velvet

    Example:

    >>> velvet(
    ... outdir = 'output_directory',
    ... # reads must be an array of tuples (even if you're only using one file)
    ... # in the following form:
    ...
    ... reads = [
    ... ('format', 'short long or paired', 'filename.fastq'), ... ]
    ...
    ... # example:
    ... reads = [
    ... ('fastq', 'paired', 'paired_reads.fastq'), ...
    ... ],
    ... kmer = 31, # see velvet readme for why this is important
    ... )
    '''
    read_ops = ['-%s -%s %s' % r for r in ops['reads'] ]

    cmd = ' '.join([
        '~/cram/bin/velveth',
        '%(outdir)s',
        '%(k)s']) % ops

    velveth = cmd + ' ' + ' '.join(read_ops) + '>/dev/null'
    velvetg = '~/cram/bin/velvetg %(outdir)s -very_clean yes > /dev/null' % ops

    ohai('running velveth: %(reads)s, k = %(k)s' % ops)
    run(velveth, generates=ops['outdir']) # run hash algorithm
    ohai('running velvetg: %(outdir)s' % ops)
    run(velvetg, generates=ops['outdir'] + '/contigs.fa') # run assembly algorithm<commit_msg>documentation<commit_after># For using Velvet de-novo assembler

from helps import *

def velvet(**ops):
    ''' Performs a de novo assembly using Velvet

    Example:

    >>> velvet(
    ... outdir = 'output_directory',
    ... # reads must be an array of tuples (even if you're only using one file)
    ... # in the following form:
    ...
    ... reads = [
    ... ('format', 'short long or paired', 'filename.fastq'), ... ]
    ...
    ... # example:
    ... reads = [
    ... ('fastq', 'paired', 'paired_reads.fastq'), ...
    ... ],
    ... kmer = 31, # see velvet readme for why this is important
    ... )

    '''
    read_ops = ['-%s -%s %s' % r for r in ops['reads'] ]

    cmd = ' '.join([
        '~/cram/bin/velveth',
        '%(outdir)s',
        '%(k)s']) % ops

    velveth = cmd + ' ' + ' '.join(read_ops) + '>/dev/null'
    velvetg = '~/cram/bin/velvetg %(outdir)s -very_clean yes > /dev/null' % ops

    ohai('running velveth: %(reads)s, k = %(k)s' % ops)
    run(velveth, generates=ops['outdir']) # run hash algorithm
    ohai('running velvetg: %(outdir)s' % ops)
    run(velvetg, generates=ops['outdir'] + '/contigs.fa') # run assembly algorithm<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
u"""
This module provides some basic Japanese NLP by wrapping the output of MeCab.
It can tokenize and normalize Japanese words, detect and remove stopwords,
and it can even respell words in kana or romaji.

This requires mecab to be installed separately. On Ubuntu:
    sudo apt-get install mecab mecab-ipadic-utf8

>>> print normalize(u'これはテストです')
テスト
>>> tag_and_stem(u'これはテストです。')
[(u'\\u3053\\u308c', 'STOP', u'\\u3053\\u308c'), (u'\\u306f', 'STOP', u'\\u306f'), (u'\\u30c6\\u30b9\\u30c8', u'\\u540d\\u8a5e', u'\\u30c6\\u30b9\\u30c8'), (u'\\u3067\\u3059', 'STOP', u'\\u3067\\u3059'), (u'\\u3002', '.', u'\\u3002')]
"""

from metanl.general import preprocess_text, untokenize
from metanl.wordlist import Wordlist, get_frequency
from metanl.extprocess import ProcessWrapper, ProcessError
import unicodedata
import re

class MeCabError(ProcessError):
    pass

# MeCab outputs the part of speech of its terms. We can simply identify
# particular (coarse or fine) parts of speech as containing stopwords.

STOPWORD_CATEGORIES = set([
    u'助詞',          # coarse: particle
    u'助動詞',        # coarse: auxiliary verb
    u'接続詞',        # coarse: conjunction
    u'フィラー',      # coarse: filler
    u'記号',          # coarse: symbol
    u'非自立',        # coarse: 'not independent'
    u'助詞類接続',    # fine: particle connection
    u'代名詞',        # fine: pronoun
    u'接尾',          # fine: suffix
])

# Forms of particular words should also be considered stopwords sometimes.
#
# A thought: Should the rare kanji version of suru not be a stopword?
# I'll need to ask someone who knows more Japanese, but it may be
# that if they're using the kanji it's for particular emphasis.
STOPWORD_ROOTS = set([
    u'する',          # suru: "to do"
    u'為る',          # suru in kanji (very rare)
    u'くる',          # kuru: "to come"
    u'来る',          # kuru in kanji
    u'いく',          # iku: "to go"
    u'行く',          # iku in kanji
    u'いる',          # iru: "to be" (animate)
    u'居る',          # iru in kanji
    u'もの',          # mono: "thing"
    u'物',            # mono in kanji
    u'よう',          # yō: "way"
    u'様',            # yō in kanji
])

class MeCabWrapper(ProcessWrapper):
    """
    Handle Japanese text using the command-line version of MeCab.
    (mecab-python is convenient, but its installer is too flaky to rely on.)

    ja_cabocha gives more sophisticated results, but requires a large number of
    additional dependencies. Using this tool for Japanese requires only
    MeCab to be installed and accepting UTF-8 text.
    """
    def _get_command(self):
        return ['mecab']

    def _get_process(self):
        try:
            proc = ProcessWrapper._get_process(self)
        except (OSError, ProcessError):
            raise MeCabError("MeCab didn't start. See README.txt for details "
                             "about installing MeCab and other Japanese NLP tools.")
        self.mecab_encoding = self._detect_mecab_encoding(proc)
        return proc

    def _detect_mecab_encoding(self, proc):
        """
        Once we've found a MeCab binary, it may be installed in UTF-8 or it
        may be installed in EUC-JP. We need to determine which one by
        experimentation.
        """
        proc.stdin.write(u'a\n')
        out = proc.stdout.readline()

        # out is now a string in an unknown encoding, and might not even be
        # valid in *any* encoding before the tab character. But after the
        # tab, it should all be in the encoding of the installed dictionary.
        encoding = 'utf-8'
        try:
            out.decode('utf-8')
        except UnicodeDecodeError:
            try:
                out.decode('euc-jp')
                encoding = 'euc-jp'
            except UnicodeDecodeError:
                raise MeCabError("I can't understand MeCab in either UTF-8 or "
                                 "EUC-JP. Check the configuration of MeCab and "
                                 "its dictionary.")
        if proc.stdout.readline() != 'EOS\n':
            raise MeCabError("Sorry! I got unexpected lines back from MeCab "
                             "and don't know what to do next.")
        return encoding

    def get_record_root(self, record):
        """
        Given a MeCab record, return the root word.
        """
        if record[7] == '*':
            return record[0]
        else:
            return record[7]

    def get_record_token(self, record):
        return record[0]

    def analyze(self, text):
        """
        Runs a line of text through MeCab, and returns the results as a
        list of lists ("records") that contain the MeCab analysis of each
        word.
        """
        try:
            self.process  # make sure things are loaded
            text = preprocess_text(text).lower()
            n_chunks = (len(text) + 1024) // 1024
            results = []
            for chunk in xrange(n_chunks):
                chunk_text = text[chunk*1024:(chunk+1)*1024].encode(self.mecab_encoding)
                self.send_input(chunk_text+'\n')
                #self.input_log.write(text+'\n')
                out_line = ''
                while True:
                    out_line = self.receive_output_line()
                    #self.output_log.write(out_line)
                    out_line = out_line.decode(self.mecab_encoding)

                    if out_line == u'EOS\n':
                        break

                    word, info = out_line.strip(u'\n').split(u'\t')
                    record = [word] + info.split(u',')

                    # special case for detecting nai -> n
                    if record[0] == u'ん' and record[5] == u'不変化型':
                        record[7] = u'ない'

                    results.append(record)
            return results
        except ProcessError:
            self.restart_process()
            return self.analyze(text)

    def is_stopword_record(self, record, common_words=True):
        """
        Determine whether a single MeCab record represents a stopword.

        By default this only strips out particles and auxiliary verbs,
        but if common_words is set to True it will also strip common verbs
        and nouns such as くる and よう.
        """
        # preserve negations
        if record[7] == u'ない':
            return False
        return (record[1] in STOPWORD_CATEGORIES or
                record[2] in STOPWORD_CATEGORIES or
                (common_words and record[7] in STOPWORD_ROOTS))

    def get_record_pos(self, record):
        """
        Given a record, get the word's part of speech.

        Here we're going to return MeCab's part of speech (written in
        Japanese), unless it's a stopword in which case we return STOP.
        """
        if self.is_stopword_record(record):
            return 'STOP'
        else:
            return record[1]

class NoStopwordMeCabWrapper(MeCabWrapper):
    """
    This version of the MeCabWrapper doesn't label anything as a stopword. It'
    used in building ConceptNet because discarding stopwords based on MeCab
    categories loses too much information.
    """
    def is_stopword_record(self, record, common_words=False):
        return False

def word_frequency(word, default_freq=0):
    """
    Looks up the word's frequency in the Leeds Internet Japanese corpus.
    """
    return get_frequency(word, 'ja', default_freq)

def get_wordlist():
    return Wordlist.load('leeds-internet-ja.txt')

# Define the classes of characters we'll be trying to transliterate
NOT_KANA, KANA, NN, SMALL, SMALL_Y, SMALL_TSU, PROLONG = range(7)

def to_kana(text):
    """
    Use MeCab to turn any text into its phonetic spelling, as katakana
    separated by spaces.
    """
    records = MECAB.analyze(text)
    kana = []
    for record in records:
        if len(record) > 9:
            kana.append(record[9])
        elif len(record) > 8:
            kana.append(record[8])
        else:
            kana.append(record[0])
    return ' '.join(k for k in kana if k)

def get_kana_info(char):
    """
    Return two things about each character:

    - Its transliterated value (in Roman characters, if it's a kana)
    - A class of characters indicating how it affects the romanization
    """
    try:
        name = unicodedata.name(char)
    except ValueError:
        return char, NOT_KANA

    # The names we're dealing with will probably look like
    # "KATAKANA CHARACTER ZI".
    if (name.startswith('HIRAGANA LETTER') or name.startswith('KATAKANA LETTER')
    or name.startswith('KATAKANA-HIRAGANA')):
        names = name.split()
        syllable = unicode(names[-1].lower())

        if name.endswith('SMALL TU'):
            # The small tsu (っ) doubles the following consonant.
            # It'll show up as 't' on its own.
            return u't', SMALL_TSU
        elif names[-1] == 'N':
            return u'n', NN
        elif names[1] == 'PROLONGED':
            # The prolongation marker doubles the previous vowel.
            # It'll show up as '_' on its own.
            return u'_', PROLONG
        elif names[-2] == 'SMALL':
            # Small characters tend to modify the sound of the previous
            # kana. If they can't modify anything, they're appended to
            # the letter 'x' instead.
            if syllable.startswith('y'):
                return u'x' + syllable, SMALL_Y
            else:
                return u'x' + syllable, SMALL

        return syllable, KANA
    else:
        if char in ROMAN_PUNCTUATION_TABLE:
            char = ROMAN_PUNCTUATION_TABLE[char]
        return char, NOT_KANA

def respell_hepburn(syllable):
    while syllable[:2] in HEPBURN_TABLE:
        syllable = HEPBURN_TABLE[syllable[:2]] + syllable[2:]
    return syllable

def romanize(text, respell=respell_hepburn):
    if respell is None:
        respell = lambda x: x

    kana = to_kana(unicode(text))
    pieces = []
    prevgroup = NOT_KANA

    for char in kana:
        roman, group = get_kana_info(char)
        if prevgroup == NN:
            # When the previous syllable is 'n' and the next syllable would
            # make it ambiguous, add an apostrophe.
            if group != KANA or roman[0] in 'aeinouy':
                if unicodedata.category(roman[0])[0] == 'L':
                    pieces[-1] += "'"

        # Determine how to spell the current character
        if group == NOT_KANA:
            pieces.append(roman)
        elif group == SMALL_TSU or group == NN:
            pieces.append(roman)
        elif group == SMALL_Y:
            if prevgroup == KANA:
                # Modify the previous syllable, if that makes sense. For example,
                # 'ni' + 'ya' becomes 'nya'.
                if not pieces[-1].endswith('i'):
                    pieces.append(roman)
                else:
                    modifier = roman[1:]
                    modified = pieces[-1]
                    pieces[-1] = modified[:-1] + modifier
            else:
                pieces.append(roman)
        elif group == SMALL:
            # Don't respell small vowels _yet_. We'll handle that at the end.
            # This may be a bit ambiguous, but nobody expects to see "tea"
            # spelled "texi".
            pieces.append(roman)
        elif group == PROLONG:
            if prevgroup in (KANA, SMALL_Y, SMALL):
                pieces[-1] = pieces[-1][:-1] + respell(pieces[-1][-1] + '_')
            else:
                pieces.append(roman)
        else:  # this is a normal kana
            if prevgroup == SMALL_TSU:
                if roman[0] in 'aeiouy':
                    # wait, there's no consonant there; cope by respelling the
                    # previous kana as 'xtu'
                    pieces[-1] = 'xtu'
                else:
                    # Turn the previous 't' into a copy of the first consonant
                    pieces[-1] = roman[0]
            elif prevgroup == NN:
                # Let Hepburn respell 'n' as 'm' in words such as 'shimbun'.
                try_respell = respell(pieces[-1] + roman[0])
                if try_respell[:-1] != pieces[-1]:
                    pieces[-1] = try_respell[:-1]
            pieces.append(roman)
        prevgroup = group

    romantext = u''.join(respell(piece) for piece in pieces)
    romantext = re.sub(r'[aeiou]x([aeiou])', r'\1', romantext)
    return untokenize(romantext)

# Hepburn romanization is the most familiar to English speakers. It involves
# respelling certain parts of romanized words to better match their
# pronunciation. For example, the name for Mount Fuji is respelled from
# "huzi-san" to "fuji-san".
HEPBURN_TABLE = {
    u'si': u'shi',
    u'sy': u'sh',
    u'ti': u'chi',
    u'ty': u'ch',
    u'tu': u'tsu',
    u'hu': u'fu',
    u'zi': u'ji',
    u'di': u'ji',
    u'zy': u'j',
    u'dy': u'j',
    u'nm': u'mm',
    u'nb': u'mb',
    u'np': u'mp',
    u'a_': u'aa',
    u'e_': u'ee',
    u'i_': u'ii',
    u'o_': u'ou',
    u'u_': u'uu'
}
ROMAN_PUNCTUATION_TABLE = {
    u'・': u'.',
    u'。': u'.',
    u'、': u',',
    u'！': u'!',
    u'「': u'``',
    u'」': u"''",
    u'？': u'?',
    u'〜': u'~'
}

# Provide externally available functions.
MECAB = MeCabWrapper()

normalize = MECAB.normalize
normalize_list = MECAB.normalize_list
tokenize = MECAB.tokenize
tokenize_list = MECAB.tokenize_list
analyze = MECAB.analyze
tag_and_stem = MECAB.tag_and_stem
is_stopword = MECAB.is_stopword
<commit_msg>More customizable stopword categories.<commit_after># -*- coding: utf-8 -*-
u"""
This module provides some basic Japanese NLP by wrapping the output of MeCab.
It can tokenize and normalize Japanese words, detect and remove stopwords,
and it can even respell words in kana or romaji.

This requires mecab to be installed separately. On Ubuntu:
    sudo apt-get install mecab mecab-ipadic-utf8

>>> print normalize(u'これはテストです')
テスト
>>> tag_and_stem(u'これはテストです。')
[(u'\\u3053\\u308c', 'STOP', u'\\u3053\\u308c'), (u'\\u306f', 'STOP', u'\\u306f'), (u'\\u30c6\\u30b9\\u30c8', u'\\u540d\\u8a5e', u'\\u30c6\\u30b9\\u30c8'), (u'\\u3067\\u3059', 'STOP', u'\\u3067\\u3059'), (u'\\u3002', '.', u'\\u3002')]
"""

from metanl.general import preprocess_text, untokenize
from metanl.wordlist import Wordlist, get_frequency
from metanl.extprocess import ProcessWrapper, ProcessError
import unicodedata
import re

class MeCabError(ProcessError):
    pass

# MeCab outputs the part of speech of its terms. We can simply identify
# particular (coarse or fine) parts of speech as containing stopwords.

STOPWORD_CATEGORIES = set([
    u'助詞',          # coarse: particle
    u'助動詞',        # coarse: auxiliary verb
    u'接続詞',        # coarse: conjunction
    u'フィラー',      # coarse: filler
    u'記号',          # coarse: symbol
    u'非自立',        # fine: 'not independent'
])

MORE_STOPWORD_CATEGORIES = set([
    u'連体詞',        # coarse: adnominal adjective ("rentaishi")
    u'助詞類接続',    # fine: particle connection
    u'代名詞',        # fine: pronoun
    u'接尾',          # fine: suffix
])

# Forms of particular words should also be considered stopwords sometimes.
#
# A thought: Should the rare kanji version of suru not be a stopword?
# I'll need to ask someone who knows more Japanese, but it may be
# that if they're using the kanji it's for particular emphasis.
STOPWORD_ROOTS = set([
    u'する',          # suru: "to do"
    u'為る',          # suru in kanji (very rare)
    u'くる',          # kuru: "to come"
    u'来る',          # kuru in kanji
    u'いく',          # iku: "to go"
    u'行く',          # iku in kanji
    u'いる',          # iru: "to be" (animate)
    u'居る',          # iru in kanji
    u'もの',          # mono: "thing"
    u'物',            # mono in kanji
    u'よう',          # yō: "way"
    u'様',            # yō in kanji
    u'れる',          # passive suffix
])

class MeCabWrapper(ProcessWrapper):
    """
    Handle Japanese text using the command-line version of MeCab.
    (mecab-python is convenient, but its installer is too flaky to rely on.)

    ja_cabocha gives more sophisticated results, but requires a large number of
    additional dependencies. Using this tool for Japanese requires only
    MeCab to be installed and accepting UTF-8 text.
    """
    def _get_command(self):
        return ['mecab']

    def _get_process(self):
        try:
            proc = ProcessWrapper._get_process(self)
        except (OSError, ProcessError):
            raise MeCabError("MeCab didn't start. See README.txt for details "
                             "about installing MeCab and other Japanese NLP tools.")
        self.mecab_encoding = self._detect_mecab_encoding(proc)
        return proc

    def _detect_mecab_encoding(self, proc):
        """
        Once we've found a MeCab binary, it may be installed in UTF-8 or it
        may be installed in EUC-JP. We need to determine which one by
        experimentation.
        """
        proc.stdin.write(u'a\n')
        out = proc.stdout.readline()

        # out is now a string in an unknown encoding, and might not even be
        # valid in *any* encoding before the tab character. But after the
        # tab, it should all be in the encoding of the installed dictionary.
        encoding = 'utf-8'
        try:
            out.decode('utf-8')
        except UnicodeDecodeError:
            try:
                out.decode('euc-jp')
                encoding = 'euc-jp'
            except UnicodeDecodeError:
                raise MeCabError("I can't understand MeCab in either UTF-8 or "
                                 "EUC-JP. Check the configuration of MeCab and "
                                 "its dictionary.")
        if proc.stdout.readline() != 'EOS\n':
            raise MeCabError("Sorry! I got unexpected lines back from MeCab "
                             "and don't know what to do next.")
        return encoding

    def get_record_root(self, record):
        """
        Given a MeCab record, return the root word.
        """
        if record[7] == '*':
            return record[0]
        else:
            return record[7]

    def get_record_token(self, record):
        return record[0]

    def analyze(self, text):
        """
        Runs a line of text through MeCab, and returns the results as a
        list of lists ("records") that contain the MeCab analysis of each
        word.
        """
        try:
            self.process  # make sure things are loaded
            text = preprocess_text(text).lower()
            n_chunks = (len(text) + 1024) // 1024
            results = []
            for chunk in xrange(n_chunks):
                chunk_text = text[chunk*1024:(chunk+1)*1024].encode(self.mecab_encoding)
                self.send_input(chunk_text+'\n')
                #self.input_log.write(text+'\n')
                out_line = ''
                while True:
                    out_line = self.receive_output_line()
                    #self.output_log.write(out_line)
                    out_line = out_line.decode(self.mecab_encoding)

                    if out_line == u'EOS\n':
                        break

                    word, info = out_line.strip(u'\n').split(u'\t')
                    record = [word] + info.split(u',')

                    # special case for detecting nai -> n
                    if record[0] == u'ん' and record[5] == u'不変化型':
                        record[7] = u'ない'

                    results.append(record)
            return results
        except ProcessError:
            self.restart_process()
            return self.analyze(text)

    def is_stopword_record(self, record, common_words=True, more_stopwords=False):
        """
        Determine whether a single MeCab record represents a stopword.

        This mostly determines words to strip based on their parts of speech.
        If common_words is set to True (default), it will also strip common verbs
        and nouns such as くる and よう. If more_stopwords is True, it will look
        at the sub-part of speech to remove more categories.
        """
        # preserve negations
        if record[7] == u'ない':
            return False
        if record[1] in STOPWORD_CATEGORIES or record[2] in STOPWORD_CATEGORIES:
            return True
        if more_stopwords and (record[1] in MORE_STOPWORD_CATEGORIES or
                               record[2] in MORE_STOPWORD_CATEGORIES):
            return True
        return (common_words and record[7] in STOPWORD_ROOTS)

    def get_record_pos(self, record):
        """
        Given a record, get the word's part of speech.

        Here we're going to return MeCab's part of speech (written in
        Japanese), unless it's a stopword in which case we return STOP.
        """
        if self.is_stopword_record(record):
            return 'STOP'
        else:
            return record[1]

class NoStopwordMeCabWrapper(MeCabWrapper):
    """
    This version of the MeCabWrapper doesn't label anything as a stopword. It'
    used in building ConceptNet because discarding stopwords based on MeCab
    categories loses too much information.
    """
    def is_stopword_record(self, record, common_words=False):
        return False

def word_frequency(word, default_freq=0):
    """
    Looks up the word's frequency in the Leeds Internet Japanese corpus.
    """
    return get_frequency(word, 'ja', default_freq)

def get_wordlist():
    return Wordlist.load('leeds-internet-ja.txt')

# Define the classes of characters we'll be trying to transliterate
NOT_KANA, KANA, NN, SMALL, SMALL_Y, SMALL_TSU, PROLONG = range(7)

def to_kana(text):
    """
    Use MeCab to turn any text into its phonetic spelling, as katakana
    separated by spaces.
    """
    records = MECAB.analyze(text)
    kana = []
    for record in records:
        if len(record) > 9:
            kana.append(record[9])
        elif len(record) > 8:
            kana.append(record[8])
        else:
            kana.append(record[0])
    return ' '.join(k for k in kana if k)

def get_kana_info(char):
    """
    Return two things about each character:

    - Its transliterated value (in Roman characters, if it's a kana)
    - A class of characters indicating how it affects the romanization
    """
    try:
        name = unicodedata.name(char)
    except ValueError:
        return char, NOT_KANA

    # The names we're dealing with will probably look like
    # "KATAKANA CHARACTER ZI".
    if (name.startswith('HIRAGANA LETTER') or name.startswith('KATAKANA LETTER')
    or name.startswith('KATAKANA-HIRAGANA')):
        names = name.split()
        syllable = unicode(names[-1].lower())

        if name.endswith('SMALL TU'):
            # The small tsu (っ) doubles the following consonant.
            # It'll show up as 't' on its own.
            return u't', SMALL_TSU
        elif names[-1] == 'N':
            return u'n', NN
        elif names[1] == 'PROLONGED':
            # The prolongation marker doubles the previous vowel.
            # It'll show up as '_' on its own.
            return u'_', PROLONG
        elif names[-2] == 'SMALL':
            # Small characters tend to modify the sound of the previous
            # kana. If they can't modify anything, they're appended to
            # the letter 'x' instead.
            if syllable.startswith('y'):
                return u'x' + syllable, SMALL_Y
            else:
                return u'x' + syllable, SMALL

        return syllable, KANA
    else:
        if char in ROMAN_PUNCTUATION_TABLE:
            char = ROMAN_PUNCTUATION_TABLE[char]
        return char, NOT_KANA

def respell_hepburn(syllable):
    while syllable[:2] in HEPBURN_TABLE:
        syllable = HEPBURN_TABLE[syllable[:2]] + syllable[2:]
    return syllable

def romanize(text, respell=respell_hepburn):
    if respell is None:
        respell = lambda x: x

    kana = to_kana(unicode(text))
    pieces = []
    prevgroup = NOT_KANA

    for char in kana:
        roman, group = get_kana_info(char)
        if prevgroup == NN:
            # When the previous syllable is 'n' and the next syllable would
            # make it ambiguous, add an apostrophe.
            if group != KANA or roman[0] in 'aeinouy':
                if unicodedata.category(roman[0])[0] == 'L':
                    pieces[-1] += "'"

        # Determine how to spell the current character
        if group == NOT_KANA:
            pieces.append(roman)
        elif group == SMALL_TSU or group == NN:
            pieces.append(roman)
        elif group == SMALL_Y:
            if prevgroup == KANA:
                # Modify the previous syllable, if that makes sense. For example,
                # 'ni' + 'ya' becomes 'nya'.
                if not pieces[-1].endswith('i'):
                    pieces.append(roman)
                else:
                    modifier = roman[1:]
                    modified = pieces[-1]
                    pieces[-1] = modified[:-1] + modifier
            else:
                pieces.append(roman)
        elif group == SMALL:
            # Don't respell small vowels _yet_. We'll handle that at the end.
            # This may be a bit ambiguous, but nobody expects to see "tea"
            # spelled "texi".
            pieces.append(roman)
        elif group == PROLONG:
            if prevgroup in (KANA, SMALL_Y, SMALL):
                pieces[-1] = pieces[-1][:-1] + respell(pieces[-1][-1] + '_')
            else:
                pieces.append(roman)
        else:  # this is a normal kana
            if prevgroup == SMALL_TSU:
                if roman[0] in 'aeiouy':
                    # wait, there's no consonant there; cope by respelling the
                    # previous kana as 'xtu'
                    pieces[-1] = 'xtu'
                else:
                    # Turn the previous 't' into a copy of the first consonant
                    pieces[-1] = roman[0]
            elif prevgroup == NN:
                # Let Hepburn respell 'n' as 'm' in words such as 'shimbun'.
                try_respell = respell(pieces[-1] + roman[0])
                if try_respell[:-1] != pieces[-1]:
                    pieces[-1] = try_respell[:-1]
            pieces.append(roman)
        prevgroup = group

    romantext = u''.join(respell(piece) for piece in pieces)
    romantext = re.sub(r'[aeiou]x([aeiou])', r'\1', romantext)
    return untokenize(romantext)

# Hepburn romanization is the most familiar to English speakers. It involves
# respelling certain parts of romanized words to better match their
# pronunciation. For example, the name for Mount Fuji is respelled from
# "huzi-san" to "fuji-san".
HEPBURN_TABLE = {
    u'si': u'shi',
    u'sy': u'sh',
    u'ti': u'chi',
    u'ty': u'ch',
    u'tu': u'tsu',
    u'hu': u'fu',
    u'zi': u'ji',
    u'di': u'ji',
    u'zy': u'j',
    u'dy': u'j',
    u'nm': u'mm',
    u'nb': u'mb',
    u'np': u'mp',
    u'a_': u'aa',
    u'e_': u'ee',
    u'i_': u'ii',
    u'o_': u'ou',
    u'u_': u'uu'
}
ROMAN_PUNCTUATION_TABLE = {
    u'・': u'.',
    u'。': u'.',
    u'、': u',',
    u'！': u'!',
    u'「': u'``',
    u'」': u"''",
    u'？': u'?',
    u'〜': u'~'
}

# Provide externally available functions.
MECAB = MeCabWrapper()

normalize = MECAB.normalize
normalize_list = MECAB.normalize_list
tokenize = MECAB.tokenize
tokenize_list = MECAB.tokenize_list
analyze = MECAB.analyze
tag_and_stem = MECAB.tag_and_stem
is_stopword = MECAB.is_stopword
<|endoftext|>


<commit_before># Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Non-deterministic dataset transformations."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensorflow.python.data.ops import dataset_ops
from tensorflow.python.data.util import nest
from tensorflow.python.data.util import sparse
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import function
from tensorflow.python.framework import ops
from tensorflow.python.ops import gen_dataset_ops
from tensorflow.python.util import deprecation

class ParallelInterleaveDataset(dataset_ops.Dataset):
  """A `Dataset` that maps a function over its input and flattens the result."""

  def __init__(self, input_dataset, map_func, cycle_length, block_length,
               sloppy):
    """See `tf.contrib.data.parallel_interleave()` for details."""
    super(ParallelInterleaveDataset, self).__init__()
    self._input_dataset = input_dataset

    @function.Defun(
        *nest.flatten(sparse.unwrap_sparse_types(input_dataset.output_types)))
    def tf_map_func(*args):
      """A wrapper for Defun that facilitates shape inference."""
      # Pass in shape information from the input_dataset.
      for arg, shape in zip(args, nest.flatten(input_dataset.output_shapes)):
        arg.set_shape(shape)

      nested_args = nest.pack_sequence_as(input_dataset.output_types, args)
      nested_args = sparse.deserialize_sparse_tensors(
          nested_args, input_dataset.output_types)
      if nest.is_sequence(nested_args):
        dataset = map_func(*nested_args)
      else:
        dataset = map_func(nested_args)

      if not isinstance(dataset, dataset_ops.Dataset):
        raise TypeError("`map_func` must return a `Dataset` object.")

      self._output_types = dataset.output_types
      self._output_shapes = dataset.output_shapes

      return dataset._as_variant_tensor()  # pylint: disable=protected-access

    self._map_func = tf_map_func
    self._map_func.add_to_graph(ops.get_default_graph())

    self._cycle_length = ops.convert_to_tensor(
        cycle_length, dtype=dtypes.int64, name="cycle_length")
    self._block_length = ops.convert_to_tensor(
        block_length, dtype=dtypes.int64, name="block_length")
    self._sloppy = ops.convert_to_tensor(
        sloppy, dtype=dtypes.bool, name="sloppy")

  def _as_variant_tensor(self):
    return gen_dataset_ops.parallel_interleave_dataset(
        self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
        self._map_func.captured_inputs,
        self._cycle_length,
        self._block_length,
        self._sloppy,
        f=self._map_func,
        output_types=nest.flatten(
            sparse.unwrap_sparse_types(self.output_types)),
        output_shapes=nest.flatten(self.output_shapes))

  @property
  def output_shapes(self):
    return self._output_shapes

  @property
  def output_types(self):
    return self._output_types

def parallel_interleave(map_func, cycle_length, block_length=1, sloppy=False):
  """A parallel version of the `Dataset.interleave()` transformation.

  `parallel_interleave()` maps `map_func` across its input to produce nested
  datasets, and outputs their elements interleaved. Unlike
  @{tf.data.Dataset.interleave}, it gets elements from `cycle_length` nested
  datasets in parallel, which increases the throughput, especially in the
  presence of stragglers. Furthermore, the `sloppy` argument can be used to
  improve performance, by relaxing the requirement that the outputs are produced
  in a deterministic order, and allowing the implementation to skip over nested
  datasets whose elements are not readily available when requested.

  Example usage:

  ```python
  # Preprocess 4 files concurrently.
  filenames = tf.data.Dataset.list_files("/path/to/data/train*.tfrecords")
  dataset = filenames.apply(
      tf.contrib.data.parallel_interleave(
          lambda filename: tf.data.TFRecordDataset(filename),
          cycle_length=4))
  ```

  WARNING: If `sloppy` is `True`, the order of produced elements is not
  deterministic.

  Args:
    map_func: A function mapping a nested structure of tensors to a `Dataset`.
    cycle_length: The number of threads to interleave from in parallel.
    block_length: The number of consecutive elements to pull from a thread
      before advancing to the next thread.
    sloppy: If false, elements are produced in deterministic order. Otherwise,
      the implementation is allowed, for the sake of expediency, to produce
      elements in a non-deterministic order.

  Returns:
    A `Dataset` transformation function, which can be passed to
    @{tf.data.Dataset.apply}.
  """
  def _apply_fn(dataset):
    return ParallelInterleaveDataset(
        dataset, map_func, cycle_length, block_length, sloppy)
  return _apply_fn

@deprecation.deprecated(
    None, "Use `tf.contrib.data.parallel_interleave(..., sloppy=True)`.")
def sloppy_interleave(map_func, cycle_length, block_length=1):
  """A non-deterministic version of the `Dataset.interleave()` transformation.

  `sloppy_interleave()` maps `map_func` across `dataset`, and
  non-deterministically interleaves the results.

  The resulting dataset is almost identical to `interleave`. The key
  difference is that if retrieving a value from a given output iterator would
  cause `get_next` to block, that iterator will be skipped, and consumed
  when next available. If consuming from all iterators would cause the
  `get_next` call to block, the `get_next` call blocks until the first value is
  available.

  If the underlying datasets produce elements as fast as they are consumed, the
  `sloppy_interleave` transformation behaves identically to `interleave`.
  However, if an underlying dataset would block the consumer,
  `sloppy_interleave` can violate the round-robin order (that `interleave`
  strictly obeys), producing an element from a different underlying
  dataset instead.

  Example usage:

  ```python
  # Preprocess 4 files concurrently.
  filenames = tf.data.Dataset.list_files("/path/to/data/train*.tfrecords")
  dataset = filenames.apply(
      tf.contrib.data.sloppy_interleave(
          lambda filename: tf.data.TFRecordDataset(filename),
          cycle_length=4))
  ```

  WARNING: The order of elements in the resulting dataset is not
  deterministic. Use `Dataset.interleave()` if you want the elements to have a
  deterministic order.

  Args:
    map_func: A function mapping a nested structure of tensors (having shapes
      and types defined by `self.output_shapes` and `self.output_types`) to a
      `Dataset`.
    cycle_length: The number of threads to interleave from in parallel.
    block_length: The number of consecutive elements to pull from a thread
      before advancing to the next thread. Note: sloppy_interleave will
      skip the remainder of elements in the block_length in order to avoid
      blocking.

  Returns:
    A `Dataset` transformation function, which can be passed to
    @{tf.data.Dataset.apply}.
  """
  def _apply_fn(dataset):
    return ParallelInterleaveDataset(
        dataset, map_func, cycle_length, block_length, sloppy=True)
  return _apply_fn
<commit_msg>Fix a bug in ParallelInterleaveDataset, which expands dictionary output to dictionary keys.<commit_after># Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Non-deterministic dataset transformations."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensorflow.python.data.ops import dataset_ops
from tensorflow.python.data.util import nest
from tensorflow.python.data.util import sparse
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import function
from tensorflow.python.framework import ops
from tensorflow.python.ops import gen_dataset_ops
from tensorflow.python.util import deprecation

class ParallelInterleaveDataset(dataset_ops.Dataset):
  """A `Dataset` that maps a function over its input and flattens the result."""

  def __init__(self, input_dataset, map_func, cycle_length, block_length,
               sloppy):
    """See `tf.contrib.data.parallel_interleave()` for details."""
    super(ParallelInterleaveDataset, self).__init__()
    self._input_dataset = input_dataset

    @function.Defun(
        *nest.flatten(sparse.unwrap_sparse_types(input_dataset.output_types)))
    def tf_map_func(*args):
      """A wrapper for Defun that facilitates shape inference."""
      # Pass in shape information from the input_dataset.
      for arg, shape in zip(args, nest.flatten(input_dataset.output_shapes)):
        arg.set_shape(shape)

      nested_args = nest.pack_sequence_as(input_dataset.output_types, args)
      nested_args = sparse.deserialize_sparse_tensors(
          nested_args, input_dataset.output_types)
      if dataset_ops._should_unpack_args(nested_args):  # pylint: disable=protected-access
        dataset = map_func(*nested_args)
      else:
        dataset = map_func(nested_args)

      if not isinstance(dataset, dataset_ops.Dataset):
        raise TypeError("`map_func` must return a `Dataset` object.")

      self._output_types = dataset.output_types
      self._output_shapes = dataset.output_shapes

      return dataset._as_variant_tensor()  # pylint: disable=protected-access

    self._map_func = tf_map_func
    self._map_func.add_to_graph(ops.get_default_graph())

    self._cycle_length = ops.convert_to_tensor(
        cycle_length, dtype=dtypes.int64, name="cycle_length")
    self._block_length = ops.convert_to_tensor(
        block_length, dtype=dtypes.int64, name="block_length")
    self._sloppy = ops.convert_to_tensor(
        sloppy, dtype=dtypes.bool, name="sloppy")

  def _as_variant_tensor(self):
    return gen_dataset_ops.parallel_interleave_dataset(
        self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
        self._map_func.captured_inputs,
        self._cycle_length,
        self._block_length,
        self._sloppy,
        f=self._map_func,
        output_types=nest.flatten(
            sparse.unwrap_sparse_types(self.output_types)),
        output_shapes=nest.flatten(self.output_shapes))

  @property
  def output_shapes(self):
    return self._output_shapes

  @property
  def output_types(self):
    return self._output_types

def parallel_interleave(map_func, cycle_length, block_length=1, sloppy=False):
  """A parallel version of the `Dataset.interleave()` transformation.

  `parallel_interleave()` maps `map_func` across its input to produce nested
  datasets, and outputs their elements interleaved. Unlike
  @{tf.data.Dataset.interleave}, it gets elements from `cycle_length` nested
  datasets in parallel, which increases the throughput, especially in the
  presence of stragglers. Furthermore, the `sloppy` argument can be used to
  improve performance, by relaxing the requirement that the outputs are produced
  in a deterministic order, and allowing the implementation to skip over nested
  datasets whose elements are not readily available when requested.

  Example usage:

  ```python
  # Preprocess 4 files concurrently.
  filenames = tf.data.Dataset.list_files("/path/to/data/train*.tfrecords")
  dataset = filenames.apply(
      tf.contrib.data.parallel_interleave(
          lambda filename: tf.data.TFRecordDataset(filename),
          cycle_length=4))
  ```

  WARNING: If `sloppy` is `True`, the order of produced elements is not
  deterministic.

  Args:
    map_func: A function mapping a nested structure of tensors to a `Dataset`.
    cycle_length: The number of threads to interleave from in parallel.
    block_length: The number of consecutive elements to pull from a thread
      before advancing to the next thread.
    sloppy: If false, elements are produced in deterministic order. Otherwise,
      the implementation is allowed, for the sake of expediency, to produce
      elements in a non-deterministic order.

  Returns:
    A `Dataset` transformation function, which can be passed to
    @{tf.data.Dataset.apply}.
  """
  def _apply_fn(dataset):
    return ParallelInterleaveDataset(
        dataset, map_func, cycle_length, block_length, sloppy)
  return _apply_fn

@deprecation.deprecated(
    None, "Use `tf.contrib.data.parallel_interleave(..., sloppy=True)`.")
def sloppy_interleave(map_func, cycle_length, block_length=1):
  """A non-deterministic version of the `Dataset.interleave()` transformation.

  `sloppy_interleave()` maps `map_func` across `dataset`, and
  non-deterministically interleaves the results.

  The resulting dataset is almost identical to `interleave`. The key
  difference is that if retrieving a value from a given output iterator would
  cause `get_next` to block, that iterator will be skipped, and consumed
  when next available. If consuming from all iterators would cause the
  `get_next` call to block, the `get_next` call blocks until the first value is
  available.

  If the underlying datasets produce elements as fast as they are consumed, the
  `sloppy_interleave` transformation behaves identically to `interleave`.
  However, if an underlying dataset would block the consumer,
  `sloppy_interleave` can violate the round-robin order (that `interleave`
  strictly obeys), producing an element from a different underlying
  dataset instead.

  Example usage:

  ```python
  # Preprocess 4 files concurrently.
  filenames = tf.data.Dataset.list_files("/path/to/data/train*.tfrecords")
  dataset = filenames.apply(
      tf.contrib.data.sloppy_interleave(
          lambda filename: tf.data.TFRecordDataset(filename),
          cycle_length=4))
  ```

  WARNING: The order of elements in the resulting dataset is not
  deterministic. Use `Dataset.interleave()` if you want the elements to have a
  deterministic order.

  Args:
    map_func: A function mapping a nested structure of tensors (having shapes
      and types defined by `self.output_shapes` and `self.output_types`) to a
      `Dataset`.
    cycle_length: The number of threads to interleave from in parallel.
    block_length: The number of consecutive elements to pull from a thread
      before advancing to the next thread. Note: sloppy_interleave will
      skip the remainder of elements in the block_length in order to avoid
      blocking.

  Returns:
    A `Dataset` transformation function, which can be passed to
    @{tf.data.Dataset.apply}.
  """
  def _apply_fn(dataset):
    return ParallelInterleaveDataset(
        dataset, map_func, cycle_length, block_length, sloppy=True)
  return _apply_fn
<|endoftext|>


<commit_before># uses ENV variables to ping the ghost inspector API and run test suite for 'SGD Acceptance.'
import os
import requests

NGROK_URL = 'http://localhost:4040/api/tunnels'

ghost_suite_id = os.environ.get('GHOST_SUITE_ID')
ghost_key = os.environ.get('GHOST_API_KEY')

# get start URL from ngrok API
try:
  ngrok_response = requests.get(NGROK_URL).json()
  start_url = ngrok_response['tunnels'][0]['public_url']
except Exception, e:
  # TODO, give an error message instructing how to setup ngrok
  raise e

# construct ghost API url and ping to run test
ghost_url = 'https://api.ghostinspector.com/v1/suites/' + ghost_suite_id + '/execute/?apiKey=' + ghost_key + '&immediate=1&startUrl=' + start_url
requests.get(ghost_url)
<commit_msg>raise exception when can't connect to ngrok on local<commit_after># uses ENV variables to ping the ghost inspector API and run test suite for 'SGD Acceptance.'
import os
import requests

NGROK_URL = 'http://localhost:4040/api/tunnels'

ghost_suite_id = os.environ.get('GHOST_SUITE_ID')
ghost_key = os.environ.get('GHOST_API_KEY')

# get start URL from ngrok API
try:
  ngrok_response = requests.get(NGROK_URL).json()
  start_url = ngrok_response['tunnels'][0]['public_url']
except Exception, e:
  raise Exception('Unable to connect to ngrok.  Make sure you have an ngrok tunnel pointing to http://localhost:6545.  https://ngrok.com/download')

# construct ghost API url and ping to run test
ghost_url = 'https://api.ghostinspector.com/v1/suites/' + ghost_suite_id + '/execute/?apiKey=' + ghost_key + '&immediate=1&startUrl=' + start_url
requests.get(ghost_url)
<|endoftext|>


<commit_before># Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""API to simulate quantization on a python graph."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensorflow.contrib.quantize.python import fold_batch_norms
from tensorflow.contrib.quantize.python import quantize
from tensorflow.python.framework import ops

def _create_graph(input_graph=None,
                  is_training=True,
                  weight_bits=8,
                  activation_bits=8,
                  quant_delay=None,
                  freeze_bn_delay=None):
  """Rewrites an input_graph in place for simulated quantization.

  The graph has fake quantization ops inserted to simulate the error
  introduced by quantization. Since the graph is transformed in place,
  the expected behavior of previously held references to nodes and tensors may
  change.

  Args:
    input_graph: The tf.Graph to be transformed, if None then defaults to the
      default graph.
    is_training: Whether quantizing training or eval graph.
    weight_bits: Number of bits to use for quantizing weights.
    activation_bits: Number of bits to use for quantizing activations.
    quant_delay: Number of steps after which weights and activations are
      quantized during training.
    freeze_bn_delay: Number of steps after which moving mean and variance are
      frozen and used instead of batch statistics during training.
      freeze_bn_delay should be greater than quant_delay and should correspond
      to the number of steps when training has almost converged

  Raises:
    ValueError: If elements contains an element that isn't a tf.Tensor or
      tf.Operation.
  """

  if input_graph is None:
    input_graph = ops.get_default_graph()
  with input_graph.as_default():
    fold_batch_norms.FoldBatchNorms(
        input_graph,
        freeze_batch_norm_delay=freeze_bn_delay,
        is_training=is_training)
    quantize.Quantize(
        input_graph,
        is_training,
        quant_delay=quant_delay,
        weight_bits=weight_bits,
        activation_bits=activation_bits)

def create_training_graph(input_graph=None, quant_delay=250000):
  """Rewrites a training input_graph in place for simulated quantization.

  The graph has fake quantization ops inserted to simulate the error
  introduced by quantization. Since the graph is transformed in place,
  the expected behavior of previously held references to nodes and tensors may
  change.

  Args:
    input_graph: The tf.Graph to be transformed.
    quant_delay: Number of steps after which weights and activations are
      quantized during training.

  Raises:
    ValueError: If elements contains an element that isn't a tf.Tensor or
      tf.Operation.
  """
  # TODO(raghuramank) Need to have freeze_bn_delay be a function of batch size
  # Currently the values below are hardcoded for mobilenetV1 on imagenet
  # Please use the experimental API if you need to tune these values.
  if quant_delay == 0:
    # Corresponds to case of restoring from a floating point checkpoint
    # In this case, we can freeze the moving mean and variance early on and
    # switch to using them during training. Therefore, freeze_bn_delay is set to
    # 200000
    freeze_bn_delay = 200000
  else:
    # If training from scratch, set freeze_bn_delay to 100 epochs after quant
    # delay. With a batch size of 64, this corresponds to 20000*100=2M steps.
    freeze_bn_delay = quant_delay + 2000000

  _create_graph(
      input_graph=input_graph,
      is_training=True,
      quant_delay=quant_delay,
      freeze_bn_delay=freeze_bn_delay)

def create_eval_graph(input_graph=None):
  """Rewrites an eval input_graph in place for simulated quantization.

  The graph has fake quantization ops inserted to simulate the error
  introduced by quantization. Since the graph is transformed in place,
  the expected behavior of previously held references to nodes and tensors may
  change.

  Args:
    input_graph: The tf.Graph to be transformed, if None then defaults to the
      default graph.

  Raises:
    ValueError: If elements contains an element that isn't a tf.Tensor or
      tf.Operation.
  """
  _create_graph(input_graph=input_graph, is_training=False)

def experimental_create_training_graph(input_graph=None,
                                       weight_bits=8,
                                       activation_bits=8,
                                       quant_delay=250000,
                                       freeze_bn_delay=500000):
  """Rewrites a training input_graph in place for simulated quantization.

  This function has additional experimental options not (yet) available to
  create_training_graph. The resulting behavior may be undefined.

  The graph has fake quantization ops inserted to simulate the error
  introduced by quantization. Since the graph is transformed in place,
  the expected behavior of previously held references to nodes and tensors may
  change.

  Args:
    input_graph: The tf.Graph to be transformed,if None then defaults to the
      default graph.
    weight_bits: Number of bits to use for quantizing weights.
    activation_bits: Number of bits to use for quantizing activations.
    quant_delay: Number of steps after which weights and activations are
      quantized during training.
    freeze_bn_delay: Number of steps after which moving mean and variance are
      frozen and used instead of batch statistics during training.
      freeze_bn_delay should be greater than quant_delay and should correspond
      to when training has almost converged

  Raises:
    ValueError: If elements contains an element that isn't a tf.Tensor or
        tf.Operation.
  """

  _create_graph(
      input_graph=input_graph,
      is_training=True,
      weight_bits=weight_bits,
      activation_bits=activation_bits,
      quant_delay=quant_delay,
      freeze_bn_delay=freeze_bn_delay)

def experimental_create_eval_graph(input_graph=None,
                                   weight_bits=8,
                                   activation_bits=8):
  """Rewrites an eval input_graph in place for simulated quantization.

  This function has additional experimental options not (yet) available to
  create_eval_graph. The resulting behavior may be undefined.

  The graph has fake quantization ops inserted to simulate the error
  introduced by quantization. Since the graph is transformed in place,
  the expected behavior of previously held references to nodes and tensors may
  change.

  Args:
    input_graph: The tf.Graph to be transformed, if None then defaults to the
      default graph.
    weight_bits: Number of bits to use for quantizing weights.
    activation_bits: Number of bits to use for quantizing activations.

  Raises:
    ValueError: If elements contains an element that isn't a tf.Tensor or
      tf.Operation.
  """
  _create_graph(
      input_graph=input_graph,
      is_training=False,
      weight_bits=weight_bits,
      activation_bits=activation_bits)
<commit_msg>Make the default values for experimental and non experimental apis match.<commit_after># Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""API to simulate quantization on a python graph."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensorflow.contrib.quantize.python import fold_batch_norms
from tensorflow.contrib.quantize.python import quantize
from tensorflow.python.framework import ops

def _create_graph(input_graph=None,
                  is_training=True,
                  weight_bits=8,
                  activation_bits=8,
                  quant_delay=None,
                  freeze_bn_delay=None):
  """Rewrites an input_graph in place for simulated quantization.

  The graph has fake quantization ops inserted to simulate the error
  introduced by quantization. Since the graph is transformed in place,
  the expected behavior of previously held references to nodes and tensors may
  change.

  Args:
    input_graph: The tf.Graph to be transformed, if None then defaults to the
      default graph.
    is_training: Whether quantizing training or eval graph.
    weight_bits: Number of bits to use for quantizing weights.
    activation_bits: Number of bits to use for quantizing activations.
    quant_delay: Number of steps after which weights and activations are
      quantized during training.
    freeze_bn_delay: Number of steps after which moving mean and variance are
      frozen and used instead of batch statistics during training.
      freeze_bn_delay should be greater than quant_delay and should correspond
      to the number of steps when training has almost converged

  Raises:
    ValueError: If elements contains an element that isn't a tf.Tensor or
      tf.Operation.
  """

  if input_graph is None:
    input_graph = ops.get_default_graph()
  with input_graph.as_default():
    fold_batch_norms.FoldBatchNorms(
        input_graph,
        freeze_batch_norm_delay=freeze_bn_delay,
        is_training=is_training)
    quantize.Quantize(
        input_graph,
        is_training,
        quant_delay=quant_delay,
        weight_bits=weight_bits,
        activation_bits=activation_bits)

def create_training_graph(input_graph=None, quant_delay=0):
  """Rewrites a training input_graph in place for simulated quantization.

  The graph has fake quantization ops inserted to simulate the error
  introduced by quantization. Since the graph is transformed in place,
  the expected behavior of previously held references to nodes and tensors may
  change.

  The default value of quant_delay is suitable for finetuning an already trained
  floating point model (recommended).
  If one wants to train a quantized model from scratch, quant_delay should be
  set to the number of steps it take the floating point model to converge.
  Quantization will be activated at this point and effectively finetune the
  model. If quant_delay is not provided when training from scratch, training can
  often fail.

  Args:
    input_graph: The tf.Graph to be transformed.
    quant_delay: Number of steps after which weights and activations are
      quantized during training.

  Raises:
    ValueError: If elements contains an element that isn't a tf.Tensor or
      tf.Operation.
  """
  # TODO(raghuramank) Need to have freeze_bn_delay be a function of batch size
  # Currently the values below are hardcoded for mobilenetV1 on imagenet
  # Please use the experimental API if you need to tune these values.
  if quant_delay == 0:
    # Corresponds to case of restoring from a floating point checkpoint
    # In this case, we can freeze the moving mean and variance early on and
    # switch to using them during training. Therefore, freeze_bn_delay is set to
    # 2e5.
    freeze_bn_delay = int(2e5)
  else:
    # If training from scratch, set freeze_bn_delay to 100 epochs after quant
    # delay. With a batch size of 64, this corresponds to 20000*100=2M steps.
    freeze_bn_delay = quant_delay + int(2e6)

  _create_graph(
      input_graph=input_graph,
      is_training=True,
      quant_delay=quant_delay,
      freeze_bn_delay=freeze_bn_delay)

def create_eval_graph(input_graph=None):
  """Rewrites an eval input_graph in place for simulated quantization.

  The graph has fake quantization ops inserted to simulate the error
  introduced by quantization. Since the graph is transformed in place,
  the expected behavior of previously held references to nodes and tensors may
  change.

  Args:
    input_graph: The tf.Graph to be transformed, if None then defaults to the
      default graph.

  Raises:
    ValueError: If elements contains an element that isn't a tf.Tensor or
      tf.Operation.
  """
  _create_graph(input_graph=input_graph, is_training=False)

def experimental_create_training_graph(input_graph=None,
                                       weight_bits=8,
                                       activation_bits=8,
                                       quant_delay=0,
                                       freeze_bn_delay=int(2e5)):
  """Rewrites a training input_graph in place for simulated quantization.

  This function has additional experimental options not (yet) available to
  create_training_graph. The resulting behavior may be undefined.

  The graph has fake quantization ops inserted to simulate the error
  introduced by quantization. Since the graph is transformed in place,
  the expected behavior of previously held references to nodes and tensors may
  change.

  The default value of quant_delay is suitable for finetuning an already trained
  floating point model (recommended).
  If one wants to train a quantized model from scratch, quant_delay should be
  set to the number of steps it take the floating point model to converge.
  Quantization will be activated at this point and effectively finetune the
  model. If quant_delay is not provided when training from scratch, training can
  often fail.

  Args:
    input_graph: The tf.Graph to be transformed,if None then defaults to the
      default graph.
    weight_bits: Number of bits to use for quantizing weights.
    activation_bits: Number of bits to use for quantizing activations.
    quant_delay: Number of steps after which weights and activations are
      quantized during training.
    freeze_bn_delay: Number of steps after which moving mean and variance are
      frozen and used instead of batch statistics during training.
      freeze_bn_delay should be greater than quant_delay and should correspond
      to when training has almost converged

  Raises:
    ValueError: If elements contains an element that isn't a tf.Tensor or
        tf.Operation.
  """

  _create_graph(
      input_graph=input_graph,
      is_training=True,
      weight_bits=weight_bits,
      activation_bits=activation_bits,
      quant_delay=quant_delay,
      freeze_bn_delay=freeze_bn_delay)

def experimental_create_eval_graph(input_graph=None,
                                   weight_bits=8,
                                   activation_bits=8):
  """Rewrites an eval input_graph in place for simulated quantization.

  This function has additional experimental options not (yet) available to
  create_eval_graph. The resulting behavior may be undefined.

  The graph has fake quantization ops inserted to simulate the error
  introduced by quantization. Since the graph is transformed in place,
  the expected behavior of previously held references to nodes and tensors may
  change.

  Args:
    input_graph: The tf.Graph to be transformed, if None then defaults to the
      default graph.
    weight_bits: Number of bits to use for quantizing weights.
    activation_bits: Number of bits to use for quantizing activations.

  Raises:
    ValueError: If elements contains an element that isn't a tf.Tensor or
      tf.Operation.
  """
  _create_graph(
      input_graph=input_graph,
      is_training=False,
      weight_bits=weight_bits,
      activation_bits=activation_bits)
<|endoftext|>