<commit_before>from django.conf.urls import patterns, include, url
from django.views.generic import TemplateView

from id import databases, requests, accounts
from id import validation, admin, tasks, errors
from core.auth import perm

from django.contrib.auth import views as auth_views
from registration.views import ActivationView

from core.views import NotificationSeen, NotificationStream
from id.views import ProfileRegistrationView, login, logout
from id.forms import ProfileRegistrationForm, FeedbackForm

js_info_dict = {
    'packages': ('id', 'ticket', 'search', 'podaci'),
}

urlpatterns = patterns('',
    url(r'^$',                              perm('any', TemplateView, template_name="splash.jinja"), name="home"),
    url(r'^about/id2/$',                    perm('any', TemplateView, template_name="about_id.jinja"), name='about_id'),
    url(r'^about/occrp/$',                  perm('any', TemplateView, template_name="about_us.jinja"), name='about_us'),

    url(r'^admin/$',                        perm('staff', admin.Panel), name='admin_panel'),
    url(r'^admin/scrapers/request/$',       perm('staff', admin.DatabaseScrapeRequestCreate), name='admin_scrapers_request'),
    url(r'^admin/budgets/$',                perm('staff', admin.Budgets), name='admin_budgets'),

    url(r'^admin/storage/$',                perm('admin', admin.Storage), name='admin_storage'),
    url(r'^admin/statistics/$',             perm('admin', admin.Statistics), name='statistics'),
    url(r'^feedback/$',                     perm('any', admin.Feedback), name='feedback'),
    url(r'^feedback/thankyou/$',            perm('any', admin.FeedbackThanks), name='feedback_thanks'),

    url(r'^databases/$',                    perm('any', databases.ExternalDatabaseList), name='externaldb_list'),
    url(r'^databases/add/$',                perm('staff', databases.ExternalDatabaseAdd), name='externaldb_add'),
    url(r'^databases/edit/(?P<id>[0-9]+)/$',perm('staff', databases.ExternalDatabaseEdit), name='externaldb_edit'),
    url(r'^databases/view/(?P<id>[0-9]+)/$',perm('any', databases.ExternalDatabaseDetail), name='externaldb_detail'),
    url(r'^databases/delete/(?P<id>[0-9]+)/$',
                                            perm('admin', databases.ExternalDatabaseDelete), name='externaldb_delete'),

    url(r'^accounts/login/$',               login, {'template_name': 'registration/login.jinja'}, name='login'),
    url(r'^accounts/logout/',               logout, {'template_name': 'registration/logout.jinja', 'fallback_redirect_url': '/accounts/login/'}, name='logout'),
    url(r'^accounts/users/$',               perm('admin', accounts.UserList), name='userprofile_list'),
    url(r'^accounts/suggest/$',             perm('loggedin', accounts.UserSuggest), name='userprofile_suggest'),
    url(r'^accounts/request/$',             perm('any', accounts.AccessRequestCreate), name='request_access_create'),
    url(r'^accounts/request/list/$',        perm('admin', accounts.AccessRequestList), name='request_access_list'),
    url(r'^accounts/request/list/approved/$',perm('admin', accounts.AccessRequestListApproved), name='request_access_list_approved'),
    url(r'^accounts/request/list/denied/$', perm('admin', accounts.AccessRequestListDenied), name='request_access_list_denied'),
    url(r'^accounts/profile/$',             perm('loggedin', accounts.ProfileUpdate), name='profile'),
    url(r'^accounts/profile/(?P<pk>[0-9]+)/$',
                                            perm('admin', accounts.ProfileUpdate), name='profile'),
    url(r'^accounts/profile/(?P<email>.+)/$',
                                            perm('admin', accounts.ProfileUpdate), name='profile'),
    url(r'^accounts/setlanguage/(?P<lang>[a-zA-Z]{2})/$',
                                            perm('any', accounts.ProfileSetLanguage), name='account_set_language'),

    url(r'^accounts/password/change/$',      auth_views.password_change, {'template_name': 'registration/password_change_form.jinja'}, name='auth_password_change'),
    url(r'^accounts/password/change/done/$', auth_views.password_change_done, {'template_name': 'registration/password_change_done.jinja'}, name='auth_password_change_done'),
    url(r'^accounts/password/reset/$',       auth_views.password_reset, {'template_name': 'registration/password_reset_form.jinja'}, name='auth_password_reset'),
    url(r'^accounts/password/reset/confirm/(?P<uidb36>[0-9A-Za-z]+)-(?P<token>.+)/$', auth_views.password_reset_confirm, {'template_name': 'registration/password_reset_confirm.jinja'}, name='auth_password_reset_confirm'),
    url(r'^accounts/password/reset/complete/$', auth_views.password_reset_complete, {'template_name': 'registration/password_reset_complete.jinja'}, name='auth_password_reset_complete'),
    url(r'^accounts/password/reset/done/$',  auth_views.password_reset_done, {'template_name': 'registration/password_reset_done.jinja'}, name='auth_password_reset_done'),

    url(r'^accounts/activate/complete/$',   TemplateView.as_view(template_name='registration/activation_complete.jinja'), name='registration_activation_complete'),
    url(r'^accounts/activate/(?P<activation_key>w+)/$', ActivationView.as_view(template_name='registration/activation_form.jinja'), name='registration_activate'),
    url(r'^accounts/register/$',            ProfileRegistrationView.as_view(template_name='registration/registration_form.jinja', form_class=ProfileRegistrationForm), name='registration_register'),
    url(r'^accounts/register/complete/$',   TemplateView.as_view(template_name='registration/registration_complete.jinja'), name='registration_complete'),
    url(r'^accounts/register/closed/$',     TemplateView.as_view(template_name='registration/registration_closed.jinja'), name='registration_disallowed'),
    url(r'^accounts/social/', include('social_auth.urls')),

    url(r'^notifications/seen/(?P<pk>([\d]+|all))/', perm('user', NotificationSeen), name='notification_seen'),
    url(r'^notifications/stream/', perm('user', NotificationStream), name='notification_stream'),
    url(r'^notifications/', perm('user', TemplateView, template_name='notifications.jinja'), name='notifications'),

    url(r'^search/', include('search.urls')),
    url(r'^ticket/', include('ticket.urls')),
    url(r'^podaci/', include('podaci.urls')),
    url(r'^projects/', perm('staff', TemplateView, template_name='projects.jinja'), name='projects'),
    url(r'^api/', include('projects.urls')),
    url(r'^robots/', include('robots.urls')),

    url(r'^i18n/', include('django.conf.urls.i18n')),
    url(r'^jsi18n/$', 'django.views.i18n.javascript_catalog', js_info_dict),
    url(r'^jsi18n/(?P<packages>\S+?)/$', 'django.views.i18n.javascript_catalog'),

    url(r'^json/all_users/$', requests.Select2AllHandler.as_view(), name='select2_all_users'),

    url(r'^captcha/', include('captcha.urls')),
)

handler400 = errors._400
handler401 = errors._401
handler403 = errors._403
handler404 = errors._403
handler500 = errors._500
<commit_msg>Removing robots.urls<commit_after>from django.conf.urls import patterns, include, url
from django.views.generic import TemplateView

from id import databases, requests, accounts
from id import validation, admin, tasks, errors
from core.auth import perm

from django.contrib.auth import views as auth_views
from registration.views import ActivationView

from core.views import NotificationSeen, NotificationStream
from id.views import ProfileRegistrationView, login, logout
from id.forms import ProfileRegistrationForm, FeedbackForm

js_info_dict = {
    'packages': ('id', 'ticket', 'search', 'podaci'),
}

urlpatterns = patterns('',
    url(r'^$',                              perm('any', TemplateView, template_name="splash.jinja"), name="home"),
    url(r'^about/id2/$',                    perm('any', TemplateView, template_name="about_id.jinja"), name='about_id'),
    url(r'^about/occrp/$',                  perm('any', TemplateView, template_name="about_us.jinja"), name='about_us'),

    url(r'^admin/$',                        perm('staff', admin.Panel), name='admin_panel'),
    url(r'^admin/scrapers/request/$',       perm('staff', admin.DatabaseScrapeRequestCreate), name='admin_scrapers_request'),
    url(r'^admin/budgets/$',                perm('staff', admin.Budgets), name='admin_budgets'),

    url(r'^admin/storage/$',                perm('admin', admin.Storage), name='admin_storage'),
    url(r'^admin/statistics/$',             perm('admin', admin.Statistics), name='statistics'),
    url(r'^feedback/$',                     perm('any', admin.Feedback), name='feedback'),
    url(r'^feedback/thankyou/$',            perm('any', admin.FeedbackThanks), name='feedback_thanks'),

    url(r'^databases/$',                    perm('any', databases.ExternalDatabaseList), name='externaldb_list'),
    url(r'^databases/add/$',                perm('staff', databases.ExternalDatabaseAdd), name='externaldb_add'),
    url(r'^databases/edit/(?P<id>[0-9]+)/$',perm('staff', databases.ExternalDatabaseEdit), name='externaldb_edit'),
    url(r'^databases/view/(?P<id>[0-9]+)/$',perm('any', databases.ExternalDatabaseDetail), name='externaldb_detail'),
    url(r'^databases/delete/(?P<id>[0-9]+)/$',
                                            perm('admin', databases.ExternalDatabaseDelete), name='externaldb_delete'),

    url(r'^accounts/login/$',               login, {'template_name': 'registration/login.jinja'}, name='login'),
    url(r'^accounts/logout/',               logout, {'template_name': 'registration/logout.jinja', 'fallback_redirect_url': '/accounts/login/'}, name='logout'),
    url(r'^accounts/users/$',               perm('admin', accounts.UserList), name='userprofile_list'),
    url(r'^accounts/suggest/$',             perm('loggedin', accounts.UserSuggest), name='userprofile_suggest'),
    url(r'^accounts/request/$',             perm('any', accounts.AccessRequestCreate), name='request_access_create'),
    url(r'^accounts/request/list/$',        perm('admin', accounts.AccessRequestList), name='request_access_list'),
    url(r'^accounts/request/list/approved/$',perm('admin', accounts.AccessRequestListApproved), name='request_access_list_approved'),
    url(r'^accounts/request/list/denied/$', perm('admin', accounts.AccessRequestListDenied), name='request_access_list_denied'),
    url(r'^accounts/profile/$',             perm('loggedin', accounts.ProfileUpdate), name='profile'),
    url(r'^accounts/profile/(?P<pk>[0-9]+)/$',
                                            perm('admin', accounts.ProfileUpdate), name='profile'),
    url(r'^accounts/profile/(?P<email>.+)/$',
                                            perm('admin', accounts.ProfileUpdate), name='profile'),
    url(r'^accounts/setlanguage/(?P<lang>[a-zA-Z]{2})/$',
                                            perm('any', accounts.ProfileSetLanguage), name='account_set_language'),

    url(r'^accounts/password/change/$',      auth_views.password_change, {'template_name': 'registration/password_change_form.jinja'}, name='auth_password_change'),
    url(r'^accounts/password/change/done/$', auth_views.password_change_done, {'template_name': 'registration/password_change_done.jinja'}, name='auth_password_change_done'),
    url(r'^accounts/password/reset/$',       auth_views.password_reset, {'template_name': 'registration/password_reset_form.jinja'}, name='auth_password_reset'),
    url(r'^accounts/password/reset/confirm/(?P<uidb36>[0-9A-Za-z]+)-(?P<token>.+)/$', auth_views.password_reset_confirm, {'template_name': 'registration/password_reset_confirm.jinja'}, name='auth_password_reset_confirm'),
    url(r'^accounts/password/reset/complete/$', auth_views.password_reset_complete, {'template_name': 'registration/password_reset_complete.jinja'}, name='auth_password_reset_complete'),
    url(r'^accounts/password/reset/done/$',  auth_views.password_reset_done, {'template_name': 'registration/password_reset_done.jinja'}, name='auth_password_reset_done'),

    url(r'^accounts/activate/complete/$',   TemplateView.as_view(template_name='registration/activation_complete.jinja'), name='registration_activation_complete'),
    url(r'^accounts/activate/(?P<activation_key>w+)/$', ActivationView.as_view(template_name='registration/activation_form.jinja'), name='registration_activate'),
    url(r'^accounts/register/$',            ProfileRegistrationView.as_view(template_name='registration/registration_form.jinja', form_class=ProfileRegistrationForm), name='registration_register'),
    url(r'^accounts/register/complete/$',   TemplateView.as_view(template_name='registration/registration_complete.jinja'), name='registration_complete'),
    url(r'^accounts/register/closed/$',     TemplateView.as_view(template_name='registration/registration_closed.jinja'), name='registration_disallowed'),
    url(r'^accounts/social/', include('social_auth.urls')),

    url(r'^notifications/seen/(?P<pk>([\d]+|all))/', perm('user', NotificationSeen), name='notification_seen'),
    url(r'^notifications/stream/', perm('user', NotificationStream), name='notification_stream'),
    url(r'^notifications/', perm('user', TemplateView, template_name='notifications.jinja'), name='notifications'),

    url(r'^search/', include('search.urls')),
    url(r'^ticket/', include('ticket.urls')),
    url(r'^podaci/', include('podaci.urls')),
    url(r'^projects/', perm('staff', TemplateView, template_name='projects.jinja'), name='projects'),
    url(r'^api/', include('projects.urls')),

    url(r'^i18n/', include('django.conf.urls.i18n')),
    url(r'^jsi18n/$', 'django.views.i18n.javascript_catalog', js_info_dict),
    url(r'^jsi18n/(?P<packages>\S+?)/$', 'django.views.i18n.javascript_catalog'),

    url(r'^json/all_users/$', requests.Select2AllHandler.as_view(), name='select2_all_users'),

    url(r'^captcha/', include('captcha.urls')),
)

handler400 = errors._400
handler401 = errors._401
handler403 = errors._403
handler404 = errors._403
handler500 = errors._500
<|endoftext|>


<commit_before>from django.conf import settings
from django.db import transaction
from django.db.models import Subquery
from django.shortcuts import get_object_or_404
from rest_framework import exceptions, status

from .mixin import ViewSetMixin
from .models import TrackedModel

class LimitExceededError(exceptions.APIException):
    status_code = status.HTTP_402_PAYMENT_REQUIRED
    default_detail = 'limit exceeded'

class LimitedModelMixin(ViewSetMixin):
    parent_key_filter = None

    def get_limit(self, deleted):
        if settings.API_LIMITS is None:
            return None

        parent_limits = settings.API_LIMITS.get(self.parent_model._meta.label)
        if not parent_limits:
            return None

        child_limit = parent_limits.get(self.queryset.model._meta.label)
        if not child_limit:
            return None

        limit = child_limit[int(deleted)]
        if not limit:
            return None

        return limit

    def _check_active_limits(self, parent):
        limit = self.get_limit(False)
        if not limit:
            return

        object_type = self.queryset.model

        child_set_name = object_type._meta.model_name + '_set'
        child_set = getattr(parent, child_set_name)

        if issubclass(object_type, TrackedModel):
            child_set = child_set.filter(deleted=False)

        if child_set.count() >= limit:
            raise LimitExceededError('exceeded limit of %d %s per %s' %
                                     (limit, object_type._meta.verbose_name_plural, parent._meta.verbose_name))

    def _evict_deleted_peers(self, instance):
        limit = self.get_limit(True)
        if not limit:
            return

        filter_kwargs = {}
        filter_kwargs[self.parent_key_filter] = getattr(instance, self.parent_key_filter)
        filter_kwargs['deleted'] = True

        delete_ids = Subquery(self.queryset.filter(**filter_kwargs).order_by('-updated', '-id')[limit:].values('id'))

        delete_objs = self.queryset.filter(id__in=delete_ids)
        delete_objs.delete()

    def locked_parent(self, parent):
        queryset = self.parent_model.objects.filter(pk=parent.pk)

        queryset = queryset.select_for_update()

        locked = get_object_or_404(queryset)

        return locked

    @transaction.atomic(savepoint=False)
    def perform_create(self, serializer):
        parent_name = self.get_parent_name()

        save_kwargs = {}

        if self.is_aggregate():
            parent = self.locked_parent(serializer.validated_data[parent_name])
        else:
            parent = self.get_parent(False, True)

            save_kwargs = {parent_name: parent}

        self._check_active_limits(parent)

        serializer.save(**save_kwargs)

    @transaction.atomic(savepoint=False)
    def perform_update(self, serializer):
        if self.is_aggregate():
            parent = self.locked_parent(serializer.validated_data[self.get_parent_name()])

            self._check_active_limits(parent)

        return super.perform_update(serializer)

    def perform_destroy(self, instance):
        super().perform_destroy(instance)

        self._evict_deleted_peers(instance)
<commit_msg>correctly call superclass method<commit_after>from django.conf import settings
from django.db import transaction
from django.db.models import Subquery
from django.shortcuts import get_object_or_404
from rest_framework import exceptions, status

from .mixin import ViewSetMixin
from .models import TrackedModel

class LimitExceededError(exceptions.APIException):
    status_code = status.HTTP_402_PAYMENT_REQUIRED
    default_detail = 'limit exceeded'

class LimitedModelMixin(ViewSetMixin):
    parent_key_filter = None

    def get_limit(self, deleted):
        if settings.API_LIMITS is None:
            return None

        parent_limits = settings.API_LIMITS.get(self.parent_model._meta.label)
        if not parent_limits:
            return None

        child_limit = parent_limits.get(self.queryset.model._meta.label)
        if not child_limit:
            return None

        limit = child_limit[int(deleted)]
        if not limit:
            return None

        return limit

    def _check_active_limits(self, parent):
        limit = self.get_limit(False)
        if not limit:
            return

        object_type = self.queryset.model

        child_set_name = object_type._meta.model_name + '_set'
        child_set = getattr(parent, child_set_name)

        if issubclass(object_type, TrackedModel):
            child_set = child_set.filter(deleted=False)

        if child_set.count() >= limit:
            raise LimitExceededError('exceeded limit of %d %s per %s' %
                                     (limit, object_type._meta.verbose_name_plural, parent._meta.verbose_name))

    def _evict_deleted_peers(self, instance):
        limit = self.get_limit(True)
        if not limit:
            return

        filter_kwargs = {}
        filter_kwargs[self.parent_key_filter] = getattr(instance, self.parent_key_filter)
        filter_kwargs['deleted'] = True

        delete_ids = Subquery(self.queryset.filter(**filter_kwargs).order_by('-updated', '-id')[limit:].values('id'))

        delete_objs = self.queryset.filter(id__in=delete_ids)
        delete_objs.delete()

    def locked_parent(self, parent):
        queryset = self.parent_model.objects.filter(pk=parent.pk)

        queryset = queryset.select_for_update()

        locked = get_object_or_404(queryset)

        return locked

    @transaction.atomic(savepoint=False)
    def perform_create(self, serializer):
        parent_name = self.get_parent_name()

        save_kwargs = {}

        if self.is_aggregate():
            parent = self.locked_parent(serializer.validated_data[parent_name])
        else:
            parent = self.get_parent(False, True)

            save_kwargs = {parent_name: parent}

        self._check_active_limits(parent)

        serializer.save(**save_kwargs)

    @transaction.atomic(savepoint=False)
    def perform_update(self, serializer):
        if self.is_aggregate():
            parent = self.locked_parent(serializer.validated_data[self.get_parent_name()])

            self._check_active_limits(parent)

        return super().perform_update(serializer)

    def perform_destroy(self, instance):
        super().perform_destroy(instance)

        self._evict_deleted_peers(instance)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# this file is a modified version of source code from the Accerciser project
# http://live.gnome.org/accerciser

"""
Backend to the console plugin.

@author: Eitan Isaacson
@organization: IBM Corporation
@copyright: Copyright (c) 2007 IBM Corporation
@license: BSD

All rights reserved. This program and the accompanying materials are made
available under the terms of the BSD which accompanies this distribution, and
is available at U{http://www.opensource.org/licenses/bsd-license.php}
"""

import gtk
import re
import sys
import os
import pango
import subprocess
from StringIO import StringIO

try:
    import IPython
except Exception, e:
    raise ImportError('Error importing IPython (%s)' % str(e))

ansi_colors = {
    '0;30': 'Black',
    '0;31': 'Red',
    '0;32': 'Green',
    '0;33': 'Brown',
    '0;34': 'Blue',
    '0;35': 'Purple',
    '0;36': 'Cyan',
    '0;37': 'LightGray',
    '1;30': 'DarkGray',
    '1;31': 'DarkRed',
    '1;32': 'SeaGreen',
    '1;33': 'Yellow',
    '1;34': 'LightBlue',
    '1;35': 'MediumPurple',
    '1;36': 'LightCyan',
    '1;37': 'White',
    }

class IterableIPShell:

    def __init__(
        self,
        argv=None,
        user_ns=None,
        user_global_ns=None,
        cin=None,
        cout=None,
        cerr=None,
        input_func=None,
        ):

        if input_func:
            IPython.iplib.raw_input_original = input_func
        if cin:
            IPython.Shell.Term.cin = cin
        if cout:
            IPython.Shell.Term.cout = cout
        if cerr:
            IPython.Shell.Term.cerr = cerr

        if argv is None:
            argv = []

        # This is to get rid of the blockage that occurs during
        # IPython.Shell.InteractiveShell.user_setup()
        IPython.iplib.raw_input = lambda x: None

        orig_stdout = sys.stdout
        orig_stderr = sys.stderr
        sys.stdout = IPython.Shell.Term.cout
        sys.stderr = IPython.Shell.Term.cerr

        self.term = IPython.genutils.IOTerm(cin=cin, cout=cout,
                cerr=cerr)
        os.environ['TERM'] = 'dumb'
        excepthook = sys.excepthook

        self.IP = IPython.Shell.make_IPython(argv, user_ns=user_ns,
                user_global_ns=user_global_ns, embedded=True,
                shell_class=IPython.Shell.InteractiveShell)
        self.IP.system = lambda cmd: \
            self.shell(self.IP.var_expand(cmd),
                       header='IPython system call: ',
                       verbose=self.IP.rc.system_verbose)
        self.IP.api.system = self.IP.system
        sys.excepthook = excepthook
        self.iter_more = 0
        self.history_level = 0
        self.complete_sep = re.compile('[\s\{\}\[\]\(\)]')
        sys.stdout = orig_stdout
        sys.stderr = orig_stderr
        self.interrupt_in_last_line = False

    def execute(self):
        self.history_level = 0
        orig_stdout = sys.stdout
        orig_stderr = sys.stderr
        sys.stdout = IPython.Shell.Term.cout
        sys.stderr = IPython.Shell.Term.cerr

        try:
            line = self.IP.raw_input(None, self.iter_more)
            if self.IP.autoindent:
                self.IP.readline_startup_hook(None)
        except KeyboardInterrupt:
            self.IP.write('KeyboardInterrupt\n')
            self.IP.resetbuffer()

            # keep cache in sync with the prompt counter:
            self.IP.outputcache.prompt_count -= 1

            if self.IP.autoindent:
                self.IP.indent_current_nsp = 0
            self.iter_more = 0
            self.interrupt_in_last_line = True
        except:
            self.IP.showtraceback()
            self.interrupt_in_last_line = True
        else:
            self.iter_more = self.IP.push(line)
            if self.IP.SyntaxTB.last_syntax_error \
                and self.IP.rc.autoedit_syntax:
                self.IP.edit_syntax_error()
                self.interrupt_in_last_line = True
            if self.IP.SyntaxTB.last_syntax_error or self.iter_more is None:
                self.interrupt_in_last_line = True
            else:
                self.interrupt_in_last_line = False
        if self.iter_more:
            self.prompt = str(self.IP.outputcache.prompt2).strip()
            if self.IP.autoindent:
                self.IP.readline_startup_hook(self.IP.pre_readline)
        else:
            self.prompt = str(self.IP.outputcache.prompt1).strip()
        sys.stdout = orig_stdout
        sys.stderr = orig_stderr

    def historyBack(self):
        self.history_level -= 1
        return self._getHistory()

    def historyForward(self):
        self.history_level += 1
        return self._getHistory()

    def _getHistory(self):
        try:
            rv = self.IP.user_ns['In'][self.history_level].strip('\n')
        except IndexError:
            self.history_level = 0
            rv = ''
        return rv

    def updateNamespace(self, ns_dict):
        self.IP.user_ns.update(ns_dict)

    def complete(self, line):
        split_line = self.complete_sep.split(line)
        possibilities = self.IP.complete(split_line[-1])
        if possibilities:
            common_prefix = reduce(self._commonPrefix, possibilities)
            completed = line[:-len(split_line[-1])] + common_prefix
        else:
            completed = line
        return completed, possibilities

    def _commonPrefix(self, str1, str2):
        for i in range(len(str1)):
            if not str2.startswith(str1[:i + 1]):
                return str1[:i]
        return str1

    def shell(
        self,
        cmd,
        verbose=0,
        debug=0,
        header='',
        ):

        stat = 0
        if verbose or debug:
            print header + cmd

        # flush stdout so we don't mangle python's buffering
        if not debug:
            # input, output = os.popen4(cmd)
            p = subprocess.Popen(
                cmd,
                shell=True,
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT,
                close_fds=True,
                )
            input, output = p.stdin, p.stdout
            print output.read()
            output.close()
            input.close()

class ConsoleView(gtk.TextView):

    def __init__(self):
        gtk.TextView.__init__(self)
        self.modify_font(pango.FontDescription('Mono'))
        self.modify_base(gtk.STATE_NORMAL,
                         gtk.gdk.color_parse(ansi_colors['0;30']))
        self.modify_text(gtk.STATE_NORMAL,
                         gtk.gdk.color_parse(ansi_colors['0;37']))
        self.set_cursor_visible(True)
        self.text_buffer = self.get_buffer()
        self.mark = self.text_buffer.create_mark('scroll_mark',
                self.text_buffer.get_end_iter(), False)
        for code in ansi_colors:
            self.text_buffer.create_tag(code,
                    foreground=ansi_colors[code], weight=700)
        self.text_buffer.create_tag('0')
        self.text_buffer.create_tag('notouch', editable=False)
        self.color_pat = re.compile('\x01?\x1b\[(.*?)m\x02?')
        self.line_start = self.text_buffer.create_mark('line_start',
                self.text_buffer.get_end_iter(), True)
        self.connect('key-press-event', self._onKeypress)
        self.last_cursor_pos = 0
        self.auto_indent = True

    def write(self, text, editable=False):
        segments = self.color_pat.split(text)
        segment = segments.pop(0)
        start_mark = self.text_buffer.create_mark(None,
                self.text_buffer.get_end_iter(), True)
        self.text_buffer.insert(self.text_buffer.get_end_iter(),
                                segment)

        if segments:
            ansi_tags = self.color_pat.findall(text)
            for tag in ansi_tags:
                i = segments.index(tag)
                self.text_buffer.insert_with_tags_by_name(self.text_buffer.get_end_iter(),
                        segments[i + 1], tag)
                segments.pop(i)
        if not editable:
            self.text_buffer.apply_tag_by_name('notouch',
                    self.text_buffer.get_iter_at_mark(start_mark),
                    self.text_buffer.get_end_iter())
        self.text_buffer.delete_mark(start_mark)
        self.scroll_mark_onscreen(self.mark)

    def showPrompt(self, prompt):
        self.write(prompt)
        self.text_buffer.move_mark(self.line_start,
                                   self.text_buffer.get_end_iter())
        if self.auto_indent:
            self.write(' ' * self.IP.indent_current_nsp, editable=True)
        self.text_buffer.place_cursor(self.text_buffer.get_end_iter())

    def changeLine(self, text):
        iter = self.text_buffer.get_iter_at_mark(self.line_start)
        iter.forward_to_line_end()
        self.text_buffer.delete(self.text_buffer.get_iter_at_mark(self.line_start),
                                iter)
        self.write(text, True)

    def getCurrentLine(self):
        rv = \
            self.text_buffer.get_slice(self.text_buffer.get_iter_at_mark(self.line_start),
                self.text_buffer.get_end_iter(), False)
        return rv

    def showReturned(self, text):
        iter = self.text_buffer.get_iter_at_mark(self.line_start)
        iter.forward_to_line_end()
        self.text_buffer.apply_tag_by_name('notouch',
                self.text_buffer.get_iter_at_mark(self.line_start),
                iter)
        self.write(text)
        if text:
            self.write('\n')
        self.showPrompt(self.prompt)

    def _onKeypress(self, obj, event):
        if not event.string:
            return
        insert_mark = self.text_buffer.get_insert()
        insert_iter = self.text_buffer.get_iter_at_mark(insert_mark)
        selection_mark = self.text_buffer.get_selection_bound()
        selection_iter = \
            self.text_buffer.get_iter_at_mark(selection_mark)
        start_iter = self.text_buffer.get_iter_at_mark(self.line_start)
        if start_iter.compare(insert_iter) <= 0 \
            and start_iter.compare(selection_iter) <= 0:
            return
        elif start_iter.compare(insert_iter) > 0 \
            and start_iter.compare(selection_iter) > 0:
            self.text_buffer.place_cursor(start_iter)
        elif insert_iter.compare(selection_iter) < 0:
            self.text_buffer.move_mark(insert_mark, start_iter)
        elif insert_iter.compare(selection_iter) > 0:
            self.text_buffer.move_mark(selection_mark, start_iter)

class IPythonView(ConsoleView, IterableIPShell):

    def __init__(self):
        ConsoleView.__init__(self)
        self.cout = StringIO()
        IterableIPShell.__init__(self, cout=self.cout, cerr=self.cout,
                                 input_func=self.raw_input)
        self.interrupt = False
        self.connect('key_press_event', self.keyPress)
        self.execute()

        self.cout.truncate(0)
        self.showPrompt(self.prompt)

    def raw_input(self, prompt=''):
        if self.interrupt:
            self.interrupt = False
            raise KeyboardInterrupt
        return self.getCurrentLine()

    def keyPress(self, widget, event):
        if event.state & gtk.gdk.CONTROL_MASK and event.keyval == 99:
            self.interrupt = True
            self._processLine()
            return True
        elif event.keyval == gtk.keysyms.Return:
            self._processLine()
            return True
        elif event.keyval == gtk.keysyms.Up:
            self.changeLine(self.historyBack())
            return True
        elif event.keyval == gtk.keysyms.Down:
            self.changeLine(self.historyForward())
            return True
        elif event.keyval == gtk.keysyms.Tab:
            if not self.getCurrentLine().strip():
                return False
            completed, possibilities = \
                self.complete(self.getCurrentLine())
            if len(possibilities) > 1:
                slice = self.getCurrentLine()
                self.write('\n')
                for symbol in possibilities:
                    self.write(symbol + '\n')
                self.showPrompt(self.prompt)
            self.changeLine(completed or slice)
            return True

    def _processLine(self):
        self.write('\n')
        self.history_level = 0
        self.execute()
        rv = self.cout.getvalue()
        if rv:
            rv = rv.strip('\n')
        self.showReturned(rv)
        self.cout.truncate(0)

<commit_msg>better error and traceback handling<commit_after># -*- coding: utf-8 -*-

# this file is a modified version of source code from the Accerciser project
# http://live.gnome.org/accerciser

"""
Backend to the console plugin.

@author: Eitan Isaacson
@organization: IBM Corporation
@copyright: Copyright (c) 2007 IBM Corporation
@license: BSD

All rights reserved. This program and the accompanying materials are made
available under the terms of the BSD which accompanies this distribution, and
is available at U{http://www.opensource.org/licenses/bsd-license.php}
"""

import gtk
import re
import sys
import os
import pango
import subprocess
from StringIO import StringIO

try:
    import IPython
except Exception, e:
    raise ImportError('Error importing IPython (%s)' % str(e))

ansi_colors = {
    '0;30': 'Black',
    '0;31': 'Red',
    '0;32': 'Green',
    '0;33': 'Brown',
    '0;34': 'Blue',
    '0;35': 'Purple',
    '0;36': 'Cyan',
    '0;37': 'LightGray',
    '1;30': 'DarkGray',
    '1;31': 'DarkRed',
    '1;32': 'SeaGreen',
    '1;33': 'Yellow',
    '1;34': 'LightBlue',
    '1;35': 'MediumPurple',
    '1;36': 'LightCyan',
    '1;37': 'White',
    }

class IterableIPShell:

    def __init__(
        self,
        argv=None,
        user_ns=None,
        user_global_ns=None,
        cin=None,
        cout=None,
        cerr=None,
        input_func=None,
        ):

        if input_func:
            IPython.iplib.raw_input_original = input_func
        if cin:
            IPython.Shell.Term.cin = cin
        if cout:
            IPython.Shell.Term.cout = cout
        if cerr:
            IPython.Shell.Term.cerr = cerr

        if argv is None:
            argv = []

        # This is to get rid of the blockage that occurs during
        # IPython.Shell.InteractiveShell.user_setup()
        IPython.iplib.raw_input = lambda x: None

        self.interrupt_in_last_line = False

        orig_stdout = sys.stdout
        orig_stderr = sys.stderr
        sys.stdout = IPython.Shell.Term.cout
        sys.stderr = IPython.Shell.Term.cerr
        try:
            self.term = IPython.genutils.IOTerm(cin=cin, cout=cout,
                    cerr=cerr)
            os.environ['TERM'] = 'dumb'
            excepthook = sys.excepthook

            self.IP = IPython.Shell.make_IPython(argv, user_ns=user_ns,
                    user_global_ns=user_global_ns, embedded=True,
                    shell_class=IPython.Shell.InteractiveShell)
            self.IP.system = lambda cmd: \
                self.shell(self.IP.var_expand(cmd),
                           header='IPython system call: ',
                           verbose=self.IP.rc.system_verbose)
            self.IP.api.system = self.IP.system
            sys.excepthook = excepthook
            self.iter_more = 0
            self.history_level = 0
            self.complete_sep = re.compile('[\s\{\}\[\]\(\)]')
        finally:
            sys.stdout = orig_stdout
            sys.stderr = orig_stderr

    def execute(self):
        self.history_level = 0
        orig_stdout = sys.stdout
        orig_stderr = sys.stderr
        sys.stdout = IPython.Shell.Term.cout
        sys.stderr = IPython.Shell.Term.cerr

        try:
            line = self.IP.raw_input(None, self.iter_more)
            if self.IP.autoindent:
                self.IP.readline_startup_hook(None)
        except KeyboardInterrupt:
            self.IP.write('KeyboardInterrupt\n')
            self.IP.resetbuffer()

            # keep cache in sync with the prompt counter:
            self.IP.outputcache.prompt_count -= 1

            if self.IP.autoindent:
                self.IP.indent_current_nsp = 0
            self.iter_more = 0
            self.interrupt_in_last_line = True
        except:
            self.IP.showtraceback()
            self.interrupt_in_last_line = True
        else:
            self.iter_more = self.IP.push(line)
            if self.IP.SyntaxTB.last_syntax_error \
                and self.IP.rc.autoedit_syntax:
                self.IP.edit_syntax_error()
                self.interrupt_in_last_line = True
            if self.IP.SyntaxTB.last_syntax_error or self.iter_more \
                is None:
                self.interrupt_in_last_line = True
            else:
                self.interrupt_in_last_line = False
        finally:
            sys.stdout = orig_stdout
            sys.stderr = orig_stderr

        if self.iter_more:
            self.prompt = str(self.IP.outputcache.prompt2).strip()
            if self.IP.autoindent:
                self.IP.readline_startup_hook(self.IP.pre_readline)
        else:
            self.prompt = str(self.IP.outputcache.prompt1).strip()

    def historyBack(self):
        self.history_level -= 1
        return self._getHistory()

    def historyForward(self):
        self.history_level += 1
        return self._getHistory()

    def _getHistory(self):
        try:
            rv = self.IP.user_ns['In'][self.history_level].strip('\n')
        except IndexError:
            self.history_level = 0
            rv = ''
        return rv

    def updateNamespace(self, ns_dict):
        self.IP.user_ns.update(ns_dict)

    def complete(self, line):
        split_line = self.complete_sep.split(line)
        possibilities = self.IP.complete(split_line[-1])
        if possibilities:
            common_prefix = reduce(self._commonPrefix, possibilities)
            completed = line[:-len(split_line[-1])] + common_prefix
        else:
            completed = line
        return completed, possibilities

    def _commonPrefix(self, str1, str2):
        for i in range(len(str1)):
            if not str2.startswith(str1[:i + 1]):
                return str1[:i]
        return str1

    def shell(
        self,
        cmd,
        verbose=0,
        debug=0,
        header='',
        ):

        stat = 0
        if verbose or debug:
            print header + cmd

        # flush stdout so we don't mangle python's buffering
        if not debug:
            # input, output = os.popen4(cmd)
            p = subprocess.Popen(
                cmd,
                shell=True,
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT,
                close_fds=True,
                )
            input, output = p.stdin, p.stdout
            print output.read()
            output.close()
            input.close()

class ConsoleView(gtk.TextView):

    def __init__(self):
        gtk.TextView.__init__(self)
        self.modify_font(pango.FontDescription('Mono'))
        self.modify_base(gtk.STATE_NORMAL,
                         gtk.gdk.color_parse(ansi_colors['0;30']))
        self.modify_text(gtk.STATE_NORMAL,
                         gtk.gdk.color_parse(ansi_colors['0;37']))
        self.set_cursor_visible(True)
        self.text_buffer = self.get_buffer()
        self.mark = self.text_buffer.create_mark('scroll_mark',
                self.text_buffer.get_end_iter(), False)
        for code in ansi_colors:
            self.text_buffer.create_tag(code,
                    foreground=ansi_colors[code], weight=700)
        self.text_buffer.create_tag('0')
        self.text_buffer.create_tag('notouch', editable=False)
        self.color_pat = re.compile('\x01?\x1b\[(.*?)m\x02?')
        self.line_start = self.text_buffer.create_mark('line_start',
                self.text_buffer.get_end_iter(), True)
        self.connect('key-press-event', self._onKeypress)
        self.last_cursor_pos = 0
        self.auto_indent = True

    def write(self, text, editable=False):
        segments = self.color_pat.split(text)
        segment = segments.pop(0)
        start_mark = self.text_buffer.create_mark(None,
                self.text_buffer.get_end_iter(), True)
        self.text_buffer.insert(self.text_buffer.get_end_iter(),
                                segment)

        if segments:
            ansi_tags = self.color_pat.findall(text)
            for tag in ansi_tags:
                i = segments.index(tag)
                self.text_buffer.insert_with_tags_by_name(self.text_buffer.get_end_iter(),
                        segments[i + 1], tag)
                segments.pop(i)
                segments.pop(i)
        if not editable:
            self.text_buffer.apply_tag_by_name('notouch',
                    self.text_buffer.get_iter_at_mark(start_mark),
                    self.text_buffer.get_end_iter())
        self.text_buffer.delete_mark(start_mark)
        self.scroll_mark_onscreen(self.mark)

    def showPrompt(self, prompt):
        self.write(prompt)
        self.text_buffer.move_mark(self.line_start,
                                   self.text_buffer.get_end_iter())
        if self.auto_indent:
            self.write(' ' * self.IP.indent_current_nsp, editable=True)
        self.text_buffer.place_cursor(self.text_buffer.get_end_iter())

    def changeLine(self, text):
        iter = self.text_buffer.get_iter_at_mark(self.line_start)
        iter.forward_to_line_end()
        self.text_buffer.delete(self.text_buffer.get_iter_at_mark(self.line_start),
                                iter)
        self.write(text, True)

    def getCurrentLine(self):
        rv = \
            self.text_buffer.get_slice(self.text_buffer.get_iter_at_mark(self.line_start),
                self.text_buffer.get_end_iter(), False)
        return rv

    def showReturned(self, text):
        iter = self.text_buffer.get_iter_at_mark(self.line_start)
        iter.forward_to_line_end()
        self.text_buffer.apply_tag_by_name('notouch',
                self.text_buffer.get_iter_at_mark(self.line_start),
                iter)
        self.write(text)
        if text:
            self.write('\n')
        self.showPrompt(self.prompt)

    def _onKeypress(self, obj, event):
        if not event.string:
            return
        insert_mark = self.text_buffer.get_insert()
        insert_iter = self.text_buffer.get_iter_at_mark(insert_mark)
        selection_mark = self.text_buffer.get_selection_bound()
        selection_iter = \
            self.text_buffer.get_iter_at_mark(selection_mark)
        start_iter = self.text_buffer.get_iter_at_mark(self.line_start)
        if start_iter.compare(insert_iter) <= 0 \
            and start_iter.compare(selection_iter) <= 0:
            return
        elif start_iter.compare(insert_iter) > 0 \
            and start_iter.compare(selection_iter) > 0:
            self.text_buffer.place_cursor(start_iter)
        elif insert_iter.compare(selection_iter) < 0:
            self.text_buffer.move_mark(insert_mark, start_iter)
        elif insert_iter.compare(selection_iter) > 0:
            self.text_buffer.move_mark(selection_mark, start_iter)

class IPythonView(ConsoleView, IterableIPShell):

    def __init__(self):
        ConsoleView.__init__(self)
        self.cout = StringIO()
        IterableIPShell.__init__(self, cout=self.cout, cerr=self.cout,
                                 input_func=self.raw_input)
        self.interrupt = False
        self.connect('key_press_event', self.keyPress)
        self.execute()

        self.cout.truncate(0)
        self.showPrompt(self.prompt)

    def raw_input(self, prompt=''):
        if self.interrupt:
            self.interrupt = False
            raise KeyboardInterrupt
        return self.getCurrentLine()

    def keyPress(self, widget, event):
        if event.state & gtk.gdk.CONTROL_MASK and event.keyval == 99:
            self.interrupt = True
            self._processLine()
            return True
        elif event.keyval == gtk.keysyms.Return:
            self._processLine()
            return True
        elif event.keyval == gtk.keysyms.Up:
            self.changeLine(self.historyBack())
            return True
        elif event.keyval == gtk.keysyms.Down:
            self.changeLine(self.historyForward())
            return True
        elif event.keyval == gtk.keysyms.Tab:
            if not self.getCurrentLine().strip():
                return False
            completed, possibilities = \
                self.complete(self.getCurrentLine())
            if len(possibilities) > 1:
                slice = self.getCurrentLine()
                self.write('\n')
                for symbol in possibilities:
                    self.write(symbol + '\n')
                self.showPrompt(self.prompt)
            self.changeLine(completed or slice)
            return True

    def _processLine(self):
        self.write('\n')
        self.history_level = 0
        self.execute()
        rv = self.cout.getvalue()
        if rv:
            rv = rv.strip('\n')
        self.showReturned(rv)
        self.cout.truncate(0)

<|endoftext|>


<commit_before># -*- coding: utf8 -*-
import DebugTag
from TedSubtitle import TedSubtitle
from TedTalk import TedTalk
import Number as Number
import sys
import ParagraphMerge as PMerge
import ParagraphRefact as PRefact
import Log
import json
import os

debugTags    = DebugTag.InitDebugTags()
DebugTagType = DebugTag.InitDebugTagTypes()

TED_ID      = sys.argv[1]
REFACT_TYPE = sys.argv[2]

enTalk = TedTalk(languageCode = 'en',    id = TED_ID)
chTalk = TedTalk(languageCode = 'zh-tw', id = TED_ID)

baseDirPath = 'practice/'
filePath = '%s/%s.txt' % (baseDirPath, str(chTalk.id))

filteredEnglishSubtitles = enTalk.subtitles
filteredChineseSubtitles = chTalk.subtitles

enLenOriginal = len(filteredEnglishSubtitles)
chLenOriginal = len(filteredChineseSubtitles)

PRefact.RefactStartOfParagraph(enTalk, chTalk, refactType = REFACT_TYPE)
filteredChineseSubtitles = chTalk.GroupToParagraph()
filteredEnglishSubtitles = enTalk.GroupToParagraph()
Log.LogInit(TED_ID, enLenOriginal, chLenOriginal, filteredEnglishSubtitles, filteredChineseSubtitles)

[filteredEnglishSubtitles, filteredChineseSubtitles] = PMerge.MergeSubtitles(filteredEnglishSubtitles, filteredChineseSubtitles)
Log.LogMerged(filteredEnglishSubtitles, filteredChineseSubtitles)

def ReadFileContent(filePath):
  fileRead = open(filePath, 'r')
  return fileRead.read().split('\n')

def WriteFileContent(filePath, content):
  fileWrite = open(filePath,'w')
  fileWrite.write(content) # python will convert \n to os.linesep
  fileWrite.close()

def GetTitleAndURL(id):
  command = "curl -s http://www.ted.com/talks/view/id/%s" % id
  output = os.popen(command).readlines()[0]
  url = output.split('"')[1]
  title = url.split('/')[-1]
  title = title.title().replace('_', ' ')
  return {"title" : title, 'url' : url}

def PrintResult( filteredEnglishSubtitles, filteredChineseSubtitles):
  obj = []
  for i in xrange(len(filteredChineseSubtitles)):
    a1 = filteredEnglishSubtitles[i].content.encode('utf8')
    b1 = filteredChineseSubtitles[i].content.encode('utf8')
    obj.append({"zh-tw" : b1, "en" : a1})

  resultObj = GetTitleAndURL(TED_ID)
  resultObj["paragraphs"] = obj
  result = json.dumps(resultObj, ensure_ascii=False)

  WriteFileContent(filePath,result)

if DebugTagType.PrintSubtitles in debugTags:
  PrintResult(filteredEnglishSubtitles, filteredChineseSubtitles)

if DebugTagType.File in debugTags:
  contentsInFile = ReadFileContent(filePath)
  lengthForChineseSubtitles = len(filteredChineseSubtitles)
  idxForChineseSubtitles = 0
  i = 0
  while i < len(contentsInFile):
    if Number.IsInt(contentsInFile[i]):
      i += 2
      hasTranslated = len(contentsInFile[i]) > 0 or True
      if hasTranslated:
        contentsInFile[i] = filteredChineseSubtitles[idxForChineseSubtitles].content.encode('utf8')
      idxForChineseSubtitles += 1
    i += 1

  WriteFileContent(filePath,'\n'.join(contentsInFile))

<commit_msg>delete unused code<commit_after># -*- coding: utf8 -*-
import DebugTag
from TedSubtitle import TedSubtitle
from TedTalk import TedTalk
import Number as Number
import sys
import ParagraphMerge as PMerge
import ParagraphRefact as PRefact
import Log
import json
import os

debugTags    = DebugTag.InitDebugTags()
DebugTagType = DebugTag.InitDebugTagTypes()

TED_ID      = sys.argv[1]
REFACT_TYPE = sys.argv[2]

enTalk = TedTalk(languageCode = 'en',    id = TED_ID)
chTalk = TedTalk(languageCode = 'zh-tw', id = TED_ID)

baseDirPath = 'practice/'
filePath = '%s/%s.json' % (baseDirPath, str(chTalk.id))

filteredEnglishSubtitles = enTalk.subtitles
filteredChineseSubtitles = chTalk.subtitles

enLenOriginal = len(filteredEnglishSubtitles)
chLenOriginal = len(filteredChineseSubtitles)

PRefact.RefactStartOfParagraph(enTalk, chTalk, refactType = REFACT_TYPE)
filteredChineseSubtitles = chTalk.GroupToParagraph()
filteredEnglishSubtitles = enTalk.GroupToParagraph()
Log.LogInit(TED_ID, enLenOriginal, chLenOriginal, filteredEnglishSubtitles, filteredChineseSubtitles)

[filteredEnglishSubtitles, filteredChineseSubtitles] = PMerge.MergeSubtitles(filteredEnglishSubtitles, filteredChineseSubtitles)
Log.LogMerged(filteredEnglishSubtitles, filteredChineseSubtitles)

def ReadFileContent(filePath):
  fileRead = open(filePath, 'r')
  return fileRead.read().split('\n')

def WriteFileContent(filePath, content):
  fileWrite = open(filePath,'w')
  fileWrite.write(content) # python will convert \n to os.linesep
  fileWrite.close()

def GetTitleAndURL(id):
  command = "curl -s http://www.ted.com/talks/view/id/%s" % id
  output = os.popen(command).readlines()[0]
  url = output.split('"')[1]
  title = url.split('/')[-1]
  title = title.title().replace('_', ' ')
  return {"title" : title, 'url' : url}

def PrintResult(filteredEnglishSubtitles, filteredChineseSubtitles):
  obj = []
  for i in xrange(len(filteredChineseSubtitles)):
    a1 = filteredEnglishSubtitles[i].content.encode('utf8')
    b1 = filteredChineseSubtitles[i].content.encode('utf8')
    obj.append({"zh-tw" : b1, "en" : a1})

  resultObj = GetTitleAndURL(TED_ID)
  resultObj["paragraphs"] = obj
  result = json.dumps(resultObj, ensure_ascii=False)

  WriteFileContent(filePath,result)

if DebugTagType.PrintSubtitles in debugTags:
  PrintResult(filteredEnglishSubtitles, filteredChineseSubtitles)

if DebugTagType.File in debugTags:
  contentsInFile = ReadFileContent(filePath)
  lengthForChineseSubtitles = len(filteredChineseSubtitles)
  idxForChineseSubtitles = 0
  i = 0
  while i < len(contentsInFile):
    if Number.IsInt(contentsInFile[i]):
      i += 2
      hasTranslated = len(contentsInFile[i]) > 0 or True
      if hasTranslated:
        contentsInFile[i] = filteredChineseSubtitles[idxForChineseSubtitles].content.encode('utf8')
      idxForChineseSubtitles += 1
    i += 1

  WriteFileContent(filePath,'\n'.join(contentsInFile))

<|endoftext|>


<commit_before>#!/usr/bin/env python

from itertools import izip

import numpy as np
from scipy.sparse import csc_matrix
from scipy.sparse import eye as sparseeye
from scipy.linalg import inv, pinv
from scipy.optimize import fmin_l_bfgs_b

from pydigree.mixedmodel.blup import blup
from pydigree.mixedmodel.likelihood import restricted_loglikelihood
from pydigree.mixedmodel.likelihood import full_loglikelihood

def is_genetic_effect(effect):
    return effect in set(['additive', 'dominance', 'mitochondrial'])

class MixedModel(object):

    """
    Fits linear models in the form of y = X * b + sum(Z_i * u_i) + e, where:
      y is the vector of outcomes
      X is a design matrix of fixed effects
      b is the vector of coefficients correstponding to those fixed effects
      Z_i is an incidence matrix corresponding to random effect i
      u_i is a vector of values corresponding to random effect i
      e is a vector of errors
    """

    def __init__(self, pedigrees, outcome=None, fixed_effects=None,
                 random_effects=None, covariance_matrices=None):
        self.maximized = False
        self.pedigrees = pedigrees
        self.outcome = outcome
        if not random_effects:
            self.random_effects = []
            self.covariance_matrices = []
        else:
            self.random_effects = random_effects
        self.fixed_effects = fixed_effects if fixed_effects else []
        self.variance_components = [None] * len(self.random_effects)
        self.obs = []
        self.V = None

    def fit_model(self):
        """
        Builds X, Z, Y, and R for the model

        Arguements: None
        Returns: Nothing
        """
        self._makeR()
        self._makey()
        self._makeX()
        self._makeZs()

    def _fit_results(self):
        self._makeV()
        self._makebeta()

    def clear_model(self):
        """ Clears all parameters from the model """
        self.random_effects = []
        self.fixed_effects = []
        self.covariance_matrices = []
        self.Zlist = []
        self.variance_components = []
        self.X = None
        self.y = None
        self.beta = None
        self.V = None
        self.R = None
        self.obs = None

    def observations(self):
        """
        Returns a list of the fully observed individuals in the model
        Fully observed individuals have observations for each fixed effect and
        and observation for the outcome variable.

        Arguements: None
        Returns: The list of fully observed individuals
        """

        def has_all_fixefs(ind, effects):
            if not effects:
                return True
            return all(ind.phenotypes[effect] is not None
                       for effect in effects)

        def has_outcome(ind):
            return ind.phenotypes[self.outcome] is not None

        obs = [x for x in self.pedigrees.individuals
               if has_all_fixefs(x, self.fixed_effects) and has_outcome(x)]
        if not self.obs:
            self.obs = obs
        return obs

    def nobs(self):
        """
        Returns the number of fully observed individuals in the pedigree
        Arguements: None
        Returns: An integer
        """
        return len(self.observations())

    def residual_variance(self):
        """ Returns the variance in y not accounted for by random effects """
        return np.var(self.y) - sum(self.variance_components)

    def _makey(self):
        """ Prepares the vector of outcome variables for model estimation """
        obs = self.observations()
        self.y = np.matrix([x.phenotypes[self.outcome]
                            for x in obs]).transpose()

    def _makeX(self):
        """
        Builds the design matrix for the fixed effects in the mixed model.
        Includes a column of ones for the intercept.

        Arguements: None
        Returns: A numpy matrix
        """
        obs = self.observations()
        xmat = [[1] * len(obs)]
        for phen in self.fixed_effects:
            xmat.append([ob.phenotypes[phen] for ob in obs])
        X = np.matrix(zip(*xmat))
        self.X = X
        return X

    def _makeZs(self):
        """
        Makes the incidence matrix for random effects

        Arguements: None
        Returns: A list of numpy matrixes
        """
        Zlist = []
        for effect_name in self.random_effects:
            allinds = self.pedigrees.individuals
            obs = frozenset(self.observations())
            obsidx = [i for i, x in enumerate(allinds) if x in obs]
            if is_genetic_effect(effect_name):
                incidence_matrix = np.matrix(np.eye(len(allinds)))[obsidx, :]
                Zlist.append(incidence_matrix)
            else:
                raise NotImplementedError('Arbitrary ranefs not implemented')
        self.Zlist = [csc_matrix(Z) for Z in Zlist]

    def _makeR(self):
        self.R = sparseeye(self.nobs(), self.nobs())

    def _makeV(self, vcs=None):
        if (not vcs) and (not self.variance_components):
            raise ValueError('Variance components not set')
        if not vcs:
            variance_components = self.variance_components
        else:
            variance_components = vcs
        V = sum(sigma * Z * A * Z.T for sigma, Z, A in
                izip(variance_components,
                     self.Zlist,
                     self.covariance_matrices))
        V = V + (self.residual_variance() * self.R)
        if vcs is not None:
            return V
        else:
            self.V = V

    def _makebeta(self):
        """
        Calculates BLUEs for the fixed effects portion of the model

        Reference:
        McCulloch & Seale. Generalized, Linear, and Mixed Models. (2001)
        Equation 6.24
        """
        vinv = inv(self.V.todense())
        self.beta = pinv(self.X.T * vinv * self.X) * self.X.T * vinv * self.y

    def set_outcome(self, outcome):
        """ Sets the outcome for the mixed model """
        self.outcome = outcome
        self._makey()

    def add_fixed_effects(self, effect):
        """ Adds a fixed effect to the model """
        self.fixed_effects.append(effect)
        self._makeX()

    def add_random_effect(self, effect, covariance_matrix):
        """ Adds a random effect to the model """
        self.random_effects.append(effect)
        self.covariance_matrices.append(covariance_matrix)
        self._makeZs()

    def add_genetic_effect(self, type='additive'):
        self.add_random_effect(type,
                               self.pedigrees.additive_relationship_matrix())

    def set_variance_components(self, variance_components):
        """
        Manually set variance components for each random effect in the model.
        Useful if you know a priori, say a heritability, and just want to
        predict breeding values for the trait.
        """
        if not all(x is not None for x in variance_components):
            raise ValueError('Not all variance components are specified')
        self.variance_components = variance_components

    def maximize(self, method='L-BFGS-B', verbose=False):
        """
        Finds the optimal values for variance components of the model by
        restricted maximum likelihood estimation.
        """
        if self.maximized == method:
            return
        starts = self.__starting_variance_components()

        def cb(x):
            if verbose:
                print 'Iteration VC estimates: %s' % \
                    ', '.join(str(y) for y in x.tolist())

        b = [(0, np.var(self.y))] * len(self.random_effects)
        r = fmin_l_bfgs_b(self.__reml_optimization_target,
                          starts, bounds=b, approx_grad=1, callback=cb)
        if verbose:
            print r
        self.variance_components = r[0].tolist()

    def likelihood(self, estimator='full', vmat=None):
        """
        Returns the likelihood of the model with the current model parameters
        """
        estimator = estimator.lower()
        if vmat is None:
            V = self.V
        else:
            V = vmat
        if estimator == 'full':
            return full_loglikelihood(self.y, V, self.X)
        elif estimator == 'restricted':
            return restricted_loglikelihood(self.y, V, self.X)

    def blup(self):
        if not all(self.variance_components):
            raise ValueError('Varcomps not specified! Maximize or set them')
        b = blup(self.y, self.X, self.Zlist,
                 self.covariance_matrices,
                 self.variance_components).transpose().tolist()[0]
        return b

    def summary(self):
        """ Prints a summary of the current model """
        self._fit_results()
        print 'Fixed effects:'
        fixefnames = ['Intercept'] + self.fixed_effects
        betas = self.beta.T.tolist()[0]
        print '\t'.join(['Name', 'Estimate'])
        for name, beta in zip(fixefnames, betas):
            print '\t'.join(str(q) for q in [name, beta])
        print
        print 'Variance components:'
        print '\t'.join(['Component', 'Variance', '% Variance'])
        for name, vc in zip(self.random_effects, self.variance_components):
            print '\t'.join(str(val) for val in [name,
                                                 vc,
                                                 100 * vc / np.var(self.y)])
        print
        print 'Loglikelihood: %s' % self.likelihood()

    def __reml_optimization_target(self, vcs):
        """ Optimization target for maximization. """
        Q = self._makeV(vcs=vcs.tolist())
        return -1.0 * self.likelihood(estimator='restricted', vmat=Q)

    def __starting_variance_components(self):
        """
        Starting variance components in optimization.
        Chooses all variance components (including residual) to be equal.
        """
        v = np.var(self.y)
        n = float(len(self.random_effects))
        return [v / (n + 1) for r in self.random_effects]
<commit_msg>copy method<commit_after>#!/usr/bin/env python

from itertools import izip
import copy

import numpy as np
from scipy.sparse import csc_matrix
from scipy.sparse import eye as sparseeye
from scipy.linalg import inv, pinv
from scipy.optimize import fmin_l_bfgs_b

from pydigree.mixedmodel.blup import blup
from pydigree.mixedmodel.likelihood import restricted_loglikelihood
from pydigree.mixedmodel.likelihood import full_loglikelihood

def is_genetic_effect(effect):
    return effect in set(['additive', 'dominance', 'mitochondrial'])

class MixedModel(object):

    """
    Fits linear models in the form of y = X * b + sum(Z_i * u_i) + e, where:
      y is the vector of outcomes
      X is a design matrix of fixed effects
      b is the vector of coefficients correstponding to those fixed effects
      Z_i is an incidence matrix corresponding to random effect i
      u_i is a vector of values corresponding to random effect i
      e is a vector of errors
    """

    def __init__(self, pedigrees, outcome=None, fixed_effects=None,
                 random_effects=None, covariance_matrices=None):
        self.maximized = False
        self.pedigrees = pedigrees
        self.outcome = outcome
        if not random_effects:
            self.random_effects = []
            self.covariance_matrices = []
        else:
            self.random_effects = random_effects
        self.fixed_effects = fixed_effects if fixed_effects else []
        self.variance_components = [None] * len(self.random_effects)
        self.obs = []
        self.V = None

    def copy(self):
        "Return a copy of the model"
        # We want to avoid copying pedigree and individual data, so
        # we'll set the pedigrees attribute to None for a sec, and then
        # change it back
        peds = self.pedigrees
        self.pedigrees = None

        newmm = copy.deepcopy(self)

        newmm.pedigrees = peds
        self.pedigrees = peds

        return newmm

    def fit_model(self):
        """
        Builds X, Z, Y, and R for the model

        Arguements: None
        Returns: Nothing
        """
        self._makeR()
        self._makey()
        self._makeX()
        self._makeZs()

    def _fit_results(self):
        self._makeV()
        self._makebeta()

    def clear_model(self):
        """ Clears all parameters from the model """
        self.random_effects = []
        self.fixed_effects = []
        self.covariance_matrices = []
        self.Zlist = []
        self.variance_components = []
        self.X = None
        self.y = None
        self.beta = None
        self.V = None
        self.R = None
        self.obs = None

    def observations(self):
        """
        Returns a list of the fully observed individuals in the model
        Fully observed individuals have observations for each fixed effect and
        and observation for the outcome variable.

        Arguements: None
        Returns: The list of fully observed individuals
        """

        def has_all_fixefs(ind, effects):
            if not effects:
                return True
            return all(ind.phenotypes[effect] is not None
                       for effect in effects)

        def has_outcome(ind):
            return ind.phenotypes[self.outcome] is not None

        obs = [x for x in self.pedigrees.individuals
               if has_all_fixefs(x, self.fixed_effects) and has_outcome(x)]
        if not self.obs:
            self.obs = obs
        return obs

    def nobs(self):
        """
        Returns the number of fully observed individuals in the pedigree
        Arguements: None
        Returns: An integer
        """
        return len(self.observations())

    def residual_variance(self):
        """ Returns the variance in y not accounted for by random effects """
        return np.var(self.y) - sum(self.variance_components)

    def _makey(self):
        """ Prepares the vector of outcome variables for model estimation """
        obs = self.observations()
        self.y = np.matrix([x.phenotypes[self.outcome]
                            for x in obs]).transpose()

    def _makeX(self):
        """
        Builds the design matrix for the fixed effects in the mixed model.
        Includes a column of ones for the intercept.

        Arguements: None
        Returns: A numpy matrix
        """
        obs = self.observations()
        xmat = [[1] * len(obs)]
        for phen in self.fixed_effects:
            xmat.append([ob.phenotypes[phen] for ob in obs])
        X = np.matrix(zip(*xmat))
        self.X = X
        return X

    def _makeZs(self):
        """
        Makes the incidence matrix for random effects

        Arguements: None
        Returns: A list of numpy matrixes
        """
        Zlist = []
        for effect_name in self.random_effects:
            allinds = self.pedigrees.individuals
            obs = frozenset(self.observations())
            obsidx = [i for i, x in enumerate(allinds) if x in obs]
            if is_genetic_effect(effect_name):
                incidence_matrix = np.matrix(np.eye(len(allinds)))[obsidx, :]
                Zlist.append(incidence_matrix)
            else:
                raise NotImplementedError('Arbitrary ranefs not implemented')
        self.Zlist = [csc_matrix(Z) for Z in Zlist]

    def _makeR(self):
        self.R = sparseeye(self.nobs(), self.nobs())

    def _makeV(self, vcs=None):
        if (not vcs) and (not self.variance_components):
            raise ValueError('Variance components not set')
        if not vcs:
            variance_components = self.variance_components
        else:
            variance_components = vcs
        V = sum(sigma * Z * A * Z.T for sigma, Z, A in
                izip(variance_components,
                     self.Zlist,
                     self.covariance_matrices))
        V = V + (self.residual_variance() * self.R)
        if vcs is not None:
            return V
        else:
            self.V = V

    def _makebeta(self):
        """
        Calculates BLUEs for the fixed effects portion of the model

        Reference:
        McCulloch & Seale. Generalized, Linear, and Mixed Models. (2001)
        Equation 6.24
        """
        vinv = inv(self.V.todense())
        self.beta = pinv(self.X.T * vinv * self.X) * self.X.T * vinv * self.y

    def set_outcome(self, outcome):
        """ Sets the outcome for the mixed model """
        self.outcome = outcome
        self._makey()

    def add_fixed_effects(self, effect):
        """ Adds a fixed effect to the model """
        self.fixed_effects.append(effect)
        self._makeX()

    def add_random_effect(self, effect, covariance_matrix):
        """ Adds a random effect to the model """
        self.random_effects.append(effect)
        self.covariance_matrices.append(covariance_matrix)
        self._makeZs()

    def add_genetic_effect(self, type='additive'):
        self.add_random_effect(type,
                               self.pedigrees.additive_relationship_matrix())

    def set_variance_components(self, variance_components):
        """
        Manually set variance components for each random effect in the model.
        Useful if you know a priori, say a heritability, and just want to
        predict breeding values for the trait.
        """
        if not all(x is not None for x in variance_components):
            raise ValueError('Not all variance components are specified')
        self.variance_components = variance_components

    def maximize(self, method='L-BFGS-B', verbose=False):
        """
        Finds the optimal values for variance components of the model by
        restricted maximum likelihood estimation.
        """
        if self.maximized == method:
            return
        starts = self.__starting_variance_components()

        def cb(x):
            if verbose:
                print 'Iteration VC estimates: %s' % \
                    ', '.join(str(y) for y in x.tolist())

        b = [(0, np.var(self.y))] * len(self.random_effects)
        r = fmin_l_bfgs_b(self.__reml_optimization_target,
                          starts, bounds=b, approx_grad=1, callback=cb)
        if verbose:
            print r
        self.variance_components = r[0].tolist()

    def likelihood(self, estimator='full', vmat=None):
        """
        Returns the likelihood of the model with the current model parameters
        """
        estimator = estimator.lower()
        if vmat is None:
            V = self.V
        else:
            V = vmat
        if estimator == 'full':
            return full_loglikelihood(self.y, V, self.X)
        elif estimator == 'restricted':
            return restricted_loglikelihood(self.y, V, self.X)

    def blup(self):
        if not all(self.variance_components):
            raise ValueError('Varcomps not specified! Maximize or set them')
        b = blup(self.y, self.X, self.Zlist,
                 self.covariance_matrices,
                 self.variance_components).transpose().tolist()[0]
        return b

    def summary(self):
        """ Prints a summary of the current model """
        self._fit_results()
        print 'Fixed effects:'
        fixefnames = ['Intercept'] + self.fixed_effects
        betas = self.beta.T.tolist()[0]
        print '\t'.join(['Name', 'Estimate'])
        for name, beta in zip(fixefnames, betas):
            print '\t'.join(str(q) for q in [name, beta])
        print
        print 'Variance components:'
        print '\t'.join(['Component', 'Variance', '% Variance'])
        for name, vc in zip(self.random_effects, self.variance_components):
            print '\t'.join(str(val) for val in [name,
                                                 vc,
                                                 100 * vc / np.var(self.y)])
        print
        print 'Loglikelihood: %s' % self.likelihood()

    def __reml_optimization_target(self, vcs):
        """ Optimization target for maximization. """
        Q = self._makeV(vcs=vcs.tolist())
        return -1.0 * self.likelihood(estimator='restricted', vmat=Q)

    def __starting_variance_components(self):
        """
        Starting variance components in optimization.
        Chooses all variance components (including residual) to be equal.
        """
        v = np.var(self.y)
        n = float(len(self.random_effects))
        return [v / (n + 1) for r in self.random_effects]
<|endoftext|>


<commit_before># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Base class for all slave-side build steps. """

from utils import misc
import config
import multiprocessing
import os
import shlex
import signal
import subprocess
import sys
import threading
import time
import traceback

DEFAULT_TIMEOUT = 2400
DEFAULT_NO_OUTPUT_TIMEOUT=1800

# multiprocessing.Value doesn't accept boolean types, so we have to use an int.
INT_TRUE = 1
INT_FALSE = 0
build_step_stdout_has_written = multiprocessing.Value('i', INT_FALSE)

class BuildStepWarning(Exception):
  pass

class BuildStepFailure(Exception):
  pass

class BuildStepTimeout(Exception):
  pass

class BuildStepLogger(object):
  """ Override stdout so that we can keep track of when anything has been
  logged.  This enables timeouts based on how long the process has gone without
  writing output.
  """
  def __init__(self):
    self.stdout = sys.stdout
    sys.stdout = self
    build_step_stdout_has_written.value = INT_FALSE

  def __del__(self):
    sys.stdout = self.stdout

  def fileno(self):
    return self.stdout.fileno()

  def write(self, data):
    build_step_stdout_has_written.value = INT_TRUE
    self.stdout.write(data)

  def flush(self):
    self.stdout.flush()

class BuildStep(multiprocessing.Process):
  def _PreRun(self):
    """ Optional preprocessing step for BuildSteps to override. """
    pass

  def __init__(self, args, attempts=1, timeout=DEFAULT_TIMEOUT,
               no_output_timeout=DEFAULT_NO_OUTPUT_TIMEOUT):
    """ Constructs a BuildStep instance.

    args: dictionary containing arguments to this BuildStep.
    attempts: how many times to try this BuildStep before giving up.
    timeout: maximum time allowed for this BuildStep.
    no_output_timeout: maximum time allowed for this BuildStep to run without
        any output.
    """
    multiprocessing.Process.__init__(self)
    self._args = args
    self._timeout = timeout
    self._no_output_timeout = no_output_timeout
    # Dimensions for BuildSteps which use tiling
    self.TILE_X = 256
    self.TILE_Y = 256

    self._configuration = args['configuration']
    self._gm_image_subdir = args['gm_image_subdir']
    self._builder_name = args['builder_name']
    self._target_platform = args['target_platform']
    self._revision = int(args['revision'])
    self._attempts = attempts
    # Figure out where we are going to store images generated by GM.
    self._gm_actual_basedir = os.path.join(os.pardir, os.pardir, 'gm', 'actual')
    self._gm_merge_basedir = os.path.join(os.pardir, os.pardir, 'gm', 'merge')
    self._gm_expected_dir = os.path.join(os.pardir, 'gm-expected', self._gm_image_subdir)
    self._gm_actual_dir = os.path.join(self._gm_actual_basedir,
                                       self._gm_image_subdir)
    self._gm_actual_svn_baseurl = '%s/%s' % (args['autogen_svn_baseurl'],
                                             'gm-actual')
    self._skp_dir = os.path.join(os.pardir, 'skp')
    self._autogen_svn_username_file = '.autogen_svn_username'
    self._autogen_svn_password_file = '.autogen_svn_password'
    self._make_flags = shlex.split(args['make_flags'].replace('"', ''))
    self._test_args = shlex.split(args['test_args'].replace('"', ''))
    self._gm_args = shlex.split(args['gm_args'].replace('"', ''))
    self._gm_args.append('--serialize')
    self._bench_args = shlex.split(args['bench_args'].replace('"', ''))

    # Figure out where we are going to store performance output.
    if args['perf_output_basedir'] != 'None':
      self._perf_data_dir = os.path.join(args['perf_output_basedir'],
                                         self._builder_name, 'data')
      self._perf_graphs_dir = os.path.join(args['perf_output_basedir'],
                                           self._builder_name, 'graphs')
    else:
      self._perf_data_dir = None
      self._perf_graphs_dir = None

  def _PathToBinary(self, binary):
    """ Returns the path to the given built executable. """
    return os.path.join('out', self._configuration, binary)

  def _Run(self):
    """ Code to be run in a given BuildStep.  No return value; throws exception
    on failure.  Override this method in subclasses.
    """
    raise Exception('Cannot instantiate abstract BuildStep')

  def run(self):
    """ Internal method used by multiprocess.Process. _Run is provided to be
    overridden instead of this method to ensure that this implementation always
    runs.
    """
    # If a BuildStep has exceeded its allotted time, the parent process needs to
    # be able to kill the BuildStep process AND any which it has spawned,
    # without harming itself. On posix platforms, the terminate() method is
    # insufficient; it fails to kill the subprocesses launched by this process.
    # So, we use use the setpgrp() function to set a new process group for the
    # BuildStep process and its children and call os.killpg() to kill the group.
    if os.name == 'posix':
      os.setpgrp()
    self._Run()

  def _WaitFunc(self, attempt):
    """ Waits a number of seconds depending upon the attempt number of a
    retry-able BuildStep before making the next attempt.  This can be overridden
    by subclasses and should be defined for attempt in [0, self._attempts - 1]

    This default implementation is exponential; we double the wait time with
    each attempt, starting with a 15-second pause between the first and second
    attempts.
    """
    base_secs = 15
    wait = base_secs * (2 ** attempt)
    print 'Retrying in %d seconds...' % wait
    time.sleep(wait)

  @staticmethod
  def KillBuildStep(step):
    """ Kills a running BuildStep.

    step: the running BuildStep instance to kill.
    """
    # On posix platforms, the terminate() method is insufficient; it fails to
    # kill the subprocesses launched by this process. So, we use use the
    # setpgrp() function to set a new process group for the BuildStep process
    # and its children and call os.killpg() to kill the group.
    if os.name == 'posix':
      os.killpg(os.getpgid(step.pid), signal.SIGTERM)
    elif os.name == 'nt':
      subprocess.call(['taskkill', '/F', '/T', '/PID', str(step.pid)])
    else:
      step.terminate()

  @staticmethod
  def RunBuildStep(StepType):
    """ Run a BuildStep, possibly making multiple attempts and handling
    timeouts.

    StepType: class type which subclasses BuildStep, indicating what step should
        be run. StepType should override _Run().
    """
    logger = BuildStepLogger()
    args = misc.ArgsToDict(sys.argv)
    attempt = 0
    while True:
      step = StepType(args)
      try:
        start_time = time.time()
        last_written_time = start_time
        step._PreRun()
        step.start()
        while step.is_alive():
          current_time = time.time()
          if current_time - start_time > step._timeout:
            BuildStep.KillBuildStep(step)
            raise BuildStepTimeout('Build step exceeded timeout of %d seconds' %
                                   step._timeout)
          elif current_time - last_written_time > step._no_output_timeout:
            BuildStep.KillBuildStep(step)
            raise BuildStepTimeout(
                'Build step exceeded %d seconds with no output' %
                step._no_output_timeout)
          time.sleep(1)
          if build_step_stdout_has_written.value == INT_TRUE:
            last_written_time = time.time()
        if step.exitcode == 0:
          return 0
        elif step.exitcode == config.Master.retcode_warnings:
          # A warning is considered to be an acceptable finishing state.
          return config.Master.retcode_warnings
        else:
          raise BuildStepFailure('Build step failed.')
      except:
        print traceback.format_exc()
        if attempt + 1 >= step._attempts:
          raise
      step._WaitFunc(attempt)
      attempt += 1
      print '**** %s, attempt %d ****' % (StepType.__name__, attempt + 1)
<commit_msg>Fix BuildStepWarning for subprocesses<commit_after># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Base class for all slave-side build steps. """

from utils import misc
import config
import multiprocessing
import os
import shlex
import signal
import subprocess
import sys
import threading
import time
import traceback

DEFAULT_TIMEOUT = 2400
DEFAULT_NO_OUTPUT_TIMEOUT=1800

# multiprocessing.Value doesn't accept boolean types, so we have to use an int.
INT_TRUE = 1
INT_FALSE = 0
build_step_stdout_has_written = multiprocessing.Value('i', INT_FALSE)

class BuildStepWarning(Exception):
  pass

class BuildStepFailure(Exception):
  pass

class BuildStepTimeout(Exception):
  pass

class BuildStepLogger(object):
  """ Override stdout so that we can keep track of when anything has been
  logged.  This enables timeouts based on how long the process has gone without
  writing output.
  """
  def __init__(self):
    self.stdout = sys.stdout
    sys.stdout = self
    build_step_stdout_has_written.value = INT_FALSE

  def __del__(self):
    sys.stdout = self.stdout

  def fileno(self):
    return self.stdout.fileno()

  def write(self, data):
    build_step_stdout_has_written.value = INT_TRUE
    self.stdout.write(data)

  def flush(self):
    self.stdout.flush()

class BuildStep(multiprocessing.Process):
  def _PreRun(self):
    """ Optional preprocessing step for BuildSteps to override. """
    pass

  def __init__(self, args, attempts=1, timeout=DEFAULT_TIMEOUT,
               no_output_timeout=DEFAULT_NO_OUTPUT_TIMEOUT):
    """ Constructs a BuildStep instance.

    args: dictionary containing arguments to this BuildStep.
    attempts: how many times to try this BuildStep before giving up.
    timeout: maximum time allowed for this BuildStep.
    no_output_timeout: maximum time allowed for this BuildStep to run without
        any output.
    """
    multiprocessing.Process.__init__(self)
    self._args = args
    self._timeout = timeout
    self._no_output_timeout = no_output_timeout
    # Dimensions for BuildSteps which use tiling
    self.TILE_X = 256
    self.TILE_Y = 256

    self._configuration = args['configuration']
    self._gm_image_subdir = args['gm_image_subdir']
    self._builder_name = args['builder_name']
    self._target_platform = args['target_platform']
    self._revision = int(args['revision'])
    self._attempts = attempts
    # Figure out where we are going to store images generated by GM.
    self._gm_actual_basedir = os.path.join(os.pardir, os.pardir, 'gm', 'actual')
    self._gm_merge_basedir = os.path.join(os.pardir, os.pardir, 'gm', 'merge')
    self._gm_expected_dir = os.path.join(os.pardir, 'gm-expected', self._gm_image_subdir)
    self._gm_actual_dir = os.path.join(self._gm_actual_basedir,
                                       self._gm_image_subdir)
    self._gm_actual_svn_baseurl = '%s/%s' % (args['autogen_svn_baseurl'],
                                             'gm-actual')
    self._skp_dir = os.path.join(os.pardir, 'skp')
    self._autogen_svn_username_file = '.autogen_svn_username'
    self._autogen_svn_password_file = '.autogen_svn_password'
    self._make_flags = shlex.split(args['make_flags'].replace('"', ''))
    self._test_args = shlex.split(args['test_args'].replace('"', ''))
    self._gm_args = shlex.split(args['gm_args'].replace('"', ''))
    self._gm_args.append('--serialize')
    self._bench_args = shlex.split(args['bench_args'].replace('"', ''))

    # Figure out where we are going to store performance output.
    if args['perf_output_basedir'] != 'None':
      self._perf_data_dir = os.path.join(args['perf_output_basedir'],
                                         self._builder_name, 'data')
      self._perf_graphs_dir = os.path.join(args['perf_output_basedir'],
                                           self._builder_name, 'graphs')
    else:
      self._perf_data_dir = None
      self._perf_graphs_dir = None

  def _PathToBinary(self, binary):
    """ Returns the path to the given built executable. """
    return os.path.join('out', self._configuration, binary)

  def _Run(self):
    """ Code to be run in a given BuildStep.  No return value; throws exception
    on failure.  Override this method in subclasses.
    """
    raise Exception('Cannot instantiate abstract BuildStep')

  def run(self):
    """ Internal method used by multiprocess.Process. _Run is provided to be
    overridden instead of this method to ensure that this implementation always
    runs.
    """
    # If a BuildStep has exceeded its allotted time, the parent process needs to
    # be able to kill the BuildStep process AND any which it has spawned,
    # without harming itself. On posix platforms, the terminate() method is
    # insufficient; it fails to kill the subprocesses launched by this process.
    # So, we use use the setpgrp() function to set a new process group for the
    # BuildStep process and its children and call os.killpg() to kill the group.
    if os.name == 'posix':
      os.setpgrp()
    try:
      self._Run()
    except BuildStepWarning as e:
      print e
      sys.exit(config.Master.retcode_warnings)

  def _WaitFunc(self, attempt):
    """ Waits a number of seconds depending upon the attempt number of a
    retry-able BuildStep before making the next attempt.  This can be overridden
    by subclasses and should be defined for attempt in [0, self._attempts - 1]

    This default implementation is exponential; we double the wait time with
    each attempt, starting with a 15-second pause between the first and second
    attempts.
    """
    base_secs = 15
    wait = base_secs * (2 ** attempt)
    print 'Retrying in %d seconds...' % wait
    time.sleep(wait)

  @staticmethod
  def KillBuildStep(step):
    """ Kills a running BuildStep.

    step: the running BuildStep instance to kill.
    """
    # On posix platforms, the terminate() method is insufficient; it fails to
    # kill the subprocesses launched by this process. So, we use use the
    # setpgrp() function to set a new process group for the BuildStep process
    # and its children and call os.killpg() to kill the group.
    if os.name == 'posix':
      os.killpg(os.getpgid(step.pid), signal.SIGTERM)
    elif os.name == 'nt':
      subprocess.call(['taskkill', '/F', '/T', '/PID', str(step.pid)])
    else:
      step.terminate()

  @staticmethod
  def RunBuildStep(StepType):
    """ Run a BuildStep, possibly making multiple attempts and handling
    timeouts.

    StepType: class type which subclasses BuildStep, indicating what step should
        be run. StepType should override _Run().
    """
    logger = BuildStepLogger()
    args = misc.ArgsToDict(sys.argv)
    attempt = 0
    while True:
      step = StepType(args)
      try:
        start_time = time.time()
        last_written_time = start_time
        step._PreRun()
        step.start()
        while step.is_alive():
          current_time = time.time()
          if current_time - start_time > step._timeout:
            BuildStep.KillBuildStep(step)
            raise BuildStepTimeout('Build step exceeded timeout of %d seconds' %
                                   step._timeout)
          elif current_time - last_written_time > step._no_output_timeout:
            BuildStep.KillBuildStep(step)
            raise BuildStepTimeout(
                'Build step exceeded %d seconds with no output' %
                step._no_output_timeout)
          time.sleep(1)
          if build_step_stdout_has_written.value == INT_TRUE:
            last_written_time = time.time()
        if step.exitcode == 0:
          return 0
        elif step.exitcode == config.Master.retcode_warnings:
          # A warning is considered to be an acceptable finishing state.
          return config.Master.retcode_warnings
        else:
          raise BuildStepFailure('Build step failed.')
      except:
        print traceback.format_exc()
        if attempt + 1 >= step._attempts:
          raise
      step._WaitFunc(attempt)
      attempt += 1
      print '**** %s, attempt %d ****' % (StepType.__name__, attempt + 1)
<|endoftext|>


<commit_before>import logging, threading, queue

from idms.lib.assertions.exceptions import SatisfactionError
from collections import defaultdict
from libdlt.depot import Depot
from libdlt.protocol import factory

log = logging.getLogger('libdlt.utils')
class ExnodeInfo(object):
    def __init__(self, ex, remote_validate=False):
        class _view(object):
            def __init__(self): self._size, self._chunks = ex.size, [[0,0]]
            def fill(self, o, s):
                self._chunks = list(sorted(self._chunks + [[o,o+s]]))
                new_chunks = [[0,0]]
                for chunk in self._chunks:
                    if chunk[0] <= new_chunks[-1][1]: new_chunks[-1][1] = chunk[1]
                    else: new_chunks.append(chunk)
                self._chunks = new_chunks
            @property
            def is_complete(self):
                return self._chunks[0][-1] == self._size
            @property
            def missing(self):
                result = [[self._chunks[i][1], self._chunks[i+1][0]] for i in range(len(self._chunks) - 1)]
                return result if self._chunks[-1][1] == ex.size else result + [[self._chunks[-1][1], ex.size]]

            def valid(self, offset):
                return any([c[0] <= offset < c[1] for c in self._chunks])

        self._allocs, self._views = [], defaultdict(_view)
        allocs = sorted(ex.extents, key=lambda x: x.offset)
        self._meta = self._validate() if remote_validate else defaultdict(lambda: True)
        for e in self._allocs:
            if self._meta[e]:
                try:
                    self._views[e.location].fill(e.offset, e.size)
                    self._allocs.append(e)
                    self._meta[e] = metadata
                except AttributeError as exp: log.warn("Bad extent - {}".format(e.id))

    @property
    def views(self):
        return self._views.items()

    def _validate(self):
        def run(x):
            _proxy = factory.makeProxy(x)
            if not hasattr(x, 'depot'): x.depot = Depot(x.location)
            try: return (x, _proxy.probe(x, timeout=0.025))
            except Exception as e:
                log.warn("Failed to connect with allocation - " + x.location)
                return (x, False)

        threads = []
        for e in self._allocs:
            t = threading.Thread(target=run, x=e)
            threads.append(t)
            t.daemon = True
            t.start()
        return dict([t.join() for t in threads])

    def is_complete(self, view=None):
        if view: return view in self._views and self._views[view].is_complete
        else: return any([v.is_complete for v in self._views.values()])

    def replicas_at(self, offset):
        return [v for v in self._views.values() if v.valid(offset)]

    def missing(self, view):
        return self._views[view].missing

    def allocs_in(self, start, end):
        for alloc in self._allocs:
            if alloc.offset < end and alloc.offset + alloc.size > start:
                yield alloc

    def fill(self, view):
        result, todo = [], self._views[view].missing
        log.debug(f"Fill requirements - {todo}")
        if not todo:
            raise SatisfactionError("No satisfying extents available to fill exnode")
        else:
            for alloc in self._allocs:
                if todo[0][0] >= todo[0][1]: todo.pop(0)
                if not todo: break
                if alloc.offset + alloc.size > todo[0][0]:
                    result.append(alloc)
                    todo[0][0] = alloc.offset + alloc.size
        log.debug(f"Recommnding fill - {[a.id for a in result]}")
        return result

    def plan_download(self, start=0, end=None):
        end = min(end or self._chunks[0][1], self._chunks[0][1])
        for alloc in self._allocs:
            if alloc.offset <= start and alloc.offset + alloc.size > start:
                yield alloc
                start  = alloc.offset + alloc.size
            if start >= end: break

    def __getitem__(self, e):
        return self._meta[e]
<commit_msg>Removed idms references<commit_after>import logging, threading, queue

from collections import defaultdict
from libdlt.depot import Depot
from libdlt.protocol import factory

log = logging.getLogger('libdlt.utils')
class ExnodeInfo(object):
    def __init__(self, ex, remote_validate=False):
        class _view(object):
            def __init__(self): self._size, self._chunks = ex.size, [[0,0]]
            def fill(self, o, s):
                self._chunks = list(sorted(self._chunks + [[o,o+s]]))
                new_chunks = [[0,0]]
                for chunk in self._chunks:
                    if chunk[0] <= new_chunks[-1][1]: new_chunks[-1][1] = chunk[1]
                    else: new_chunks.append(chunk)
                self._chunks = new_chunks
            @property
            def is_complete(self):
                return self._chunks[0][-1] == self._size
            @property
            def missing(self):
                result = [[self._chunks[i][1], self._chunks[i+1][0]] for i in range(len(self._chunks) - 1)]
                return result if self._chunks[-1][1] == ex.size else result + [[self._chunks[-1][1], ex.size]]

            def valid(self, offset):
                return any([c[0] <= offset < c[1] for c in self._chunks])

        self._allocs, self._views = [], defaultdict(_view)
        allocs = sorted(ex.extents, key=lambda x: x.offset)
        self._meta = self._validate() if remote_validate else defaultdict(lambda: True)
        for e in self._allocs:
            if self._meta[e]:
                try:
                    self._views[e.location].fill(e.offset, e.size)
                    self._allocs.append(e)
                    self._meta[e] = metadata
                except AttributeError as exp: log.warn("Bad extent - {}".format(e.id))

    @property
    def views(self):
        return self._views.items()

    def _validate(self):
        def run(x):
            _proxy = factory.makeProxy(x)
            if not hasattr(x, 'depot'): x.depot = Depot(x.location)
            try: return (x, _proxy.probe(x, timeout=0.025))
            except Exception as e:
                log.warn("Failed to connect with allocation - " + x.location)
                return (x, False)

        threads = []
        for e in self._allocs:
            t = threading.Thread(target=run, x=e)
            threads.append(t)
            t.daemon = True
            t.start()
        return dict([t.join() for t in threads])

    def is_complete(self, view=None):
        if view: return view in self._views and self._views[view].is_complete
        else: return any([v.is_complete for v in self._views.values()])

    def replicas_at(self, offset):
        return [v for v in self._views.values() if v.valid(offset)]

    def missing(self, view):
        return self._views[view].missing

    def allocs_in(self, start, end):
        for alloc in self._allocs:
            if alloc.offset < end and alloc.offset + alloc.size > start:
                yield alloc

    def fill(self, view):
        result, todo = [], self._views[view].missing
        log.debug(f"Fill requirements - {todo}")
        if not todo:
            raise ValueError("No satisfying extents available to fill exnode")
        else:
            for alloc in self._allocs:
                if todo[0][0] >= todo[0][1]: todo.pop(0)
                if not todo: break
                if alloc.offset + alloc.size > todo[0][0]:
                    result.append(alloc)
                    todo[0][0] = alloc.offset + alloc.size
        log.debug(f"Recommnding fill - {[a.id for a in result]}")
        return result

    def plan_download(self, start=0, end=None):
        end = min(end or self._chunks[0][1], self._chunks[0][1])
        for alloc in self._allocs:
            if alloc.offset <= start and alloc.offset + alloc.size > start:
                yield alloc
                start  = alloc.offset + alloc.size
            if start >= end: break

    def __getitem__(self, e):
        return self._meta[e]
<|endoftext|>