<commit_before>#
# Newfies-Dialer License
# http://www.newfies-dialer.org
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Copyright (C) 2011-2013 Star2Billing S.L.
#
# The Initial Developer of the Original Code is
# Arezqui Belaid <info@star2billing.com>
#

from django.contrib import admin
from django.contrib.admin.options import IncorrectLookupParameters
from django.contrib.admin.views.main import ERROR_FLAG
from django.conf.urls import patterns
from django.conf import settings
from django.core.urlresolvers import reverse
from django.template import RequestContext
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import render_to_response
from django.utils.translation import ugettext_lazy as _
from django.utils.translation import ungettext
from django.db.models import Sum, Avg, Count
from dialer_cdr.models import Callrequest, VoIPCall
from dialer_cdr.forms import AdminVoipSearchForm
from dialer_cdr.function_def import voipcall_record_common_fun, \
    voipcall_search_admin_form_fun
from common.common_functions import getvar
from common.app_label_renamer import AppLabelRenamer
from genericadmin.admin import GenericAdminModelAdmin
from datetime import datetime
import tablib

AppLabelRenamer(native_app_label=u'dialer_cdr', app_label=_('Dialer CDR')).main()
APP_LABEL = _('VoIP report')

class CallrequestAdmin(GenericAdminModelAdmin):
    """Allows the administrator to view and modify certain attributes
    of a Callrequest."""
    content_type_whitelist = ('survey/survey', )
    fieldsets = (
        (_('standard options').capitalize(), {
            'fields': ('user', 'request_uuid', 'call_time', 'campaign',
                       'status', 'hangup_cause', 'callerid', 'phone_number',
                       'timeout', 'timelimit', 'call_type', 'aleg_gateway',
                       'content_type', 'object_id', ),
        }),
        (_('advanced options').capitalize(), {
            'classes': ('collapse',),
            'fields': ('extra_data', 'extra_dial_string', 'subscriber', 'completed'),
        }),
    )
    #If we try to display user / content_type low the performance
    list_display = ('id', 'request_uuid', 'aleg_uuid', 'call_time',
                    'status', 'callerid', 'phone_number', 'call_type',
                    'completed', 'num_attempt', 'last_attempt_time',)
    list_display_links = ('id', 'request_uuid', )
    list_filter = ['callerid', 'call_time', 'status', 'call_type', 'campaign']
    ordering = ('-id', )
    search_fields = ('request_uuid', )

admin.site.register(Callrequest, CallrequestAdmin)

class VoIPCallAdmin(admin.ModelAdmin):
    """Allows the administrator to view and modify certain attributes
    of a VoIPCall."""
    can_add = False
    detail_title = _("call report").title()
    list_display = ('id', 'leg_type', 'callid', 'callerid', 'phone_number',
                    'starting_date', 'min_duration', 'billsec', 'disposition',
                    'hangup_cause', 'hangup_cause_q850')
    valid_lookups = ('callrequest__campaign_id',)
    if settings.AMD:
        list_display += ('amd_status',)
    ordering = ('-id', )

    def lookup_allowed(self, lookup, *args, **kwargs):
        if lookup.startswith(self.valid_lookups):
            return True
        return super(VoIPCallAdmin, self).lookup_allowed(lookup, *args, **kwargs)

    def user_link(self, obj):
        """User link to user profile"""

        if obj.user.is_staff:
            url = reverse('admin:auth_staff_change', args=(obj.user_id,))
        else:
            url = reverse('admin:auth_customer_change', args=(obj.user_id,))
        return '%s' % (url, obj.user)
    user_link.allow_tags = True
    user_link.short_description = _('user').capitalize()

    def used_gateway_link(self, obj):
        """Used gateway link to edit gateway detail"""
        if obj.used_gateway:
            url = reverse('admin:dialer_gateway_gateway_change',
                          args=(obj.used_gateway.id,))
            return '%s' % (url, obj.used_gateway)
    used_gateway_link.allow_tags = True
    used_gateway_link.short_description = _('gateway used').capitalize()

    def has_add_permission(self, request):
        """Remove add permission on VoIP Call Report model

        **Logic Description**:

            * Override django admin has_add_permission method to remove add
              permission on VoIP Call Report model
        """
        if not self.can_add:
            return False
        return super(VoIPCallAdmin, self).has_add_permission(request)

    def get_urls(self):
        urls = super(VoIPCallAdmin, self).get_urls()
        my_urls = patterns('',
            (r'^$', self.admin_site.admin_view(self.changelist_view)),
            (r'^voip_daily_report/$',
             self.admin_site.admin_view(self.voip_daily_report)),
            (r'^export_voip_report/$',
             self.admin_site.admin_view(self.export_voip_report)),
        )
        return my_urls + urls

    def changelist_view(self, request, extra_context=None):
        """
        Override changelist_view method of django-admin for search parameters

        **Attributes**:

            * ``form`` - AdminVoipSearchForm
            * ``template`` - admin/dialer_cdr/voipcall/change_list.html

        **Logic Description**:

            * VoIP report Record Listing with search option & Daily Call Report
              search Parameters: by date, by status and by billed.
        """
        opts = VoIPCall._meta
        query_string = ''
        form = AdminVoipSearchForm()
        if request.method == 'POST':
            # Session variable get record set with searched option into export file
            request.session['admin_voipcall_record_kwargs'] = voipcall_record_common_fun(request)

            query_string = voipcall_search_admin_form_fun(request)
            return HttpResponseRedirect("/admin/%s/%s/?%s"
                % (opts.app_label, opts.object_name.lower(), query_string))
        else:
            status = ''
            from_date = ''
            to_date = ''
            campaign_id = ''
            leg_type = ''

            from_date = getvar(request, 'starting_date__gte')
            to_date = getvar(request, 'starting_date__lte')[0:10]
            status = getvar(request, 'disposition__exact')
            campaign_id = getvar(request, 'callrequest__campaign_id')
            leg_type = getvar(request, 'leg_type__exact')

            form = AdminVoipSearchForm(initial={'status': status,
                                                'from_date': from_date,
                                                'to_date': to_date,
                                                'campaign_id': campaign_id,
                                                'leg_type': leg_type})

        ChangeList = self.get_changelist(request)
        try:
            cl = ChangeList(request, self.model, self.list_display,
                self.list_display_links, self.list_filter, self.date_hierarchy,
                self.search_fields, self.list_select_related,
                self.list_per_page, self.list_max_show_all, self.list_editable,
                self)
        except IncorrectLookupParameters:
            if ERROR_FLAG in request.GET.keys():
                return render_to_response('admin/invalid_setup.html',
                        {'title': _('Database error')})
            return HttpResponseRedirect('%s?%s=1' % (request.path, ERROR_FLAG))

        if request.META['QUERY_STRING'] == '':
            # Default
            # Session variable get record set with searched option into export file
            request.session['admin_voipcall_record_kwargs'] = voipcall_record_common_fun(request)

            query_string = voipcall_search_admin_form_fun(request)
            return HttpResponseRedirect("/admin/%s/%s/?%s"
                % (opts.app_label, opts.object_name.lower(), query_string))

        cl.formset = None

        selection_note_all = ungettext('%(total_count)s selected',
            'All %(total_count)s selected', cl.result_count)

        ctx = {
            'selection_note': _('0 of %(cnt)s selected') % {'cnt': len(cl.result_list)},
            'selection_note_all': selection_note_all % {'total_count': cl.result_count},
            'cl': cl,
            'form': form,
            'opts': opts,
            'model_name': opts.object_name.lower(),
            'app_label': APP_LABEL,
            'title': _('call report'),
        }
        return super(VoIPCallAdmin, self).changelist_view(request, extra_context=ctx)

    def voip_daily_report(self, request):
        opts = VoIPCall._meta
        kwargs = {}
        if request.method == 'POST':
            form = AdminVoipSearchForm(request.POST)
            kwargs = voipcall_record_common_fun(request)
        else:
            kwargs = voipcall_record_common_fun(request)
            tday = datetime.today()
            form = AdminVoipSearchForm(initial={"from_date": tday.strftime("%Y-%m-%d"),
                                                "to_date": tday.strftime("%Y-%m-%d")})
            if len(kwargs) == 0:
                kwargs['starting_date__gte'] = datetime(tday.year, tday.month, tday.day,
                                                        0, 0, 0, 0)

        select_data = {"starting_date": "SUBSTR(CAST(starting_date as CHAR(30)),1,10)"}
        # Get Total Records from VoIPCall Report table for Daily Call Report
        total_data = VoIPCall.objects.extra(select=select_data)\
            .values('starting_date')\
            .filter(**kwargs)\
            .annotate(Count('starting_date'))\
            .annotate(Sum('duration'))\
            .annotate(Avg('duration'))\
            .order_by('-starting_date')

        # Following code will count total voip calls, duration
        if total_data:
            max_duration = max([x['duration__sum'] for x in total_data])
            total_duration = sum([x['duration__sum'] for x in total_data])
            total_calls = sum([x['starting_date__count'] for x in total_data])
            total_avg_duration = (sum([x['duration__avg']
                    for x in total_data])) / total_calls
        else:
            max_duration = 0
            total_duration = 0
            total_calls = 0
            total_avg_duration = 0

        ctx = RequestContext(request, {
            'form': form,
            'total_data': total_data,
            'total_duration': total_duration,
            'total_calls': total_calls,
            'total_avg_duration': total_avg_duration,
            'max_duration': max_duration,
            'opts': opts,
            'model_name': opts.object_name.lower(),
            'app_label': APP_LABEL,
            'title': _('call aggregate report'),
        })

        return render_to_response('admin/dialer_cdr/voipcall/voip_report.html',
               context_instance=ctx)

    def export_voip_report(self, request):
        """Export a CSV file of VoIP call records

        **Important variable**:

            * request.session['admin_voipcall_record_kwargs'] - stores voipcall kwargs

        **Exported fields**: [user, callid, callerid, phone_number,
                              starting_date, duration, disposition,
                              used_gateway]
        """
        # get the response object, this can be used as a stream.
        format = request.GET['format']
        response = HttpResponse(mimetype='text/' + format)
        # force download.
        response['Content-Disposition'] = 'attachment;filename=export.' + format

        # super(VoIPCall_ReportAdmin, self).queryset(request)
        kwargs = request.session['admin_voipcall_record_kwargs']
        qs = VoIPCall.objects.filter(**kwargs)

        amd_status = ''
        if settings.AMD:
            amd_status = 'amd_status'

        headers = ('user', 'callid', 'callerid', 'phone_number',
                   'starting_date', 'duration', 'billsec',
                   'disposition', 'used_gateway', amd_status)

        list_val = []
        for i in qs:
            gateway_used = i.used_gateway.name if i.used_gateway else ''
            amd_status = i.amd_status if settings.AMD else ''

            starting_date = i.starting_date
            if format == 'json':
                starting_date = str(i.starting_date)

            list_val.append((i.user.username,
                             i.callid,
                             i.callerid,
                             i.phone_number,
                             starting_date,
                             i.duration,
                             i.billsec,
                             i.disposition,
                             gateway_used,
                             amd_status))

        data = tablib.Dataset(*list_val, headers=headers)

        if format == 'xls':
            response.write(data.xls)

        if format == 'csv':
            response.write(data.csv)

        if format == 'json':
            response.write(data.json)

        return response

admin.site.register(VoIPCall, VoIPCallAdmin)
<commit_msg>issue<commit_after>#
# Newfies-Dialer License
# http://www.newfies-dialer.org
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Copyright (C) 2011-2013 Star2Billing S.L.
#
# The Initial Developer of the Original Code is
# Arezqui Belaid <info@star2billing.com>
#

from django.contrib import admin
from django.contrib.admin.options import IncorrectLookupParameters
from django.contrib.admin.views.main import ERROR_FLAG
from django.conf.urls import patterns
from django.conf import settings
from django.core.urlresolvers import reverse
from django.template import RequestContext
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import render_to_response
from django.utils.translation import ugettext_lazy as _
from django.utils.translation import ungettext
from django.db.models import Sum, Avg, Count
from dialer_cdr.models import Callrequest, VoIPCall
from dialer_cdr.forms import AdminVoipSearchForm
from dialer_cdr.function_def import voipcall_record_common_fun, \
    voipcall_search_admin_form_fun
from common.common_functions import getvar
from common.app_label_renamer import AppLabelRenamer
from genericadmin.admin import GenericAdminModelAdmin
from datetime import datetime
import tablib

# AppLabelRenamer(native_app_label=u'dialer_cdr', app_label=_('Dialer CDR')).main()
APP_LABEL = _('VoIP report')

class CallrequestAdmin(GenericAdminModelAdmin):
    """Allows the administrator to view and modify certain attributes
    of a Callrequest."""
    content_type_whitelist = ('survey/survey', )
    fieldsets = (
        (_('standard options').capitalize(), {
            'fields': ('user', 'request_uuid', 'call_time', 'campaign',
                       'status', 'hangup_cause', 'callerid', 'phone_number',
                       'timeout', 'timelimit', 'call_type', 'aleg_gateway',
                       'content_type', 'object_id', ),
        }),
        (_('advanced options').capitalize(), {
            'classes': ('collapse',),
            'fields': ('extra_data', 'extra_dial_string', 'subscriber', 'completed'),
        }),
    )
    #If we try to display user / content_type low the performance
    list_display = ('id', 'request_uuid', 'aleg_uuid', 'call_time',
                    'status', 'callerid', 'phone_number', 'call_type',
                    'completed', 'num_attempt', 'last_attempt_time',)
    list_display_links = ('id', 'request_uuid', )
    list_filter = ['callerid', 'call_time', 'status', 'call_type', 'campaign']
    ordering = ('-id', )
    search_fields = ('request_uuid', )

admin.site.register(Callrequest, CallrequestAdmin)

class VoIPCallAdmin(admin.ModelAdmin):
    """Allows the administrator to view and modify certain attributes
    of a VoIPCall."""
    can_add = False
    detail_title = _("call report").title()
    list_display = ('id', 'leg_type', 'callid', 'callerid', 'phone_number',
                    'starting_date', 'min_duration', 'billsec', 'disposition',
                    'hangup_cause', 'hangup_cause_q850')
    valid_lookups = ('callrequest__campaign_id',)
    if settings.AMD:
        list_display += ('amd_status',)
    ordering = ('-id', )

    def lookup_allowed(self, lookup, *args, **kwargs):
        if lookup.startswith(self.valid_lookups):
            return True
        return super(VoIPCallAdmin, self).lookup_allowed(lookup, *args, **kwargs)

    def user_link(self, obj):
        """User link to user profile"""

        if obj.user.is_staff:
            url = reverse('admin:auth_staff_change', args=(obj.user_id,))
        else:
            url = reverse('admin:auth_customer_change', args=(obj.user_id,))
        return '%s' % (url, obj.user)
    user_link.allow_tags = True
    user_link.short_description = _('user').capitalize()

    def used_gateway_link(self, obj):
        """Used gateway link to edit gateway detail"""
        if obj.used_gateway:
            url = reverse('admin:dialer_gateway_gateway_change',
                          args=(obj.used_gateway.id,))
            return '%s' % (url, obj.used_gateway)
    used_gateway_link.allow_tags = True
    used_gateway_link.short_description = _('gateway used').capitalize()

    def has_add_permission(self, request):
        """Remove add permission on VoIP Call Report model

        **Logic Description**:

            * Override django admin has_add_permission method to remove add
              permission on VoIP Call Report model
        """
        if not self.can_add:
            return False
        return super(VoIPCallAdmin, self).has_add_permission(request)

    def get_urls(self):
        urls = super(VoIPCallAdmin, self).get_urls()
        my_urls = patterns('',
            (r'^$', self.admin_site.admin_view(self.changelist_view)),
            (r'^voip_daily_report/$',
             self.admin_site.admin_view(self.voip_daily_report)),
            (r'^export_voip_report/$',
             self.admin_site.admin_view(self.export_voip_report)),
        )
        return my_urls + urls

    def changelist_view(self, request, extra_context=None):
        """
        Override changelist_view method of django-admin for search parameters

        **Attributes**:

            * ``form`` - AdminVoipSearchForm
            * ``template`` - admin/dialer_cdr/voipcall/change_list.html

        **Logic Description**:

            * VoIP report Record Listing with search option & Daily Call Report
              search Parameters: by date, by status and by billed.
        """
        opts = VoIPCall._meta
        query_string = ''
        form = AdminVoipSearchForm()
        if request.method == 'POST':
            # Session variable get record set with searched option into export file
            request.session['admin_voipcall_record_kwargs'] = voipcall_record_common_fun(request)

            query_string = voipcall_search_admin_form_fun(request)
            return HttpResponseRedirect("/admin/%s/%s/?%s"
                % (opts.app_label, opts.object_name.lower(), query_string))
        else:
            status = ''
            from_date = ''
            to_date = ''
            campaign_id = ''
            leg_type = ''

            from_date = getvar(request, 'starting_date__gte')
            to_date = getvar(request, 'starting_date__lte')[0:10]
            status = getvar(request, 'disposition__exact')
            campaign_id = getvar(request, 'callrequest__campaign_id')
            leg_type = getvar(request, 'leg_type__exact')

            form = AdminVoipSearchForm(initial={'status': status,
                                                'from_date': from_date,
                                                'to_date': to_date,
                                                'campaign_id': campaign_id,
                                                'leg_type': leg_type})

        ChangeList = self.get_changelist(request)
        try:
            cl = ChangeList(request, self.model, self.list_display,
                self.list_display_links, self.list_filter, self.date_hierarchy,
                self.search_fields, self.list_select_related,
                self.list_per_page, self.list_max_show_all, self.list_editable,
                self)
        except IncorrectLookupParameters:
            if ERROR_FLAG in request.GET.keys():
                return render_to_response('admin/invalid_setup.html',
                        {'title': _('Database error')})
            return HttpResponseRedirect('%s?%s=1' % (request.path, ERROR_FLAG))

        if request.META['QUERY_STRING'] == '':
            # Default
            # Session variable get record set with searched option into export file
            request.session['admin_voipcall_record_kwargs'] = voipcall_record_common_fun(request)

            query_string = voipcall_search_admin_form_fun(request)
            return HttpResponseRedirect("/admin/%s/%s/?%s"
                % (opts.app_label, opts.object_name.lower(), query_string))

        cl.formset = None

        selection_note_all = ungettext('%(total_count)s selected',
            'All %(total_count)s selected', cl.result_count)

        ctx = {
            'selection_note': _('0 of %(cnt)s selected') % {'cnt': len(cl.result_list)},
            'selection_note_all': selection_note_all % {'total_count': cl.result_count},
            'cl': cl,
            'form': form,
            'opts': opts,
            'model_name': opts.object_name.lower(),
            'app_label': APP_LABEL,
            'title': _('call report'),
        }
        return super(VoIPCallAdmin, self).changelist_view(request, extra_context=ctx)

    def voip_daily_report(self, request):
        opts = VoIPCall._meta
        kwargs = {}
        if request.method == 'POST':
            form = AdminVoipSearchForm(request.POST)
            kwargs = voipcall_record_common_fun(request)
        else:
            kwargs = voipcall_record_common_fun(request)
            tday = datetime.today()
            form = AdminVoipSearchForm(initial={"from_date": tday.strftime("%Y-%m-%d"),
                                                "to_date": tday.strftime("%Y-%m-%d")})
            if len(kwargs) == 0:
                kwargs['starting_date__gte'] = datetime(tday.year, tday.month, tday.day,
                                                        0, 0, 0, 0)

        select_data = {"starting_date": "SUBSTR(CAST(starting_date as CHAR(30)),1,10)"}
        # Get Total Records from VoIPCall Report table for Daily Call Report
        total_data = VoIPCall.objects.extra(select=select_data)\
            .values('starting_date')\
            .filter(**kwargs)\
            .annotate(Count('starting_date'))\
            .annotate(Sum('duration'))\
            .annotate(Avg('duration'))\
            .order_by('-starting_date')

        # Following code will count total voip calls, duration
        if total_data:
            max_duration = max([x['duration__sum'] for x in total_data])
            total_duration = sum([x['duration__sum'] for x in total_data])
            total_calls = sum([x['starting_date__count'] for x in total_data])
            total_avg_duration = (sum([x['duration__avg']
                    for x in total_data])) / total_calls
        else:
            max_duration = 0
            total_duration = 0
            total_calls = 0
            total_avg_duration = 0

        ctx = RequestContext(request, {
            'form': form,
            'total_data': total_data,
            'total_duration': total_duration,
            'total_calls': total_calls,
            'total_avg_duration': total_avg_duration,
            'max_duration': max_duration,
            'opts': opts,
            'model_name': opts.object_name.lower(),
            'app_label': APP_LABEL,
            'title': _('call aggregate report'),
        })

        return render_to_response('admin/dialer_cdr/voipcall/voip_report.html',
               context_instance=ctx)

    def export_voip_report(self, request):
        """Export a CSV file of VoIP call records

        **Important variable**:

            * request.session['admin_voipcall_record_kwargs'] - stores voipcall kwargs

        **Exported fields**: [user, callid, callerid, phone_number,
                              starting_date, duration, disposition,
                              used_gateway]
        """
        # get the response object, this can be used as a stream.
        format = request.GET['format']
        response = HttpResponse(mimetype='text/' + format)
        # force download.
        response['Content-Disposition'] = 'attachment;filename=export.' + format

        # super(VoIPCall_ReportAdmin, self).queryset(request)
        kwargs = request.session['admin_voipcall_record_kwargs']
        qs = VoIPCall.objects.filter(**kwargs)

        amd_status = ''
        if settings.AMD:
            amd_status = 'amd_status'

        headers = ('user', 'callid', 'callerid', 'phone_number',
                   'starting_date', 'duration', 'billsec',
                   'disposition', 'used_gateway', amd_status)

        list_val = []
        for i in qs:
            gateway_used = i.used_gateway.name if i.used_gateway else ''
            amd_status = i.amd_status if settings.AMD else ''

            starting_date = i.starting_date
            if format == 'json':
                starting_date = str(i.starting_date)

            list_val.append((i.user.username,
                             i.callid,
                             i.callerid,
                             i.phone_number,
                             starting_date,
                             i.duration,
                             i.billsec,
                             i.disposition,
                             gateway_used,
                             amd_status))

        data = tablib.Dataset(*list_val, headers=headers)

        if format == 'xls':
            response.write(data.xls)

        if format == 'csv':
            response.write(data.csv)

        if format == 'json':
            response.write(data.json)

        return response

admin.site.register(VoIPCall, VoIPCallAdmin)
<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-
#
# Desc:    This file is part of the ecromedos Document Preparation System
# Author:  Tobias Koch <tobias@tobijk.de>
# License: MIT
# URL:     http://www.ecromedos.net
#

book = """<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE book SYSTEM "http://www.ecromedos.net/dtd/2.0/ecromedos.dtd">
<book lang="en_US" fontsize="12pt" papersize="a4paper" div="14" bcor="0cm"
    secnumdepth="3" secsplitdepth="1">

        <subject>Subject</subject>
        <title>Title</title>
        <subtitle>Subtitle</subtitle>
        <author>Author</author>
        <date>Date</date>
        <publisher>Publisher</publisher>

    <legal>

        Legal info.

    </legal>

    <make-toc depth="3" lof="no" lot="no" lol="no"/>

    <preface>
        <title>Preface</title>

            First paragraph in first preface...

    </preface>

    <chapter>
        <title>Chapter Title</title>

            First paragraph in first chapter...

    </chapter>

    <appendix>
        <title>First Appendix</title>

            First paragraph in first appendix...

    </appendix>

    <make-glossary locale="en_US" alphabet="[Symbols],A,B,C,D,E,F,G,H,I,J,
      K,L,M,N,O,P,Q,R,S,T,U,V,W,X,Y,Z" tocentry="yes"/>

    <biblio number="yes">
        <bibitem id="EXAMPLE" label="LABEL">Bibliography entry...</bibitem>
    </biblio>

    <make-index group="default" locale="en_US" alphabet="[Symbols],A,B,C,
        D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U,V,W,X,Y,Z" tocentry="yes"
        columns="2"/>

</book>
"""

report = """<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE book SYSTEM "http://www.ecromedos.net/dtd/2.0/ecromedos.dtd">
<report lang="en_US" fontsize="12pt" papersize="a4paper" div="14" bcor="0cm"
    secnumdepth="3" secsplitdepth="1">

        <subject>Subject</subject>
        <title>Title</title>
        <subtitle>Subtitle</subtitle>
        <author>Author</author>
        <date>Date</date>
        <publisher>Publisher</publisher>

    <legal>

        Legal info.

    </legal>

    <make-toc depth="3" lof="no" lot="no" lol="no"/>

    <preface>
        <title>Preface</title>

            First paragraph in first preface...

    </preface>

    <chapter>
        <title>Chapter Title</title>

            First paragraph in first chapter...

    </chapter>

    <appendix>
        <title>First Appendix</title>

            First paragraph in first appendix...

    </appendix>

    <make-glossary locale="en_US" alphabet="[Symbols],A,B,C,D,E,F,G,H,I,J,
      K,L,M,N,O,P,Q,R,S,T,U,V,W,X,Y,Z" tocentry="yes"/>

    <biblio number="yes">
        <bibitem id="EXAMPLE" label="LABEL">Bibliography entry...</bibitem>
    </biblio>

    <make-index group="default" locale="en_US" alphabet="[Symbols],A,B,C,
        D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U,V,W,X,Y,Z" tocentry="yes"
        columns="2"/>

</report>
"""

article = """<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article SYSTEM "http://www.ecromedos.net/dtd/2.0/ecromedos.dtd">

        <subject>Subject</subject>
        <title>Title</title>
        <subtitle>Subtitle</subtitle>
        <author>Author</author>
        <date>Date</date>
        <publisher>Publisher</publisher>

    <make-toc depth="3" lof="no" lot="no" lol="no"/>

    <abstract>

            Document summary...

    </abstract>

        <title>Section Title</title>

            First paragraph in first section...

    <biblio number="yes">
        <bibitem id="EXAMPLE" label="LABEL">Bibliography entry...</bibitem>
    </biblio>

"""

<commit_msg>Templates use DTD v3<commit_after># -*- coding: UTF-8 -*-
#
# Desc:    This file is part of the ecromedos Document Preparation System
# Author:  Tobias Koch <tobias@tobijk.de>
# License: MIT
# URL:     http://www.ecromedos.net
#

book = """<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE book SYSTEM "http://www.ecromedos.net/dtd/3.0/ecromedos.dtd">
<book lang="en_US" fontsize="12pt" papersize="a4paper" div="14" bcor="0cm"
    secnumdepth="3" secsplitdepth="1">

        <subject>Subject</subject>
        <title>Title</title>
        <subtitle>Subtitle</subtitle>
        <author>Author</author>
        <date>Date</date>
        <publisher>Publisher</publisher>

    <legal>

        Legal info.

    </legal>

    <make-toc depth="3" lof="no" lot="no" lol="no"/>

    <preface>
        <title>Preface</title>

            First paragraph in first preface...

    </preface>

    <chapter>
        <title>Chapter Title</title>

            First paragraph in first chapter...

    </chapter>

    <appendix>
        <title>First Appendix</title>

            First paragraph in first appendix...

    </appendix>

    <make-glossary locale="en_US" alphabet="[Symbols],A,B,C,D,E,F,G,H,I,J,
      K,L,M,N,O,P,Q,R,S,T,U,V,W,X,Y,Z" tocentry="yes"/>

    <biblio number="yes">
        <bibitem id="EXAMPLE" label="LABEL">Bibliography entry...</bibitem>
    </biblio>

    <make-index group="default" locale="en_US" alphabet="[Symbols],A,B,C,
        D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U,V,W,X,Y,Z" tocentry="yes"
        columns="2"/>

</book>
"""

report = """<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE book SYSTEM "http://www.ecromedos.net/dtd/3.0/ecromedos.dtd">
<report lang="en_US" fontsize="12pt" papersize="a4paper" div="14" bcor="0cm"
    secnumdepth="3" secsplitdepth="1">

        <subject>Subject</subject>
        <title>Title</title>
        <subtitle>Subtitle</subtitle>
        <author>Author</author>
        <date>Date</date>
        <publisher>Publisher</publisher>

    <legal>

        Legal info.

    </legal>

    <make-toc depth="3" lof="no" lot="no" lol="no"/>

    <preface>
        <title>Preface</title>

            First paragraph in first preface...

    </preface>

    <chapter>
        <title>Chapter Title</title>

            First paragraph in first chapter...

    </chapter>

    <appendix>
        <title>First Appendix</title>

            First paragraph in first appendix...

    </appendix>

    <make-glossary locale="en_US" alphabet="[Symbols],A,B,C,D,E,F,G,H,I,J,
      K,L,M,N,O,P,Q,R,S,T,U,V,W,X,Y,Z" tocentry="yes"/>

    <biblio number="yes">
        <bibitem id="EXAMPLE" label="LABEL">Bibliography entry...</bibitem>
    </biblio>

    <make-index group="default" locale="en_US" alphabet="[Symbols],A,B,C,
        D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U,V,W,X,Y,Z" tocentry="yes"
        columns="2"/>

</report>
"""

article = """<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article SYSTEM "http://www.ecromedos.net/dtd/3.0/ecromedos.dtd">

        <subject>Subject</subject>
        <title>Title</title>
        <subtitle>Subtitle</subtitle>
        <author>Author</author>
        <date>Date</date>
        <publisher>Publisher</publisher>

    <make-toc depth="3" lof="no" lot="no" lol="no"/>

    <abstract>

            Document summary...

    </abstract>

        <title>Section Title</title>

            First paragraph in first section...

    <biblio number="yes">
        <bibitem id="EXAMPLE" label="LABEL">Bibliography entry...</bibitem>
    </biblio>

"""

<|endoftext|>


<commit_before>"""
Clustering of LINEAR data
-------------------------
"""
# Author: Jake VanderPlas <vanderplas@astro.washington.edu>
# License: BSD
#   The figure produced by this code is published in the textbook
#   "Statistics, Data Mining, and Machine Learning in Astronomy" (2013)
#   For more information, see http://astroML.github.com
import numpy as np
from matplotlib import pyplot as plt

from sklearn.mixture import GMM

from astroML.decorators import pickle_results
from astroML.datasets import fetch_LINEAR_geneva

#------------------------------------------------------------
# Get the Geneva periods data
data = fetch_LINEAR_geneva()

#----------------------------------------------------------------------
# compute Gaussian Mixture models

filetemplate = 'gmm_res_%i_%i.pkl'
attributes = [('gi', 'logP'),
              ('ug', 'gi', 'iK', 'JK', 'logP', 'amp', 'skew')]
labels = ['$u-g$', '$g-i$', '$i-K$', '$J-K$',
          r'$\log(P)$', 'amplitude', 'skew']
components = np.arange(1, 21)

#------------------------------------------------------------
# Create attribute arrays
Xarrays = []
for attr in attributes:
    Xarrays.append(np.vstack([data[a] for a in attr]).T)

#------------------------------------------------------------
# Compute the results (and save to pickle file)
@pickle_results('LINEAR_clustering.pkl')
def compute_GMM_results(components, attributes):
    clfs = []

    for attr, X in zip(attributes, Xarrays):
        clfs_i = []

        for comp in components:
            print "  - %i component fit" % comp
            clf = GMM(comp, covariance_type='full',
                      random_state=0, n_iter=500)
            clf.fit(X)
            clfs_i.append(clf)

            if not clf.converged_:
                print "           NOT CONVERGED!"

        clfs.append(clfs_i)
    return clfs

clfs = compute_GMM_results(components, attributes)

#------------------------------------------------------------
# Plot the results
fig = plt.figure(figsize=(8, 8))
fig.subplots_adjust(hspace=0.1, wspace=0.1)

for i in range(2):
    # Grab the best classifier, based on the BIC
    X = Xarrays[i]
    BIC = [c.bic(X) for c in clfs[i]]
    i_best = np.argmin(BIC)
    clf = clfs[i][i_best]
    n_components = clf.n_components

    # Predict the cluster labels with the classifier
    c = clf.predict(X)
    classes = np.unique(c)

    # sort the cluster by normalized density of points
    counts = np.sum(c == classes[:, None], 1)
    size = np.array([np.linalg.det(C) for C in clf.covars_])
    weights = clf.weights_
    density = counts * 1. / weights / size

    # Clusters with very few points are less interesting:
    # set their density to zero so they'll go to the end of the list
    density[counts < 5] = 0
    isort = np.argsort(density)[::-1]

    # find statistics of the top 6 clusters
    Nclusters = 6

    means = []
    stdevs = []
    counts = []

    names = [name for name in data.dtype.names[2:] if name != 'LINEARobjectID']
    i_logP = names.index('logP')

    for j in range(Nclusters):
        flag = (c == isort[j])
        counts.append(np.sum(flag))
        means.append([np.mean(data[n][flag]) for n in names])
        stdevs.append([data[n][flag].std() for n in names])

    counts = np.array(counts)
    means = np.array(means)
    stdevs = np.array(stdevs)

    # define colors based on median of logP
    j_ordered = np.argsort(-means[:, i_logP])
    print j_ordered

    # tweak colors by hand
    if i == 1:
        j_ordered[3], j_ordered[2] = j_ordered[2], j_ordered[3]

    color = np.zeros(c.shape)
    for j in range(Nclusters):
        flag = (c == isort[j_ordered[j]])
        color[flag] = j + 1

    # separate into foureground and background
    back = (color == 0)
    fore = ~back

    # Plot the resulting clusters
    ax1 = fig.add_subplot(221 + 2 * i)
    ax1.scatter(data['gi'][back], data['logP'][back],
                c='gray', s=4, linewidths=0)
    ax1.scatter(data['gi'][fore], data['logP'][fore],
                c=color[fore], s=4, linewidths=0)

    ax1.set_ylabel(r'$\log(P)$')

    ax2 = plt.subplot(222 + 2 * i)
    ax2.scatter(data['amp'][back], data['logP'][back],
                c='gray', s=4, lw=0)
    ax2.scatter(data['amp'][fore], data['logP'][fore],
                c=color[fore], s=4, lw=0)

    #------------------------------
    # set axis limits
    ax1.set_xlim(-0.6, 2.1)
    ax2.set_xlim(0.1, 1.5)
    ax1.set_ylim(-1.5, 0.5)
    ax2.set_ylim(-1.5, 0.5)

    ax2.yaxis.set_major_formatter(plt.NullFormatter())
    if i == 0:
        ax1.xaxis.set_major_formatter(plt.NullFormatter())
        ax2.xaxis.set_major_formatter(plt.NullFormatter())
    else:
        ax1.set_xlabel(r'$g-i$')
        ax2.set_xlabel(r'$A$')

    #------------------------------
    # print table of means and medians directly to LaTeX format
    print r"\begin{tabular}{|l|lllllll|}"
    print r"   \hline"
    for j in range(len(labels)):
        print '   &', labels[j],
    print r"\\"
    print r"   \hline"

    for j in range(Nclusters):
        print "   %i " % (j + 1),
        for k in range(len(names)):
            print " & $%.2f \pm %.2f$ " % (means[j, k], stdevs[j, k]),
        print r"\\"

    print r"\hline"
    print r"\end{tabular}"

#------------------------------------------------------------
# Second figure
fig = plt.figure(figsize=(8, 8))
fig.subplots_adjust(left=0.11, right=0.95, wspace=0.3)

attrs = ['skew', 'ug', 'iK', 'JK']
labels = ['skew', '$u-g$', '$i-K$', '$J-K$']
ylims = [(-1.8, 2.2), (0.6, 2.9), (0.1, 2.6), (-0.2, 1.2)]

for i in range(4):
    ax = fig.add_subplot(221 + i)
    ax.scatter(data['gi'][back], data[attrs[i]][back],
               c='gray', s=4, lw=0)
    ax.scatter(data['gi'][fore], data[attrs[i]][fore],
               c=color[fore], s=4, lw=0)
    ax.set_xlabel('$g-i$')
    ax.set_ylabel(labels[i])

    ax.set_xlim(-0.6, 2.1)
    ax.set_ylim(ylims[i])

plt.show()
<commit_msg>remove unnecessary print statement<commit_after>"""
Clustering of LINEAR data
-------------------------
"""
# Author: Jake VanderPlas <vanderplas@astro.washington.edu>
# License: BSD
#   The figure produced by this code is published in the textbook
#   "Statistics, Data Mining, and Machine Learning in Astronomy" (2013)
#   For more information, see http://astroML.github.com
import numpy as np
from matplotlib import pyplot as plt

from sklearn.mixture import GMM

from astroML.decorators import pickle_results
from astroML.datasets import fetch_LINEAR_geneva

#------------------------------------------------------------
# Get the Geneva periods data
data = fetch_LINEAR_geneva()

#----------------------------------------------------------------------
# compute Gaussian Mixture models

filetemplate = 'gmm_res_%i_%i.pkl'
attributes = [('gi', 'logP'),
              ('ug', 'gi', 'iK', 'JK', 'logP', 'amp', 'skew')]
labels = ['$u-g$', '$g-i$', '$i-K$', '$J-K$',
          r'$\log(P)$', 'amplitude', 'skew']
components = np.arange(1, 21)

#------------------------------------------------------------
# Create attribute arrays
Xarrays = []
for attr in attributes:
    Xarrays.append(np.vstack([data[a] for a in attr]).T)

#------------------------------------------------------------
# Compute the results (and save to pickle file)
@pickle_results('LINEAR_clustering.pkl')
def compute_GMM_results(components, attributes):
    clfs = []

    for attr, X in zip(attributes, Xarrays):
        clfs_i = []

        for comp in components:
            print "  - %i component fit" % comp
            clf = GMM(comp, covariance_type='full',
                      random_state=0, n_iter=500)
            clf.fit(X)
            clfs_i.append(clf)

            if not clf.converged_:
                print "           NOT CONVERGED!"

        clfs.append(clfs_i)
    return clfs

clfs = compute_GMM_results(components, attributes)

#------------------------------------------------------------
# Plot the results
fig = plt.figure(figsize=(8, 8))
fig.subplots_adjust(hspace=0.1, wspace=0.1)

for i in range(2):
    # Grab the best classifier, based on the BIC
    X = Xarrays[i]
    BIC = [c.bic(X) for c in clfs[i]]
    i_best = np.argmin(BIC)
    clf = clfs[i][i_best]
    n_components = clf.n_components

    # Predict the cluster labels with the classifier
    c = clf.predict(X)
    classes = np.unique(c)

    # sort the cluster by normalized density of points
    counts = np.sum(c == classes[:, None], 1)
    size = np.array([np.linalg.det(C) for C in clf.covars_])
    weights = clf.weights_
    density = counts * 1. / weights / size

    # Clusters with very few points are less interesting:
    # set their density to zero so they'll go to the end of the list
    density[counts < 5] = 0
    isort = np.argsort(density)[::-1]

    # find statistics of the top 6 clusters
    Nclusters = 6

    means = []
    stdevs = []
    counts = []

    names = [name for name in data.dtype.names[2:] if name != 'LINEARobjectID']
    i_logP = names.index('logP')

    for j in range(Nclusters):
        flag = (c == isort[j])
        counts.append(np.sum(flag))
        means.append([np.mean(data[n][flag]) for n in names])
        stdevs.append([data[n][flag].std() for n in names])

    counts = np.array(counts)
    means = np.array(means)
    stdevs = np.array(stdevs)

    # define colors based on median of logP
    j_ordered = np.argsort(-means[:, i_logP])

    # tweak colors by hand
    if i == 1:
        j_ordered[3], j_ordered[2] = j_ordered[2], j_ordered[3]

    color = np.zeros(c.shape)
    for j in range(Nclusters):
        flag = (c == isort[j_ordered[j]])
        color[flag] = j + 1

    # separate into foureground and background
    back = (color == 0)
    fore = ~back

    # Plot the resulting clusters
    ax1 = fig.add_subplot(221 + 2 * i)
    ax1.scatter(data['gi'][back], data['logP'][back],
                c='gray', s=4, linewidths=0)
    ax1.scatter(data['gi'][fore], data['logP'][fore],
                c=color[fore], s=4, linewidths=0)

    ax1.set_ylabel(r'$\log(P)$')

    ax2 = plt.subplot(222 + 2 * i)
    ax2.scatter(data['amp'][back], data['logP'][back],
                c='gray', s=4, lw=0)
    ax2.scatter(data['amp'][fore], data['logP'][fore],
                c=color[fore], s=4, lw=0)

    #------------------------------
    # set axis limits
    ax1.set_xlim(-0.6, 2.1)
    ax2.set_xlim(0.1, 1.5)
    ax1.set_ylim(-1.5, 0.5)
    ax2.set_ylim(-1.5, 0.5)

    ax2.yaxis.set_major_formatter(plt.NullFormatter())
    if i == 0:
        ax1.xaxis.set_major_formatter(plt.NullFormatter())
        ax2.xaxis.set_major_formatter(plt.NullFormatter())
    else:
        ax1.set_xlabel(r'$g-i$')
        ax2.set_xlabel(r'$A$')

    #------------------------------
    # print table of means and medians directly to LaTeX format
    print r"\begin{tabular}{|l|lllllll|}"
    print r"   \hline"
    for j in range(len(labels)):
        print '   &', labels[j],
    print r"\\"
    print r"   \hline"

    for j in range(Nclusters):
        print "   %i " % (j + 1),
        for k in range(len(names)):
            print " & $%.2f \pm %.2f$ " % (means[j, k], stdevs[j, k]),
        print r"\\"

    print r"\hline"
    print r"\end{tabular}"

#------------------------------------------------------------
# Second figure
fig = plt.figure(figsize=(8, 8))
fig.subplots_adjust(left=0.11, right=0.95, wspace=0.3)

attrs = ['skew', 'ug', 'iK', 'JK']
labels = ['skew', '$u-g$', '$i-K$', '$J-K$']
ylims = [(-1.8, 2.2), (0.6, 2.9), (0.1, 2.6), (-0.2, 1.2)]

for i in range(4):
    ax = fig.add_subplot(221 + i)
    ax.scatter(data['gi'][back], data[attrs[i]][back],
               c='gray', s=4, lw=0)
    ax.scatter(data['gi'][fore], data[attrs[i]][fore],
               c=color[fore], s=4, lw=0)
    ax.set_xlabel('$g-i$')
    ax.set_ylabel(labels[i])

    ax.set_xlim(-0.6, 2.1)
    ax.set_ylim(ylims[i])

plt.show()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright (c) 2012 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import os
import os.path
import generate_protocol_externs
import re
import shutil
import subprocess
import sys
import tempfile
try:
    import json
except ImportError:
    import simplejson as json

scripts_path = os.path.dirname(os.path.abspath(__file__))
devtools_path = os.path.dirname(scripts_path)
inspector_path = os.path.dirname(devtools_path) + "/core/inspector"
devtools_frontend_path = devtools_path + "/front_end"
protocol_externs_file = devtools_frontend_path + "/protocol_externs.js"
webgl_rendering_context_idl_path = os.path.dirname(devtools_path) + "/core/html/canvas/WebGLRenderingContextBase.idl"
closure_compiler_jar = scripts_path + "/closure/compiler.jar"
closure_runner_jar = scripts_path + "/compiler-runner/closure-runner.jar"
jsdoc_validator_jar = scripts_path + "/jsdoc-validator/jsdoc-validator.jar"
java_exec = "java -Xms1024m -server -XX:+TieredCompilation"

generate_protocol_externs.generate_protocol_externs(protocol_externs_file, devtools_path + "/protocol.json")

jsmodule_name_prefix = "jsmodule_"
js_modules_name = "frontend_modules.json"

try:
    with open(os.path.join(scripts_path, js_modules_name), "rt") as js_modules_file:
        modules = json.loads(js_modules_file.read())
except:
    print "ERROR: Failed to read %s" % js_modules_name
    raise

# `importScript` function must not be used in any files
# except module headers. Refer to devtools.gyp file for
# the module header list.
allowed_import_statements_files = [
    "utilities.js",
    "AdvancedSearchView.js",
    "ConsolePanel.js",
    "ElementsPanel.js",
    "ResourcesPanel.js",
    "NetworkPanel.js",
    "SourcesPanel.js",
    "TimelinePanel.js",
    "ProfilesPanel.js",
    "AuditsPanel.js",
    "LayersPanel.js",
    "CodeMirrorTextEditor.js",
    "ExtensionServer.js",
]

type_checked_jsdoc_tags_list = ["param", "return", "type", "enum"]

type_checked_jsdoc_tags_or = "|".join(type_checked_jsdoc_tags_list)

# Basic regex for invalid JsDoc types: an object type name ([A-Z][A-Za-z0-9.]+[A-Za-z0-9]) not preceded by '!', '?', ':' (this, new), or '.' (object property).
invalid_type_regex = re.compile(r"@(?:" + type_checked_jsdoc_tags_or + r")\s*\{.*(?<![!?:.A-Za-z0-9])([A-Z][A-Za-z0-9.]+[A-Za-z0-9])[^/]*\}")

invalid_type_designator_regex = re.compile(r"@(?:" + type_checked_jsdoc_tags_or + r")\s*.*([?!])=?\}")

error_warning_regex = re.compile(r"(?:WARNING|ERROR)")

errors_found = False

def hasErrors(output):
    return re.search(error_warning_regex, output) != None

def verify_importScript_usage():
    errors_found = False
    for module in modules:
        for file_name in module['sources']:
            if file_name in allowed_import_statements_files:
                continue
            sourceFile = open(devtools_frontend_path + "/" + file_name, "r")
            source = sourceFile.read()
            sourceFile.close()
            if "importScript(" in source:
                print "ERROR: importScript function is allowed in module header files only (found in %s)" % file_name
                errors_found = True
    return errors_found

def dump_all_checked_files():
    files = {}
    for module in modules:
        for source in module["sources"]:
            files[devtools_frontend_path + "/" + source] = True
    return " ".join(files.keys())

def verify_jsdoc_extra():
    return subprocess.Popen("%s -jar %s %s" % (java_exec, jsdoc_validator_jar, dump_all_checked_files()), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)

def verify_jsdoc():
    errors_found = False
    for module in modules:
        for file_name in module['sources']:
            lineIndex = 0
            full_file_name = devtools_frontend_path + "/" + file_name
            with open(full_file_name, "r") as sourceFile:
                for line in sourceFile:
                    line = line.rstrip()
                    lineIndex += 1
                    if not line:
                        continue
                    if verify_jsdoc_line(full_file_name, lineIndex, line):
                        errors_found = True
    return errors_found

def verify_jsdoc_line(fileName, lineIndex, line):
    def print_error(message, errorPosition):
        print "%s:%s: ERROR - %s\n%s\n%s\n" % (fileName, lineIndex, message, line, " " * errorPosition + "^")

    errors_found = False
    match = re.search(invalid_type_regex, line)
    if match:
        print_error("Type '%s' nullability not marked explicitly with '?' (nullable) or '!' (non-nullable)" % match.group(1), match.start(1))
        errors_found = True

    match = re.search(invalid_type_designator_regex, line)
    if (match):
        print_error("Type nullability indicator misplaced, should precede type", match.start(1))
        errors_found = True
    return errors_found

def check_java_path():
    proc = subprocess.Popen("which java", stdout=subprocess.PIPE, shell=True)
    (javaPath, _) = proc.communicate()

    if proc.returncode != 0:
        print "Cannot find java ('which java' return code = %d, should be 0)" % proc.returncode
        sys.exit(1)
    print "Java executable: " + re.sub(r"\n$", "", javaPath)

check_java_path()

print "Verifying 'importScript' function usage..."
errors_found |= verify_importScript_usage()

print "Verifying JSDoc comments..."
errors_found |= verify_jsdoc()
jsdocValidatorProc = verify_jsdoc_extra()

modules_dir = tempfile.mkdtemp()
common_closure_args = " --summary_detail_level 3 --compilation_level SIMPLE_OPTIMIZATIONS --warning_level VERBOSE --language_in ECMASCRIPT5 --accept_const_keyword --module_output_path_prefix %s/" % modules_dir

compiler_args_file = tempfile.NamedTemporaryFile(mode='wt', delete=False)
closure_runner_command = "%s -jar %s --compiler-args-file %s" % (java_exec, closure_runner_jar, compiler_args_file.name)

spawned_compiler_command = "%s -jar %s %s \\\n" % (java_exec, closure_compiler_jar, common_closure_args)

modules_by_name = {}
for module in modules:
    modules_by_name[module["name"]] = module

dependents_by_module_name = {}
for module in modules:
    name = module["name"]
    for dep in module["dependencies"]:
        list = dependents_by_module_name.get(dep)
        if not list:
            list = []
            dependents_by_module_name[dep] = list
        list.append(name)

standalone_modules = []
for module in modules:
    if "standalone" in module:
        standalone_modules.append(module)

def verify_standalone_modules():
    standalone_module_names = {}
    for standalone_module in standalone_modules:
        standalone_module_names[standalone_module["name"]] = True
    for module in modules:
        for dependency in module["dependencies"]:
            if dependency in standalone_module_names:
                print "ERROR: Standalone module %s cannot be in dependencies of %s" % (dependency, module["name"])
                errors_found = True

verify_standalone_modules()

def dump_module(name, recursively, processed_modules):
    if name in processed_modules:
        return ""
    processed_modules[name] = True
    module = modules_by_name[name]
    command = ""
    if recursively:
        for dependency in module["dependencies"]:
            command += dump_module(dependency, recursively, processed_modules)
    command += " --module " + jsmodule_name_prefix + module["name"] + ":"
    command += str(len(module["sources"]))
    firstDependency = True
    for dependency in module["dependencies"]:
        if firstDependency:
            command += ":"
        else:
            command += ","
        firstDependency = False
        command += jsmodule_name_prefix + dependency
    for script in module["sources"]:
        command += " --js " + devtools_frontend_path + "/" + script
    return command

print "Compiling frontend..."

for module in modules:
    closure_args = common_closure_args
    closure_args += " --externs " + devtools_frontend_path + "/externs.js"
    closure_args += " --externs " + protocol_externs_file
    closure_args += dump_module(module["name"], True, {})
    compiler_args_file.write("%s %s\n" % (module["name"], closure_args))

compiler_args_file.close()
modular_compiler_proc = subprocess.Popen(closure_runner_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)

def unclosure_injected_script(sourceFileName, outFileName):
    sourceFile = open(sourceFileName, "r")
    source = sourceFile.read()
    sourceFile.close()

    def replace_function(matchobj):
        return re.sub(r"@param", "param", matchobj.group(1) or "") + "\n//" + matchobj.group(2)

    # Comment out the closure function and its jsdocs
    source = re.sub(r"(/\*\*(?:[\s\n]*\*\s*@param[^\n]+\n)+\s*\*/\s*)?\n(\(function)", replace_function, source, count=1)

    # Comment out its return statement
    source = re.sub(r"\n(\s*return\s+[^;]+;\s*\n\}\)\s*)$", "\n/*\\1*/", source)

    outFileName = open(outFileName, "w")
    outFileName.write(source)
    outFileName.close()

injectedScriptSourceTmpFile = inspector_path + "/" + "InjectedScriptSourceTmp.js"
injectedScriptCanvasModuleSourceTmpFile = inspector_path + "/" + "InjectedScriptCanvasModuleSourceTmp.js"

unclosure_injected_script(inspector_path + "/" + "InjectedScriptSource.js", injectedScriptSourceTmpFile)
unclosure_injected_script(inspector_path + "/" + "InjectedScriptCanvasModuleSource.js", injectedScriptCanvasModuleSourceTmpFile)

print "Compiling InjectedScriptSource.js and InjectedScriptCanvasModuleSource.js..."
command = spawned_compiler_command
command += "    --externs " + inspector_path + "/" + "InjectedScriptExterns.js" + " \\\n"
command += "    --externs " + protocol_externs_file + " \\\n"
command += "    --module " + jsmodule_name_prefix + "injected_script" + ":1" + " \\\n"
command += "        --js " + injectedScriptSourceTmpFile + " \\\n"
command += "    --module " + jsmodule_name_prefix + "injected_canvas_script" + ":1:" + jsmodule_name_prefix + "injected_script" + " \\\n"
command += "        --js " + injectedScriptCanvasModuleSourceTmpFile + " \\\n"
command += "\n"

injectedScriptCompileProc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)

print "Checking generated code in InjectedScriptCanvasModuleSource.js..."
check_injected_webgl_calls_command = "%s/check_injected_webgl_calls_info.py %s %s/InjectedScriptCanvasModuleSource.js" % (scripts_path, webgl_rendering_context_idl_path, inspector_path)
canvasModuleCompileProc = subprocess.Popen(check_injected_webgl_calls_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)

print

(jsdocValidatorOut, _) = jsdocValidatorProc.communicate()
if jsdocValidatorOut:
    print ("JSDoc validator output:\n%s" % jsdocValidatorOut)
    errors_found = True

(moduleCompileOut, _) = modular_compiler_proc.communicate()
print "Modular compilation output:"

start_module_regex = re.compile(r"^@@ START_MODULE:(.+) @@$")
end_module_regex = re.compile(r"^@@ END_MODULE @@$")

in_module = False
skipped_modules = {}
error_count = 0

def skip_dependents(module_name):
    for skipped_module in dependents_by_module_name.get(module_name, []):
        skipped_modules[skipped_module] = True

# pylint: disable=E1103
for line in moduleCompileOut.splitlines():
    if not in_module:
        match = re.search(start_module_regex, line)
        if not match:
            continue
        in_module = True
        module_name = match.group(1)
        skip_module = skipped_modules.get(module_name)
        if skip_module:
            skip_dependents(module_name)
        module_output = ["Skipping module %s...\n" % module_name if skip_module else "Module %s:" % module_name]
    else:
        match = re.search(end_module_regex, line)
        if not match:
            if not skip_module:
                module_output.append(line)
                if hasErrors(line):
                    error_count += 1
                    skip_dependents(module_name)
            continue

        in_module = False
        print os.linesep.join(module_output)

if error_count:
    print "Total Closure errors: %d\n" % error_count
    errors_found = True

(injectedScriptCompileOut, _) = injectedScriptCompileProc.communicate()
print "InjectedScriptSource.js and InjectedScriptCanvasModuleSource.js compilation output:\n", injectedScriptCompileOut
errors_found |= hasErrors(injectedScriptCompileOut)

(canvasModuleCompileOut, _) = canvasModuleCompileProc.communicate()
print "InjectedScriptCanvasModuleSource.js generated code check output:\n", canvasModuleCompileOut
errors_found |= hasErrors(canvasModuleCompileOut)

if errors_found:
    print "ERRORS DETECTED"

os.remove(injectedScriptSourceTmpFile)
os.remove(injectedScriptCanvasModuleSourceTmpFile)
os.remove(compiler_args_file.name)
os.remove(protocol_externs_file)
shutil.rmtree(modules_dir, True)
<commit_msg>DevTools: Make compiler output shorter, remove some unneeded new lines.<commit_after>#!/usr/bin/env python
# Copyright (c) 2012 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import os
import os.path
import generate_protocol_externs
import re
import shutil
import subprocess
import sys
import tempfile
try:
    import json
except ImportError:
    import simplejson as json

scripts_path = os.path.dirname(os.path.abspath(__file__))
devtools_path = os.path.dirname(scripts_path)
inspector_path = os.path.dirname(devtools_path) + "/core/inspector"
devtools_frontend_path = devtools_path + "/front_end"
protocol_externs_file = devtools_frontend_path + "/protocol_externs.js"
webgl_rendering_context_idl_path = os.path.dirname(devtools_path) + "/core/html/canvas/WebGLRenderingContextBase.idl"
closure_compiler_jar = scripts_path + "/closure/compiler.jar"
closure_runner_jar = scripts_path + "/compiler-runner/closure-runner.jar"
jsdoc_validator_jar = scripts_path + "/jsdoc-validator/jsdoc-validator.jar"
java_exec = "java -Xms1024m -server -XX:+TieredCompilation"

generate_protocol_externs.generate_protocol_externs(protocol_externs_file, devtools_path + "/protocol.json")

jsmodule_name_prefix = "jsmodule_"
js_modules_name = "frontend_modules.json"

try:
    with open(os.path.join(scripts_path, js_modules_name), "rt") as js_modules_file:
        modules = json.loads(js_modules_file.read())
except:
    print "ERROR: Failed to read %s" % js_modules_name
    raise

# `importScript` function must not be used in any files
# except module headers. Refer to devtools.gyp file for
# the module header list.
allowed_import_statements_files = [
    "utilities.js",
    "AdvancedSearchView.js",
    "ConsolePanel.js",
    "ElementsPanel.js",
    "ResourcesPanel.js",
    "NetworkPanel.js",
    "SourcesPanel.js",
    "TimelinePanel.js",
    "ProfilesPanel.js",
    "AuditsPanel.js",
    "LayersPanel.js",
    "CodeMirrorTextEditor.js",
    "ExtensionServer.js",
]

type_checked_jsdoc_tags_list = ["param", "return", "type", "enum"]

type_checked_jsdoc_tags_or = "|".join(type_checked_jsdoc_tags_list)

# Basic regex for invalid JsDoc types: an object type name ([A-Z][A-Za-z0-9.]+[A-Za-z0-9]) not preceded by '!', '?', ':' (this, new), or '.' (object property).
invalid_type_regex = re.compile(r"@(?:" + type_checked_jsdoc_tags_or + r")\s*\{.*(?<![!?:.A-Za-z0-9])([A-Z][A-Za-z0-9.]+[A-Za-z0-9])[^/]*\}")

invalid_type_designator_regex = re.compile(r"@(?:" + type_checked_jsdoc_tags_or + r")\s*.*([?!])=?\}")

error_warning_regex = re.compile(r"(?:WARNING|ERROR)")

errors_found = False

def hasErrors(output):
    return re.search(error_warning_regex, output) != None

def verify_importScript_usage():
    errors_found = False
    for module in modules:
        for file_name in module['sources']:
            if file_name in allowed_import_statements_files:
                continue
            sourceFile = open(devtools_frontend_path + "/" + file_name, "r")
            source = sourceFile.read()
            sourceFile.close()
            if "importScript(" in source:
                print "ERROR: importScript function is allowed in module header files only (found in %s)" % file_name
                errors_found = True
    return errors_found

def dump_all_checked_files():
    files = {}
    for module in modules:
        for source in module["sources"]:
            files[devtools_frontend_path + "/" + source] = True
    return " ".join(files.keys())

def verify_jsdoc_extra():
    return subprocess.Popen("%s -jar %s %s" % (java_exec, jsdoc_validator_jar, dump_all_checked_files()), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)

def verify_jsdoc():
    errors_found = False
    for module in modules:
        for file_name in module['sources']:
            lineIndex = 0
            full_file_name = devtools_frontend_path + "/" + file_name
            with open(full_file_name, "r") as sourceFile:
                for line in sourceFile:
                    line = line.rstrip()
                    lineIndex += 1
                    if not line:
                        continue
                    if verify_jsdoc_line(full_file_name, lineIndex, line):
                        errors_found = True
    return errors_found

def verify_jsdoc_line(fileName, lineIndex, line):
    def print_error(message, errorPosition):
        print "%s:%s: ERROR - %s\n%s\n%s\n" % (fileName, lineIndex, message, line, " " * errorPosition + "^")

    errors_found = False
    match = re.search(invalid_type_regex, line)
    if match:
        print_error("Type '%s' nullability not marked explicitly with '?' (nullable) or '!' (non-nullable)" % match.group(1), match.start(1))
        errors_found = True

    match = re.search(invalid_type_designator_regex, line)
    if (match):
        print_error("Type nullability indicator misplaced, should precede type", match.start(1))
        errors_found = True
    return errors_found

def check_java_path():
    proc = subprocess.Popen("which java", stdout=subprocess.PIPE, shell=True)
    (javaPath, _) = proc.communicate()

    if proc.returncode != 0:
        print "Cannot find java ('which java' return code = %d, should be 0)" % proc.returncode
        sys.exit(1)
    print "Java executable: " + re.sub(r"\n$", "", javaPath)

check_java_path()

print "Verifying 'importScript' function usage..."
errors_found |= verify_importScript_usage()

print "Verifying JSDoc comments..."
errors_found |= verify_jsdoc()
jsdocValidatorProc = verify_jsdoc_extra()

modules_dir = tempfile.mkdtemp()
common_closure_args = " --summary_detail_level 3 --compilation_level SIMPLE_OPTIMIZATIONS --warning_level VERBOSE --language_in ECMASCRIPT5 --accept_const_keyword --module_output_path_prefix %s/" % modules_dir

compiler_args_file = tempfile.NamedTemporaryFile(mode='wt', delete=False)
closure_runner_command = "%s -jar %s --compiler-args-file %s" % (java_exec, closure_runner_jar, compiler_args_file.name)

spawned_compiler_command = "%s -jar %s %s \\\n" % (java_exec, closure_compiler_jar, common_closure_args)

modules_by_name = {}
for module in modules:
    modules_by_name[module["name"]] = module

dependents_by_module_name = {}
for module in modules:
    name = module["name"]
    for dep in module["dependencies"]:
        list = dependents_by_module_name.get(dep)
        if not list:
            list = []
            dependents_by_module_name[dep] = list
        list.append(name)

standalone_modules = []
for module in modules:
    if "standalone" in module:
        standalone_modules.append(module)

def verify_standalone_modules():
    standalone_module_names = {}
    for standalone_module in standalone_modules:
        standalone_module_names[standalone_module["name"]] = True
    for module in modules:
        for dependency in module["dependencies"]:
            if dependency in standalone_module_names:
                print "ERROR: Standalone module %s cannot be in dependencies of %s" % (dependency, module["name"])
                errors_found = True

verify_standalone_modules()

def dump_module(name, recursively, processed_modules):
    if name in processed_modules:
        return ""
    processed_modules[name] = True
    module = modules_by_name[name]
    command = ""
    if recursively:
        for dependency in module["dependencies"]:
            command += dump_module(dependency, recursively, processed_modules)
    command += " --module " + jsmodule_name_prefix + module["name"] + ":"
    command += str(len(module["sources"]))
    firstDependency = True
    for dependency in module["dependencies"]:
        if firstDependency:
            command += ":"
        else:
            command += ","
        firstDependency = False
        command += jsmodule_name_prefix + dependency
    for script in module["sources"]:
        command += " --js " + devtools_frontend_path + "/" + script
    return command

print "Compiling frontend..."

for module in modules:
    closure_args = common_closure_args
    closure_args += " --externs " + devtools_frontend_path + "/externs.js"
    closure_args += " --externs " + protocol_externs_file
    closure_args += dump_module(module["name"], True, {})
    compiler_args_file.write("%s %s\n" % (module["name"], closure_args))

compiler_args_file.close()
modular_compiler_proc = subprocess.Popen(closure_runner_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)

def unclosure_injected_script(sourceFileName, outFileName):
    sourceFile = open(sourceFileName, "r")
    source = sourceFile.read()
    sourceFile.close()

    def replace_function(matchobj):
        return re.sub(r"@param", "param", matchobj.group(1) or "") + "\n//" + matchobj.group(2)

    # Comment out the closure function and its jsdocs
    source = re.sub(r"(/\*\*(?:[\s\n]*\*\s*@param[^\n]+\n)+\s*\*/\s*)?\n(\(function)", replace_function, source, count=1)

    # Comment out its return statement
    source = re.sub(r"\n(\s*return\s+[^;]+;\s*\n\}\)\s*)$", "\n/*\\1*/", source)

    outFileName = open(outFileName, "w")
    outFileName.write(source)
    outFileName.close()

injectedScriptSourceTmpFile = inspector_path + "/" + "InjectedScriptSourceTmp.js"
injectedScriptCanvasModuleSourceTmpFile = inspector_path + "/" + "InjectedScriptCanvasModuleSourceTmp.js"

unclosure_injected_script(inspector_path + "/" + "InjectedScriptSource.js", injectedScriptSourceTmpFile)
unclosure_injected_script(inspector_path + "/" + "InjectedScriptCanvasModuleSource.js", injectedScriptCanvasModuleSourceTmpFile)

print "Compiling InjectedScriptSource.js and InjectedScriptCanvasModuleSource.js..."
command = spawned_compiler_command
command += "    --externs " + inspector_path + "/" + "InjectedScriptExterns.js" + " \\\n"
command += "    --externs " + protocol_externs_file + " \\\n"
command += "    --module " + jsmodule_name_prefix + "injected_script" + ":1" + " \\\n"
command += "        --js " + injectedScriptSourceTmpFile + " \\\n"
command += "    --module " + jsmodule_name_prefix + "injected_canvas_script" + ":1:" + jsmodule_name_prefix + "injected_script" + " \\\n"
command += "        --js " + injectedScriptCanvasModuleSourceTmpFile + " \\\n"
command += "\n"

injectedScriptCompileProc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)

print "Checking generated code in InjectedScriptCanvasModuleSource.js..."
check_injected_webgl_calls_command = "%s/check_injected_webgl_calls_info.py %s %s/InjectedScriptCanvasModuleSource.js" % (scripts_path, webgl_rendering_context_idl_path, inspector_path)
canvasModuleCompileProc = subprocess.Popen(check_injected_webgl_calls_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)

print

(jsdocValidatorOut, _) = jsdocValidatorProc.communicate()
if jsdocValidatorOut:
    print ("JSDoc validator output:\n%s" % jsdocValidatorOut)
    errors_found = True

(moduleCompileOut, _) = modular_compiler_proc.communicate()
print "Modular compilation output:"

start_module_regex = re.compile(r"^@@ START_MODULE:(.+) @@$")
end_module_regex = re.compile(r"^@@ END_MODULE @@$")

in_module = False
skipped_modules = {}
error_count = 0

def skip_dependents(module_name):
    for skipped_module in dependents_by_module_name.get(module_name, []):
        skipped_modules[skipped_module] = True

# pylint: disable=E1103
for line in moduleCompileOut.splitlines():
    if not in_module:
        match = re.search(start_module_regex, line)
        if not match:
            continue
        in_module = True
        module_error_count = 0
        module_output = []
        module_name = match.group(1)
        skip_module = skipped_modules.get(module_name)
        if skip_module:
            skip_dependents(module_name)
    else:
        match = re.search(end_module_regex, line)
        if not match:
            if not skip_module:
                module_output.append(line)
                if hasErrors(line):
                    error_count += 1
                    module_error_count += 1
                    skip_dependents(module_name)
            continue

        in_module = False
        if skip_module:
            print "Skipping module %s..." % module_name
        elif not module_error_count:
            print "Module %s compiled successfully: %s" % (module_name, module_output[0])
        else:
            print "Module %s compile failed: %s errors\n" % (module_name, module_error_count)
            print os.linesep.join(module_output)

if error_count:
    print "Total Closure errors: %d\n" % error_count
    errors_found = True

(injectedScriptCompileOut, _) = injectedScriptCompileProc.communicate()
print "InjectedScriptSource.js and InjectedScriptCanvasModuleSource.js compilation output:\n", injectedScriptCompileOut
errors_found |= hasErrors(injectedScriptCompileOut)

(canvasModuleCompileOut, _) = canvasModuleCompileProc.communicate()
print "InjectedScriptCanvasModuleSource.js generated code check output:\n", canvasModuleCompileOut
errors_found |= hasErrors(canvasModuleCompileOut)

if errors_found:
    print "ERRORS DETECTED"

os.remove(injectedScriptSourceTmpFile)
os.remove(injectedScriptCanvasModuleSourceTmpFile)
os.remove(compiler_args_file.name)
os.remove(protocol_externs_file)
shutil.rmtree(modules_dir, True)
<|endoftext|>


<commit_before>import numpy as np
import pytest

from opt_einsum import contract, helpers, contract_expression, backends

try:
    import tensorflow as tf
    # needed so tensorflow doesn't allocate all gpu mem
    _TF_CONFIG = tf.ConfigProto()
    _TF_CONFIG.gpu_options.allow_growth = True
    found_tensorflow = True
except ImportError:
    found_tensorflow = False

try:
    import os
    os.environ['MKL_THREADING_LAYER'] = 'GNU'
    import theano
    found_theano = True
except ImportError:
    found_theano = False

tests = [
    'ab,bc->ca',
    'abc,bcd,dea',
    'abc,def->fedcba',
    # test 'prefer einsum' ops
    'ijk,ikj',
    'i,j->ij',
    'ijk,k->ij',
]

@pytest.mark.skipif(not found_tensorflow, reason="Tensorflow not installed.")
@pytest.mark.parametrize("string", tests)
def test_tensorflow(string):
    views = helpers.build_views(string)
    ein = contract(string, *views, optimize=False, use_blas=False)
    opt = np.empty_like(ein)

    shps = [v.shape for v in views]
    expr = contract_expression(string, *shps, optimize=True)

    sess = tf.Session(config=_TF_CONFIG)
    with sess.as_default():
        expr(*views, backend='tensorflow', out=opt)
    sess.close()

    assert np.allclose(ein, opt)

    # test non-conversion mode
    tensorflow_views = [backends.to_tensorflow(view) for view in views]
    expr(*tensorflow_views, backend='tensorflow')

@pytest.mark.skipif(not found_tensorflow, reason="Tensorflow not installed.")
def test_tensorflow_with_constants():
    eq = 'ij,jk,kl->li'
    shapes = (2, 3), (3, 4), (4, 5)
    constants = {0, 2}
    ops = [np.random.rand(*shp) if i in constants else shp for i, shp in enumerate(shapes)]
    var = np.random.rand(*shapes[1])

    res_exp = contract(eq, ops[0], var, ops[2])

    expr = contract_expression(eq, *ops, constants=constants)

    # check tensorflow
    sess = tf.Session(config=_TF_CONFIG)
    with sess.as_default():
        res_got = expr(var, backend='tensorflow')
    sess.close()
    assert 'tensorflow' in expr._evaluated_constants
    assert np.allclose(res_exp, res_got)

    # check can call with numpy still
    res_got2 = expr(var, backend='numpy')
    assert np.allclose(res_exp, res_got2)

    # check tensorflow call returns tensorflow still
    res_got3 = expr(backends.to_tensorflow(var), backend='tensorflow')
    assert isinstance(res_got3, tf.Tensor)

@pytest.mark.skipif(not found_theano, reason="Theano not installed.")
@pytest.mark.parametrize("string", tests)
def test_theano(string):
    views = helpers.build_views(string)
    ein = contract(string, *views, optimize=False, use_blas=False)
    shps = [v.shape for v in views]

    expr = contract_expression(string, *shps, optimize=True)

    opt = expr(*views, backend='theano')
    assert np.allclose(ein, opt)

    # test non-conversion mode
    theano_views = [backends.to_theano(view) for view in views]
    theano_opt = expr(*theano_views, backend='theano')
    assert isinstance(theano_opt, theano.tensor.TensorVariable)

@pytest.mark.skipif(not found_theano, reason="theano not installed.")
def test_theano_with_constants():
    eq = 'ij,jk,kl->li'
    shapes = (2, 3), (3, 4), (4, 5)
    constants = {0, 2}
    ops = [np.random.rand(*shp) if i in constants else shp for i, shp in enumerate(shapes)]
    var = np.random.rand(*shapes[1])

    res_exp = contract(eq, ops[0], var, ops[2])

    expr = contract_expression(eq, *ops, constants=constants)

    # check theano
    res_got = expr(var, backend='theano')
    assert 'theano' in expr._evaluated_constants
    assert np.allclose(res_exp, res_got)

    # check can call with numpy still
    res_got2 = expr(var, backend='numpy')
    assert np.allclose(res_exp, res_got2)

    # check theano call returns theano still
    res_got3 = expr(backends.to_theano(var), backend='theano')
    assert isinstance(res_got3, theano.tensor.TensorVariable)

@pytest.mark.parametrize("string", tests)
def test_cupy(string):  # pragma: no cover
    cupy = pytest.importorskip("cupy")

    views = helpers.build_views(string)
    ein = contract(string, *views, optimize=False, use_blas=False)
    shps = [v.shape for v in views]

    expr = contract_expression(string, *shps, optimize=True)

    opt = expr(*views, backend='cupy')
    assert np.allclose(ein, opt)

    # test non-conversion mode
    cupy_views = [backends.to_cupy(view) for view in views]
    cupy_opt = expr(*cupy_views, backend='cupy')
    assert isinstance(cupy_opt, cupy.ndarray)
    assert np.allclose(ein, cupy.asnumpy(cupy_opt))

def test_cupy_with_constants():  # pragma: no cover
    cupy = pytest.importorskip("cupy")

    eq = 'ij,jk,kl->li'
    shapes = (2, 3), (3, 4), (4, 5)
    constants = {0, 2}
    ops = [np.random.rand(*shp) if i in constants else shp for i, shp in enumerate(shapes)]
    var = np.random.rand(*shapes[1])

    res_exp = contract(eq, ops[0], var, ops[2])

    expr = contract_expression(eq, *ops, constants=constants)

    # check cupy
    res_got = expr(var, backend='cupy')
    assert 'cupy' in expr._evaluated_constants
    assert np.allclose(res_exp, res_got)

    # check can call with numpy still
    res_got2 = expr(var, backend='numpy')
    assert np.allclose(res_exp, res_got2)

    # check cupy call returns cupy still
    res_got3 = expr(cupy.asarray(var), backend='cupy')
    assert isinstance(res_got3, cupy.ndarray)
    assert np.allclose(res_exp, res_got3.get())

@pytest.mark.parametrize("string", tests)
def test_dask(string):
    da = pytest.importorskip("dask.array")

    views = helpers.build_views(string)
    ein = contract(string, *views, optimize=False, use_blas=False)
    shps = [v.shape for v in views]
    expr = contract_expression(string, *shps, optimize=True)

    # test non-conversion mode
    da_views = [da.from_array(x, chunks=(2)) for x in views]
    da_opt = expr(*da_views, backend='dask')

    # check type is maintained when not using numpy arrays
    assert isinstance(da_opt, da.Array)

    assert np.allclose(ein, np.array(da_opt))

    # try raw contract
    da_opt = contract(string, *da_views, backend='dask')
    assert isinstance(da_opt, da.Array)
    assert np.allclose(ein, np.array(da_opt))

@pytest.mark.parametrize("string", tests)
def test_sparse(string):
    sparse = pytest.importorskip("sparse")

    views = helpers.build_views(string)

    # sparsify views so they don't become dense during contraction
    for view in views:
        np.random.seed(42)
        mask = np.random.choice([False, True], view.shape, True, [0.05, 0.95])
        view[mask] = 0

    ein = contract(string, *views, optimize=False, use_blas=False)
    shps = [v.shape for v in views]
    expr = contract_expression(string, *shps, optimize=True)

    # test non-conversion mode
    sparse_views = [sparse.COO.from_numpy(x) for x in views]
    sparse_opt = expr(*sparse_views, backend='sparse')

    # check type is maintained when not using numpy arrays
    assert isinstance(sparse_opt, sparse.COO)

    assert np.allclose(ein, sparse_opt.todense())

    # try raw contract
    sparse_opt = contract(string, *sparse_views, backend='sparse')
    assert isinstance(sparse_opt, sparse.COO)
    assert np.allclose(ein, sparse_opt.todense())

@pytest.mark.parametrize("string", tests)
def test_torch(string):
    torch = pytest.importorskip("torch")

    views = helpers.build_views(string)
    ein = contract(string, *views, optimize=False, use_blas=False)
    shps = [v.shape for v in views]

    expr = contract_expression(string, *shps, optimize=True)

    opt = expr(*views, backend='torch')
    assert np.allclose(ein, opt)

    # test non-conversion mode
    torch_views = [backends.to_torch(view) for view in views]
    torch_opt = expr(*torch_views, backend='torch')
    assert isinstance(torch_opt, torch.Tensor)
    assert np.allclose(ein, torch_opt.cpu().numpy())

def test_torch_with_constants():
    torch = pytest.importorskip("torch")

    eq = 'ij,jk,kl->li'
    shapes = (2, 3), (3, 4), (4, 5)
    constants = {0, 2}
    ops = [np.random.rand(*shp) if i in constants else shp for i, shp in enumerate(shapes)]
    var = np.random.rand(*shapes[1])

    res_exp = contract(eq, ops[0], var, ops[2])

    expr = contract_expression(eq, *ops, constants=constants)

    # check torch
    res_got = expr(var, backend='torch')
    assert 'torch' in expr._evaluated_constants
    assert np.allclose(res_exp, res_got)

    # check can call with numpy still
    res_got2 = expr(var, backend='numpy')
    assert np.allclose(res_exp, res_got2)

    # check torch call returns torch still
    res_got3 = expr(backends.to_torch(var), backend='torch')
    assert isinstance(res_got3, torch.Tensor)
    res_got3 = res_got3.numpy() if res_got3.device.type == 'cpu' else res_got3.cpu().numpy()
    assert np.allclose(res_exp, res_got3)
<commit_msg>travis: fix torch<commit_after>import numpy as np
import pytest

from opt_einsum import contract, helpers, contract_expression, backends

try:
    import tensorflow as tf
    # needed so tensorflow doesn't allocate all gpu mem
    _TF_CONFIG = tf.ConfigProto()
    _TF_CONFIG.gpu_options.allow_growth = True
    found_tensorflow = True
except ImportError:
    found_tensorflow = False

try:
    import os
    os.environ['MKL_THREADING_LAYER'] = 'GNU'
    import theano
    found_theano = True
except ImportError:
    found_theano = False

try:
    import torch
    found_torch = True
except ImportError:
    found_torch = False

tests = [
    'ab,bc->ca',
    'abc,bcd,dea',
    'abc,def->fedcba',
    # test 'prefer einsum' ops
    'ijk,ikj',
    'i,j->ij',
    'ijk,k->ij',
]

@pytest.mark.skipif(not found_tensorflow, reason="Tensorflow not installed.")
@pytest.mark.parametrize("string", tests)
def test_tensorflow(string):
    views = helpers.build_views(string)
    ein = contract(string, *views, optimize=False, use_blas=False)
    opt = np.empty_like(ein)

    shps = [v.shape for v in views]
    expr = contract_expression(string, *shps, optimize=True)

    sess = tf.Session(config=_TF_CONFIG)
    with sess.as_default():
        expr(*views, backend='tensorflow', out=opt)
    sess.close()

    assert np.allclose(ein, opt)

    # test non-conversion mode
    tensorflow_views = [backends.to_tensorflow(view) for view in views]
    expr(*tensorflow_views, backend='tensorflow')

@pytest.mark.skipif(not found_tensorflow, reason="Tensorflow not installed.")
def test_tensorflow_with_constants():
    eq = 'ij,jk,kl->li'
    shapes = (2, 3), (3, 4), (4, 5)
    constants = {0, 2}
    ops = [np.random.rand(*shp) if i in constants else shp for i, shp in enumerate(shapes)]
    var = np.random.rand(*shapes[1])

    res_exp = contract(eq, ops[0], var, ops[2])

    expr = contract_expression(eq, *ops, constants=constants)

    # check tensorflow
    sess = tf.Session(config=_TF_CONFIG)
    with sess.as_default():
        res_got = expr(var, backend='tensorflow')
    sess.close()
    assert 'tensorflow' in expr._evaluated_constants
    assert np.allclose(res_exp, res_got)

    # check can call with numpy still
    res_got2 = expr(var, backend='numpy')
    assert np.allclose(res_exp, res_got2)

    # check tensorflow call returns tensorflow still
    res_got3 = expr(backends.to_tensorflow(var), backend='tensorflow')
    assert isinstance(res_got3, tf.Tensor)

@pytest.mark.skipif(not found_theano, reason="Theano not installed.")
@pytest.mark.parametrize("string", tests)
def test_theano(string):
    views = helpers.build_views(string)
    ein = contract(string, *views, optimize=False, use_blas=False)
    shps = [v.shape for v in views]

    expr = contract_expression(string, *shps, optimize=True)

    opt = expr(*views, backend='theano')
    assert np.allclose(ein, opt)

    # test non-conversion mode
    theano_views = [backends.to_theano(view) for view in views]
    theano_opt = expr(*theano_views, backend='theano')
    assert isinstance(theano_opt, theano.tensor.TensorVariable)

@pytest.mark.skipif(not found_theano, reason="theano not installed.")
def test_theano_with_constants():
    eq = 'ij,jk,kl->li'
    shapes = (2, 3), (3, 4), (4, 5)
    constants = {0, 2}
    ops = [np.random.rand(*shp) if i in constants else shp for i, shp in enumerate(shapes)]
    var = np.random.rand(*shapes[1])

    res_exp = contract(eq, ops[0], var, ops[2])

    expr = contract_expression(eq, *ops, constants=constants)

    # check theano
    res_got = expr(var, backend='theano')
    assert 'theano' in expr._evaluated_constants
    assert np.allclose(res_exp, res_got)

    # check can call with numpy still
    res_got2 = expr(var, backend='numpy')
    assert np.allclose(res_exp, res_got2)

    # check theano call returns theano still
    res_got3 = expr(backends.to_theano(var), backend='theano')
    assert isinstance(res_got3, theano.tensor.TensorVariable)

@pytest.mark.parametrize("string", tests)
def test_cupy(string):  # pragma: no cover
    cupy = pytest.importorskip("cupy")

    views = helpers.build_views(string)
    ein = contract(string, *views, optimize=False, use_blas=False)
    shps = [v.shape for v in views]

    expr = contract_expression(string, *shps, optimize=True)

    opt = expr(*views, backend='cupy')
    assert np.allclose(ein, opt)

    # test non-conversion mode
    cupy_views = [backends.to_cupy(view) for view in views]
    cupy_opt = expr(*cupy_views, backend='cupy')
    assert isinstance(cupy_opt, cupy.ndarray)
    assert np.allclose(ein, cupy.asnumpy(cupy_opt))

def test_cupy_with_constants():  # pragma: no cover
    cupy = pytest.importorskip("cupy")

    eq = 'ij,jk,kl->li'
    shapes = (2, 3), (3, 4), (4, 5)
    constants = {0, 2}
    ops = [np.random.rand(*shp) if i in constants else shp for i, shp in enumerate(shapes)]
    var = np.random.rand(*shapes[1])

    res_exp = contract(eq, ops[0], var, ops[2])

    expr = contract_expression(eq, *ops, constants=constants)

    # check cupy
    res_got = expr(var, backend='cupy')
    assert 'cupy' in expr._evaluated_constants
    assert np.allclose(res_exp, res_got)

    # check can call with numpy still
    res_got2 = expr(var, backend='numpy')
    assert np.allclose(res_exp, res_got2)

    # check cupy call returns cupy still
    res_got3 = expr(cupy.asarray(var), backend='cupy')
    assert isinstance(res_got3, cupy.ndarray)
    assert np.allclose(res_exp, res_got3.get())

@pytest.mark.parametrize("string", tests)
def test_dask(string):
    da = pytest.importorskip("dask.array")

    views = helpers.build_views(string)
    ein = contract(string, *views, optimize=False, use_blas=False)
    shps = [v.shape for v in views]
    expr = contract_expression(string, *shps, optimize=True)

    # test non-conversion mode
    da_views = [da.from_array(x, chunks=(2)) for x in views]
    da_opt = expr(*da_views, backend='dask')

    # check type is maintained when not using numpy arrays
    assert isinstance(da_opt, da.Array)

    assert np.allclose(ein, np.array(da_opt))

    # try raw contract
    da_opt = contract(string, *da_views, backend='dask')
    assert isinstance(da_opt, da.Array)
    assert np.allclose(ein, np.array(da_opt))

@pytest.mark.parametrize("string", tests)
def test_sparse(string):
    sparse = pytest.importorskip("sparse")

    views = helpers.build_views(string)

    # sparsify views so they don't become dense during contraction
    for view in views:
        np.random.seed(42)
        mask = np.random.choice([False, True], view.shape, True, [0.05, 0.95])
        view[mask] = 0

    ein = contract(string, *views, optimize=False, use_blas=False)
    shps = [v.shape for v in views]
    expr = contract_expression(string, *shps, optimize=True)

    # test non-conversion mode
    sparse_views = [sparse.COO.from_numpy(x) for x in views]
    sparse_opt = expr(*sparse_views, backend='sparse')

    # check type is maintained when not using numpy arrays
    assert isinstance(sparse_opt, sparse.COO)

    assert np.allclose(ein, sparse_opt.todense())

    # try raw contract
    sparse_opt = contract(string, *sparse_views, backend='sparse')
    assert isinstance(sparse_opt, sparse.COO)
    assert np.allclose(ein, sparse_opt.todense())

@pytest.mark.skipif(not found_torch, reason="Torch not installed.")
@pytest.mark.parametrize("string", tests)
def test_torch(string):

    views = helpers.build_views(string)
    ein = contract(string, *views, optimize=False, use_blas=False)
    shps = [v.shape for v in views]

    expr = contract_expression(string, *shps, optimize=True)

    opt = expr(*views, backend='torch')
    assert np.allclose(ein, opt)

    # test non-conversion mode
    torch_views = [backends.to_torch(view) for view in views]
    torch_opt = expr(*torch_views, backend='torch')
    assert isinstance(torch_opt, torch.Tensor)
    assert np.allclose(ein, torch_opt.cpu().numpy())

@pytest.mark.skipif(not found_torch, reason="Torch not installed.")
def test_torch_with_constants():

    eq = 'ij,jk,kl->li'
    shapes = (2, 3), (3, 4), (4, 5)
    constants = {0, 2}
    ops = [np.random.rand(*shp) if i in constants else shp for i, shp in enumerate(shapes)]
    var = np.random.rand(*shapes[1])

    res_exp = contract(eq, ops[0], var, ops[2])

    expr = contract_expression(eq, *ops, constants=constants)

    # check torch
    res_got = expr(var, backend='torch')
    assert 'torch' in expr._evaluated_constants
    assert np.allclose(res_exp, res_got)

    # check can call with numpy still
    res_got2 = expr(var, backend='numpy')
    assert np.allclose(res_exp, res_got2)

    # check torch call returns torch still
    res_got3 = expr(backends.to_torch(var), backend='torch')
    assert isinstance(res_got3, torch.Tensor)
    res_got3 = res_got3.numpy() if res_got3.device.type == 'cpu' else res_got3.cpu().numpy()
    assert np.allclose(res_exp, res_got3)
<|endoftext|>