<commit_before># Copyright 2012 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
# Copyright 2013 NTT MCL Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import json

from django.conf import settings
from django.core.urlresolvers import reverse
from django.core.urlresolvers import reverse_lazy
from django.http import HttpResponse  # noqa
from django.views.generic import TemplateView  # noqa
from django.views.generic import View  # noqa

from openstack_dashboard import api

from openstack_dashboard.dashboards.project.network_topology.instances \
    import tables as instances_tables
from openstack_dashboard.dashboards.project.network_topology.ports \
    import tables as ports_tables
from openstack_dashboard.dashboards.project.network_topology.routers \
    import tables as routers_tables

from openstack_dashboard.dashboards.project.instances import\
    views as i_views
from openstack_dashboard.dashboards.project.instances.workflows import\
    create_instance as i_workflows
from openstack_dashboard.dashboards.project.networks import\
    views as n_views
from openstack_dashboard.dashboards.project.networks import\
    workflows as n_workflows
from openstack_dashboard.dashboards.project.routers import\
    views as r_views

class NTCreateRouterView(r_views.CreateView):
    template_name = 'project/network_topology/create_router.html'
    success_url = reverse_lazy("horizon:project:network_topology:index")

class NTCreateNetwork(n_workflows.CreateNetwork):
    def get_success_url(self):
        return reverse("horizon:project:network_topology:index")

    def get_failure_url(self):
        return reverse("horizon:project:network_topology:index")

class NTCreateNetworkView(n_views.CreateView):
    workflow_class = NTCreateNetwork

class NTLaunchInstance(i_workflows.LaunchInstance):
    success_url = "horizon:project:network_topology:index"

class NTLaunchInstanceView(i_views.LaunchInstanceView):
    workflow_class = NTLaunchInstance

class InstanceView(i_views.IndexView):
    table_class = instances_tables.InstancesTable
    template_name = 'project/network_topology/iframe.html'

class RouterView(r_views.IndexView):
    table_class = routers_tables.RoutersTable
    template_name = 'project/network_topology/iframe.html'

class RouterDetailView(r_views.DetailView):
    table_classes = (ports_tables.PortsTable, )
    template_name = 'project/network_topology/iframe.html'

    def get_interfaces_data(self):
        pass

class NetworkTopologyView(TemplateView):
    template_name = 'project/network_topology/index.html'

    def _has_permission(self, policy):
        has_permission = True
        policy_check = getattr(settings, "POLICY_CHECK_FUNCTION", None)

        if policy_check:
            has_permission = policy_check(policy, self.request)

        return has_permission

    def get_context_data(self, **kwargs):
        context = super(NetworkTopologyView, self).get_context_data(**kwargs)
        network_config = getattr(settings, 'OPENSTACK_NEUTRON_NETWORK', {})

        context['launch_instance_allowed'] = self._has_permission(
            (("compute", "compute:create"),))
        context['create_network_allowed'] = self._has_permission(
            (("network", "create_network"),))
        context['create_router_allowed'] = (
            network_config.get('enable_router', True) and
            self._has_permission((("network", "create_router"),)))
        context['console_type'] = getattr(
            settings, 'CONSOLE_TYPE', 'AUTO')
        return context

class JSONView(View):

    @property
    def is_router_enabled(self):
        network_config = getattr(settings, 'OPENSTACK_NEUTRON_NETWORK', {})
        return network_config.get('enable_router', True)

    def add_resource_url(self, view, resources):
        tenant_id = self.request.user.tenant_id
        for resource in resources:
            if (resource.get('tenant_id')
                    and tenant_id != resource.get('tenant_id')):
                continue
            resource['url'] = reverse(view, None, [str(resource['id'])])

    def _check_router_external_port(self, ports, router_id, network_id):
        for port in ports:
            if (port['network_id'] == network_id
                    and port['device_id'] == router_id):
                return True
        return False

    def _get_servers(self, request):
        # Get nova data
        try:
            servers, more = api.nova.server_list(request)
        except Exception:
            servers = []
        console_type = getattr(settings, 'CONSOLE_TYPE', 'AUTO')
        if console_type == 'SPICE':
            console = 'spice'
        else:
            console = 'vnc'
        data = [{'name': server.name,
                 'status': server.status,
                 'console': console,
                 'task': getattr(server, 'OS-EXT-STS:task_state'),
                 'id': server.id} for server in servers]
        self.add_resource_url('horizon:project:instances:detail', data)
        return data

    def _get_networks(self, request):
        # Get neutron data
        # if we didn't specify tenant_id, all networks shown as admin user.
        # so it is need to specify the networks. However there is no need to
        # specify tenant_id for subnet. The subnet which belongs to the public
        # network is needed to draw subnet information on public network.
        try:
            neutron_networks = api.neutron.network_list_for_tenant(
                request,
                request.user.tenant_id)
        except Exception:
            neutron_networks = []
        networks = [{'name': network.name,
                     'id': network.id,
                     'subnets': [{'cidr': subnet.cidr}
                                 for subnet in network.subnets],
                     'router:external': network['router:external']}
                    for network in neutron_networks]
        self.add_resource_url('horizon:project:networks:detail',
                              networks)

        # Add public networks to the networks list
        if self.is_router_enabled:
            try:
                neutron_public_networks = api.neutron.network_list(
                    request,
                    **{'router:external': True})
            except Exception:
                neutron_public_networks = []
            my_network_ids = [net['id'] for net in networks]
            for publicnet in neutron_public_networks:
                if publicnet.id in my_network_ids:
                    continue
                try:
                    subnets = [{'cidr': subnet.cidr}
                               for subnet in publicnet.subnets]
                except Exception:
                    subnets = []
                networks.append({
                    'name': publicnet.name,
                    'id': publicnet.id,
                    'subnets': subnets,
                    'router:external': publicnet['router:external']})

        return sorted(networks,
                      key=lambda x: x.get('router:external'),
                      reverse=True)

    def _get_routers(self, request):
        if not self.is_router_enabled:
            return []
        try:
            neutron_routers = api.neutron.router_list(
                request,
                tenant_id=request.user.tenant_id)
        except Exception:
            neutron_routers = []

        routers = [{'id': router.id,
                    'name': router.name,
                    'status': router.status,
                    'external_gateway_info': router.external_gateway_info}
            for router in neutron_routers]
        self.add_resource_url('horizon:project:routers:detail', routers)
        return routers

    def _get_ports(self, request):
        try:
            neutron_ports = api.neutron.port_list(request)
        except Exception:
            neutron_ports = []

        ports = [{'id': port.id,
                  'network_id': port.network_id,
                  'device_id': port.device_id,
                  'fixed_ips': port.fixed_ips,
                  'device_owner': port.device_owner,
                  'status': port.status}
                 for port in neutron_ports]
        self.add_resource_url('horizon:project:networks:ports:detail',
                              ports)
        return ports

    def _prepare_gateway_ports(self, routers, ports):
        # user can't see port on external network. so we are
        # adding fake port based on router information
        for router in routers:
            external_gateway_info = router.get('external_gateway_info')
            if not external_gateway_info:
                continue
            external_network = external_gateway_info.get(
                'network_id')
            if not external_network:
                continue
            if self._check_router_external_port(ports,
                                                router['id'],
                                                external_network):
                continue
            fake_port = {'id': 'gateway%s' % external_network,
                         'network_id': external_network,
                         'device_id': router['id'],
                         'fixed_ips': []}
            ports.append(fake_port)

    def get(self, request, *args, **kwargs):
        data = {'servers': self._get_servers(request),
                'networks': self._get_networks(request),
                'ports': self._get_ports(request),
                'routers': self._get_routers(request)}
        self._prepare_gateway_ports(data['routers'], data['ports'])
        json_string = json.dumps(data, ensure_ascii=False)
        return HttpResponse(json_string, content_type='text/json')
<commit_msg>Removing HA interfaces from port list<commit_after># Copyright 2012 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
# Copyright 2013 NTT MCL Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import json

from django.conf import settings
from django.core.urlresolvers import reverse
from django.core.urlresolvers import reverse_lazy
from django.http import HttpResponse  # noqa
from django.views.generic import TemplateView  # noqa
from django.views.generic import View  # noqa

from openstack_dashboard import api

from openstack_dashboard.dashboards.project.network_topology.instances \
    import tables as instances_tables
from openstack_dashboard.dashboards.project.network_topology.ports \
    import tables as ports_tables
from openstack_dashboard.dashboards.project.network_topology.routers \
    import tables as routers_tables

from openstack_dashboard.dashboards.project.instances import\
    views as i_views
from openstack_dashboard.dashboards.project.instances.workflows import\
    create_instance as i_workflows
from openstack_dashboard.dashboards.project.networks import\
    views as n_views
from openstack_dashboard.dashboards.project.networks import\
    workflows as n_workflows
from openstack_dashboard.dashboards.project.routers import\
    views as r_views

class NTCreateRouterView(r_views.CreateView):
    template_name = 'project/network_topology/create_router.html'
    success_url = reverse_lazy("horizon:project:network_topology:index")

class NTCreateNetwork(n_workflows.CreateNetwork):
    def get_success_url(self):
        return reverse("horizon:project:network_topology:index")

    def get_failure_url(self):
        return reverse("horizon:project:network_topology:index")

class NTCreateNetworkView(n_views.CreateView):
    workflow_class = NTCreateNetwork

class NTLaunchInstance(i_workflows.LaunchInstance):
    success_url = "horizon:project:network_topology:index"

class NTLaunchInstanceView(i_views.LaunchInstanceView):
    workflow_class = NTLaunchInstance

class InstanceView(i_views.IndexView):
    table_class = instances_tables.InstancesTable
    template_name = 'project/network_topology/iframe.html'

class RouterView(r_views.IndexView):
    table_class = routers_tables.RoutersTable
    template_name = 'project/network_topology/iframe.html'

class RouterDetailView(r_views.DetailView):
    table_classes = (ports_tables.PortsTable, )
    template_name = 'project/network_topology/iframe.html'

    def get_interfaces_data(self):
        pass

class NetworkTopologyView(TemplateView):
    template_name = 'project/network_topology/index.html'

    def _has_permission(self, policy):
        has_permission = True
        policy_check = getattr(settings, "POLICY_CHECK_FUNCTION", None)

        if policy_check:
            has_permission = policy_check(policy, self.request)

        return has_permission

    def get_context_data(self, **kwargs):
        context = super(NetworkTopologyView, self).get_context_data(**kwargs)
        network_config = getattr(settings, 'OPENSTACK_NEUTRON_NETWORK', {})

        context['launch_instance_allowed'] = self._has_permission(
            (("compute", "compute:create"),))
        context['create_network_allowed'] = self._has_permission(
            (("network", "create_network"),))
        context['create_router_allowed'] = (
            network_config.get('enable_router', True) and
            self._has_permission((("network", "create_router"),)))
        context['console_type'] = getattr(
            settings, 'CONSOLE_TYPE', 'AUTO')
        return context

class JSONView(View):

    @property
    def is_router_enabled(self):
        network_config = getattr(settings, 'OPENSTACK_NEUTRON_NETWORK', {})
        return network_config.get('enable_router', True)

    def add_resource_url(self, view, resources):
        tenant_id = self.request.user.tenant_id
        for resource in resources:
            if (resource.get('tenant_id')
                    and tenant_id != resource.get('tenant_id')):
                continue
            resource['url'] = reverse(view, None, [str(resource['id'])])

    def _check_router_external_port(self, ports, router_id, network_id):
        for port in ports:
            if (port['network_id'] == network_id
                    and port['device_id'] == router_id):
                return True
        return False

    def _get_servers(self, request):
        # Get nova data
        try:
            servers, more = api.nova.server_list(request)
        except Exception:
            servers = []
        console_type = getattr(settings, 'CONSOLE_TYPE', 'AUTO')
        if console_type == 'SPICE':
            console = 'spice'
        else:
            console = 'vnc'
        data = [{'name': server.name,
                 'status': server.status,
                 'console': console,
                 'task': getattr(server, 'OS-EXT-STS:task_state'),
                 'id': server.id} for server in servers]
        self.add_resource_url('horizon:project:instances:detail', data)
        return data

    def _get_networks(self, request):
        # Get neutron data
        # if we didn't specify tenant_id, all networks shown as admin user.
        # so it is need to specify the networks. However there is no need to
        # specify tenant_id for subnet. The subnet which belongs to the public
        # network is needed to draw subnet information on public network.
        try:
            neutron_networks = api.neutron.network_list_for_tenant(
                request,
                request.user.tenant_id)
        except Exception:
            neutron_networks = []
        networks = [{'name': network.name,
                     'id': network.id,
                     'subnets': [{'cidr': subnet.cidr}
                                 for subnet in network.subnets],
                     'router:external': network['router:external']}
                    for network in neutron_networks]
        self.add_resource_url('horizon:project:networks:detail',
                              networks)

        # Add public networks to the networks list
        if self.is_router_enabled:
            try:
                neutron_public_networks = api.neutron.network_list(
                    request,
                    **{'router:external': True})
            except Exception:
                neutron_public_networks = []
            my_network_ids = [net['id'] for net in networks]
            for publicnet in neutron_public_networks:
                if publicnet.id in my_network_ids:
                    continue
                try:
                    subnets = [{'cidr': subnet.cidr}
                               for subnet in publicnet.subnets]
                except Exception:
                    subnets = []
                networks.append({
                    'name': publicnet.name,
                    'id': publicnet.id,
                    'subnets': subnets,
                    'router:external': publicnet['router:external']})

        return sorted(networks,
                      key=lambda x: x.get('router:external'),
                      reverse=True)

    def _get_routers(self, request):
        if not self.is_router_enabled:
            return []
        try:
            neutron_routers = api.neutron.router_list(
                request,
                tenant_id=request.user.tenant_id)
        except Exception:
            neutron_routers = []

        routers = [{'id': router.id,
                    'name': router.name,
                    'status': router.status,
                    'external_gateway_info': router.external_gateway_info}
            for router in neutron_routers]
        self.add_resource_url('horizon:project:routers:detail', routers)
        return routers

    def _get_ports(self, request):
        try:
            neutron_ports = api.neutron.port_list(request)
        except Exception:
            neutron_ports = []

        ports = [{'id': port.id,
                  'network_id': port.network_id,
                  'device_id': port.device_id,
                  'fixed_ips': port.fixed_ips,
                  'device_owner': port.device_owner,
                  'status': port.status}
                 for port in neutron_ports
                 if port.device_owner != 'network:router_ha_interface']
        self.add_resource_url('horizon:project:networks:ports:detail',
                              ports)
        return ports

    def _prepare_gateway_ports(self, routers, ports):
        # user can't see port on external network. so we are
        # adding fake port based on router information
        for router in routers:
            external_gateway_info = router.get('external_gateway_info')
            if not external_gateway_info:
                continue
            external_network = external_gateway_info.get(
                'network_id')
            if not external_network:
                continue
            if self._check_router_external_port(ports,
                                                router['id'],
                                                external_network):
                continue
            fake_port = {'id': 'gateway%s' % external_network,
                         'network_id': external_network,
                         'device_id': router['id'],
                         'fixed_ips': []}
            ports.append(fake_port)

    def get(self, request, *args, **kwargs):
        data = {'servers': self._get_servers(request),
                'networks': self._get_networks(request),
                'ports': self._get_ports(request),
                'routers': self._get_routers(request)}
        self._prepare_gateway_ports(data['routers'], data['ports'])
        json_string = json.dumps(data, ensure_ascii=False)
        return HttpResponse(json_string, content_type='text/json')
<|endoftext|>


<commit_before>from django.contrib import admin
from publications.models import Licence, EstimatedImpactFactor, KeyWord, Publication, Comment, Reviewer
# Register your models here.

class LicenceAdmin(admin.ModelAdmin):
    list_display = ['short_name']
    list_filter = ['short_name']
    ordering = ['short_name']
    search_fields = ['short_name']

class EstimatedImpactFactorAdmin(admin.ModelAdmin):
    list_display = ['scientist', 'publication', 'estimated_impact_factor']
    list_filter = ['scientist', 'publication', 'estimated_impact_factor']
    ordering = ['scientist', 'publication', 'estimated_impact_factor']
    search_fields = ['scientist', 'publication', 'estimated_impact_factor']

class KeyWordAdmin(admin.ModelAdmin):
    list_display = ['tag']
    list_filter = ['tag']
    ordering = ['tag']
    search_fields = ['tag']

class PublicationAdmin(admin.ModelAdmin):
    list_display = ['editor', 'creation_date', 'payment_date', 'validation_date',
                    'title', 'publication_score', 'estimated_impact_factor', 'status', 'licence']
    list_filter = ['editor', 'creation_date', 'payment_date', 'validation_date',
                   'title', 'publication_score', 'estimated_impact_factor', 'status', 'licence']
    ordering = ['editor', 'creation_date', 'payment_date', 'validation_date',
                'title', 'publication_score', 'estimated_impact_factor', 'status', 'licence']
    search_fields = ['editor', 'creation_date', 'payment_date', 'validation_date',
                     'title', 'publication_score', 'estimated_impact_factor', 'status', 'licence']

class CommentAdmin(admin.ModelAdmin):
    list_display = ['publication', 'author', 'comment_type', 'seriousness', 'validated', 'corrected', 'licence']
    list_filter = ['publication', 'author', 'comment_type', 'seriousness', 'validated', 'corrected', 'licence']
    ordering = ['publication', 'author', 'comment_type', 'seriousness', 'validated', 'corrected', 'licence']
    search_fields = ['publication', 'author', 'comment_type', 'seriousness', 'validated', 'corrected', 'licence']

class ReviewerAdmin(admin.ModelAdmin):
    list_display = ['scientist', 'publication']
    list_filter = ['scientist', 'publication']
    ordering = ['scientist', 'publication']
    search_fields = ['scientist', 'publication']

admin.site.register(Licence, LicenceAdmin)
<commit_msg>admin publication done<commit_after>from django.contrib import admin
from publications.models import Licence, EstimatedImpactFactor, KeyWord, Publication, Comment, Reviewer
# Register your models here.

class LicenceAdmin(admin.ModelAdmin):
    list_display = ['short_name']
    list_filter = ['short_name']
    ordering = ['short_name']
    search_fields = ['short_name']

class EstimatedImpactFactorAdmin(admin.ModelAdmin):
    list_display = ['scientist', 'publication', 'estimated_impact_factor']
    list_filter = ['scientist', 'publication', 'estimated_impact_factor']
    ordering = ['scientist', 'publication', 'estimated_impact_factor']
    search_fields = ['scientist', 'publication', 'estimated_impact_factor']

class KeyWordAdmin(admin.ModelAdmin):
    list_display = ['tag']
    list_filter = ['tag']
    ordering = ['tag']
    search_fields = ['tag']

class PublicationAdmin(admin.ModelAdmin):
    list_display = ['editor', 'creation_date', 'payment_date', 'validation_date',
                    'title', 'publication_score', 'estimated_impact_factor', 'status', 'licence']
    list_filter = ['editor', 'creation_date', 'payment_date', 'validation_date',
                   'title', 'publication_score', 'estimated_impact_factor', 'status', 'licence']
    ordering = ['editor', 'creation_date', 'payment_date', 'validation_date',
                'title', 'publication_score', 'estimated_impact_factor', 'status', 'licence']
    search_fields = ['editor', 'creation_date', 'payment_date', 'validation_date',
                     'title', 'publication_score', 'estimated_impact_factor', 'status', 'licence']

class CommentAdmin(admin.ModelAdmin):
    list_display = ['publication', 'author', 'comment_type', 'seriousness', 'validated', 'corrected', 'licence']
    list_filter = ['publication', 'author', 'comment_type', 'seriousness', 'validated', 'corrected', 'licence']
    ordering = ['publication', 'author', 'comment_type', 'seriousness', 'validated', 'corrected', 'licence']
    search_fields = ['publication', 'author', 'comment_type', 'seriousness', 'validated', 'corrected', 'licence']

class ReviewerAdmin(admin.ModelAdmin):
    list_display = ['scientist', 'publication']
    list_filter = ['scientist', 'publication']
    ordering = ['scientist', 'publication']
    search_fields = ['scientist', 'publication']

admin.site.register(Licence, LicenceAdmin)
admin.site.register(EstimatedImpactFactor, EstimatedImpactFactorAdmin)
admin.site.register(KeyWord, KeyWordAdmin)
admin.site.register(Publication, PublicationAdmin)
admin.site.register(Comment, CommentAdmin)
admin.site.register(Reviewer, ReviewerAdmin)<|endoftext|>


<commit_before>import pathlib
import pytest
import sklearn

def test_files_generated_by_templates_are_git_ignored():
    """Check the consistence of the files generated from template files."""
    gitignore_file = pathlib.Path(sklearn.__file__).parent.parent / ".gitignore"
    if not gitignore_file.exists():
        pytest.skip("Tests are not run from the source folder")

    base_dir = pathlib.Path(sklearn.__file__).parent
    ignored_files = open(gitignore_file, "r").readlines()
    ignored_files = list(map(lambda line: line.strip("\n"), ignored_files))

    for filename in base_dir.glob("**/*.tp"):
        filename = filename.relative_to(base_dir.parent)
        # From "path/to/template.p??.tp" to "path/to/template.p??"
        filename_wo_tempita_suffix = filename.with_suffix("")
        assert str(filename_wo_tempita_suffix) in ignored_files
<commit_msg>FIX Fixes test for checking gitignore and Cython templates (#20997)<commit_after>import pathlib
import pytest
import sklearn

def test_files_generated_by_templates_are_git_ignored():
    """Check the consistence of the files generated from template files."""
    gitignore_file = pathlib.Path(sklearn.__file__).parent.parent / ".gitignore"
    if not gitignore_file.exists():
        pytest.skip("Tests are not run from the source folder")

    base_dir = pathlib.Path(sklearn.__file__).parent
    ignored_files = gitignore_file.read_text().split("\n")
    ignored_files = [pathlib.Path(line) for line in ignored_files]

    for filename in base_dir.glob("**/*.tp"):
        filename = filename.relative_to(base_dir.parent)
        # From "path/to/template.p??.tp" to "path/to/template.p??"
        filename_wo_tempita_suffix = filename.with_suffix("")
        assert filename_wo_tempita_suffix in ignored_files
<|endoftext|>


<commit_before>#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****

import sys
import os

# load modules from parent dir
sys.path.insert(1, os.path.dirname(sys.path[0]))

# import the guts
from mozharness.base.config import parse_config_file
from mozharness.base.errors import MakefileErrorList
from mozharness.base.log import ERROR, FATAL
from mozharness.base.script import BaseScript
from mozharness.base.vcs.vcsbase import VCSMixin
from mozharness.mozilla.l10n.locales import GaiaLocalesMixin, LocalesMixin

class B2gMultilocale(LocalesMixin, BaseScript, VCSMixin, GaiaLocalesMixin):
    """ This is a helper script that requires MercurialBuildFactory
        logic to work.  We may eventually make this a standalone
        script.

        We could inherit MercurialScript instead of BaseScript + VCSMixin
    """
    config_options = [
        [["--locale"], {
            "action": "extend",
            "dest": "locales",
            "type": "string",
            "help": "Specify the locale(s) to repack"
        }],
        [["--gaia-languages-file"], {
            "dest": "gaia_languages_file",
            "help": "languages file for gaia multilocale profile",
        }],
        [["--gecko-languages-file"], {
            "dest": "locales_file",
            "help": "languages file for gecko multilocale",
        }],
        [["--gecko-l10n-root"], {
            "dest": "hg_l10n_base",
            "help": "root location for gecko l10n repos",
        }],
        [["--gecko-l10n-base-dir"], {
            "dest": "l10n_dir",
            "help": "dir to clone gecko l10n repos into, relative to the work directory",
        }],
        [["--merge-locales"], {
            "dest": "merge_locales",
            "help": "Dummy option to keep from burning. We now always merge",
        }],
        [["--gaia-l10n-root"], {
            "dest": "gaia_l10n_root",
            "help": "root location for gaia l10n repos",
        }],
        [["--gaia-l10n-base-dir"], {
            "dest": "gaia_l10n_base_dir",
            "default": "build-gaia-l10n",
            "help": "dir to clone l10n repos into, relative to the work directory",
        }],
        [["--gaia-l10n-vcs"], {
            "dest": "gaia_l10n_vcs",
            "help": "vcs to use for gaia l10n",
        }],
    ]

    def __init__(self, require_config_file=False):
        LocalesMixin.__init__(self)
        BaseScript.__init__(self,
                            config_options=self.config_options,
                            all_actions=[
                                'pull',
                                'build',
                                'summary',
                            ],
                            require_config_file=require_config_file,

                            # Default configuration
                            config={
                                'gaia_l10n_vcs': 'hg',
                                'vcs_share_base': os.environ.get('HG_SHARE_BASE_DIR'),
                                'locales_dir': 'b2g/locales',
                                'l10n_dir': 'gecko-l10n',
                                # I forget what this was for.  Copied from the Android multilocale stuff
                                'ignore_locales': ["en-US", "multi"],
                                # This only has 2 locales in it.  We probably need files that mirror gaia's locale lists
                                # We need 2 sets of locales files because the locale names in gaia are different than gecko, e.g. 'es' vs 'es-ES'
                                # We'll need to override this for localizer buidls
                                'locales_file': 'build/b2g/locales/all-locales',
                                'mozilla_dir': 'build',
                                'objdir': 'obj-firefox',
                                'merge_locales': True,
                                'work_dir': '.',
                            },
                            )

    def _pre_config_lock(self, rw_config):
        super(B2gMultilocale, self)._pre_config_lock(rw_config)

        if 'pull' in self.actions:
            message = ""
            if 'gaia_languages_file' not in self.config:
                message += 'Must specify --gaia-languages-file!\n'
            if 'gaia_l10n_root' not in self.config:
                message += 'Must specify --gaia-l10n-root!\n'
            if 'locales_file' not in self.config:
                message += 'Must specify --gecko-languages-file!\n'
            if 'hg_l10n_base' not in self.config:
                message += 'Must specify --gecko-l10n-root!\n'
            if message:
                self.fatal(message)

    def query_abs_dirs(self):
        if self.abs_dirs:
            return self.abs_dirs
        abs_dirs = LocalesMixin.query_abs_dirs(self)

        c = self.config
        dirs = {
            'src': os.path.join(c['work_dir'], 'gecko'),
            'work_dir': abs_dirs['abs_work_dir'],
            'gaia_l10n_base_dir': os.path.join(abs_dirs['abs_work_dir'], self.config['gaia_l10n_base_dir']),
            'abs_compare_locales_dir': os.path.join(abs_dirs['base_work_dir'], 'compare-locales'),
        }

        abs_dirs.update(dirs)
        self.abs_dirs = abs_dirs
        return self.abs_dirs

    # Actions {{{2
    def pull(self):
        """ Clone gaia and gecko locale repos
        """
        languages_file = self.config['gaia_languages_file']
        l10n_base_dir = self.query_abs_dirs()['gaia_l10n_base_dir']
        l10n_config = {
            'root': self.config['gaia_l10n_root'],
            'vcs': self.config['gaia_l10n_vcs'],
        }

        self.pull_gaia_locale_source(l10n_config, parse_config_file(languages_file).keys(), l10n_base_dir)
        self.pull_locale_source()
        gecko_locales = self.query_locales()
        # populate b2g/overrides, which isn't in gecko atm
        dirs = self.query_abs_dirs()
        for locale in gecko_locales:
            self.mkdir_p(os.path.join(dirs['abs_l10n_dir'], locale, 'b2g', 'chrome', 'overrides'))
            self.copytree(os.path.join(dirs['abs_l10n_dir'], locale, 'mobile', 'overrides'),
                          os.path.join(dirs['abs_l10n_dir'], locale, 'b2g', 'chrome', 'overrides'),
                          error_level=FATAL)

    def build(self):
        """ Do the multilocale portion of the build + packaging.
        """
        dirs = self.query_abs_dirs()
        gecko_locales = self.query_locales()
        make = self.query_exe('make', return_type='string')
        env = self.query_env(
            partial_env={
                'LOCALE_MERGEDIR': dirs['abs_merge_dir'],
                'MOZ_CHROME_MULTILOCALE': 'en-US ' + ' '.join(gecko_locales),
            }
        )
        merge_env = self.query_env(
            partial_env={
                'PATH': '%(PATH)s' + os.pathsep + os.path.join(dirs['abs_compare_locales_dir'], 'scripts'),
                'PYTHONPATH': os.path.join(dirs['abs_compare_locales_dir'],
                                           'lib'),

            }
        )
        for locale in gecko_locales:
            command = make + ' merge-%s L10NBASEDIR=%s LOCALE_MERGEDIR=%s' % (locale, dirs['abs_l10n_dir'], dirs['abs_merge_dir'])
            status = self.run_command(command,
                                      cwd=dirs['abs_locales_dir'],
                                      error_list=MakefileErrorList,
                                      env=merge_env)
            command = make + ' chrome-%s L10NBASEDIR=%s LOCALE_MERGEDIR=%s' % (locale, dirs['abs_l10n_dir'], dirs['abs_merge_dir'])
            status = self.run_command(command,
                                      cwd=dirs['abs_locales_dir'],
                                      error_list=MakefileErrorList)
            if status:
                self.add_summary("Failed to add locale %s!" % locale, level=ERROR)
        command = make + ' -C app'
        self.run_command(command,
                         cwd=os.path.join(dirs['abs_objdir'], 'b2g'),
                         error_list=MakefileErrorList,
                         halt_on_failure=True)
        command = make + ' package AB_CD=multi'
        self.run_command(command, cwd=dirs['abs_objdir'],
                         error_list=MakefileErrorList, env=env,
                         halt_on_failure=True)
        command = make + ' package-tests AB_CD=multi'
        self.run_command(command, cwd=dirs['abs_objdir'],
                         error_list=MakefileErrorList, env=env,
                         halt_on_failure=True)

# main {{{1
if __name__ == '__main__':
    myScript = B2gMultilocale()
    myScript.run_and_exit()
<commit_msg>Bug 1113108: Add a 10 minute timeout for vcs operations r=jlund<commit_after>#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****

import sys
import os

# load modules from parent dir
sys.path.insert(1, os.path.dirname(sys.path[0]))

# import the guts
from mozharness.base.config import parse_config_file
from mozharness.base.errors import MakefileErrorList
from mozharness.base.log import ERROR, FATAL
from mozharness.base.script import BaseScript
from mozharness.base.vcs.vcsbase import VCSMixin
from mozharness.mozilla.l10n.locales import GaiaLocalesMixin, LocalesMixin

class B2gMultilocale(LocalesMixin, BaseScript, VCSMixin, GaiaLocalesMixin):
    """ This is a helper script that requires MercurialBuildFactory
        logic to work.  We may eventually make this a standalone
        script.

        We could inherit MercurialScript instead of BaseScript + VCSMixin
    """
    config_options = [
        [["--locale"], {
            "action": "extend",
            "dest": "locales",
            "type": "string",
            "help": "Specify the locale(s) to repack"
        }],
        [["--gaia-languages-file"], {
            "dest": "gaia_languages_file",
            "help": "languages file for gaia multilocale profile",
        }],
        [["--gecko-languages-file"], {
            "dest": "locales_file",
            "help": "languages file for gecko multilocale",
        }],
        [["--gecko-l10n-root"], {
            "dest": "hg_l10n_base",
            "help": "root location for gecko l10n repos",
        }],
        [["--gecko-l10n-base-dir"], {
            "dest": "l10n_dir",
            "help": "dir to clone gecko l10n repos into, relative to the work directory",
        }],
        [["--merge-locales"], {
            "dest": "merge_locales",
            "help": "Dummy option to keep from burning. We now always merge",
        }],
        [["--gaia-l10n-root"], {
            "dest": "gaia_l10n_root",
            "help": "root location for gaia l10n repos",
        }],
        [["--gaia-l10n-base-dir"], {
            "dest": "gaia_l10n_base_dir",
            "default": "build-gaia-l10n",
            "help": "dir to clone l10n repos into, relative to the work directory",
        }],
        [["--gaia-l10n-vcs"], {
            "dest": "gaia_l10n_vcs",
            "help": "vcs to use for gaia l10n",
        }],
    ]

    def __init__(self, require_config_file=False):
        LocalesMixin.__init__(self)
        BaseScript.__init__(self,
                            config_options=self.config_options,
                            all_actions=[
                                'pull',
                                'build',
                                'summary',
                            ],
                            require_config_file=require_config_file,

                            # Default configuration
                            config={
                                'gaia_l10n_vcs': 'hg',
                                'vcs_share_base': os.environ.get('HG_SHARE_BASE_DIR'),
                                'locales_dir': 'b2g/locales',
                                'l10n_dir': 'gecko-l10n',
                                # I forget what this was for.  Copied from the Android multilocale stuff
                                'ignore_locales': ["en-US", "multi"],
                                # This only has 2 locales in it.  We probably need files that mirror gaia's locale lists
                                # We need 2 sets of locales files because the locale names in gaia are different than gecko, e.g. 'es' vs 'es-ES'
                                # We'll need to override this for localizer buidls
                                'locales_file': 'build/b2g/locales/all-locales',
                                'mozilla_dir': 'build',
                                'objdir': 'obj-firefox',
                                'merge_locales': True,
                                'work_dir': '.',
                                'vcs_output_timeout': 600,  # 10 minutes should be enough for anyone!
                            },
                            )

    def _pre_config_lock(self, rw_config):
        super(B2gMultilocale, self)._pre_config_lock(rw_config)

        if 'pull' in self.actions:
            message = ""
            if 'gaia_languages_file' not in self.config:
                message += 'Must specify --gaia-languages-file!\n'
            if 'gaia_l10n_root' not in self.config:
                message += 'Must specify --gaia-l10n-root!\n'
            if 'locales_file' not in self.config:
                message += 'Must specify --gecko-languages-file!\n'
            if 'hg_l10n_base' not in self.config:
                message += 'Must specify --gecko-l10n-root!\n'
            if message:
                self.fatal(message)

    def query_abs_dirs(self):
        if self.abs_dirs:
            return self.abs_dirs
        abs_dirs = LocalesMixin.query_abs_dirs(self)

        c = self.config
        dirs = {
            'src': os.path.join(c['work_dir'], 'gecko'),
            'work_dir': abs_dirs['abs_work_dir'],
            'gaia_l10n_base_dir': os.path.join(abs_dirs['abs_work_dir'], self.config['gaia_l10n_base_dir']),
            'abs_compare_locales_dir': os.path.join(abs_dirs['base_work_dir'], 'compare-locales'),
        }

        abs_dirs.update(dirs)
        self.abs_dirs = abs_dirs
        return self.abs_dirs

    # Actions {{{2
    def pull(self):
        """ Clone gaia and gecko locale repos
        """
        languages_file = self.config['gaia_languages_file']
        l10n_base_dir = self.query_abs_dirs()['gaia_l10n_base_dir']
        l10n_config = {
            'root': self.config['gaia_l10n_root'],
            'vcs': self.config['gaia_l10n_vcs'],
        }

        self.pull_gaia_locale_source(l10n_config, parse_config_file(languages_file).keys(), l10n_base_dir)
        self.pull_locale_source()
        gecko_locales = self.query_locales()
        # populate b2g/overrides, which isn't in gecko atm
        dirs = self.query_abs_dirs()
        for locale in gecko_locales:
            self.mkdir_p(os.path.join(dirs['abs_l10n_dir'], locale, 'b2g', 'chrome', 'overrides'))
            self.copytree(os.path.join(dirs['abs_l10n_dir'], locale, 'mobile', 'overrides'),
                          os.path.join(dirs['abs_l10n_dir'], locale, 'b2g', 'chrome', 'overrides'),
                          error_level=FATAL)

    def build(self):
        """ Do the multilocale portion of the build + packaging.
        """
        dirs = self.query_abs_dirs()
        gecko_locales = self.query_locales()
        make = self.query_exe('make', return_type='string')
        env = self.query_env(
            partial_env={
                'LOCALE_MERGEDIR': dirs['abs_merge_dir'],
                'MOZ_CHROME_MULTILOCALE': 'en-US ' + ' '.join(gecko_locales),
            }
        )
        merge_env = self.query_env(
            partial_env={
                'PATH': '%(PATH)s' + os.pathsep + os.path.join(dirs['abs_compare_locales_dir'], 'scripts'),
                'PYTHONPATH': os.path.join(dirs['abs_compare_locales_dir'],
                                           'lib'),

            }
        )
        for locale in gecko_locales:
            command = make + ' merge-%s L10NBASEDIR=%s LOCALE_MERGEDIR=%s' % (locale, dirs['abs_l10n_dir'], dirs['abs_merge_dir'])
            status = self.run_command(command,
                                      cwd=dirs['abs_locales_dir'],
                                      error_list=MakefileErrorList,
                                      env=merge_env)
            command = make + ' chrome-%s L10NBASEDIR=%s LOCALE_MERGEDIR=%s' % (locale, dirs['abs_l10n_dir'], dirs['abs_merge_dir'])
            status = self.run_command(command,
                                      cwd=dirs['abs_locales_dir'],
                                      error_list=MakefileErrorList)
            if status:
                self.add_summary("Failed to add locale %s!" % locale, level=ERROR)
        command = make + ' -C app'
        self.run_command(command,
                         cwd=os.path.join(dirs['abs_objdir'], 'b2g'),
                         error_list=MakefileErrorList,
                         halt_on_failure=True)
        command = make + ' package AB_CD=multi'
        self.run_command(command, cwd=dirs['abs_objdir'],
                         error_list=MakefileErrorList, env=env,
                         halt_on_failure=True)
        command = make + ' package-tests AB_CD=multi'
        self.run_command(command, cwd=dirs['abs_objdir'],
                         error_list=MakefileErrorList, env=env,
                         halt_on_failure=True)

# main {{{1
if __name__ == '__main__':
    myScript = B2gMultilocale()
    myScript.run_and_exit()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
#    Copyright (C) 2010 OpenERP s.a. (<http://openerp.com>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

{
    'name': 'Base',
    'version': '1.2',
    'category': 'Generic Modules/Base',
    'description': """The kernel of OpenERP, needed for all installation.""",
    'author': 'OpenERP s.a.',
    'maintainer': 'OpenERP s.a.',
    'website': 'http://www.openerp.com',
    'depends': [],
    'init_xml': [
        'base_data.xml',
        'security/base_security.xml',
        'base_menu.xml',
        'res/res_security.xml',
        'res/res_config.xml',
        'data/res.country.state.csv'
    ],
    'update_xml': [
        'base_update.xml',
        'ir/wizard/wizard_menu_view.xml',
        'ir/ir.xml',
        'ir/workflow/workflow_view.xml',
        'module/module_wizard.xml',
        'module/module_view.xml',
        'module/module_web_view.xml',
        'module/module_data.xml',
        'module/module_report.xml',
        'res/res_request_view.xml',
        'res/res_lang_view.xml',
        'res/res_log_view.xml',
        'res/partner/partner_report.xml',
        'res/partner/partner_view.xml',
        'res/partner/partner_wizard.xml',
        'res/bank_view.xml',
        'res/country_view.xml',
        'res/res_currency_view.xml',
        'res/partner/crm_view.xml',
        'res/partner/wizard/partner_sms_send_view.xml',
        'res/partner/partner_data.xml',
        'res/ir_property_view.xml',
        'security/base_security.xml',
        'maintenance/maintenance_view.xml',
        'security/ir.model.access.csv'

    ],
    'demo_xml': [
        'base_demo.xml',
        'res/partner/partner_demo.xml',
        'res/partner/crm_demo.xml',
    ],
    'test': [
        'test/base_test.xml',
        #'test/base_test.yml'
        'test/test_context.xml',
        'test/bug_lp541545.xml',
    ],
    'installable': True,
    'active': True,
    'certificate': '0076807797149',
}
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>[RENAME] Tiny ERP to OpenERP SA in all places<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
#    Copyright (C) 2010 OpenERP s.a. (<http://openerp.com>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

{
    'name': 'Base',
    'version': '1.2',
    'category': 'Generic Modules/Base',
    'description': """The kernel of OpenERP, needed for all installation.""",
    'author': 'OpenERP SA',
    'maintainer': 'OpenERP SA',
    'website': 'http://www.openerp.com',
    'depends': [],
    'init_xml': [
        'base_data.xml',
        'security/base_security.xml',
        'base_menu.xml',
        'res/res_security.xml',
        'res/res_config.xml',
        'data/res.country.state.csv'
    ],
    'update_xml': [
        'base_update.xml',
        'ir/wizard/wizard_menu_view.xml',
        'ir/ir.xml',
        'ir/workflow/workflow_view.xml',
        'module/module_wizard.xml',
        'module/module_view.xml',
        'module/module_web_view.xml',
        'module/module_data.xml',
        'module/module_report.xml',
        'res/res_request_view.xml',
        'res/res_lang_view.xml',
        'res/res_log_view.xml',
        'res/partner/partner_report.xml',
        'res/partner/partner_view.xml',
        'res/partner/partner_wizard.xml',
        'res/bank_view.xml',
        'res/country_view.xml',
        'res/res_currency_view.xml',
        'res/partner/crm_view.xml',
        'res/partner/wizard/partner_sms_send_view.xml',
        'res/partner/partner_data.xml',
        'res/ir_property_view.xml',
        'security/base_security.xml',
        'maintenance/maintenance_view.xml',
        'security/ir.model.access.csv'

    ],
    'demo_xml': [
        'base_demo.xml',
        'res/partner/partner_demo.xml',
        'res/partner/crm_demo.xml',
    ],
    'test': [
        'test/base_test.xml',
        #'test/base_test.yml'
        'test/test_context.xml',
        'test/bug_lp541545.xml',
    ],
    'installable': True,
    'active': True,
    'certificate': '0076807797149',
}
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# Interpreter version: python 2.7
#
# Imports =====================================================================
from __future__ import unicode_literals

import sys
import os.path
from os.path import join
from os.path import dirname
from functools import partial
from urllib import unquote_plus

from bottle import run
from bottle import abort
from bottle import error
from bottle import route
from bottle import HTTPError
from bottle import static_file
from bottle import SimpleTemplate

import mime
from bottle import auth_basic

sys.path.insert(0, join(dirname(__file__), "../src/edeposit/amqp"))

try:
    from storage import DBPublication
    from storage.tree_handler import tree_handler
    from storage.publication_storage import search_publications

    from storage import settings
    from storage import web_tools
except ImportError:
    from edeposit.amqp.storage import DBPublication
    from edeposit.amqp.storage.tree_handler import tree_handler
    from edeposit.amqp.storage.publication_storage import search_publications

    from edeposit.amqp.storage import settings
    from edeposit.amqp.storage import web_tools

# Variables ===================================================================
INDEX_TEMPLATE = """<!DOCTYPE html>

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
    <title>Seznam veřejně přístupných publikací</title>

Seznam veřejně přístupných publikací projektu E-deposit

% for pub in publications:

          {{pub.title}}

        Autor{{delimiter}}
        {{pub.author}}

        ISBN{{delimiter}}
        {{pub.isbn}}

        Rok vydání{{delimiter}}
        {{pub.pub_year}}

        URN:NBN{{delimiter}}
        {{pub.urnnbn}}

% end

"""

PRIVATE_ACCESS_MSG = """<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">

    <title>Veřejně nepřístupný dokument</title>
    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">

Chyba: Veřejně nepřístupný dokument

      Dokument `{{name}}` s UUID <tt>`{{uuid}}`</tt> bohužel není
      veřejně přístupný.

      Tato publikace je zpřístupněna pouze na terminálech v prostorech
      Národní knihovny ČR.

"""

TREES_TEMPLATE = """<!DOCTYPE html>

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
    <title>Seznam publikací periodika</title>

% if link_up:
    [Zpět]
% end

% for tree in trees:

Seznam publikací periodika

    {{ !render_tree(tree, 1) }}

% end

"""

TREE_TEMPLATE = """

{{tree.name}}

% for sub_tree in tree.sub_trees:
    {{!render_tree(sub_tree, ind+1)}}
% end

% if tree.sub_publications:

%   for sub_publication_uuid in tree.sub_publications:

{{ sub_publication_uuid }}

%   end

"""

# Functions & classes =========================================================
@error(403)
def error403(error):
    tb = error.traceback

    if isinstance(tb, dict) and "name" in tb and "uuid" in tb:
        return SimpleTemplate(PRIVATE_ACCESS_MSG).render(
            name=error.traceback["name"],
            uuid=error.traceback["uuid"]
        )

    return "Access denied!"

@route(join("/", settings.UUID_DOWNLOAD_KEY, "<uuid>"))
def fetch_by_uuid(uuid):
    """
    Serve publications by UUID.
    """
    # fetch all - private and public - publications
    all_pubs = [
        pub
        for pub in search_publications(DBPublication(uuid=uuid))
    ]

    if not all_pubs:
        abort(404, "Dokument s UUID '%s' není dostupný." % (uuid))

    public_pubs = [
        pub
        for pub in all_pubs
        if pub.is_public
    ]

    if not public_pubs:
        name = all_pubs[0].title
        raise HTTPError(
            403,
            body="Forbidden!",
            traceback={"name": name.decode("utf-8"), "uuid": uuid}
        )

    if len(public_pubs) > 1:
        abort(500, "Inkonzistence databáze - vráceno vícero UUID.")

    pub = public_pubs[0]

    if settings.PUBLIC_DIR not in pub.file_pointer:
        abort(500, "Pokus o neoprávněný přístup!")

    down_fn = os.path.basename(pub.filename)
    local_fn = pub.file_pointer.replace(settings.PUBLIC_DIR, "")

    mime_type = mime.Types.of(down_fn)[0].content_type
    if mime_type == "inode/symlink":
        mime_type = "auto"

    response = static_file(
        local_fn,
        root=settings.PUBLIC_DIR,
        download=down_fn,
        mimetype=mime_type,
    )

    # Bottle doesn't allow you to specify custom filename AND to say, that you
    # want to have this file for view in browser instead of download. This
    # happens because it uses `download` parameter as both bool flag and
    # alternative filename. - this hack allows it.
    if mime_type == "application/pdf":
        disposition = 'inline; filename="%s"' % down_fn
        response.headers[str("Content-Disposition")] = str(disposition)

    return response

def render_trees(trees, path_composer):
    def render_tree(tree, ind=1):
        if not tree.is_public:
            return ""

        rendered_tree = SimpleTemplate(TREE_TEMPLATE).render(
            tree=tree,
            render_tree=render_tree,
            ind=ind,
            path_composer=path_composer,
        )

        # keep nice indentation
        ind_txt = ind * "  "
        return ind_txt + ("\n" + ind_txt).join(rendered_tree.splitlines())

    trees = list(trees)
    first_tree = trees[0]
    parent = tree_handler().get_parent(first_tree)
    link_up = path_composer(parent) if parent else None

    return SimpleTemplate(TREES_TEMPLATE).render(
        trees=trees,
        render_tree=render_tree,
        link_up=link_up,
    )

@route(join("/", settings.ISSN_DOWNLOAD_KEY, "<issn>"))
def show_periodical_tree_by_issn(issn):
    trees = tree_handler().trees_by_issn(issn)

    if not trees:
        abort(404, "Dokument s ISSN '%s' není dostupný." % issn)

    return render_trees(
        trees,
        partial(web_tools.compose_tree_path, issn=True)
    )

@route(join("/", settings.PATH_DOWNLOAD_KEY, "<path:path>"))
def show_periodical_tree_by_path(path):
    path = unquote_plus(path)
    trees = tree_handler().trees_by_path(path)

    if not trees:
        path = path.decode("utf-8")
        abort(404, "Dokument s názvem '%s' není dostupný." % path)

    return render_trees(
        trees,
        partial(web_tools.compose_tree_path, issn=False)
    )

def list_publications():
    """
    Return list of all publications in basic graphic HTML render.
    """
    publications = search_publications(
        DBPublication(is_public=True)
    )

    return SimpleTemplate(INDEX_TEMPLATE).render(
        publications=publications,
        compose_path=web_tools.compose_path,
        delimiter=":",
    )

def check_auth(username, password):
    """
    This function is used to check `username` and `password` in case that
    :attr:`.settings.PRIVATE_INDEX` is set to ``True`` (default is ``False``).
    """
    return (username == settings.PRIVATE_INDEX_USERNAME and
            settings.PRIVATE_INDEX_PASSWORD and
            password == settings.PRIVATE_INDEX_PASSWORD)

@auth_basic(check_auth)
def private_index():
    """
    Private index in case that login is enabled.
    """
    return list_publications()

@route("/")
def index():
    """
    Handle index of the project.
    """
    if not settings.PRIVATE_INDEX:
        return list_publications()

    return private_index()

# Main program ================================================================
if __name__ == '__main__':
    run(
        server=settings.WEB_SERVER,
        host=settings.WEB_ADDR,
        port=settings.WEB_PORT,
        debug=settings.WEB_DEBUG,
        reloader=settings.WEB_RELOADER,
    )
<commit_msg>#47: Added support for periodical trees.<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# Interpreter version: python 2.7
#
# Imports =====================================================================
from __future__ import unicode_literals

import sys
import os.path
from os.path import join
from os.path import dirname
from functools import partial
from urllib import unquote_plus

from bottle import run
from bottle import abort
from bottle import error
from bottle import route
from bottle import HTTPError
from bottle import static_file
from bottle import SimpleTemplate

import mime
from bottle import auth_basic

sys.path.insert(0, join(dirname(__file__), "../src/edeposit/amqp"))

try:
    from storage import DBPublication
    from storage.tree_handler import tree_handler
    from storage.publication_storage import search_publications
    from storage.publication_storage import search_pubs_by_uuid

    from storage import settings
    from storage import web_tools
except ImportError:
    from edeposit.amqp.storage import DBPublication
    from edeposit.amqp.storage.tree_handler import tree_handler
    from edeposit.amqp.storage.publication_storage import search_publications
    from edeposit.amqp.storage.publication_storage import search_pubs_by_uuid

    from edeposit.amqp.storage import settings
    from edeposit.amqp.storage import web_tools

# Variables ===================================================================
INDEX_TEMPLATE = """<!DOCTYPE html>

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
    <title>Seznam veřejně přístupných publikací</title>

Seznam veřejně přístupných publikací projektu E-deposit

% for pub in publications:

          {{pub.title}}

        Autor{{delimiter}}
        {{pub.author}}

        ISBN{{delimiter}}
        {{pub.isbn}}

        Rok vydání{{delimiter}}
        {{pub.pub_year}}

        URN:NBN{{delimiter}}
        {{pub.urnnbn}}

% end

"""

PRIVATE_ACCESS_MSG = """<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">

    <title>Veřejně nepřístupný dokument</title>
    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">

Chyba: Veřejně nepřístupný dokument

      Dokument `{{name}}` s UUID <tt>`{{uuid}}`</tt> bohužel není
      veřejně přístupný.

      Tato publikace je zpřístupněna pouze na terminálech v prostorech
      Národní knihovny ČR.

"""

TREES_TEMPLATE = """<!DOCTYPE html>

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
    <title>Seznam publikací periodika</title>

% if link_up:
    [Zpět]
% end

% for tree in trees:

Seznam publikací periodika {{tree.name}}

    {{ !render_tree(tree, 1) }}

% end

"""

TREE_TEMPLATE = """

{{tree.name}}

% for sub_tree in tree.sub_trees:
    {{!render_tree(sub_tree, ind+1)}}
% end

% if tree.sub_publications:

%   for sub_publication_uuid in tree.sub_publications:
%     pub = pub_cache[sub_publication_uuid]

%     if pub.is_public:

{{ pub.title }}

%     else:

{{ pub.title }} (neveřejný zdroj)

%     end
%   end

"""

# Functions & classes =========================================================
@error(403)
def error403(error):
    tb = error.traceback

    if isinstance(tb, dict) and "name" in tb and "uuid" in tb:
        return SimpleTemplate(PRIVATE_ACCESS_MSG).render(
            name=error.traceback["name"],
            uuid=error.traceback["uuid"]
        )

    return "Access denied!"

@route(join("/", settings.UUID_DOWNLOAD_KEY, "<uuid>"))
def fetch_by_uuid(uuid):
    """
    Serve publications by UUID.
    """
    # fetch all - private and public - publications
    all_pubs = [
        pub
        for pub in search_publications(DBPublication(uuid=uuid))
    ]

    if not all_pubs:
        abort(404, "Dokument s UUID '%s' není dostupný." % (uuid))

    public_pubs = [
        pub
        for pub in all_pubs
        if pub.is_public
    ]

    if not public_pubs:
        name = all_pubs[0].title
        raise HTTPError(
            403,
            body="Forbidden!",
            traceback={"name": name.decode("utf-8"), "uuid": uuid}
        )

    if len(public_pubs) > 1:
        abort(500, "Inkonzistence databáze - vráceno vícero UUID.")

    pub = public_pubs[0]

    if settings.PUBLIC_DIR not in pub.file_pointer:
        abort(500, "Pokus o neoprávněný přístup!")

    down_fn = os.path.basename(pub.filename)
    local_fn = pub.file_pointer.replace(settings.PUBLIC_DIR, "")

    mime_type = mime.Types.of(down_fn)[0].content_type
    if mime_type == "inode/symlink":
        mime_type = "auto"

    response = static_file(
        local_fn,
        root=settings.PUBLIC_DIR,
        download=down_fn,
        mimetype=mime_type,
    )

    # Bottle doesn't allow you to specify custom filename AND to say, that you
    # want to have this file for view in browser instead of download. This
    # happens because it uses `download` parameter as both bool flag and
    # alternative filename. - this hack allows it.
    if mime_type == "application/pdf":
        disposition = 'inline; filename="%s"' % down_fn
        response.headers[str("Content-Disposition")] = str(disposition)

    return response

def render_trees(trees, path_composer):
    trees = list(trees)  # by default, this is set

    def create_pub_cache(trees):
        sub_pubs_uuids = sum((x.collect_publications() for x in trees), [])

        uuid_mapping = {
            uuid: search_pubs_by_uuid(uuid)
            for uuid in set(sub_pubs_uuids)
        }

        # cleaned dict without blank matches
        return {
            uuid: pub[0]
            for uuid, pub in uuid_mapping.iteritems()
            if pub
        }

    pub_cache = create_pub_cache(trees)

    def render_tree(tree, ind=1):
        if not tree.is_public:
            return ""

        rendered_tree = SimpleTemplate(TREE_TEMPLATE).render(
            tree=tree,
            render_tree=render_tree,
            ind=ind,
            path_composer=path_composer,
            pub_cache=pub_cache,
        )

        # keep nice indentation
        ind_txt = ind * "  "
        return ind_txt + ("\n" + ind_txt).join(rendered_tree.splitlines())

    parent = tree_handler().get_parent(trees[0])
    link_up = path_composer(parent) if parent else None

    return SimpleTemplate(TREES_TEMPLATE).render(
        trees=trees,
        render_tree=render_tree,
        link_up=link_up,
    )

@route(join("/", settings.ISSN_DOWNLOAD_KEY, "<issn>"))
def show_periodical_tree_by_issn(issn):
    trees = tree_handler().trees_by_issn(issn)

    if not trees:
        abort(404, "Dokument s ISSN '%s' není dostupný." % issn)

    return render_trees(
        trees,
        partial(web_tools.compose_tree_path, issn=True)
    )

@route(join("/", settings.PATH_DOWNLOAD_KEY, "<path:path>"))
def show_periodical_tree_by_path(path):
    path = unquote_plus(path)
    trees = tree_handler().trees_by_path(path)

    if not trees:
        path = path.decode("utf-8")
        abort(404, "Dokument s názvem '%s' není dostupný." % path)

    return render_trees(
        trees,
        partial(web_tools.compose_tree_path, issn=False)
    )

def list_publications():
    """
    Return list of all publications in basic graphic HTML render.
    """
    publications = search_publications(
        DBPublication(is_public=True)
    )

    return SimpleTemplate(INDEX_TEMPLATE).render(
        publications=publications,
        compose_path=web_tools.compose_path,
        delimiter=":",
    )

def check_auth(username, password):
    """
    This function is used to check `username` and `password` in case that
    :attr:`.settings.PRIVATE_INDEX` is set to ``True`` (default is ``False``).
    """
    return (username == settings.PRIVATE_INDEX_USERNAME and
            settings.PRIVATE_INDEX_PASSWORD and
            password == settings.PRIVATE_INDEX_PASSWORD)

@auth_basic(check_auth)
def private_index():
    """
    Private index in case that login is enabled.
    """
    return list_publications()

@route("/")
def index():
    """
    Handle index of the project.
    """
    if not settings.PRIVATE_INDEX:
        return list_publications()

    return private_index()

# Main program ================================================================
if __name__ == '__main__':
    run(
        server=settings.WEB_SERVER,
        host=settings.WEB_ADDR,
        port=settings.WEB_PORT,
        debug=settings.WEB_DEBUG,
        reloader=settings.WEB_RELOADER,
    )
<|endoftext|>


<commit_before>#!/usr/bin/env python3

# Copyright (c) 2016, Ruslan Baratov
# All rights reserved.

import argparse
import hashlib
import os
import platform
import requests
import shutil
import stat
import subprocess
import sys
import tarfile
import time
import zipfile

print(
    'Python version: {}.{}'.format(
        sys.version_info.major, sys.version_info.minor
     )
)

parser = argparse.ArgumentParser(
    description='Install dependencies for CI testing'
)

args = parser.parse_args()

class FileToDownload:
  def __init__(self, url, sha1, local_path, unpack_dir):
    self.url = url
    self.sha1 = sha1
    self.local_path = local_path
    self.unpack_dir = unpack_dir

    self.download()
    self.unpack()

  def download(self):
    ok = self.hash_match()
    if ok:
      print('File already downloaded: {}'.format(self.local_path))
    else:
      self.real_file_download()
      assert(self.hash_match() == True)

  def hash_match(self):
    if not os.path.exists(self.local_path):
      print('File not exists: {}'.format(self.local_path))
      return False
    sha1_of_file = hashlib.sha1(open(self.local_path, 'rb').read()).hexdigest()
    ok = (sha1_of_file == self.sha1)
    if ok:
      return True
    else:
      print('SHA1 mismatch for file {}:'.format(self.local_path))
      print('  {} (real)'.format(sha1_of_file))
      print('  {} (expected)'.format(self.sha1))
      return False

  def real_file_download(self):
    max_retry = 3
    for i in range(max_retry):
      try:
        self.real_file_download_once()
        print('Done')
        return
      except Exception as exc:
        print('Exception catched ({}), retry... ({} of {})'.format(exc, i+1, max_retry))
        time.sleep(60)
    sys.exit('Download failed')

  # http://stackoverflow.com/a/16696317/2288008
  def real_file_download_once(self):
    print('Downloading:\n  {}\n  -> {}'.format(self.url, self.local_path))
    r = requests.get(self.url, stream=True)
    if not r.ok:
      raise Exception('Downloading failed: {}'.format(self.url))
    with open(self.local_path, 'wb') as f:
      for chunk in r.iter_content(chunk_size=16*1024):
        if chunk:
          f.write(chunk)

  def unpack(self):
    print('Unpacking {}'.format(self.local_path))
    last_cwd = os.getcwd()
    os.chdir(self.unpack_dir)
    if self.url.endswith('.tar.gz'):
      tar_archive = tarfile.open(self.local_path)
      tar_archive.extractall(path=self.unpack_dir)
      tar_archive.close()
    elif self.url.endswith('.zip'):
      # Can't use ZipFile module because permissions will be lost, see bug:
      # * https://bugs.python.org/issue15795
      subprocess.check_call(['unzip', self.local_path])
    elif self.url.endswith('.bin'):
      os.chmod(self.local_path, os.stat(self.local_path).st_mode | stat.S_IEXEC)
      devnull = open(os.devnull, 'w') # subprocess.DEVNULL is not available for Python 3.2
      subprocess.check_call(self.local_path, stdout=devnull)
    else:
      sys.exit('Unknown archive format')
    os.chdir(last_cwd)

### Parse toolchain name

toolchain = os.getenv('TOOLCHAIN')
if toolchain is None:
  toolchain = ''
  print('** WARNING ** Environment variable TOOLCHAIN is empty')

def get_android_full_version_url():
  if toolchain.startswith('android-ndk-r10e-'):
    if platform.system() == 'Darwin':
      return 'http://dl.google.com/android/ndk/android-ndk-r10e-darwin-x86_64.bin', 'b57c2b9213251180dcab794352bfc9a241bf2557',
    if platform.system() == 'Linux':
      return 'http://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin', 'c685e5f106f8daa9b5449d0a4f21ee8c0afcb2f6',
  if toolchain.startswith('android-ndk-r11c-'):
    if platform.system() == 'Darwin':
      return 'http://dl.google.com/android/repository/android-ndk-r11c-darwin-x86_64.zip', '4ce8e7ed8dfe08c5fe58aedf7f46be2a97564696',
    if platform.system() == 'Linux':
      return 'http://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip', 'de5ce9bddeee16fb6af2b9117e9566352aa7e279',
  if toolchain.startswith('android-ndk-r15c-'):
    if platform.system() == 'Darwin':
      return 'https://dl.google.com/android/repository/android-ndk-r15c-darwin-x86_64.zip', 'ea4b5d76475db84745aa8828000d009625fc1f98',
    if platform.system() == 'Linux':
      return 'https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip', '0bf02d4e8b85fd770fd7b9b2cdec57f9441f27a2',
  sys.exit('Android supported only for Linux and OSX')

def get_android_url():
  if not os.getenv('TRAVIS'):
    return get_android_full_version_url()
  if toolchain == 'android-ndk-r10e-api-19-armeabi-v7a-neon':
    if platform.system() == 'Linux':
      return 'https://github.com/hunter-packages/android-ndk/releases/download/v1.0.0/android-ndk-r10e-arm-linux-androideabi-4.9-gnu-libstdc.-4.9-armeabi-v7a-android-19-arch-arm-Linux.tar.gz', '847177799b0fe4f7480f910bbf1815c3e3fed0da'
    if platform.system() == 'Darwin':
      return 'https://github.com/hunter-packages/android-ndk/releases/download/v1.0.0/android-ndk-r10e-arm-linux-androideabi-4.9-gnu-libstdc.-4.9-armeabi-v7a-android-19-arch-arm-Darwin.tar.gz', 'e568e9a8f562e7d1bc06f93e6f7cc7f44df3ded2'
  if toolchain == 'android-ndk-r11c-api-19-armeabi-v7a-neon':
    if platform.system() == 'Linux':
      return 'https://github.com/hunter-packages/android-ndk/releases/download/v1.0.1/android-ndk-r11c-arm-linux-androideabi-4.9-gnu-libstdc.-4.9-armeabi-v7a-android-19-arch-arm-Linux.tar.gz', '2e0da01961e0031bfd7d8db6ce4a15372bd8c3e8'
    if platform.system() == 'Darwin':
      return 'https://github.com/hunter-packages/android-ndk/releases/download/v1.0.1/android-ndk-r11c-arm-linux-androideabi-4.9-gnu-libstdc.-4.9-armeabi-v7a-android-19-arch-arm-Darwin.tar.gz', '664b3c8104142de2af16f887c19d1b2e618725cb'
  if toolchain == 'android-ndk-r15c-api-21-armeabi-v7a-neon-clang-libcxx':
    if platform.system() == 'Linux':
      return 'https://github.com/hunter-packages/android-ndk/releases/download/v1.0.1/android-ndk-r15c-arm-linux-androideabi-4.9-llvm-libc.-android-21-arch-arm-Linux.tar.gz', '952403abedc3960b6d6eee35aeed940d935baaea'
    if platform.system() == 'Darwin':
      return 'https://github.com/hunter-packages/android-ndk/releases/download/v1.0.1/android-ndk-r15c-arm-linux-androideabi-4.9-llvm-libc.-android-21-arch-arm-Darwin.tar.gz', '978ef8b724dc3691a128d8f48a8440172478d82b'
  if toolchain == 'android-ndk-r16b-api-24-armeabi-v7a-neon-clang-libcxx':
    if platform.system() == 'Linux':
      return 'https://github.com/hunter-packages/android-ndk/releases/download/v1.0.1/android-ndk-r16b-arm-linux-androideabi-4.9-llvm-libc.-android-24-arch-arm-Linux.tar.gz', 'b9ee32e31376fd5fe090172169f14faf50af6b68'
  return get_android_full_version_url()

def get_cmake_url():
  if platform.system() == 'Darwin':
    return (
        'https://github.com/ruslo/CMake/releases/download/v3.10.2/cmake-3.10.2-Darwin-x86_64.tar.gz',
        'f25b70bb1ea5e83223cc0fceb499113ba5f19d5c'
    )
  elif platform.system() == 'Linux':
    return (
        'https://github.com/ruslo/CMake/releases/download/v3.10.2/cmake-3.10.2-Linux-x86_64.tar.gz',
        '3bdedf32c5780a0aa3e06d08df3b7236b17fa243'
    )
  elif platform.system() == 'Windows':
    return (
        'https://github.com/ruslo/CMake/releases/download/v3.10.2/cmake-3.10.2-win64-x64.zip',
        'd9008462a011f1379416855f78f9a5935262d5c6'
    )
  else:
    sys.exit('Unknown system: {}'.format(platform.system()))

is_android = toolchain.startswith('android-')
is_ninja = toolchain.startswith('ninja-')

### Prepare directories

ci_dir = os.path.join(os.getcwd(), '_ci')

if not os.path.exists(ci_dir):
  os.mkdir(ci_dir)

cmake_url, cmake_sha1 = get_cmake_url()
cmake_archive_local = cmake_url.split('/')[-1]
cmake_archive_local = os.path.join(ci_dir, cmake_archive_local)

ninja_archive_local = os.path.join(ci_dir, 'ninja.zip')

if is_android:
  url, sha1 = get_android_url()
  android_archive_local = url.split('/')[-1]
else:
  android_archive_local = 'android.bin'
android_archive_local = os.path.join(ci_dir, android_archive_local)

expected_files = [
    cmake_archive_local, android_archive_local, ninja_archive_local
]

for i in os.listdir(ci_dir):
  dir_item = os.path.join(ci_dir, i)
  expected = (dir_item in expected_files)
  if os.path.isdir(dir_item):
    print('Removing directory: {}'.format(dir_item))
    shutil.rmtree(dir_item)
  elif not expected:
    print('Removing file: {}'.format(dir_item))
    os.remove(dir_item)

cmake_dir = os.path.join(ci_dir, 'cmake')
ninja_dir = os.path.join(ci_dir, 'ninja')

### Downloading files

# https://cmake.org/download/

FileToDownload(cmake_url, cmake_sha1, cmake_archive_local, ci_dir)

if is_android:
  url, sha1 = get_android_url()
  FileToDownload(url, sha1, android_archive_local, ci_dir)

if is_ninja:
  ninja = FileToDownload(
      'https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-win.zip',
      '637cc6e144f5cc7c6388a30f3c32ad81b2e0442e',
      ninja_archive_local,
      ci_dir
  )

### Unify directories

for i in os.listdir(ci_dir):
  src = os.path.join(ci_dir, i)
  if i.startswith('cmake') and os.path.isdir(src):
    macosx_contents = os.path.join(src, 'CMake.app', 'Contents')
    if os.path.isdir(macosx_contents):
      os.rename(macosx_contents, cmake_dir)
    else:
      os.rename(src, cmake_dir)

  if i == 'ninja.exe':
    os.mkdir(ninja_dir)
    os.rename(src, os.path.join(ninja_dir, i))
<commit_msg>add full android-nd-r16b downloads<commit_after>#!/usr/bin/env python3

# Copyright (c) 2016, Ruslan Baratov
# All rights reserved.

import argparse
import hashlib
import os
import platform
import requests
import shutil
import stat
import subprocess
import sys
import tarfile
import time
import zipfile

print(
    'Python version: {}.{}'.format(
        sys.version_info.major, sys.version_info.minor
     )
)

parser = argparse.ArgumentParser(
    description='Install dependencies for CI testing'
)

args = parser.parse_args()

class FileToDownload:
  def __init__(self, url, sha1, local_path, unpack_dir):
    self.url = url
    self.sha1 = sha1
    self.local_path = local_path
    self.unpack_dir = unpack_dir

    self.download()
    self.unpack()

  def download(self):
    ok = self.hash_match()
    if ok:
      print('File already downloaded: {}'.format(self.local_path))
    else:
      self.real_file_download()
      assert(self.hash_match() == True)

  def hash_match(self):
    if not os.path.exists(self.local_path):
      print('File not exists: {}'.format(self.local_path))
      return False
    sha1_of_file = hashlib.sha1(open(self.local_path, 'rb').read()).hexdigest()
    ok = (sha1_of_file == self.sha1)
    if ok:
      return True
    else:
      print('SHA1 mismatch for file {}:'.format(self.local_path))
      print('  {} (real)'.format(sha1_of_file))
      print('  {} (expected)'.format(self.sha1))
      return False

  def real_file_download(self):
    max_retry = 3
    for i in range(max_retry):
      try:
        self.real_file_download_once()
        print('Done')
        return
      except Exception as exc:
        print('Exception catched ({}), retry... ({} of {})'.format(exc, i+1, max_retry))
        time.sleep(60)
    sys.exit('Download failed')

  # http://stackoverflow.com/a/16696317/2288008
  def real_file_download_once(self):
    print('Downloading:\n  {}\n  -> {}'.format(self.url, self.local_path))
    r = requests.get(self.url, stream=True)
    if not r.ok:
      raise Exception('Downloading failed: {}'.format(self.url))
    with open(self.local_path, 'wb') as f:
      for chunk in r.iter_content(chunk_size=16*1024):
        if chunk:
          f.write(chunk)

  def unpack(self):
    print('Unpacking {}'.format(self.local_path))
    last_cwd = os.getcwd()
    os.chdir(self.unpack_dir)
    if self.url.endswith('.tar.gz'):
      tar_archive = tarfile.open(self.local_path)
      tar_archive.extractall(path=self.unpack_dir)
      tar_archive.close()
    elif self.url.endswith('.zip'):
      # Can't use ZipFile module because permissions will be lost, see bug:
      # * https://bugs.python.org/issue15795
      subprocess.check_call(['unzip', self.local_path])
    elif self.url.endswith('.bin'):
      os.chmod(self.local_path, os.stat(self.local_path).st_mode | stat.S_IEXEC)
      devnull = open(os.devnull, 'w') # subprocess.DEVNULL is not available for Python 3.2
      subprocess.check_call(self.local_path, stdout=devnull)
    else:
      sys.exit('Unknown archive format')
    os.chdir(last_cwd)

### Parse toolchain name

toolchain = os.getenv('TOOLCHAIN')
if toolchain is None:
  toolchain = ''
  print('** WARNING ** Environment variable TOOLCHAIN is empty')

def get_android_full_version_url():
  if toolchain.startswith('android-ndk-r10e-'):
    if platform.system() == 'Darwin':
      return 'http://dl.google.com/android/ndk/android-ndk-r10e-darwin-x86_64.bin', 'b57c2b9213251180dcab794352bfc9a241bf2557',
    if platform.system() == 'Linux':
      return 'http://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin', 'c685e5f106f8daa9b5449d0a4f21ee8c0afcb2f6',
  if toolchain.startswith('android-ndk-r11c-'):
    if platform.system() == 'Darwin':
      return 'http://dl.google.com/android/repository/android-ndk-r11c-darwin-x86_64.zip', '4ce8e7ed8dfe08c5fe58aedf7f46be2a97564696',
    if platform.system() == 'Linux':
      return 'http://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip', 'de5ce9bddeee16fb6af2b9117e9566352aa7e279',
  if toolchain.startswith('android-ndk-r15c-'):
    if platform.system() == 'Darwin':
      return 'https://dl.google.com/android/repository/android-ndk-r15c-darwin-x86_64.zip', 'ea4b5d76475db84745aa8828000d009625fc1f98',
    if platform.system() == 'Linux':
      return 'https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip', '0bf02d4e8b85fd770fd7b9b2cdec57f9441f27a2',
  if toolchain.startswith('android-ndk-r16b-'):
    if platform.system() == 'Darwin':
      return 'https://dl.google.com/android/repository/android-ndk-r16b-darwin-x86_64.zip', 'e51e615449b98c716cf912057e2682e75d55e2de',
    if platform.system() == 'Linux':
      return 'https://dl.google.com/android/repository/android-ndk-r16b-linux-x86_64.zip', '42aa43aae89a50d1c66c3f9fdecd676936da6128',
  sys.exit('Android supported only for Linux and OSX')

def get_android_url():
  if not os.getenv('TRAVIS'):
    return get_android_full_version_url()
  if toolchain == 'android-ndk-r10e-api-19-armeabi-v7a-neon':
    if platform.system() == 'Linux':
      return 'https://github.com/hunter-packages/android-ndk/releases/download/v1.0.0/android-ndk-r10e-arm-linux-androideabi-4.9-gnu-libstdc.-4.9-armeabi-v7a-android-19-arch-arm-Linux.tar.gz', '847177799b0fe4f7480f910bbf1815c3e3fed0da'
    if platform.system() == 'Darwin':
      return 'https://github.com/hunter-packages/android-ndk/releases/download/v1.0.0/android-ndk-r10e-arm-linux-androideabi-4.9-gnu-libstdc.-4.9-armeabi-v7a-android-19-arch-arm-Darwin.tar.gz', 'e568e9a8f562e7d1bc06f93e6f7cc7f44df3ded2'
  if toolchain == 'android-ndk-r11c-api-19-armeabi-v7a-neon':
    if platform.system() == 'Linux':
      return 'https://github.com/hunter-packages/android-ndk/releases/download/v1.0.1/android-ndk-r11c-arm-linux-androideabi-4.9-gnu-libstdc.-4.9-armeabi-v7a-android-19-arch-arm-Linux.tar.gz', '2e0da01961e0031bfd7d8db6ce4a15372bd8c3e8'
    if platform.system() == 'Darwin':
      return 'https://github.com/hunter-packages/android-ndk/releases/download/v1.0.1/android-ndk-r11c-arm-linux-androideabi-4.9-gnu-libstdc.-4.9-armeabi-v7a-android-19-arch-arm-Darwin.tar.gz', '664b3c8104142de2af16f887c19d1b2e618725cb'
  if toolchain == 'android-ndk-r15c-api-21-armeabi-v7a-neon-clang-libcxx':
    if platform.system() == 'Linux':
      return 'https://github.com/hunter-packages/android-ndk/releases/download/v1.0.1/android-ndk-r15c-arm-linux-androideabi-4.9-llvm-libc.-android-21-arch-arm-Linux.tar.gz', '952403abedc3960b6d6eee35aeed940d935baaea'
    if platform.system() == 'Darwin':
      return 'https://github.com/hunter-packages/android-ndk/releases/download/v1.0.1/android-ndk-r15c-arm-linux-androideabi-4.9-llvm-libc.-android-21-arch-arm-Darwin.tar.gz', '978ef8b724dc3691a128d8f48a8440172478d82b'
  if toolchain == 'android-ndk-r16b-api-24-armeabi-v7a-neon-clang-libcxx':
    if platform.system() == 'Linux':
      return 'https://github.com/hunter-packages/android-ndk/releases/download/v1.0.1/android-ndk-r16b-arm-linux-androideabi-4.9-llvm-libc.-android-24-arch-arm-Linux.tar.gz', 'b9ee32e31376fd5fe090172169f14faf50af6b68'
  return get_android_full_version_url()

def get_cmake_url():
  if platform.system() == 'Darwin':
    return (
        'https://github.com/ruslo/CMake/releases/download/v3.10.2/cmake-3.10.2-Darwin-x86_64.tar.gz',
        'f25b70bb1ea5e83223cc0fceb499113ba5f19d5c'
    )
  elif platform.system() == 'Linux':
    return (
        'https://github.com/ruslo/CMake/releases/download/v3.10.2/cmake-3.10.2-Linux-x86_64.tar.gz',
        '3bdedf32c5780a0aa3e06d08df3b7236b17fa243'
    )
  elif platform.system() == 'Windows':
    return (
        'https://github.com/ruslo/CMake/releases/download/v3.10.2/cmake-3.10.2-win64-x64.zip',
        'd9008462a011f1379416855f78f9a5935262d5c6'
    )
  else:
    sys.exit('Unknown system: {}'.format(platform.system()))

is_android = toolchain.startswith('android-')
is_ninja = toolchain.startswith('ninja-')

### Prepare directories

ci_dir = os.path.join(os.getcwd(), '_ci')

if not os.path.exists(ci_dir):
  os.mkdir(ci_dir)

cmake_url, cmake_sha1 = get_cmake_url()
cmake_archive_local = cmake_url.split('/')[-1]
cmake_archive_local = os.path.join(ci_dir, cmake_archive_local)

ninja_archive_local = os.path.join(ci_dir, 'ninja.zip')

if is_android:
  url, sha1 = get_android_url()
  android_archive_local = url.split('/')[-1]
else:
  android_archive_local = 'android.bin'
android_archive_local = os.path.join(ci_dir, android_archive_local)

expected_files = [
    cmake_archive_local, android_archive_local, ninja_archive_local
]

for i in os.listdir(ci_dir):
  dir_item = os.path.join(ci_dir, i)
  expected = (dir_item in expected_files)
  if os.path.isdir(dir_item):
    print('Removing directory: {}'.format(dir_item))
    shutil.rmtree(dir_item)
  elif not expected:
    print('Removing file: {}'.format(dir_item))
    os.remove(dir_item)

cmake_dir = os.path.join(ci_dir, 'cmake')
ninja_dir = os.path.join(ci_dir, 'ninja')

### Downloading files

# https://cmake.org/download/

FileToDownload(cmake_url, cmake_sha1, cmake_archive_local, ci_dir)

if is_android:
  url, sha1 = get_android_url()
  FileToDownload(url, sha1, android_archive_local, ci_dir)

if is_ninja:
  ninja = FileToDownload(
      'https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-win.zip',
      '637cc6e144f5cc7c6388a30f3c32ad81b2e0442e',
      ninja_archive_local,
      ci_dir
  )

### Unify directories

for i in os.listdir(ci_dir):
  src = os.path.join(ci_dir, i)
  if i.startswith('cmake') and os.path.isdir(src):
    macosx_contents = os.path.join(src, 'CMake.app', 'Contents')
    if os.path.isdir(macosx_contents):
      os.rename(macosx_contents, cmake_dir)
    else:
      os.rename(src, cmake_dir)

  if i == 'ninja.exe':
    os.mkdir(ninja_dir)
    os.rename(src, os.path.join(ninja_dir, i))
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
jupyter-notebook-memory.py: get memory for each jupyter notebook

Usage:
  jupyter-notebook-memory.py [options]
  jupyter-notebook-memory.py -h | --help
  jupyter-notebook-memory.py --version

Options:
  -u=<u>        Show notebooks for only one user.
  -n=<n>        Number of parallel processes.
                [default: 10]
  --ports=<p>   Port range to check (comma-separated).
                [default: 1000,32000]
  --version     Show version.
  -h --help     Show this screen.

Description:
  Get a tab-delimited table of how much rss memory is used by each
  Jupyter notebook kernel. The ports with the `--ports` range will
  be checked for notebooks using any of those ports. This is needed
  to map the memory usage info on the kernel to the file name of the
  notebook.

  To view a nicely formated version of the output table in bash:
  `jupyter-notebook-memory.py | column -t -s $'\\t' | less -S`

"""

from docopt import docopt
import sys
import os
import pwd
import psutil
import re
import string
import json
import requests
from itertools import chain
import multiprocessing as mp
import signal
import errno
from functools import wraps
import numpy as np
import pandas as pd

class TimeoutError(Exception):
    pass

def timeout(seconds=10, error_message=os.strerror(errno.ETIME)):
    def decorator(func):
        def _handle_timeout(signum, frame):
            raise TimeoutError(error_message)

        def wrapper(*args, **kwargs):
            signal.signal(signal.SIGALRM, _handle_timeout)
            signal.alarm(seconds)
            try:
                result = func(*args, **kwargs)
            finally:
                signal.alarm(0)
            return result

        return wraps(func)(wrapper)

    return decorator

def nb_mem(user=None):
    # memory info from psutil.Process
    UID   = 1
    EUID  = 2
    regex = re.compile(r'.+kernel-(.+)\.json')

    pids = [pid for pid in os.listdir('/proc') if pid.isdigit()]

    df_mem = []
    for pid in pids:
        try:
            ret = open(os.path.join('/proc', pid, 'cmdline'), 'rb').read()
        except IOError: # proc has already terminated
            continue

        # jupyter notebook processes
        if len(ret) > 0 and 'share/jupyter/runtime' in ret:
            # kernel
            kernel_ID = re.sub(regex, r'\1', ret)
            kernel_ID = filter(lambda x: x in string.printable, kernel_ID)

            # memory
            process = psutil.Process(int(pid))
            mem = round(process.memory_info()[0] / float(1e9), 3)

            # user name for pid
            for ln in open('/proc/{}/status'.format(int(pid))):
                if ln.startswith('Uid:'):
                    uid = int(ln.split()[UID])
                    uname = pwd.getpwuid(uid).pw_name

            if user is not None and uname != user:
                continue
            else:
                # user, pid, memory, proc_desc
                df_mem.append([uname, pid, mem, kernel_ID])

    df_mem = pd.DataFrame(df_mem)
    if df_mem.shape[0] == 0:
        sys.exit('No notebooks found!')
    df_mem.columns = ['user', 'pid', 'memory_GB', 'kernel_ID']
    return df_mem

@timeout(1)
def _get_requests(port):
    try:
        url = 'http://127.0.0.1:{}/api/sessions'.format(port)
        reqs = requests.get(url)
    except requests.ConnectionError:
        reqs = None
    return reqs

def _np_port(port):
    try:
        reqs = _get_requests(port)
    except TimeoutError:
        sys.stderr.write('port {}: timeout\n'.format(port))
        reqs = None

    ret = []
    if reqs:
        try:
            reqs = json.loads(reqs.content)
        except (AttributeError, ValueError) as e:
            pass
        for req in reqs:
            kernel_ID = req['kernel']['id']
            notebook_path = req['notebook']['path']
            ret.append([port, kernel_ID, notebook_path])
    return ret

def nb_port(ports, nprocs=1):
    p = mp.Pool(processes=int(nprocs))
    df_nb = p.map(_np_port, range(ports[0],ports[1]+1))
    df_nb = chain.from_iterable(df_nb)
    df_nb = [x for x in df_nb if len(x) > 0]
    df_nb = pd.DataFrame(df_nb)
    if df_nb.shape[0] == 0:
        sys.exit('No notebooks found!')
    df_nb.columns = ['port', 'kernel_ID', 'notebook_path']
    return df_nb

def main(user, ports, nprocs=1):
    # making tables of notebook info
    df_mem = nb_mem(user=user)
    df_nb = nb_port(ports=ports, nprocs=nprocs)
    # joining tables
    df = pd.merge(df_nb, df_mem, on=['kernel_ID'], how='inner')
    df = df.sort(['memory_GB'], ascending=False)

    # reordering df
    df = df[['user', 'memory_GB', 'port', 'notebook_path', 'pid', 'kernel_ID']]

    # writing
    df.to_csv(sys.stdout, sep='\t', index=False)

if __name__ == '__main__':
    uargs = docopt(__doc__, version='0.1')
    ports = [int(x) for x in uargs['--ports'].split(',')]
    main(user=uargs['-u'], ports=ports, nprocs=uargs['-n'])

<commit_msg>timeout bug fix<commit_after>#!/usr/bin/env python

"""
jupyter-notebook-memory.py: get memory for each jupyter notebook

Usage:
  jupyter-notebook-memory.py [options]
  jupyter-notebook-memory.py -h | --help
  jupyter-notebook-memory.py --version

Options:
  -u=<u>        Show notebooks for only one user.
  -n=<n>        Number of parallel processes.
                [default: 10]
  --ports=<p>   Port range to check (comma-separated).
                [default: 1000,32000]
  --version     Show version.
  -h --help     Show this screen.

Description:
  Get a tab-delimited table of how much rss memory is used by each
  Jupyter notebook kernel. The ports with the `--ports` range will
  be checked for notebooks using any of those ports. This is needed
  to map the memory usage info on the kernel to the file name of the
  notebook.

  To view a nicely formated version of the output table in bash:
  `jupyter-notebook-memory.py | column -t -s $'\\t' | less -S`

  Note: notebooks may be on ports >32000 (up to ~65000),
  but this requests in odd url requests issues that take a lot longer.

"""

from docopt import docopt
import sys
import os
import pwd
import psutil
import re
import string
import json
import requests
from itertools import chain
import multiprocessing as mp
import signal
import errno
from functools import wraps
import numpy as np
import pandas as pd

class TimeoutError(Exception):
    pass

def timeout(seconds=10, error_message=os.strerror(errno.ETIME)):
    def decorator(func):
        def _handle_timeout(signum, frame):
            raise TimeoutError(error_message)

        def wrapper(*args, **kwargs):
            signal.signal(signal.SIGALRM, _handle_timeout)
            signal.alarm(seconds)
            try:
                result = func(*args, **kwargs)
            finally:
                signal.alarm(0)
            return result

        return wraps(func)(wrapper)

    return decorator

def nb_mem(user=None):
    # memory info from psutil.Process
    UID   = 1
    EUID  = 2
    regex = re.compile(r'.+kernel-(.+)\.json')

    pids = [pid for pid in os.listdir('/proc') if pid.isdigit()]

    df_mem = []
    for pid in pids:
        try:
            ret = open(os.path.join('/proc', pid, 'cmdline'), 'rb').read()
        except IOError: # proc has already terminated
            continue

        # jupyter notebook processes
        if len(ret) > 0 and 'share/jupyter/runtime' in ret:
            # kernel
            kernel_ID = re.sub(regex, r'\1', ret)
            kernel_ID = filter(lambda x: x in string.printable, kernel_ID)

            # memory
            process = psutil.Process(int(pid))
            mem = round(process.memory_info()[0] / float(1e9), 3)

            # user name for pid
            for ln in open('/proc/{}/status'.format(int(pid))):
                if ln.startswith('Uid:'):
                    uid = int(ln.split()[UID])
                    uname = pwd.getpwuid(uid).pw_name

            if user is not None and uname != user:
                continue
            else:
                # user, pid, memory, proc_desc
                df_mem.append([uname, pid, mem, kernel_ID])

    df_mem = pd.DataFrame(df_mem)
    if df_mem.shape[0] == 0:
        sys.exit('No notebooks found!')
    df_mem.columns = ['user', 'pid', 'memory_GB', 'kernel_ID']
    return df_mem

@timeout(2)
def _get_requests(port):
    try:
        url = 'http://127.0.0.1:{}/api/sessions'.format(port)
        reqs = requests.get(url)
    except requests.ConnectionError:
        reqs = None
    return reqs

def _np_port(port):
    try:
        reqs = _get_requests(port)
    except TimeoutError:
        sys.stderr.write('port {}: timeout\n'.format(port))
        reqs = None

    ret = []
    if reqs:
        try:
            reqs = json.loads(reqs.content)
        except (AttributeError, ValueError) as e:
            pass
        for req in reqs:
            kernel_ID = req['kernel']['id']
            notebook_path = req['notebook']['path']
            ret.append([port, kernel_ID, notebook_path])
    return ret

def nb_port(ports, nprocs=1):
    p = mp.Pool(processes=int(nprocs))
    df_nb = p.map(_np_port, range(ports[0],ports[1]+1))
    df_nb = chain.from_iterable(df_nb)
    df_nb = [x for x in df_nb if len(x) > 0]
    df_nb = pd.DataFrame(df_nb)
    if df_nb.shape[0] == 0:
        sys.exit('No notebooks found!')
    df_nb.columns = ['port', 'kernel_ID', 'notebook_path']
    return df_nb

def main(user, ports, nprocs=1):
    # making tables of notebook info
    df_mem = nb_mem(user=user)
    df_nb = nb_port(ports=ports, nprocs=nprocs)
    # joining tables
    df = pd.merge(df_nb, df_mem, on=['kernel_ID'], how='inner')
    df = df.sort(['memory_GB'], ascending=False)

    # reordering df
    df = df[['user', 'memory_GB', 'port', 'notebook_path', 'pid', 'kernel_ID']]

    # writing
    df.to_csv(sys.stdout, sep='\t', index=False)

if __name__ == '__main__':
    uargs = docopt(__doc__, version='0.1')
    ports = [int(x) for x in uargs['--ports'].split(',')]
    main(user=uargs['-u'], ports=ports, nprocs=uargs['-n'])

<|endoftext|>