<commit_before>import time

from django.core.management.base import BaseCommand

from treeherder.perf.models import PerformanceSignature

class Command(BaseCommand):
    help = "migrate legacy signature data to new schema, which has the new extra_options field"

    def add_arguments(self, parser):
        parser.add_argument(
            '--interval',
            dest='interval',
            help='Wait specified interval between signature migrations',
            type=float,
            default=0.0
        )

    def _migrate_signature(self, signature, sleep_interval):
        extra_options = ' '.join(sorted(signature.extra_properties['test_options']))
        signature.extra_options = extra_options
        signature.save()
        time.sleep(sleep_interval)

    def handle(self, *args, **options):
        try:
            perf_signatures = PerformanceSignature.objects.all()
        except PerformanceSignature.DoesNotExist:
            print("PerformanceSignature does not exist")
            raise

        print("Migrating PerformanceSignatures...")

        for perf_signature in perf_signatures:
            self._migrate_signature(perf_signature, options['interval'])

        print("Done")
<commit_msg>Bug 1310386 - Fix migration script<commit_after>import time

from django.core.management.base import BaseCommand

from treeherder.perf.models import PerformanceSignature

class Command(BaseCommand):
    help = "migrate legacy signature data to new schema, which has the new extra_options field"

    def add_arguments(self, parser):
        parser.add_argument(
            '--interval',
            dest='interval',
            help='Wait specified interval between signature migrations',
            type=float,
            default=0.0
        )

    def _migrate_signature(self, signature, sleep_interval):
        extra_options = ' '.join(sorted(signature.extra_properties.get('test_options', [])))
        signature.extra_options = extra_options
        signature.save()
        time.sleep(sleep_interval)

    def handle(self, *args, **options):
        try:
            perf_signatures = PerformanceSignature.objects.all()
        except PerformanceSignature.DoesNotExist:
            print("PerformanceSignature does not exist")
            raise

        print("Migrating PerformanceSignatures...")

        for perf_signature in perf_signatures:
            self._migrate_signature(perf_signature, options['interval'])

        print("Done")
<|endoftext|>


<commit_before>from __future__ import absolute_import, unicode_literals
from future.builtins import int, open, str

import os

from json import dumps
try:
    from urllib.parse import urljoin, urlparse
except ImportError:
    from urlparse import urljoin, urlparse

from django.apps import apps
from django.contrib import admin
from django.contrib.admin.views.decorators import staff_member_required
from django.contrib.admin.options import ModelAdmin
from django.contrib.staticfiles import finders
from django.core.exceptions import PermissionDenied
from django.core.urlresolvers import reverse
from django.http import (HttpResponse, HttpResponseServerError,
                         HttpResponseNotFound)
from django.shortcuts import redirect
from django.template import RequestContext
from django.template.loader import get_template
from django.utils.translation import ugettext_lazy as _
from django.views.decorators.csrf import requires_csrf_token

from mezzanine.conf import settings
from mezzanine.core.forms import get_edit_form
from mezzanine.core.models import Displayable, SitePermission
from mezzanine.utils.cache import add_cache_bypass
from mezzanine.utils.views import is_editable, paginate, render, set_cookie
from mezzanine.utils.sites import has_site_permission
from mezzanine.utils.urls import next_url

def set_device(request, device=""):
    """
    Sets a device name in a cookie when a user explicitly wants to go
    to the site for a particular device (eg mobile).
    """
    response = redirect(add_cache_bypass(next_url(request) or "/"))
    set_cookie(response, "mezzanine-device", device, 60 * 60 * 24 * 365)
    return response

@staff_member_required
def set_site(request):
    """
    Put the selected site ID into the session - posted to from
    the "Select site" drop-down in the header of the admin. The
    site ID is then used in favour of the current request's
    domain in ``mezzanine.core.managers.CurrentSiteManager``.
    """
    site_id = int(request.GET["site_id"])
    if not request.user.is_superuser:
        try:
            SitePermission.objects.get(user=request.user, sites=site_id)
        except SitePermission.DoesNotExist:
            raise PermissionDenied
    request.session["site_id"] = site_id
    admin_url = reverse("admin:index")
    next = next_url(request) or admin_url
    # Don't redirect to a change view for an object that won't exist
    # on the selected site - go to its list view instead.
    if next.startswith(admin_url):
        parts = next.split("/")
        if len(parts) > 4 and parts[4].isdigit():
            next = "/".join(parts[:4])
    return redirect(next)

def direct_to_template(request, template, extra_context=None, **kwargs):
    """
    Replacement for Django's ``direct_to_template`` that uses
    ``TemplateResponse`` via ``mezzanine.utils.views.render``.
    """
    context = extra_context or {}
    context["params"] = kwargs
    for (key, value) in context.items():
        if callable(value):
            context[key] = value()
    return render(request, template, context)

@staff_member_required
def edit(request):
    """
    Process the inline editing form.
    """
    model = apps.get_model(request.POST["app"], request.POST["model"])
    obj = model.objects.get(id=request.POST["id"])
    form = get_edit_form(obj, request.POST["fields"], data=request.POST,
                         files=request.FILES)
    if not (is_editable(obj, request) and has_site_permission(request.user)):
        response = _("Permission denied")
    elif form.is_valid():
        form.save()
        model_admin = ModelAdmin(model, admin.site)
        message = model_admin.construct_change_message(request, form, None)
        model_admin.log_change(request, obj, message)
        response = ""
    else:
        response = list(form.errors.values())[0][0]
    return HttpResponse(response)

def search(request, template="search_results.html", extra_context=None):
    """
    Display search results. Takes an optional "contenttype" GET parameter
    in the form "app-name.ModelName" to limit search results to a single model.
    """
    query = request.GET.get("q", "")
    page = request.GET.get("page", 1)
    per_page = settings.SEARCH_PER_PAGE
    max_paging_links = settings.MAX_PAGING_LINKS
    try:
        parts = request.GET.get("type", "").split(".", 1)
        search_model = apps.get_model(*parts)
        search_model.objects.search  # Attribute check
    except (ValueError, TypeError, LookupError, AttributeError):
        search_model = Displayable
        search_type = _("Everything")
    else:
        search_type = search_model._meta.verbose_name_plural.capitalize()
    results = search_model.objects.search(query, for_user=request.user)
    paginated = paginate(results, page, per_page, max_paging_links)
    context = {"query": query, "results": paginated,
               "search_type": search_type}
    context.update(extra_context or {})
    return render(request, template, context)

@staff_member_required
def static_proxy(request):
    """
    Serves TinyMCE plugins inside the inline popups and the uploadify
    SWF, as these are normally static files, and will break with
    cross-domain JavaScript errors if ``STATIC_URL`` is an external
    host. URL for the file is passed in via querystring in the inline
    popup plugin template, and we then attempt to pull out the relative
    path to the file, so that we can serve it locally via Django.
    """
    normalize = lambda u: ("//" + u.split("://")[-1]) if "://" in u else u
    url = normalize(request.GET["u"])
    host = "//" + request.get_host()
    static_url = normalize(settings.STATIC_URL)
    for prefix in (host, static_url, "/"):
        if url.startswith(prefix):
            url = url.replace(prefix, "", 1)
    response = ""
    content_type = ""
    path = finders.find(url)
    if path:
        if isinstance(path, (list, tuple)):
            path = path[0]
        if url.endswith(".htm"):
            # Inject <base href="{{ STATIC_URL }}"> into TinyMCE
            # plugins, since the path static files in these won't be
            # on the same domain.
            static_url = settings.STATIC_URL + os.path.split(url)[0] + "/"
            if not urlparse(static_url).scheme:
                static_url = urljoin(host, static_url)
            base_tag = "<base href='%s'>" % static_url
            content_type = "text/html"
            with open(path, "r") as f:
                response = f.read().replace("<head>", "<head>" + base_tag)
        else:
            content_type = "application/octet-stream"
            with open(path, "rb") as f:
                response = f.read()
    return HttpResponse(response, content_type=content_type)

def displayable_links_js(request):
    """
    Renders a list of url/title pairs for all ``Displayable`` subclass
    instances into JSON that's used to populate a list of links in
    TinyMCE.
    """
    links = []
    if "mezzanine.pages" in settings.INSTALLED_APPS:
        from mezzanine.pages.models import Page
        is_page = lambda obj: isinstance(obj, Page)
    else:
        is_page = lambda obj: False
    # For each item's title, we use its model's verbose_name, but in the
    # case of Page subclasses, we just use "Page", and then sort the items
    # by whether they're a Page subclass or not, then by their URL.
    for url, obj in Displayable.objects.url_map(for_user=request.user).items():
        title = getattr(obj, "titles", obj.title)
        real = hasattr(obj, "id")
        page = is_page(obj)
        if real:
            verbose_name = _("Page") if page else obj._meta.verbose_name
            title = "%s: %s" % (verbose_name, title)
        links.append((not page and real, {"title": str(title), "value": url}))
    return HttpResponse(dumps([link[1] for link in sorted(links)]))

@requires_csrf_token
def page_not_found(request, template_name="errors/404.html"):
    """
    Mimics Django's 404 handler but with a different template path.
    """
    context = RequestContext(request, {
        "STATIC_URL": settings.STATIC_URL,
        "request_path": request.path,
    })
    t = get_template(template_name)
    return HttpResponseNotFound(t.render(context))

@requires_csrf_token
def server_error(request, template_name="errors/500.html"):
    """
    Mimics Django's error handler but adds ``STATIC_URL`` to the
    context.
    """
    context = RequestContext(request, {"STATIC_URL": settings.STATIC_URL})
    t = get_template(template_name)
    return HttpResponseServerError(t.render(context))
<commit_msg>Fix #1351 - exception in displayable_links_js<commit_after>from __future__ import absolute_import, unicode_literals
from future.builtins import int, open, str

import os

from json import dumps
try:
    from urllib.parse import urljoin, urlparse
except ImportError:
    from urlparse import urljoin, urlparse

from django.apps import apps
from django.contrib import admin
from django.contrib.admin.views.decorators import staff_member_required
from django.contrib.admin.options import ModelAdmin
from django.contrib.staticfiles import finders
from django.core.exceptions import PermissionDenied
from django.core.urlresolvers import reverse
from django.http import (HttpResponse, HttpResponseServerError,
                         HttpResponseNotFound)
from django.shortcuts import redirect
from django.template import RequestContext
from django.template.loader import get_template
from django.utils.translation import ugettext_lazy as _
from django.views.decorators.csrf import requires_csrf_token

from mezzanine.conf import settings
from mezzanine.core.forms import get_edit_form
from mezzanine.core.models import Displayable, SitePermission
from mezzanine.utils.cache import add_cache_bypass
from mezzanine.utils.views import is_editable, paginate, render, set_cookie
from mezzanine.utils.sites import has_site_permission
from mezzanine.utils.urls import next_url

def set_device(request, device=""):
    """
    Sets a device name in a cookie when a user explicitly wants to go
    to the site for a particular device (eg mobile).
    """
    response = redirect(add_cache_bypass(next_url(request) or "/"))
    set_cookie(response, "mezzanine-device", device, 60 * 60 * 24 * 365)
    return response

@staff_member_required
def set_site(request):
    """
    Put the selected site ID into the session - posted to from
    the "Select site" drop-down in the header of the admin. The
    site ID is then used in favour of the current request's
    domain in ``mezzanine.core.managers.CurrentSiteManager``.
    """
    site_id = int(request.GET["site_id"])
    if not request.user.is_superuser:
        try:
            SitePermission.objects.get(user=request.user, sites=site_id)
        except SitePermission.DoesNotExist:
            raise PermissionDenied
    request.session["site_id"] = site_id
    admin_url = reverse("admin:index")
    next = next_url(request) or admin_url
    # Don't redirect to a change view for an object that won't exist
    # on the selected site - go to its list view instead.
    if next.startswith(admin_url):
        parts = next.split("/")
        if len(parts) > 4 and parts[4].isdigit():
            next = "/".join(parts[:4])
    return redirect(next)

def direct_to_template(request, template, extra_context=None, **kwargs):
    """
    Replacement for Django's ``direct_to_template`` that uses
    ``TemplateResponse`` via ``mezzanine.utils.views.render``.
    """
    context = extra_context or {}
    context["params"] = kwargs
    for (key, value) in context.items():
        if callable(value):
            context[key] = value()
    return render(request, template, context)

@staff_member_required
def edit(request):
    """
    Process the inline editing form.
    """
    model = apps.get_model(request.POST["app"], request.POST["model"])
    obj = model.objects.get(id=request.POST["id"])
    form = get_edit_form(obj, request.POST["fields"], data=request.POST,
                         files=request.FILES)
    if not (is_editable(obj, request) and has_site_permission(request.user)):
        response = _("Permission denied")
    elif form.is_valid():
        form.save()
        model_admin = ModelAdmin(model, admin.site)
        message = model_admin.construct_change_message(request, form, None)
        model_admin.log_change(request, obj, message)
        response = ""
    else:
        response = list(form.errors.values())[0][0]
    return HttpResponse(response)

def search(request, template="search_results.html", extra_context=None):
    """
    Display search results. Takes an optional "contenttype" GET parameter
    in the form "app-name.ModelName" to limit search results to a single model.
    """
    query = request.GET.get("q", "")
    page = request.GET.get("page", 1)
    per_page = settings.SEARCH_PER_PAGE
    max_paging_links = settings.MAX_PAGING_LINKS
    try:
        parts = request.GET.get("type", "").split(".", 1)
        search_model = apps.get_model(*parts)
        search_model.objects.search  # Attribute check
    except (ValueError, TypeError, LookupError, AttributeError):
        search_model = Displayable
        search_type = _("Everything")
    else:
        search_type = search_model._meta.verbose_name_plural.capitalize()
    results = search_model.objects.search(query, for_user=request.user)
    paginated = paginate(results, page, per_page, max_paging_links)
    context = {"query": query, "results": paginated,
               "search_type": search_type}
    context.update(extra_context or {})
    return render(request, template, context)

@staff_member_required
def static_proxy(request):
    """
    Serves TinyMCE plugins inside the inline popups and the uploadify
    SWF, as these are normally static files, and will break with
    cross-domain JavaScript errors if ``STATIC_URL`` is an external
    host. URL for the file is passed in via querystring in the inline
    popup plugin template, and we then attempt to pull out the relative
    path to the file, so that we can serve it locally via Django.
    """
    normalize = lambda u: ("//" + u.split("://")[-1]) if "://" in u else u
    url = normalize(request.GET["u"])
    host = "//" + request.get_host()
    static_url = normalize(settings.STATIC_URL)
    for prefix in (host, static_url, "/"):
        if url.startswith(prefix):
            url = url.replace(prefix, "", 1)
    response = ""
    content_type = ""
    path = finders.find(url)
    if path:
        if isinstance(path, (list, tuple)):
            path = path[0]
        if url.endswith(".htm"):
            # Inject <base href="{{ STATIC_URL }}"> into TinyMCE
            # plugins, since the path static files in these won't be
            # on the same domain.
            static_url = settings.STATIC_URL + os.path.split(url)[0] + "/"
            if not urlparse(static_url).scheme:
                static_url = urljoin(host, static_url)
            base_tag = "<base href='%s'>" % static_url
            content_type = "text/html"
            with open(path, "r") as f:
                response = f.read().replace("<head>", "<head>" + base_tag)
        else:
            content_type = "application/octet-stream"
            with open(path, "rb") as f:
                response = f.read()
    return HttpResponse(response, content_type=content_type)

def displayable_links_js(request):
    """
    Renders a list of url/title pairs for all ``Displayable`` subclass
    instances into JSON that's used to populate a list of links in
    TinyMCE.
    """
    links = []
    if "mezzanine.pages" in settings.INSTALLED_APPS:
        from mezzanine.pages.models import Page
        is_page = lambda obj: isinstance(obj, Page)
    else:
        is_page = lambda obj: False
    # For each item's title, we use its model's verbose_name, but in the
    # case of Page subclasses, we just use "Page", and then sort the items
    # by whether they're a Page subclass or not, then by their URL.
    for url, obj in Displayable.objects.url_map(for_user=request.user).items():
        title = getattr(obj, "titles", obj.title)
        real = hasattr(obj, "id")
        page = is_page(obj)
        if real:
            verbose_name = _("Page") if page else obj._meta.verbose_name
            title = "%s: %s" % (verbose_name, title)
        links.append((not page and real, {"title": str(title), "value": url}))
    sorted_links = sorted(links, key=lambda link: (link[0], link[1]['value']))
    return HttpResponse(dumps([link[1] for link in sorted_links]))

@requires_csrf_token
def page_not_found(request, template_name="errors/404.html"):
    """
    Mimics Django's 404 handler but with a different template path.
    """
    context = RequestContext(request, {
        "STATIC_URL": settings.STATIC_URL,
        "request_path": request.path,
    })
    t = get_template(template_name)
    return HttpResponseNotFound(t.render(context))

@requires_csrf_token
def server_error(request, template_name="errors/500.html"):
    """
    Mimics Django's error handler but adds ``STATIC_URL`` to the
    context.
    """
    context = RequestContext(request, {"STATIC_URL": settings.STATIC_URL})
    t = get_template(template_name)
    return HttpResponseServerError(t.render(context))
<|endoftext|>


<commit_before># Copyright 2020 Planet Labs, Inc.
# Copyright 2022 Planet Labs PBC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging

import pytest

from planet import specs

LOGGER = logging.getLogger(__name__)

TEST_PRODUCT_BUNDLE = 'visual'
# must be a valid item type for TEST_PRODUCT_BUNDLE
TEST_ITEM_TYPE = 'PSScene'
ALL_ITEM_TYPES = [
    'PSOrthoTile',
    'Sentinel1',
    'REOrthoTile',
    'PSScene',
    'PSScene4Band',
    'Landsat8L1G',
    'PSScene3Band',
    'REScene',
    'MOD09GA',
    'MYD09GA',
    'MOD09GQ',
    'SkySatCollect',
    'Sentinel2L1C',
    'MYD09GQ',
    'SkySatScene'
]

def test_get_type_match():
    spec_list = ['Locket', 'drop', 'DEER']

    test_entry = 'locket'
    assert 'Locket' == specs.get_match(test_entry, spec_list)

    with pytest.raises(specs.NoMatchException):
        specs.get_match('a', ['b'])

def test_validate_bundle_supported():
    assert 'analytic' == specs.validate_bundle('ANALYTIC')

def test_validate_bundle_notsupported():
    with pytest.raises(specs.SpecificationException):
        specs.validate_bundle('notsupported')

def test_validate_item_type_supported():
    assert 'PSOrthoTile' == specs.validate_item_type('psorthotile', 'analytic')

def test_validate_item_type_notsupported_bundle():
    with pytest.raises(specs.SpecificationException):
        specs.validate_item_type('psorthotile', 'wha')

def test_validate_item_type_notsupported_itemtype():
    with pytest.raises(specs.SpecificationException):
        specs.validate_item_type('notsupported', 'analytic')

def test_validate_order_type_supported():
    assert 'full' == specs.validate_order_type('FULL')

def test_validate_order_type_notsupported():
    with pytest.raises(specs.SpecificationException):
        specs.validate_order_type('notsupported')

def test_validate_arhive_type_supported():
    assert 'zip' == specs.validate_archive_type('ZIP')

def test_validate_arhive_type_notsupported():
    with pytest.raises(specs.SpecificationException):
        specs.validate_archive_type('notsupported')

def test_validate_file_format_supported():
    assert 'COG' == specs.validate_file_format('cog')

def test_validate_file_format_notsupported():
    with pytest.raises(specs.SpecificationException):
        specs.validate_archive_type('notsupported')

def test_get_product_bundles():
    bundles = specs.get_product_bundles()
    assert TEST_PRODUCT_BUNDLE in bundles

def test_get_item_types_with_bundle():
    item_types = specs.get_item_types(TEST_PRODUCT_BUNDLE)
    assert TEST_ITEM_TYPE in item_types

def test_get_item_types_without_bundle():
    item_types = specs.get_item_types()
    for item in item_types:
        assert item in ALL_ITEM_TYPES
<commit_msg>Update tests/unit/test_specs.py<commit_after># Copyright 2020 Planet Labs, Inc.
# Copyright 2022 Planet Labs PBC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging

import pytest

from planet import specs

LOGGER = logging.getLogger(__name__)

TEST_PRODUCT_BUNDLE = 'visual'
# must be a valid item type for TEST_PRODUCT_BUNDLE
TEST_ITEM_TYPE = 'PSScene'
ALL_ITEM_TYPES = [
    'PSOrthoTile',
    'Sentinel1',
    'REOrthoTile',
    'PSScene',
    'PSScene4Band',
    'Landsat8L1G',
    'PSScene3Band',
    'REScene',
    'MOD09GA',
    'MYD09GA',
    'MOD09GQ',
    'SkySatCollect',
    'Sentinel2L1C',
    'MYD09GQ',
    'SkySatScene'
]

def test_get_type_match():
    spec_list = ['Locket', 'drop', 'DEER']

    test_entry = 'locket'
    assert 'Locket' == specs.get_match(test_entry, spec_list)

    with pytest.raises(specs.NoMatchException):
        specs.get_match('a', ['b'])

def test_validate_bundle_supported():
    assert 'analytic' == specs.validate_bundle('ANALYTIC')

def test_validate_bundle_notsupported():
    with pytest.raises(specs.SpecificationException):
        specs.validate_bundle('notsupported')

def test_validate_item_type_supported():
    assert 'PSOrthoTile' == specs.validate_item_type('psorthotile', 'analytic')

def test_validate_item_type_notsupported_bundle():
    with pytest.raises(specs.SpecificationException):
        specs.validate_item_type('psorthotile', 'wha')

def test_validate_item_type_notsupported_itemtype():
    with pytest.raises(specs.SpecificationException):
        specs.validate_item_type('notsupported', 'analytic')

def test_validate_order_type_supported():
    assert 'full' == specs.validate_order_type('FULL')

def test_validate_order_type_notsupported():
    with pytest.raises(specs.SpecificationException):
        specs.validate_order_type('notsupported')

def test_validate_arhive_type_supported():
    assert 'zip' == specs.validate_archive_type('ZIP')

def test_validate_arhive_type_notsupported():
    with pytest.raises(specs.SpecificationException):
        specs.validate_archive_type('notsupported')

def test_validate_file_format_supported():
    assert 'COG' == specs.validate_file_format('cog')

def test_validate_file_format_notsupported():
    with pytest.raises(specs.SpecificationException):
        specs.validate_archive_type('notsupported')

def test_get_product_bundles():
    bundles = specs.get_product_bundles()
    assert TEST_PRODUCT_BUNDLE in bundles

def test_get_item_types_with_bundle():
    item_types = specs.get_item_types(product_bundle=TEST_PRODUCT_BUNDLE)
    assert TEST_ITEM_TYPE in item_types

def test_get_item_types_without_bundle():
    item_types = specs.get_item_types()
    for item in item_types:
        assert item in ALL_ITEM_TYPES
<|endoftext|>


<commit_before>#### 4-ticks transmission frames
# carrier : 1000
# zero: 1100
# one: 1110

import sys,time
import RPi.GPIO as GPIO

#### #### #### ####
CARRIER=2
ERROR=-1

#### #### #### ####

# decode a frame
def readFrame(cyclecount0, cyclecount1):
	if cyclecount0 < cyclecount1 :
		if cyclecount1 < 2 * cyclecount0 :
			return 0 # c0 < c1 < 2.c0 : 1100
		return 1 # 2.c0 < c1 : 1110
	else :
		if cyclecount0 < 2 * cyclecount1 :
			return 0 # c1 < c0 < 2.c1 : 1100
		return CARRIER # 2.c1 < c0 : 1000
	return ERROR # never reach

# receive a frame
def getFrame(pin):
	starttime = time.time()
	cyclecount1=0
	cyclecount0=0
	while GPIO.input(pin):
		cyclecount1 = cyclecount1 + 1
		if time.time() > starttime + 30 :
			return ERROR
	while not GPIO.input(pin):
		cyclecount0 = cyclecount0 + 1
                if time.time() > starttime + 30 :
                        return ERROR
	return readFrame(cyclecount0, cyclecount1)

#### #### #### ####

def sendCarrier(pin, sleepms):
	GPIO.output(pin, True)
	time.sleep(0.001 * sleepms)
	GPIO.output(pin, False)
	time.sleep(0.003 * sleepms)

def send0(pin, sleepms):
	GPIO.output(pin, True)
	time.sleep(0.002 * sleepms)
	GPIO.output(pin, False)
	time.sleep(0.002 * sleepms)

def send1(pin, sleepms):
	GPIO.output(pin, True)
	time.sleep(0.003 * sleepms)
	GPIO.output(pin, False)
	time.sleep(0.001 * sleepms)

#### #### #### ####

<commit_msg>gpio_RX: wait for edge instead of polling<commit_after>#### 4-ticks transmission frames
# carrier : 1000
# zero: 1100
# one: 1110

import sys,time
import RPi.GPIO as GPIO

#### #### #### ####
CARRIER=2
ERROR=-1

#### #### #### ####

# decode a frame
def readFrame(cyclecount0, cyclecount1):
	if cyclecount0 < cyclecount1 :
		if cyclecount1 < 2 * cyclecount0 :
			return 0 # c0 < c1 < 2.c0 : 1100
		return 1 # 2.c0 < c1 : 1110
	else :
		if cyclecount0 < 2 * cyclecount1 :
			return 0 # c1 < c0 < 2.c1 : 1100
		return CARRIER # 2.c1 < c0 : 1000
	return ERROR # never reach

# receive a frame
def getFrame(pin):
	if not GPIO.input(pin):
		GPIO.wait_for_edge(pin, GPIO.RISING)
	start1 = time.time()
	GPIO.wait_for_edge(pin, GPIO.FALLING)
	start0 = time.time()
	cyclecount1 = start0 - start1
	GPIO.wait_for_edge(pin, GPIO.RISING)
	cyclecount0 = time.time() - start0
	return readFrame(cyclecount0, cyclecount1)

#### #### #### ####

def sendCarrier(pin, sleepms):
	GPIO.output(pin, True)
	time.sleep(0.001 * sleepms)
	GPIO.output(pin, False)
	time.sleep(0.003 * sleepms)

def send0(pin, sleepms):
	GPIO.output(pin, True)
	time.sleep(0.002 * sleepms)
	GPIO.output(pin, False)
	time.sleep(0.002 * sleepms)

def send1(pin, sleepms):
	GPIO.output(pin, True)
	time.sleep(0.003 * sleepms)
	GPIO.output(pin, False)
	time.sleep(0.001 * sleepms)

#### #### #### ####

<|endoftext|>


<commit_before>from __future__ import absolute_import, unicode_literals
from future.builtins import int, open

import os
try:
    from urllib.parse import urljoin, urlparse
except ImportError:
    from urlparse import urljoin, urlparse

from django.contrib import admin
from django.contrib.admin.views.decorators import staff_member_required
from django.contrib.admin.options import ModelAdmin
from django.contrib.staticfiles import finders
from django.core.exceptions import PermissionDenied
from django.core.urlresolvers import reverse
from django.db.models import get_model
from django.http import (HttpResponse, HttpResponseServerError,
                         HttpResponseNotFound)
from django.shortcuts import redirect
from django.template import RequestContext
from django.template.loader import get_template
from django.utils.translation import ugettext_lazy as _
from django.views.decorators.csrf import requires_csrf_token

from mezzanine.conf import settings
from mezzanine.core.forms import get_edit_form
from mezzanine.core.models import Displayable, SitePermission
from mezzanine.utils.cache import add_cache_bypass
from mezzanine.utils.views import is_editable, paginate, render, set_cookie
from mezzanine.utils.sites import has_site_permission
from mezzanine.utils.urls import next_url

def set_device(request, device=""):
    """
    Sets a device name in a cookie when a user explicitly wants to go
    to the site for a particular device (eg mobile).
    """
    response = redirect(add_cache_bypass(next_url(request) or "/"))
    set_cookie(response, "mezzanine-device", device, 60 * 60 * 24 * 365)
    return response

@staff_member_required
def set_site(request):
    """
    Put the selected site ID into the session - posted to from
    the "Select site" drop-down in the header of the admin. The
    site ID is then used in favour of the current request's
    domain in ``mezzanine.core.managers.CurrentSiteManager``.
    """
    site_id = int(request.GET["site_id"])
    if not request.user.is_superuser:
        try:
            SitePermission.objects.get(user=request.user, sites=site_id)
        except SitePermission.DoesNotExist:
            raise PermissionDenied
    request.session["site_id"] = site_id
    admin_url = reverse("admin:index")
    next = next_url(request) or admin_url
    # Don't redirect to a change view for an object that won't exist
    # on the selected site - go to its list view instead.
    if next.startswith(admin_url):
        parts = next.split("/")
        if len(parts) > 4 and parts[4].isdigit():
            next = "/".join(parts[:4])
    return redirect(next)

def direct_to_template(request, template, extra_context=None, **kwargs):
    """
    Replacement for Django's ``direct_to_template`` that uses
    ``TemplateResponse`` via ``mezzanine.utils.views.render``.
    """
    context = extra_context or {}
    context["params"] = kwargs
    for (key, value) in context.items():
        if callable(value):
            context[key] = value()
    return render(request, template, context)

@staff_member_required
def edit(request):
    """
    Process the inline editing form.
    """
    model = get_model(request.POST["app"], request.POST["model"])
    obj = model.objects.get(id=request.POST["id"])
    form = get_edit_form(obj, request.POST["fields"], data=request.POST,
                         files=request.FILES)
    if not (is_editable(obj, request) and has_site_permission(request.user)):
        response = _("Permission denied")
    elif form.is_valid():
        form.save()
        model_admin = ModelAdmin(model, admin.site)
        message = model_admin.construct_change_message(request, form, None)
        model_admin.log_change(request, obj, message)
        response = ""
    else:
        response = list(form.errors.values())[0][0]
    return HttpResponse(response)

def search(request, template="search_results.html"):
    """
    Display search results. Takes an optional "contenttype" GET parameter
    in the form "app-name.ModelName" to limit search results to a single model.
    """
    settings.use_editable()
    query = request.GET.get("q", "")
    page = request.GET.get("page", 1)
    per_page = settings.SEARCH_PER_PAGE
    max_paging_links = settings.MAX_PAGING_LINKS
    try:
        search_model = get_model(*request.GET.get("type", "").split(".", 1))
        if not issubclass(search_model, Displayable):
            raise TypeError
    except TypeError:
        search_model = Displayable
        search_type = _("Everything")
    else:
        search_type = search_model._meta.verbose_name_plural.capitalize()
    results = search_model.objects.search(query, for_user=request.user)
    paginated = paginate(results, page, per_page, max_paging_links)
    context = {"query": query, "results": paginated,
               "search_type": search_type}
    return render(request, template, context)

@staff_member_required
def static_proxy(request):
    """
    Serves TinyMCE plugins inside the inline popups and the uploadify
    SWF, as these are normally static files, and will break with
    cross-domain JavaScript errors if ``STATIC_URL`` is an external
    host. URL for the file is passed in via querystring in the inline
    popup plugin template.
    """
    # Get the relative URL after STATIC_URL.
    url = request.GET["u"]
    protocol = "http" if not request.is_secure() else "https"
    host = protocol + "://" + request.get_host()
    generic_host = "//" + request.get_host()
    # STATIC_URL often contains host or generic_host, so remove it
    # first otherwise the replacement loop below won't work.
    static_url = settings.STATIC_URL.replace(host, "", 1)
    static_url = static_url.replace(generic_host, "", 1)
    for prefix in (host, generic_host, static_url, "/"):
        if url.startswith(prefix):
            url = url.replace(prefix, "", 1)
    response = ""
    mimetype = ""
    path = finders.find(url)
    if path:
        if isinstance(path, (list, tuple)):
            path = path[0]
        if url.endswith(".htm"):
            # Inject <base href="{{ STATIC_URL }}"> into TinyMCE
            # plugins, since the path static files in these won't be
            # on the same domain.
            static_url = settings.STATIC_URL + os.path.split(url)[0] + "/"
            if not urlparse(static_url).scheme:
                static_url = urljoin(host, static_url)
            base_tag = "<base href='%s'>" % static_url
            mimetype = "text/html"
            with open(path, "r") as f:
                response = f.read().replace("<head>", "<head>" + base_tag)
        else:
            mimetype = "application/octet-stream"
            with open(path, "rb") as f:
                response = f.read()
    return HttpResponse(response, mimetype=mimetype)

def displayable_links_js(request, template_name="admin/displayable_links.js"):
    """
    Renders a list of url/title pairs for all ``Displayable`` subclass
    instances into JavaScript that's used to populate a list of links
    in TinyMCE.
    """
    links = []
    if "mezzanine.pages" in settings.INSTALLED_APPS:
        from mezzanine.pages.models import Page
        is_page = lambda obj: isinstance(obj, Page)
    else:
        is_page = lambda obj: False
    # For each item's title, we use its model's verbose_name, but in the
    # case of Page subclasses, we just use "Page", and then sort the items
    # by whether they're a Page subclass or not, then by their URL.
    for url, obj in Displayable.objects.url_map(for_user=request.user).items():
        title = getattr(obj, "titles", obj.title)
        real = hasattr(obj, "id")
        page = is_page(obj)
        if real:
            verbose_name = _("Page") if page else obj._meta.verbose_name
            title = "%s: %s" % (verbose_name, title)
        links.append((not page and real, url, title))
    context = {"links": [link[1:] for link in sorted(links)]}
    return render(request, template_name, context, mimetype="text/javascript")

@requires_csrf_token
def page_not_found(request, template_name="errors/404.html"):
    """
    Mimics Django's 404 handler but with a different template path.
    """
    context = RequestContext(request, {
        "STATIC_URL": settings.STATIC_URL,
        "request_path": request.path,
    })
    t = get_template(template_name)
    return HttpResponseNotFound(t.render(context))

@requires_csrf_token
def server_error(request, template_name="errors/500.html"):
    """
    Mimics Django's error handler but adds ``STATIC_URL`` to the
    context.
    """
    context = RequestContext(request, {"STATIC_URL": settings.STATIC_URL})
    t = get_template(template_name)
    return HttpResponseServerError(t.render(context))
<commit_msg>Fix handling for schemeless static urls in static proxy.<commit_after>from __future__ import absolute_import, unicode_literals
from future.builtins import int, open

import os
try:
    from urllib.parse import urljoin, urlparse
except ImportError:
    from urlparse import urljoin, urlparse

from django.contrib import admin
from django.contrib.admin.views.decorators import staff_member_required
from django.contrib.admin.options import ModelAdmin
from django.contrib.staticfiles import finders
from django.core.exceptions import PermissionDenied
from django.core.urlresolvers import reverse
from django.db.models import get_model
from django.http import (HttpResponse, HttpResponseServerError,
                         HttpResponseNotFound)
from django.shortcuts import redirect
from django.template import RequestContext
from django.template.loader import get_template
from django.utils.translation import ugettext_lazy as _
from django.views.decorators.csrf import requires_csrf_token

from mezzanine.conf import settings
from mezzanine.core.forms import get_edit_form
from mezzanine.core.models import Displayable, SitePermission
from mezzanine.utils.cache import add_cache_bypass
from mezzanine.utils.views import is_editable, paginate, render, set_cookie
from mezzanine.utils.sites import has_site_permission
from mezzanine.utils.urls import next_url

def set_device(request, device=""):
    """
    Sets a device name in a cookie when a user explicitly wants to go
    to the site for a particular device (eg mobile).
    """
    response = redirect(add_cache_bypass(next_url(request) or "/"))
    set_cookie(response, "mezzanine-device", device, 60 * 60 * 24 * 365)
    return response

@staff_member_required
def set_site(request):
    """
    Put the selected site ID into the session - posted to from
    the "Select site" drop-down in the header of the admin. The
    site ID is then used in favour of the current request's
    domain in ``mezzanine.core.managers.CurrentSiteManager``.
    """
    site_id = int(request.GET["site_id"])
    if not request.user.is_superuser:
        try:
            SitePermission.objects.get(user=request.user, sites=site_id)
        except SitePermission.DoesNotExist:
            raise PermissionDenied
    request.session["site_id"] = site_id
    admin_url = reverse("admin:index")
    next = next_url(request) or admin_url
    # Don't redirect to a change view for an object that won't exist
    # on the selected site - go to its list view instead.
    if next.startswith(admin_url):
        parts = next.split("/")
        if len(parts) > 4 and parts[4].isdigit():
            next = "/".join(parts[:4])
    return redirect(next)

def direct_to_template(request, template, extra_context=None, **kwargs):
    """
    Replacement for Django's ``direct_to_template`` that uses
    ``TemplateResponse`` via ``mezzanine.utils.views.render``.
    """
    context = extra_context or {}
    context["params"] = kwargs
    for (key, value) in context.items():
        if callable(value):
            context[key] = value()
    return render(request, template, context)

@staff_member_required
def edit(request):
    """
    Process the inline editing form.
    """
    model = get_model(request.POST["app"], request.POST["model"])
    obj = model.objects.get(id=request.POST["id"])
    form = get_edit_form(obj, request.POST["fields"], data=request.POST,
                         files=request.FILES)
    if not (is_editable(obj, request) and has_site_permission(request.user)):
        response = _("Permission denied")
    elif form.is_valid():
        form.save()
        model_admin = ModelAdmin(model, admin.site)
        message = model_admin.construct_change_message(request, form, None)
        model_admin.log_change(request, obj, message)
        response = ""
    else:
        response = list(form.errors.values())[0][0]
    return HttpResponse(response)

def search(request, template="search_results.html"):
    """
    Display search results. Takes an optional "contenttype" GET parameter
    in the form "app-name.ModelName" to limit search results to a single model.
    """
    settings.use_editable()
    query = request.GET.get("q", "")
    page = request.GET.get("page", 1)
    per_page = settings.SEARCH_PER_PAGE
    max_paging_links = settings.MAX_PAGING_LINKS
    try:
        search_model = get_model(*request.GET.get("type", "").split(".", 1))
        if not issubclass(search_model, Displayable):
            raise TypeError
    except TypeError:
        search_model = Displayable
        search_type = _("Everything")
    else:
        search_type = search_model._meta.verbose_name_plural.capitalize()
    results = search_model.objects.search(query, for_user=request.user)
    paginated = paginate(results, page, per_page, max_paging_links)
    context = {"query": query, "results": paginated,
               "search_type": search_type}
    return render(request, template, context)

@staff_member_required
def static_proxy(request):
    """
    Serves TinyMCE plugins inside the inline popups and the uploadify
    SWF, as these are normally static files, and will break with
    cross-domain JavaScript errors if ``STATIC_URL`` is an external
    host. URL for the file is passed in via querystring in the inline
    popup plugin template, and we then attempt to pull out the relative
    path to the file, so that we can serve it locally via Django.
    """
    normalize = lambda u: "//" + u.split("://")[-1]
    url = normalize(request.GET["u"])
    host = normalize(request.get_host())
    static_url = settings.STATIC_URL
    if "://" in static_url:
        static_url = normalize(static_url)
    for prefix in (host, static_url, "/"):
        if url.startswith(prefix):
            url = url.replace(prefix, "", 1)
    response = ""
    mimetype = ""
    path = finders.find(url)
    if path:
        if isinstance(path, (list, tuple)):
            path = path[0]
        if url.endswith(".htm"):
            # Inject <base href="{{ STATIC_URL }}"> into TinyMCE
            # plugins, since the path static files in these won't be
            # on the same domain.
            static_url = settings.STATIC_URL + os.path.split(url)[0] + "/"
            if not urlparse(static_url).scheme:
                static_url = urljoin(host, static_url)
            base_tag = "<base href='%s'>" % static_url
            mimetype = "text/html"
            with open(path, "r") as f:
                response = f.read().replace("<head>", "<head>" + base_tag)
        else:
            mimetype = "application/octet-stream"
            with open(path, "rb") as f:
                response = f.read()
    return HttpResponse(response, mimetype=mimetype)

def displayable_links_js(request, template_name="admin/displayable_links.js"):
    """
    Renders a list of url/title pairs for all ``Displayable`` subclass
    instances into JavaScript that's used to populate a list of links
    in TinyMCE.
    """
    links = []
    if "mezzanine.pages" in settings.INSTALLED_APPS:
        from mezzanine.pages.models import Page
        is_page = lambda obj: isinstance(obj, Page)
    else:
        is_page = lambda obj: False
    # For each item's title, we use its model's verbose_name, but in the
    # case of Page subclasses, we just use "Page", and then sort the items
    # by whether they're a Page subclass or not, then by their URL.
    for url, obj in Displayable.objects.url_map(for_user=request.user).items():
        title = getattr(obj, "titles", obj.title)
        real = hasattr(obj, "id")
        page = is_page(obj)
        if real:
            verbose_name = _("Page") if page else obj._meta.verbose_name
            title = "%s: %s" % (verbose_name, title)
        links.append((not page and real, url, title))
    context = {"links": [link[1:] for link in sorted(links)]}
    return render(request, template_name, context, mimetype="text/javascript")

@requires_csrf_token
def page_not_found(request, template_name="errors/404.html"):
    """
    Mimics Django's 404 handler but with a different template path.
    """
    context = RequestContext(request, {
        "STATIC_URL": settings.STATIC_URL,
        "request_path": request.path,
    })
    t = get_template(template_name)
    return HttpResponseNotFound(t.render(context))

@requires_csrf_token
def server_error(request, template_name="errors/500.html"):
    """
    Mimics Django's error handler but adds ``STATIC_URL`` to the
    context.
    """
    context = RequestContext(request, {"STATIC_URL": settings.STATIC_URL})
    t = get_template(template_name)
    return HttpResponseServerError(t.render(context))
<|endoftext|>


<commit_before><commit_msg>tests: Fix update-manpages test.<commit_after><|endoftext|>


<commit_before>from __future__ import unicode_literals

import os
import sys
from warnings import warn

from django import VERSION
from django.conf import global_settings as defaults
from django.template.base import add_to_builtins

from mezzanine.utils.importing import path_for_import
from mezzanine.utils.timezone import get_best_local_timezone

class SitesAllowedHosts(object):
    """
    This is a fallback for Django 1.5's ALLOWED_HOSTS setting
    which is required when DEBUG is False. It looks up the
    ``Site`` model and uses any domains added to it, the
    first time the setting is accessed.
    """
    def __iter__(self):
        if getattr(self, "_hosts", None) is None:
            from django.contrib.sites.models import Site
            self._hosts = [s.domain.split(":")[0] for s in Site.objects.all()]
        return iter(self._hosts)

def set_dynamic_settings(s):
    """
    Called at the end of the project's settings module, and is passed
    its globals dict for updating with some final tweaks for settings
    that generally aren't specified, but can be given some better
    defaults based on other settings that have been specified. Broken
    out into its own function so that the code need not be replicated
    in the settings modules of other project-based apps that leverage
    Mezzanine's settings module.
    """

    # Moves an existing list setting value to a different position.
    move = lambda n, k, i: s[n].insert(i, s[n].pop(s[n].index(k)))
    # Add a value to the end of a list setting if not in the list.
    append = lambda n, k: s[n].append(k) if k not in s[n] else None
    # Add a value to the start of a list setting if not in the list.
    prepend = lambda n, k: s[n].insert(0, k) if k not in s[n] else None
    # Remove a value from a list setting if in the list.
    remove = lambda n, k: s[n].remove(k) if k in s[n] else None

    s["TEMPLATE_DEBUG"] = s.get("TEMPLATE_DEBUG", s.get("DEBUG", False))
    add_to_builtins("mezzanine.template.loader_tags")

    if not s.get("ALLOWED_HOSTS", []):
        warn("You haven't defined the ALLOWED_HOSTS settings, which "
             "Django 1.5 requires. Will fall back to the domains "
             "configured as sites.")
        s["ALLOWED_HOSTS"] = SitesAllowedHosts()

    if s.get("TIME_ZONE", None) is None:
        tz = get_best_local_timezone()
        s["TIME_ZONE"] = tz
        warn("TIME_ZONE setting is not set, using closest match: %s" % tz)

    # Define some settings based on management command being run.
    management_command = sys.argv[1] if len(sys.argv) > 1 else ""
    # Some kind of testing is running via test or testserver.
    s["TESTING"] = management_command in ("test", "testserver")
    # Some kind of development server is running via runserver,
    # runserver_plus or harvest (lettuce)
    s["DEV_SERVER"] = management_command.startswith(("runserver", "harvest"))

    # Change tuple settings to lists for easier manipulation.
    s.setdefault("AUTHENTICATION_BACKENDS", defaults.AUTHENTICATION_BACKENDS)
    s.setdefault("STATICFILES_FINDERS", defaults.STATICFILES_FINDERS)
    tuple_list_settings = ["AUTHENTICATION_BACKENDS", "INSTALLED_APPS",
                           "MIDDLEWARE_CLASSES", "STATICFILES_FINDERS",
                           "LANGUAGES", "TEMPLATE_CONTEXT_PROCESSORS"]
    for setting in tuple_list_settings[:]:
        if not isinstance(s.get(setting, []), list):
            s[setting] = list(s[setting])
        else:
            # Setting is already a list, so we'll exclude it from
            # the list of settings we'll revert back to tuples.
            tuple_list_settings.remove(setting)

    # From Mezzanine 3.1.2 and onward we added the context processor
    # for handling the page variable in templates - here we help
    # upgrading by adding it if missing, with a warning. This helper
    # can go away eventually.
    cp = "mezzanine.pages.context_processors.page"
    if ("mezzanine.pages" in s["INSTALLED_APPS"] and
            cp not in s["TEMPLATE_CONTEXT_PROCESSORS"]):
        warn("%s is required in the TEMPLATE_CONTEXT_PROCESSORS setting. "
             "Adding it now, but you should update settings.py to "
             "explicitly include it." % cp)
        append("TEMPLATE_CONTEXT_PROCESSORS", cp)

    # Set up cookie messaging if none defined.
    storage = "django.contrib.messages.storage.cookie.CookieStorage"
    s.setdefault("MESSAGE_STORAGE", storage)

    if s["TESTING"]:
        # Following bits are work-arounds for some assumptions that
        # Django 1.5's tests make.

        # contrib.auth tests fail without its own auth backend installed.
        append("AUTHENTICATION_BACKENDS",
               "django.contrib.auth.backends.ModelBackend")

        # Tests in contrib.redirects simply don't work with a
        # catch-all urlpattern such as Mezzanine's pages app.
        remove("INSTALLED_APPS", "django.contrib.redirects")
        remove("MIDDLEWARE_CLASSES",
            "django.contrib.redirects.middleware.RedirectFallbackMiddleware")

    else:
        # Setup for optional apps.
        optional = list(s.get("OPTIONAL_APPS", []))
        if s.get("USE_SOUTH") and VERSION < (1, 7):
            optional.append("south")
        elif not s.get("USE_SOUTH", True) and "south" in s["INSTALLED_APPS"]:
            s["INSTALLED_APPS"].remove("south")
        for app in optional:
            if app not in s["INSTALLED_APPS"]:
                try:
                    __import__(app)
                except ImportError:
                    pass
                else:
                    s["INSTALLED_APPS"].append(app)

    # To support migrations for both Django 1.7 and South, Sotuh's old
    # migrations for each app were moved into "app.migrations.south"
    # packages. Here we assign each of these to SOUTH_MIGRATION_MODULES
    # allowing South to find them.
    if "south" in s["INSTALLED_APPS"]:
        s.setdefault("SOUTH_MIGRATION_MODULES", {})
        for app in s["INSTALLED_APPS"]:
            # We need to verify the path to the custom migrations
            # package exists for each app. We can't simply try
            # and import it, for some apps this causes side effects,
            # so we need to import something higher up to get at its
            # filesystem path - this can't be the actual app either,
            # side effects again, but we can generally import the
            # top-level package for apps that are contained within
            # one, which covers Mezzanine, Cartridge, Drum.
            if "." not in app:
                continue
            migrations = "%s.migrations.south" % app
            parts = migrations.split(".", 1)
            root = path_for_import(parts[0])
            other = parts[1].replace(".", os.sep)
            if os.path.exists(os.path.join(root, other)):
                s["SOUTH_MIGRATION_MODULES"][app.split(".")[-1]] = migrations

    if "debug_toolbar" in s["INSTALLED_APPS"]:
        debug_mw = "debug_toolbar.middleware.DebugToolbarMiddleware"
        append("MIDDLEWARE_CLASSES", debug_mw)

    # If compressor installed, ensure it's configured and make
    # Mezzanine's settings available to its offline context,
    # since jQuery is configured via a setting.
    if "compressor" in s["INSTALLED_APPS"]:
        append("STATICFILES_FINDERS", "compressor.finders.CompressorFinder")
        s.setdefault("COMPRESS_OFFLINE_CONTEXT", {
            "MEDIA_URL": s.get("MEDIA_URL", ""),
            "STATIC_URL": s.get("STATIC_URL", ""),
        })

        def mezzanine_settings():
            from mezzanine.conf import settings
            return settings
        s["COMPRESS_OFFLINE_CONTEXT"]["settings"] = mezzanine_settings

    # Ensure the Mezzanine auth backend is enabled if
    # mezzanine.accounts is being used.
    if "mezzanine.accounts" in s["INSTALLED_APPS"]:
        auth_backend = "mezzanine.core.auth_backends.MezzanineBackend"
        s.setdefault("AUTHENTICATION_BACKENDS", [])
        prepend("AUTHENTICATION_BACKENDS", auth_backend)

    # Ensure Grappelli is after Mezzanine in app order so that
    # admin templates are loaded in the correct order.
    grappelli_name = s.get("PACKAGE_NAME_GRAPPELLI")
    if s["TESTING"]:
        # Optional apps aren't installed when testing, but we need
        # grappelli to perform some admin tests for certain HTML.
        try:
            __import__(grappelli_name)
        except ImportError:
            pass
        else:
            append("INSTALLED_APPS", grappelli_name)
    try:
        move("INSTALLED_APPS", grappelli_name, len(s["INSTALLED_APPS"]))
    except ValueError:
        s["GRAPPELLI_INSTALLED"] = False
    else:
        s["GRAPPELLI_INSTALLED"] = True

    # Ensure admin is at the bottom of the app order so that admin
    # templates are loaded in the correct order, and that staticfiles
    # is also at the end so its runserver can be overridden.
    apps = ["django.contrib.admin"]
    if VERSION >= (1, 7):
        apps += ["django.contrib.staticfiles"]
    for app in apps:
        try:
            move("INSTALLED_APPS", app, len(s["INSTALLED_APPS"]))
        except ValueError:
            pass

    # Ensure we have a test runner (removed in Django 1.6)
    s.setdefault("TEST_RUNNER", "django.test.simple.DjangoTestSuiteRunner")

    # Add missing apps if existing apps depend on them.
    if "mezzanine.blog" in s["INSTALLED_APPS"]:
        append("INSTALLED_APPS", "mezzanine.generic")
    if "mezzanine.generic" in s["INSTALLED_APPS"]:
        s.setdefault("COMMENTS_APP", "mezzanine.generic")
        append("INSTALLED_APPS", "django.contrib.comments")

    # Ensure mezzanine.boot is first.
    try:
        move("INSTALLED_APPS", "mezzanine.boot", 0)
    except ValueError:
        pass

    # Remove caching middleware if no backend defined.
    if not (s.get("CACHE_BACKEND") or s.get("CACHES")):
        s["MIDDLEWARE_CLASSES"] = [mw for mw in s["MIDDLEWARE_CLASSES"] if not
                                   (mw.endswith("UpdateCacheMiddleware") or
                                    mw.endswith("FetchFromCacheMiddleware"))]

    # If only LANGUAGE_CODE has been defined, ensure the other required
    # settings for translations are configured.
    if (s.get("LANGUAGE_CODE") and len(s.get("LANGUAGES", [])) == 1 and
            s["LANGUAGE_CODE"] != s["LANGUAGES"][0][0]):
        s["USE_I18N"] = True
        s["LANGUAGES"] = [(s["LANGUAGE_CODE"], "")]

    # Revert tuple settings back to tuples.
    for setting in tuple_list_settings:
        s[setting] = tuple(s[setting])

    # Some settings tweaks for different DB engines.
    for (key, db) in s["DATABASES"].items():
        shortname = db["ENGINE"].split(".")[-1]
        if shortname == "sqlite3" and os.sep not in db["NAME"]:
            # If the Sqlite DB name doesn't contain a path, assume
            # it's in the project directory and add the path to it.
            db_path = os.path.join(s.get("PROJECT_ROOT", ""), db["NAME"])
            s["DATABASES"][key]["NAME"] = db_path
        elif shortname == "mysql":
            # Required MySQL collation for tests.
            s["DATABASES"][key]["TEST_COLLATION"] = "utf8_general_ci"
<commit_msg>Let sqlite configuration not contain a ``NAME`` at all (as with some settings environments).<commit_after>from __future__ import unicode_literals

import os
import sys
from warnings import warn

from django import VERSION
from django.conf import global_settings as defaults
from django.template.base import add_to_builtins

from mezzanine.utils.importing import path_for_import
from mezzanine.utils.timezone import get_best_local_timezone

class SitesAllowedHosts(object):
    """
    This is a fallback for Django 1.5's ALLOWED_HOSTS setting
    which is required when DEBUG is False. It looks up the
    ``Site`` model and uses any domains added to it, the
    first time the setting is accessed.
    """
    def __iter__(self):
        if getattr(self, "_hosts", None) is None:
            from django.contrib.sites.models import Site
            self._hosts = [s.domain.split(":")[0] for s in Site.objects.all()]
        return iter(self._hosts)

def set_dynamic_settings(s):
    """
    Called at the end of the project's settings module, and is passed
    its globals dict for updating with some final tweaks for settings
    that generally aren't specified, but can be given some better
    defaults based on other settings that have been specified. Broken
    out into its own function so that the code need not be replicated
    in the settings modules of other project-based apps that leverage
    Mezzanine's settings module.
    """

    # Moves an existing list setting value to a different position.
    move = lambda n, k, i: s[n].insert(i, s[n].pop(s[n].index(k)))
    # Add a value to the end of a list setting if not in the list.
    append = lambda n, k: s[n].append(k) if k not in s[n] else None
    # Add a value to the start of a list setting if not in the list.
    prepend = lambda n, k: s[n].insert(0, k) if k not in s[n] else None
    # Remove a value from a list setting if in the list.
    remove = lambda n, k: s[n].remove(k) if k in s[n] else None

    s["TEMPLATE_DEBUG"] = s.get("TEMPLATE_DEBUG", s.get("DEBUG", False))
    add_to_builtins("mezzanine.template.loader_tags")

    if not s.get("ALLOWED_HOSTS", []):
        warn("You haven't defined the ALLOWED_HOSTS settings, which "
             "Django 1.5 requires. Will fall back to the domains "
             "configured as sites.")
        s["ALLOWED_HOSTS"] = SitesAllowedHosts()

    if s.get("TIME_ZONE", None) is None:
        tz = get_best_local_timezone()
        s["TIME_ZONE"] = tz
        warn("TIME_ZONE setting is not set, using closest match: %s" % tz)

    # Define some settings based on management command being run.
    management_command = sys.argv[1] if len(sys.argv) > 1 else ""
    # Some kind of testing is running via test or testserver.
    s["TESTING"] = management_command in ("test", "testserver")
    # Some kind of development server is running via runserver,
    # runserver_plus or harvest (lettuce)
    s["DEV_SERVER"] = management_command.startswith(("runserver", "harvest"))

    # Change tuple settings to lists for easier manipulation.
    s.setdefault("AUTHENTICATION_BACKENDS", defaults.AUTHENTICATION_BACKENDS)
    s.setdefault("STATICFILES_FINDERS", defaults.STATICFILES_FINDERS)
    tuple_list_settings = ["AUTHENTICATION_BACKENDS", "INSTALLED_APPS",
                           "MIDDLEWARE_CLASSES", "STATICFILES_FINDERS",
                           "LANGUAGES", "TEMPLATE_CONTEXT_PROCESSORS"]
    for setting in tuple_list_settings[:]:
        if not isinstance(s.get(setting, []), list):
            s[setting] = list(s[setting])
        else:
            # Setting is already a list, so we'll exclude it from
            # the list of settings we'll revert back to tuples.
            tuple_list_settings.remove(setting)

    # From Mezzanine 3.1.2 and onward we added the context processor
    # for handling the page variable in templates - here we help
    # upgrading by adding it if missing, with a warning. This helper
    # can go away eventually.
    cp = "mezzanine.pages.context_processors.page"
    if ("mezzanine.pages" in s["INSTALLED_APPS"] and
            cp not in s["TEMPLATE_CONTEXT_PROCESSORS"]):
        warn("%s is required in the TEMPLATE_CONTEXT_PROCESSORS setting. "
             "Adding it now, but you should update settings.py to "
             "explicitly include it." % cp)
        append("TEMPLATE_CONTEXT_PROCESSORS", cp)

    # Set up cookie messaging if none defined.
    storage = "django.contrib.messages.storage.cookie.CookieStorage"
    s.setdefault("MESSAGE_STORAGE", storage)

    if s["TESTING"]:
        # Following bits are work-arounds for some assumptions that
        # Django 1.5's tests make.

        # contrib.auth tests fail without its own auth backend installed.
        append("AUTHENTICATION_BACKENDS",
               "django.contrib.auth.backends.ModelBackend")

        # Tests in contrib.redirects simply don't work with a
        # catch-all urlpattern such as Mezzanine's pages app.
        remove("INSTALLED_APPS", "django.contrib.redirects")
        remove("MIDDLEWARE_CLASSES",
            "django.contrib.redirects.middleware.RedirectFallbackMiddleware")

    else:
        # Setup for optional apps.
        optional = list(s.get("OPTIONAL_APPS", []))
        if s.get("USE_SOUTH") and VERSION < (1, 7):
            optional.append("south")
        elif not s.get("USE_SOUTH", True) and "south" in s["INSTALLED_APPS"]:
            s["INSTALLED_APPS"].remove("south")
        for app in optional:
            if app not in s["INSTALLED_APPS"]:
                try:
                    __import__(app)
                except ImportError:
                    pass
                else:
                    s["INSTALLED_APPS"].append(app)

    # To support migrations for both Django 1.7 and South, Sotuh's old
    # migrations for each app were moved into "app.migrations.south"
    # packages. Here we assign each of these to SOUTH_MIGRATION_MODULES
    # allowing South to find them.
    if "south" in s["INSTALLED_APPS"]:
        s.setdefault("SOUTH_MIGRATION_MODULES", {})
        for app in s["INSTALLED_APPS"]:
            # We need to verify the path to the custom migrations
            # package exists for each app. We can't simply try
            # and import it, for some apps this causes side effects,
            # so we need to import something higher up to get at its
            # filesystem path - this can't be the actual app either,
            # side effects again, but we can generally import the
            # top-level package for apps that are contained within
            # one, which covers Mezzanine, Cartridge, Drum.
            if "." not in app:
                continue
            migrations = "%s.migrations.south" % app
            parts = migrations.split(".", 1)
            root = path_for_import(parts[0])
            other = parts[1].replace(".", os.sep)
            if os.path.exists(os.path.join(root, other)):
                s["SOUTH_MIGRATION_MODULES"][app.split(".")[-1]] = migrations

    if "debug_toolbar" in s["INSTALLED_APPS"]:
        debug_mw = "debug_toolbar.middleware.DebugToolbarMiddleware"
        append("MIDDLEWARE_CLASSES", debug_mw)

    # If compressor installed, ensure it's configured and make
    # Mezzanine's settings available to its offline context,
    # since jQuery is configured via a setting.
    if "compressor" in s["INSTALLED_APPS"]:
        append("STATICFILES_FINDERS", "compressor.finders.CompressorFinder")
        s.setdefault("COMPRESS_OFFLINE_CONTEXT", {
            "MEDIA_URL": s.get("MEDIA_URL", ""),
            "STATIC_URL": s.get("STATIC_URL", ""),
        })

        def mezzanine_settings():
            from mezzanine.conf import settings
            return settings
        s["COMPRESS_OFFLINE_CONTEXT"]["settings"] = mezzanine_settings

    # Ensure the Mezzanine auth backend is enabled if
    # mezzanine.accounts is being used.
    if "mezzanine.accounts" in s["INSTALLED_APPS"]:
        auth_backend = "mezzanine.core.auth_backends.MezzanineBackend"
        s.setdefault("AUTHENTICATION_BACKENDS", [])
        prepend("AUTHENTICATION_BACKENDS", auth_backend)

    # Ensure Grappelli is after Mezzanine in app order so that
    # admin templates are loaded in the correct order.
    grappelli_name = s.get("PACKAGE_NAME_GRAPPELLI")
    if s["TESTING"]:
        # Optional apps aren't installed when testing, but we need
        # grappelli to perform some admin tests for certain HTML.
        try:
            __import__(grappelli_name)
        except ImportError:
            pass
        else:
            append("INSTALLED_APPS", grappelli_name)
    try:
        move("INSTALLED_APPS", grappelli_name, len(s["INSTALLED_APPS"]))
    except ValueError:
        s["GRAPPELLI_INSTALLED"] = False
    else:
        s["GRAPPELLI_INSTALLED"] = True

    # Ensure admin is at the bottom of the app order so that admin
    # templates are loaded in the correct order, and that staticfiles
    # is also at the end so its runserver can be overridden.
    apps = ["django.contrib.admin"]
    if VERSION >= (1, 7):
        apps += ["django.contrib.staticfiles"]
    for app in apps:
        try:
            move("INSTALLED_APPS", app, len(s["INSTALLED_APPS"]))
        except ValueError:
            pass

    # Ensure we have a test runner (removed in Django 1.6)
    s.setdefault("TEST_RUNNER", "django.test.simple.DjangoTestSuiteRunner")

    # Add missing apps if existing apps depend on them.
    if "mezzanine.blog" in s["INSTALLED_APPS"]:
        append("INSTALLED_APPS", "mezzanine.generic")
    if "mezzanine.generic" in s["INSTALLED_APPS"]:
        s.setdefault("COMMENTS_APP", "mezzanine.generic")
        append("INSTALLED_APPS", "django.contrib.comments")

    # Ensure mezzanine.boot is first.
    try:
        move("INSTALLED_APPS", "mezzanine.boot", 0)
    except ValueError:
        pass

    # Remove caching middleware if no backend defined.
    if not (s.get("CACHE_BACKEND") or s.get("CACHES")):
        s["MIDDLEWARE_CLASSES"] = [mw for mw in s["MIDDLEWARE_CLASSES"] if not
                                   (mw.endswith("UpdateCacheMiddleware") or
                                    mw.endswith("FetchFromCacheMiddleware"))]

    # If only LANGUAGE_CODE has been defined, ensure the other required
    # settings for translations are configured.
    if (s.get("LANGUAGE_CODE") and len(s.get("LANGUAGES", [])) == 1 and
            s["LANGUAGE_CODE"] != s["LANGUAGES"][0][0]):
        s["USE_I18N"] = True
        s["LANGUAGES"] = [(s["LANGUAGE_CODE"], "")]

    # Revert tuple settings back to tuples.
    for setting in tuple_list_settings:
        s[setting] = tuple(s[setting])

    # Some settings tweaks for different DB engines.
    for (key, db) in s["DATABASES"].items():
        shortname = db["ENGINE"].split(".")[-1]
        if shortname == "sqlite3":
            # If the Sqlite DB name doesn't contain a path, assume
            # it's in the project directory and add the path to it.
            if "NAME" in db and os.sep not in db["NAME"]:
                db_path = os.path.join(s.get("PROJECT_ROOT", ""), db["NAME"])
                s["DATABASES"][key]["NAME"] = db_path
        elif shortname == "mysql":
            # Required MySQL collation for tests.
            s["DATABASES"][key]["TEST_COLLATION"] = "utf8_general_ci"
<|endoftext|>


<commit_before>#!/usr/bin/python3
# -*- coding: utf-8 -*-
##################################################################################
# Access information about binary and source packages in multiple
# (independent) apt-repositories utilizing libapt / python-apt/
# apt_pkg without the need to change the local system and it's apt-setup.
#
# Copyright (C) 2018  Christoph Lutz
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
##################################################################################
import logging
import sys
import configparser
import urllib3
import io
import codecs
import re
import subprocess
import os
import apt_pkg
import tempfile
from html.parser import HTMLParser
from urllib.parse import urljoin, urlparse
from urllib3.exceptions import MaxRetryError

logger = logging.getLogger(__name__)

def scanRepository(url, suites=None):
    logger.debug("scanRepository('{}', {})".format(url, suites))
    url = url + ("/" if not url.endswith("/") else "")
    res = list()
    if suites:
        for s in suites:
            try:
                res.append(scanReleaseFile(urljoin(url, os.path.join('dists', s, 'Release')), url))
            except Exception:
                logger.warn("Could not resolve suite {} of repository {}".format(s, url))
    else:
        try:
            res.extend(scanReleases(urljoin(url, "dists/"), url))
        except Exception as e:
            logger.warn("Could not resolve repository {}".format(url))
    return res

def scanReleases(url, repoUrl, recursive=True):
    '''
       return suites found at url and all it's relevant subfolders if recursive==True
    '''
    logger.debug("scanReleases('{}', {})".format(url, recursive))
    suites = list()
    ignoreFolders = list(['by-hash'])

    p = urlparse(url)
    if p.scheme == "file":
        index = LocalFilesystemScanner(url)
    else:
        index = HtmlIndexParser(url)

    if index.release:
        suite = scanReleaseFile(index.release, repoUrl)
        if suite:
          suites.append(suite)
          ignoreFolders.extend(suite['components'])

    if recursive:
        for subfolder, suburl in sorted(index.getSubfolders().items()):
            if not subfolder in ignoreFolders:
                suites.extend(scanReleases(suburl, repoUrl))

    return suites

def scanReleaseFile(url, repoUrl):
    logger.debug("scanReleaseFile('{}')".format(url))
    data = getFromURL(url)
    with tempfile.TemporaryFile() as fp:
        fp.write(data)
        fp.seek(0)
        with apt_pkg.TagFile(fp) as tagfile:
            try:
                for section in tagfile:
                    components = section.get('Components').split(" ") if section.get('Components') else list()
                    architectures = section.get('Architectures').split(" ") if section.get('Architectures') else list()
                    md5sum = section.get('MD5Sum').split("\n") if section.get('Md5Sum') else list()
                    files=[re.sub(" +", " ", s.strip()).split(" ")[2] for s in md5sum]
                    hasSources=suiteHasSources(files)
                    return {
                        'repoUrl': repoUrl,
                        'releaseUrl': url,
                        'suite': section.get('Suite'),
                        'codename': section.get('Codename'),
                        'components': components,
                        'architectures': architectures,
                        'hasSources': hasSources
                        #'files':files
                    }
            except SystemError:
                raise Exception("invalid release file or no suite found at {}".format(url))

def suiteHasSources(files):
    for f in files:
        if ( f.endswith('/source/Sources') or
             f.endswith('/source/Sources.xz') or
             f.endswith('/source/Sources.gz')):
            return True
    return False

class HtmlIndexParser(HTMLParser):
    def __init__(self, baseurl):
        HTMLParser.__init__(self)
        self.baseurl = urljoin(baseurl, "./")
        self.release = None
        self.inRelease = None
        self.subfolders = dict()
        self.feed(getHttp(self.baseurl).decode('utf8'))

    def handle_starttag(self, tag, attrs):
        if tag.upper() == 'A':
            for attr, value in attrs:
                if attr.lower() == 'href':
                    href = urljoin(self.baseurl, value)
                    if href.startswith(self.baseurl):
                        self.handle_suburl(href)

    def handle_suburl(self, url):
        path = os.path.relpath(urlparse(url).path, urlparse(self.baseurl).path)
        if path == 'Release':
            self.release = url
        if path == 'InRelease':
            self.inRelease = url
        if url.endswith('/'):
            self.subfolders[path] = url

    def getSubfolders(self):
        return self.subfolders

class LocalFilesystemScanner():
    def __init__(self, baseurl):
        self.release = None
        self.inRelease = None
        self.subfolders = dict()
        dir = urlparse(baseurl).path
        if not os.path.isdir(dir):
            raise IOError("No such directory '{}'".format(dir))
        logger.debug("scanning local directory '{}'".format(baseurl))
        for file in os.listdir(dir):
            url = urljoin(baseurl + "/", file)
            if file == 'Release':
                self.release = url
            if file == 'InRelease':
                self.inRelease = url
            if os.path.isdir(os.path.join(dir, file)):
                self.subfolders[file] = url

    def getSubfolders(self):
        return self.subfolders

def getFromURL(url):
    '''
        encapsulates url access for http:// and file:// urls
    '''
    p = urlparse(url)
    if p.scheme == "file":
        data = None
        with open(p.path.replace("%20", " "), "rb") as input:
            data = input.read()
        return data
    if p.scheme == "http":
        return getHttp(url)

def getHttp(url):
    http = urllib3.PoolManager()
    try:
        req = http.request('GET', url)
        if req.status != 200:
            raise Exception("http-request to url {} failed with status code {}".format(url, req.status))
        return req.data
    except MaxRetryError as e:
        raise Exception("http-request to url {} failed".format(url), e)
<commit_msg>RepositoryScanner ensures that Release-File defines a Suite<commit_after>#!/usr/bin/python3
# -*- coding: utf-8 -*-
##################################################################################
# Access information about binary and source packages in multiple
# (independent) apt-repositories utilizing libapt / python-apt/
# apt_pkg without the need to change the local system and it's apt-setup.
#
# Copyright (C) 2018  Christoph Lutz
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
##################################################################################
import logging
import sys
import configparser
import urllib3
import io
import codecs
import re
import subprocess
import os
import apt_pkg
import tempfile
from html.parser import HTMLParser
from urllib.parse import urljoin, urlparse
from urllib3.exceptions import MaxRetryError

logger = logging.getLogger(__name__)

def scanRepository(url, suites=None):
    logger.debug("scanRepository('{}', {})".format(url, suites))
    url = url + ("/" if not url.endswith("/") else "")
    res = list()
    if suites:
        for s in suites:
            try:
                res.append(scanReleaseFile(urljoin(url, os.path.join('dists', s, 'Release')), url))
            except Exception:
                logger.warn("Could not resolve suite {} of repository {}".format(s, url))
    else:
        try:
            res.extend(scanReleases(urljoin(url, "dists/"), url))
        except Exception as e:
            logger.warn("Could not resolve repository {}".format(url))
    return res

def scanReleases(url, repoUrl, recursive=True):
    '''
       return suites found at url and all it's relevant subfolders if recursive==True
    '''
    logger.debug("scanReleases('{}', {})".format(url, recursive))
    suites = list()
    ignoreFolders = list(['by-hash'])

    p = urlparse(url)
    if p.scheme == "file":
        index = LocalFilesystemScanner(url)
    else:
        index = HtmlIndexParser(url)

    if index.release:
        suite = scanReleaseFile(index.release, repoUrl)
        if suite:
          suites.append(suite)
          ignoreFolders.extend(suite['components'])

    if recursive:
        for subfolder, suburl in sorted(index.getSubfolders().items()):
            if not subfolder in ignoreFolders:
                suites.extend(scanReleases(suburl, repoUrl))

    return suites

def scanReleaseFile(url, repoUrl):
    logger.debug("scanReleaseFile('{}')".format(url))
    data = getFromURL(url)
    with tempfile.TemporaryFile() as fp:
        fp.write(data)
        fp.seek(0)
        with apt_pkg.TagFile(fp) as tagfile:
            try:
                for section in tagfile:
                    if not section.get('Suite'):
                        continue
                    components = section.get('Components').split(" ") if section.get('Components') else list()
                    architectures = section.get('Architectures').split(" ") if section.get('Architectures') else list()
                    md5sum = section.get('MD5Sum').split("\n") if section.get('Md5Sum') else list()
                    files=[re.sub(" +", " ", s.strip()).split(" ")[2] for s in md5sum]
                    hasSources=suiteHasSources(files)
                    return {
                        'repoUrl': repoUrl,
                        'releaseUrl': url,
                        'suite': section.get('Suite'),
                        'codename': section.get('Codename'),
                        'components': components,
                        'architectures': architectures,
                        'hasSources': hasSources
                        #'files':files
                    }
            except SystemError:
                raise Exception("invalid release file or no suite found at {}".format(url))

def suiteHasSources(files):
    for f in files:
        if ( f.endswith('/source/Sources') or
             f.endswith('/source/Sources.xz') or
             f.endswith('/source/Sources.gz')):
            return True
    return False

class HtmlIndexParser(HTMLParser):
    def __init__(self, baseurl):
        HTMLParser.__init__(self)
        self.baseurl = urljoin(baseurl, "./")
        self.release = None
        self.inRelease = None
        self.subfolders = dict()
        self.feed(getHttp(self.baseurl).decode('utf8'))

    def handle_starttag(self, tag, attrs):
        if tag.upper() == 'A':
            for attr, value in attrs:
                if attr.lower() == 'href':
                    href = urljoin(self.baseurl, value)
                    if href.startswith(self.baseurl):
                        self.handle_suburl(href)

    def handle_suburl(self, url):
        path = os.path.relpath(urlparse(url).path, urlparse(self.baseurl).path)
        if path == 'Release':
            self.release = url
        if path == 'InRelease':
            self.inRelease = url
        if url.endswith('/'):
            self.subfolders[path] = url

    def getSubfolders(self):
        return self.subfolders

class LocalFilesystemScanner():
    def __init__(self, baseurl):
        self.release = None
        self.inRelease = None
        self.subfolders = dict()
        dir = urlparse(baseurl).path
        if not os.path.isdir(dir):
            raise IOError("No such directory '{}'".format(dir))
        logger.debug("scanning local directory '{}'".format(baseurl))
        for file in os.listdir(dir):
            url = urljoin(baseurl + "/", file)
            if file == 'Release':
                self.release = url
            if file == 'InRelease':
                self.inRelease = url
            if os.path.isdir(os.path.join(dir, file)):
                self.subfolders[file] = url

    def getSubfolders(self):
        return self.subfolders

def getFromURL(url):
    '''
        encapsulates url access for http:// and file:// urls
    '''
    p = urlparse(url)
    if p.scheme == "file":
        data = None
        with open(p.path.replace("%20", " "), "rb") as input:
            data = input.read()
        return data
    if p.scheme == "http":
        return getHttp(url)

def getHttp(url):
    http = urllib3.PoolManager()
    try:
        req = http.request('GET', url)
        if req.status != 200:
            raise Exception("http-request to url {} failed with status code {}".format(url, req.status))
        return req.data
    except MaxRetryError as e:
        raise Exception("http-request to url {} failed".format(url), e)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Script for releasing Kunagi

import sys
import os
import shutil

def fail(message):
    print message
    print ''
    print '*** Releasing Kunagi FAILED ****'
    exit(1)

def execute(cmd, dir=None):
    previousDir = os.getcwd()
    if dir:
        os.chdir(dir)
    pipe = os.popen('{ ' + cmd + '; } 2>&1', 'r')
    text = pipe.read()
    sts = pipe.close()
    if dir:
        os.chdir(previousDir)
    if sts is None: sts = 0
    if text[-1:] == '\n': text = text[:-1]
    dirinfo = ''
    if dir:
        dirinfo = '\n  work-dir: ' + dir
    if sts != 0:
        fail('Executing command failed: ' + cmd + dirinfo + '\n' + text)
    return text

# ------------ main -----------------

# check parameters
if len(sys.argv) < 2:
    fail('Missing parameter: release-label')

releaseLabel = sys.argv[1]
branchName = 'r' + releaseLabel
print 'Releasing Kunagi ' + releaseLabel

# configuration
os.chdir('/home/scrum/')
workDir = os.path.abspath('kunagi-release-workdir')
artifactsDestinationHomeDir = '/var/www/kunagi.org/releases'
githubUser = 'git://github.com/Kunagi'
buildDir = workDir + '/kunagi/build'
packageDir = buildDir + '/package-content/kunagi'
packageWar = buildDir + '/kunagi.war'
packageZip = buildDir + '/kunagi-' + releaseLabel + '.zip'
packageTar = buildDir + '/kunagi-' + releaseLabel + '.tar.gz'

# cleanup previous release
print '  Clean work direcotry: ' + workDir
if os.path.exists(workDir):
    shutil.rmtree(workDir)

# checkout
print '  Clone git repositories from ' + githubUser
execute('git clone ' + githubUser + '/kunagi.git ' + workDir + '/kunagi')
execute('git checkout ' + branchName, workDir + '/kunagi')
execute('git clone ' + githubUser + '/ilarkesto.git ' + workDir + '/ilarkesto')
execute('git checkout kunagi-' + branchName, workDir + '/ilarkesto')

#update libs
print '  Updating libs'
execute('./update-libs.bsh', workDir + '/ilarkesto')

# write build.properties file
print '  Update build.properties'
buildPropertiesFilePath = workDir + '/kunagi/src/main/java/scrum/server/build.properties'
buildPropertiesFile = f = open(buildPropertiesFilePath, 'a')
f.write('release.label=' + releaseLabel + '\n')
f.close()

# build
print '  Build & Test'
execute('ant package', workDir + '/kunagi')

# pack
print '  Pack'
if not os.path.exists(packageDir):
    fail('Missing package directory: ' + packageDir)
newPackageDir = packageDir + '-' + releaseLabel
shutil.move(packageDir, newPackageDir)
packageDir = newPackageDir
packageDirParentDir = os.path.abspath(packageDir + '/../')
execute('tar -czf ../kunagi-' + releaseLabel + '.tar.gz kunagi-' + releaseLabel, packageDirParentDir)
execute('zip -r9 ../kunagi-' + releaseLabel + '.zip kunagi-' + releaseLabel, packageDirParentDir)

# debianize
print '  Create debian package'
debianDir = buildDir + '/debian'
debianDocDir = debianDir + '/usr/share/doc/kunagi'
packageDeb = buildDir + '/kunagi_' + releaseLabel + '.deb'
shutil.copytree(workDir + '/kunagi/src/debian', buildDir + '/debian')
os.makedirs(debianDir + '/var/lib/tomcat6/webapps')
shutil.copyfile(packageWar, debianDir + '/var/lib/tomcat6/webapps/kunagi.war')
os.makedirs(debianDocDir)
shutil.copyfile(workDir + '/kunagi/README', debianDocDir + '/README')
execute('fakeroot dpkg-deb --build debian', buildDir)
shutil.move(buildDir + '/debian.deb', packageDeb)

# check files and directories
print '  Check artifacts'
artifactsDestinationDir = artifactsDestinationHomeDir + '/' + releaseLabel
#if os.path.exists(artifactsDestinationDir):
#    fail('Release directory already exists: ' + artifactsDestinationDir)
if not os.path.exists(packageWar):
    fail('Missing war package: ' + packageWar)
if not os.path.exists(packageZip):
    fail('Missing zip package: ' + packageZip)
if not os.path.exists(packageTar):
    fail('Missing tar package: ' + packageTar)

# update homepage
print '  Update homepage'
execute('ant releaseHomepage', workDir + '/kunagi')

# upload to sourceforge
print '  Upload artifacts to SourceForge'
sourceForgePath = 'koczewski,kunagi@frs.sourceforge.net:/home/frs/project/k/ku/kunagi/' + releaseLabel
execute('scp ' + packageWar + ' ' + sourceForgePath + '/kunagi.war')
execute('scp ' + packageDeb + ' ' + sourceForgePath + '/kunagi_' + releaseLabel + '.deb')
execute('scp ' + packageTar + ' ' + sourceForgePath + '/kunagi-' + releaseLabel + '.tar.gz')
execute('scp ' + packageZip + ' ' + sourceForgePath + '/kunagi-' + releaseLabel + '.zip')

# copy artifacts
print '  Copy artifacts to ' + artifactsDestinationDir
os.mkdir(artifactsDestinationDir)
shutil.copyfile(packageWar, artifactsDestinationDir + '/kunagi.war')
shutil.copyfile(packageDeb, artifactsDestinationDir + '/kunagi_' + releaseLabel + '.deb')
shutil.copyfile(packageTar, artifactsDestinationDir + '/kunagi-' + releaseLabel + '.tar.gz')
shutil.copyfile(packageZip, artifactsDestinationDir + '/kunagi-' + releaseLabel + '.zip')

# deploy
print '  Deploy demo'
shutil.copyfile(packageWar, '/home/kunagi-demo/tomcat/webapps/kunagi-demo.war')
#shutil.copyfile(packageWar, '/home/scrum/tomcat/webapps/scrum.war')

print 'Kunagi ' + releaseLabel + ' released'
<commit_msg>release script for kunagi.exe<commit_after>#!/usr/bin/env python

# Script for releasing Kunagi

import sys
import os
import shutil

def fail(message):
    print message
    print ''
    print '*** Releasing Kunagi FAILED ****'
    exit(1)

def execute(cmd, dir=None):
    previousDir = os.getcwd()
    if dir:
        os.chdir(dir)
    pipe = os.popen('{ ' + cmd + '; } 2>&1', 'r')
    text = pipe.read()
    sts = pipe.close()
    if dir:
        os.chdir(previousDir)
    if sts is None: sts = 0
    if text[-1:] == '\n': text = text[:-1]
    dirinfo = ''
    if dir:
        dirinfo = '\n  work-dir: ' + dir
    if sts != 0:
        fail('Executing command failed: ' + cmd + dirinfo + '\n' + text)
    return text

# ------------ main -----------------

# check parameters
if len(sys.argv) < 2:
    fail('Missing parameter: release-label')

releaseLabel = sys.argv[1]
branchName = 'r' + releaseLabel
print 'Releasing Kunagi ' + releaseLabel

# configuration
os.chdir('/home/scrum/')
workDir = os.path.abspath('kunagi-release-workdir')
artifactsDestinationHomeDir = '/var/www/kunagi.org/releases'
githubUser = 'git://github.com/Kunagi'
buildDir = workDir + '/kunagi/build'
packageDir = buildDir + '/package-content/kunagi'
packageWar = buildDir + '/kunagi.war'
packageZip = buildDir + '/kunagi-' + releaseLabel + '.zip'
packageTar = buildDir + '/kunagi-' + releaseLabel + '.tar.gz'
packageExe = artifactsDestinationHomeDir + '/kunagi.exe'

# cleanup previous release
print '  Clean work direcotry: ' + workDir
if os.path.exists(workDir):
    shutil.rmtree(workDir)

# checkout
print '  Clone git repositories from ' + githubUser
execute('git clone ' + githubUser + '/kunagi.git ' + workDir + '/kunagi')
execute('git checkout ' + branchName, workDir + '/kunagi')
execute('git clone ' + githubUser + '/ilarkesto.git ' + workDir + '/ilarkesto')
execute('git checkout kunagi-' + branchName, workDir + '/ilarkesto')

#update libs
print '  Updating libs'
execute('./update-libs.bsh', workDir + '/ilarkesto')

# write build.properties file
print '  Update build.properties'
buildPropertiesFilePath = workDir + '/kunagi/src/main/java/scrum/server/build.properties'
buildPropertiesFile = f = open(buildPropertiesFilePath, 'a')
f.write('release.label=' + releaseLabel + '\n')
f.close()

# build
print '  Build & Test'
execute('ant package', workDir + '/kunagi')

# pack
print '  Pack'
if not os.path.exists(packageDir):
    fail('Missing package directory: ' + packageDir)
newPackageDir = packageDir + '-' + releaseLabel
shutil.move(packageDir, newPackageDir)
packageDir = newPackageDir
packageDirParentDir = os.path.abspath(packageDir + '/../')
execute('tar -czf ../kunagi-' + releaseLabel + '.tar.gz kunagi-' + releaseLabel, packageDirParentDir)
execute('zip -r9 ../kunagi-' + releaseLabel + '.zip kunagi-' + releaseLabel, packageDirParentDir)

# debianize
print '  Create debian package'
debianDir = buildDir + '/debian'
debianDocDir = debianDir + '/usr/share/doc/kunagi'
packageDeb = buildDir + '/kunagi_' + releaseLabel + '.deb'
shutil.copytree(workDir + '/kunagi/src/debian', buildDir + '/debian')
os.makedirs(debianDir + '/var/lib/tomcat6/webapps')
shutil.copyfile(packageWar, debianDir + '/var/lib/tomcat6/webapps/kunagi.war')
os.makedirs(debianDocDir)
shutil.copyfile(workDir + '/kunagi/README', debianDocDir + '/README')
execute('fakeroot dpkg-deb --build debian', buildDir)
shutil.move(buildDir + '/debian.deb', packageDeb)

# check files and directories
print '  Check artifacts'
artifactsDestinationDir = artifactsDestinationHomeDir + '/' + releaseLabel
#if os.path.exists(artifactsDestinationDir):
#    fail('Release directory already exists: ' + artifactsDestinationDir)
if not os.path.exists(packageWar):
    fail('Missing war package: ' + packageWar)
if not os.path.exists(packageZip):
    fail('Missing zip package: ' + packageZip)
if not os.path.exists(packageTar):
    fail('Missing tar package: ' + packageTar)
if not os.path.exists(packageExe):
    fail('Missing exe package: ' + packageExe)

# update homepage
print '  Update homepage'
execute('ant releaseHomepage', workDir + '/kunagi')

# upload to sourceforge
print '  Upload artifacts to SourceForge'
sourceForgePath = 'koczewski,kunagi@frs.sourceforge.net:/home/frs/project/k/ku/kunagi/' + releaseLabel
execute('scp ' + packageWar + ' ' + sourceForgePath + '/kunagi.war')
execute('scp ' + packageExe + ' ' + sourceForgePath + '/kunagi.exe')
execute('scp ' + packageDeb + ' ' + sourceForgePath + '/kunagi_' + releaseLabel + '.deb')
execute('scp ' + packageTar + ' ' + sourceForgePath + '/kunagi-' + releaseLabel + '.tar.gz')
execute('scp ' + packageZip + ' ' + sourceForgePath + '/kunagi-' + releaseLabel + '.zip')

# copy artifacts
print '  Copy artifacts to ' + artifactsDestinationDir
os.mkdir(artifactsDestinationDir)
shutil.copyfile(packageWar, artifactsDestinationDir + '/kunagi.war')
shutil.copyfile(packageExe, artifactsDestinationDir + '/kunagi.exe')
shutil.copyfile(packageDeb, artifactsDestinationDir + '/kunagi_' + releaseLabel + '.deb')
shutil.copyfile(packageTar, artifactsDestinationDir + '/kunagi-' + releaseLabel + '.tar.gz')
shutil.copyfile(packageZip, artifactsDestinationDir + '/kunagi-' + releaseLabel + '.zip')

# deploy
print '  Deploy demo'
shutil.copyfile(packageWar, '/home/kunagi-demo/tomcat/webapps/kunagi-demo.war')
#shutil.copyfile(packageWar, '/home/scrum/tomcat/webapps/scrum.war')

print 'Kunagi ' + releaseLabel + ' released'
<|endoftext|>