<commit_before>import numpy as np
import pandas as pd
from copy import deepcopy

def main(net):
  '''
  calculate pvalue of category closeness
  '''

  # calculate the distance between the cols within the same category and compare
  # to null distribution

  inst_cols = deepcopy(net.dat['nodes']['col'])

  inst_index = deepcopy(net.dat['node_info']['col']['clust'])
  # reorder based on clustered order
  inst_cols = [ inst_cols[i] for i in inst_index]

  dm = dist_matrix_lattice(inst_cols)

  tmp_dict = net.dat['node_info']['col']['dict_cat_0']

  print(dm)

  for inst_cat in tmp_dict:
    print(tmp_dict[inst_cat])

    # calc_mean_dist_subset(dm, )

def dist_matrix_lattice(names):
  from scipy.spatial.distance import pdist, squareform

  lattice_size = len(names)
  mat = np.zeros([lattice_size, 1])
  mat[:,0] = range(lattice_size)

  inst_dm = pdist(mat, metric='euclidean')

  inst_dm[inst_dm < 0] = float(0)

  inst_dm = squareform(inst_dm)

  df = pd.DataFrame(data=inst_dm, columns=names, index=names)

  return df

def calc_mean_dist_subset(df, subset):
  pass

<commit_msg>calculating histograms of distances, will use this to estimate the pvalue of category grouping<commit_after>import numpy as np
import pandas as pd
from copy import deepcopy

def main(net):
  '''
  calculate pvalue of category closeness
  '''

  # calculate the distance between the cols within the same category and compare
  # to null distribution

  inst_cols = deepcopy(net.dat['nodes']['col'])

  inst_index = deepcopy(net.dat['node_info']['col']['clust'])
  # reorder based on clustered order
  inst_cols = [ inst_cols[i] for i in inst_index]

  # make distance matrix dataframe
  dm = dist_matrix_lattice(inst_cols)

  tmp_dict = net.dat['node_info']['col']['dict_cat_0']

  for inst_cat in tmp_dict:
    subset = tmp_dict[inst_cat]
    print(subset)
    inst_mean = calc_mean_dist_subset(dm, subset)
    print(inst_mean)

    hist = calc_hist_distances(dm, subset, inst_cols)
    print(hist['prob'])
    print(hist['bins'])
    print('\n\n')

def dist_matrix_lattice(names):
  from scipy.spatial.distance import pdist, squareform

  lattice_size = len(names)
  mat = np.zeros([lattice_size, 1])
  mat[:,0] = range(lattice_size)

  inst_dm = pdist(mat, metric='euclidean')

  inst_dm[inst_dm < 0] = float(0)

  inst_dm = squareform(inst_dm)

  df = pd.DataFrame(data=inst_dm, columns=names, index=names)

  return df

def calc_mean_dist_subset(dm, subset):
  return np.mean(dm[subset].ix[subset].values)

def calc_hist_distances(dm, subset, inst_cols):
  np.random.seed(100)

  num_null = 1000
  num_points = len(subset)

  mean_dist = []
  for i in range(num_null):
    tmp = np.random.choice(inst_cols, num_points, replace=False)
    mean_dist.append( np.mean(dm[tmp].ix[tmp].values)  )

  tmp_dist = sorted(deepcopy(mean_dist))
  print('lowest distances')
  print(tmp_dist[0:4])

  mean_dist = np.asarray(mean_dist)
  s1 = pd.Series(mean_dist)
  hist = np.histogram(s1, bins=20)

  H = {}
  H['prob'] = hist[0]/np.float(num_null)
  H['bins'] = hist[1]

  return H<|endoftext|>


<commit_before>"""
Modified version to use request.build_absolute_uri()
This module contains helper functions for controlling caching. It does so by
managing the "Vary" header of responses. It includes functions to patch the
header of response objects directly and decorators that change functions to do
that header-patching themselves.

For information on the Vary header, see:

    http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.44

Essentially, the "Vary" HTTP header defines which headers a cache should take
into account when building its cache key. Requests with the same path but
different header content for headers named in "Vary" need to get different
cache keys to prevent delivery of wrong content.

An example: i18n middleware would need to distinguish caches by the
"Accept-language" header.
"""
from __future__ import unicode_literals

import hashlib
import re
import time

from django.conf import settings
from django.core.cache import get_cache
from django.utils.encoding import iri_to_uri, force_bytes, force_text
from django.utils.http import http_date
from django.utils.timezone import get_current_timezone_name
from django.utils.translation import get_language

from django.utils.cache import cc_delim_re, _i18n_cache_key_suffix

def _generate_cache_key(request, method, headerlist, key_prefix):
    """Returns a cache key from the headers given in the header list."""
    ctx = hashlib.md5()
    for header in headerlist:
        value = request.META.get(header, None)
        if value is not None:
            ctx.update(force_bytes(value))
    path = hashlib.md5(force_bytes(iri_to_uri(request.build_absolute_uri())))
    cache_key = 'views.decorators.cache.cache_page.%s.%s.%s.%s' % (
        key_prefix, method, path.hexdigest(), ctx.hexdigest())
    return _i18n_cache_key_suffix(request, cache_key)

def _generate_cache_header_key(key_prefix, request):
    """Returns a cache key for the header cache."""
    path = hashlib.md5(force_bytes(iri_to_uri(request.build_absolute_uri())))
    cache_key = 'views.decorators.cache.cache_header.%s.%s' % (
        key_prefix, path.hexdigest())
    return _i18n_cache_key_suffix(request, cache_key)

def get_cache_key(request, key_prefix=None, method='GET', cache=None):
    """
    Returns a cache key based on the request path and query. It can be used
    in the request phase because it pulls the list of headers to take into
    account from the global path registry and uses those to build a cache key
    to check against.

    If there is no headerlist stored, the page needs to be rebuilt, so this
    function returns None.
    """
    if key_prefix is None:
        key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX
    cache_key = _generate_cache_header_key(key_prefix, request)
    if cache is None:
        cache = get_cache(settings.CACHE_MIDDLEWARE_ALIAS)
    headerlist = cache.get(cache_key, None)
    if headerlist is not None:
        return _generate_cache_key(request, method, headerlist, key_prefix)
    else:
        return None

def learn_cache_key(request, response, cache_timeout=None, key_prefix=None, cache=None):
    """
    Learns what headers to take into account for some request path from the
    response object. It stores those headers in a global path registry so that
    later access to that path will know what headers to take into account
    without building the response object itself. The headers are named in the
    Vary header of the response, but we want to prevent response generation.

    The list of headers to use for cache key generation is stored in the same
    cache as the pages themselves. If the cache ages some data out of the
    cache, this just means that we have to build the response once to get at
    the Vary header and so at the list of headers to use for the cache key.
    """
    if key_prefix is None:
        key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX
    if cache_timeout is None:
        cache_timeout = settings.CACHE_MIDDLEWARE_SECONDS
    cache_key = _generate_cache_header_key(key_prefix, request)
    if cache is None:
        cache = get_cache(settings.CACHE_MIDDLEWARE_ALIAS)
    if response.has_header('Vary'):
        headerlist = ['HTTP_'+header.upper().replace('-', '_')
                      for header in cc_delim_re.split(response['Vary'])]
        cache.set(cache_key, headerlist, cache_timeout)
        return _generate_cache_key(request, request.method, headerlist, key_prefix)
    else:
        # if there is no Vary header, we still need a cache key
        # for the request.get_full_path()
        cache.set(cache_key, [], cache_timeout)
        return _generate_cache_key(request, request.method, [], key_prefix)

<commit_msg>fixes get_cache<commit_after>"""
Modified version to use request.build_absolute_uri()
This module contains helper functions for controlling caching. It does so by
managing the "Vary" header of responses. It includes functions to patch the
header of response objects directly and decorators that change functions to do
that header-patching themselves.

For information on the Vary header, see:

    http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.44

Essentially, the "Vary" HTTP header defines which headers a cache should take
into account when building its cache key. Requests with the same path but
different header content for headers named in "Vary" need to get different
cache keys to prevent delivery of wrong content.

An example: i18n middleware would need to distinguish caches by the
"Accept-language" header.
"""
from __future__ import unicode_literals

import hashlib
import re
import time

from django.conf import settings
from django.core.cache import caches
from django.utils.encoding import iri_to_uri, force_bytes, force_text
from django.utils.http import http_date
from django.utils.timezone import get_current_timezone_name
from django.utils.translation import get_language

from django.utils.cache import cc_delim_re, _i18n_cache_key_suffix

def _generate_cache_key(request, method, headerlist, key_prefix):
    """Returns a cache key from the headers given in the header list."""
    ctx = hashlib.md5()
    for header in headerlist:
        value = request.META.get(header, None)
        if value is not None:
            ctx.update(force_bytes(value))
    path = hashlib.md5(force_bytes(iri_to_uri(request.build_absolute_uri())))
    cache_key = 'views.decorators.cache.cache_page.%s.%s.%s.%s' % (
        key_prefix, method, path.hexdigest(), ctx.hexdigest())
    return _i18n_cache_key_suffix(request, cache_key)

def _generate_cache_header_key(key_prefix, request):
    """Returns a cache key for the header cache."""
    path = hashlib.md5(force_bytes(iri_to_uri(request.build_absolute_uri())))
    cache_key = 'views.decorators.cache.cache_header.%s.%s' % (
        key_prefix, path.hexdigest())
    return _i18n_cache_key_suffix(request, cache_key)

def get_cache_key(request, key_prefix=None, method='GET', cache=None):
    """
    Returns a cache key based on the request path and query. It can be used
    in the request phase because it pulls the list of headers to take into
    account from the global path registry and uses those to build a cache key
    to check against.

    If there is no headerlist stored, the page needs to be rebuilt, so this
    function returns None.
    """
    if key_prefix is None:
        key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX
    cache_key = _generate_cache_header_key(key_prefix, request)
    if cache is None:
        cache = caches[settings.CACHE_MIDDLEWARE_ALIAS]
    headerlist = cache.get(cache_key, None)
    if headerlist is not None:
        return _generate_cache_key(request, method, headerlist, key_prefix)
    else:
        return None

def learn_cache_key(request, response, cache_timeout=None, key_prefix=None, cache=None):
    """
    Learns what headers to take into account for some request path from the
    response object. It stores those headers in a global path registry so that
    later access to that path will know what headers to take into account
    without building the response object itself. The headers are named in the
    Vary header of the response, but we want to prevent response generation.

    The list of headers to use for cache key generation is stored in the same
    cache as the pages themselves. If the cache ages some data out of the
    cache, this just means that we have to build the response once to get at
    the Vary header and so at the list of headers to use for the cache key.
    """
    if key_prefix is None:
        key_prefix = settings.CACHE_MIDDLEWARE_KEY_PREFIX
    if cache_timeout is None:
        cache_timeout = settings.CACHE_MIDDLEWARE_SECONDS
    cache_key = _generate_cache_header_key(key_prefix, request)
    if cache is None:
        cache = caches[settings.CACHE_MIDDLEWARE_ALIAS]
    if response.has_header('Vary'):
        headerlist = ['HTTP_'+header.upper().replace('-', '_')
                      for header in cc_delim_re.split(response['Vary'])]
        cache.set(cache_key, headerlist, cache_timeout)
        return _generate_cache_key(request, request.method, headerlist, key_prefix)
    else:
        # if there is no Vary header, we still need a cache key
        # for the request.get_full_path()
        cache.set(cache_key, [], cache_timeout)
        return _generate_cache_key(request, request.method, [], key_prefix)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from django.urls import reverse
from django.http import HttpResponseRedirect, HttpResponse
from django.shortcuts import get_object_or_404
from django.template import loader
from django.forms import inlineformset_factory
from django.db.models import Count

from Instanssi.common.http import Http403
from Instanssi.common.auth import staff_access_required
from Instanssi.admin_base.misc.custom_render import admin_render
from Instanssi.admin_store.forms import StoreItemForm, TaItemExportForm, StoreItemVariantForm
from Instanssi.store.models import StoreItem, StoreItemVariant, StoreTransaction, TransactionItem
from Instanssi.kompomaatti.models import Event

# Logging related
import logging
logger = logging.getLogger(__name__)

@staff_access_required
def index(request):
    return admin_render(request, "admin_store/index.html", {})

@staff_access_required
def export(request):
    if request.method == 'POST':
        form = TaItemExportForm(request.POST)
        if form.is_valid():
            return HttpResponseRedirect(reverse('manage-store:transactions_csv', args=(form.cleaned_data['event'],)))
    else:
        form = TaItemExportForm()

    return admin_render(request, "admin_store/export.html", {'form': form})

@staff_access_required
def amounts(request):
    item_tree = []

    # TODO: This is a really quickly made thing; needs optimizing.

    for event in Event.objects.iterator():
        counts = TransactionItem.objects\
            .filter(item__event=event)\
            .exclude(transaction__time_paid=None)\
            .values('item')\
            .annotate(Count('item'))
        if not counts:
            continue

        item_list = []
        for c in counts:
            if not c['item']:
                continue

            # Find item description
            item = StoreItem.objects.get(pk=c['item'])

            # Find available variants (if any) and count them
            variants = TransactionItem.objects\
                .filter(item=c['item']) \
                .exclude(transaction__time_paid=None)\
                .values('variant')\
                .annotate(Count('variant'))
            variant_list = []
            for v in variants:
                if not v['variant']:
                    continue
                variant = StoreItemVariant.objects.get(pk=v['variant'])
                variant_list.append({
                    'sold_variant': variant,
                    'count': v['variant__count']
                })

            # Add everything to a list for template
            item_list.append({
                'sold_item': item,
                'count': c['item__count'],
                'variants': variant_list,
            })

        # Add the event & item list to outgoing template data
        item_tree.append({
            'event': event,
            'items': item_list
        })

    # Render response
    return admin_render(request, "admin_store/amounts.html", {
        'item_tree': item_tree
    })

@staff_access_required
def items(request):
    StoreItemFormSet = inlineformset_factory(
        parent_model=StoreItem, model=StoreItemVariant, form=StoreItemVariantForm, extra=5)

    # Handle form data
    if request.method == 'POST':
        if not request.user.has_perm('store.add_storeitem'):
            raise Http403

        item_form = StoreItemForm(request.POST, request.FILES)
        variant_formset = StoreItemFormSet(request.POST, prefix="nested", instance=item_form.instance)
        if item_form.is_valid() and variant_formset.is_valid():
            item = item_form.save()
            variant_formset.save()
            logger.info('Store Item "{}" added.'.format(item.name), extra={'user': request.user})
            return HttpResponseRedirect(reverse('manage-store:items'))
    else:
        item_form = StoreItemForm()
        variant_formset = StoreItemFormSet(prefix="nested", instance=item_form.instance)

    # Get items
    m_items = StoreItem.objects.all()

    # Render response
    return admin_render(request, "admin_store/items.html", {
        'items': m_items,
        'item_form': item_form,
        'variant_formset': variant_formset
    })

@staff_access_required
def status(request):
    if not request.user.has_perm('store.view_storetransaction'):
        raise Http403

    transactions = StoreTransaction.objects.all()

    # Render response
    return admin_render(request, "admin_store/status.html", {
        'transactions': transactions,
    })

@staff_access_required
def tis_csv(request, event_id):
    if not request.user.has_perm('store.view_storetransaction'):
        raise Http403

    response = HttpResponse(content_type='text/csv')
    response['Content-Disposition'] = 'attachment; filename="instanssi_store.csv"'
    t = loader.get_template('admin_store/tis_csv.txt')
    c = {
        'data': TransactionItem.objects.filter(item__event=event_id, time_delivered__isnull=False),
    }
    response.write(t.render(c))
    return response

@staff_access_required
def tis(request):
    if not request.user.has_perm('store.view_storetransaction'):
        raise Http403

    items = TransactionItem.objects.all()

    # Render response
    return admin_render(request, "admin_store/tis.html", {
        'items': items,
    })

@staff_access_required
def transaction_status(request, transaction_id):
    if not request.user.has_perm('store.view_storetransaction'):
        raise Http403

    # Get transaction
    transaction = get_object_or_404(StoreTransaction, pk=transaction_id)

    # Get items
    items = transaction.get_transaction_items()

    # Render response
    return admin_render(request, "admin_store/transactionstatus.html", {
        'transaction_id': int(transaction_id),
        'transaction': transaction,
        'items': items,
    })

@staff_access_required
def edit_item(request, item_id):
    if not request.user.has_perm('store.change_storeitem'):
        raise Http403

    StoreItemFormSet = inlineformset_factory(
        parent_model=StoreItem, model=StoreItemVariant, form=StoreItemVariantForm, extra=3)

    # Get Item
    item = get_object_or_404(StoreItem, pk=item_id)

    # Handle form data
    if request.method == 'POST':
        variant_formset = StoreItemFormSet(request.POST, instance=item)
        item_form = StoreItemForm(request.POST, request.FILES, instance=item)
        if item_form.is_valid() and variant_formset.is_valid():
            item_form.save()
            variant_formset.save()
            logger.info('Store Item "{}" edited.'.format(item.name), extra={'user': request.user})
            return HttpResponseRedirect(reverse('manage-store:edit_item', args=(item.id,)))
    else:
        item_form = StoreItemForm(instance=item)
        variant_formset = StoreItemFormSet(instance=item)

    # Render response
    return admin_render(request, "admin_store/itemedit.html", {
        'item_form': item_form,
        'variant_formset': variant_formset
    })

@staff_access_required
def delete_item(request, item_id):
    # Check for permissions
    if not request.user.has_perm('store.delete_storeitem'):
        raise Http403

    # Delete entry
    try:
        item = StoreItem.objects.get(id=item_id)
        if item.num_sold() == 0:
            item.delete()
            logger.info('Store Item "{}" deleted.'.format(item.name), extra={'user': request.user})
    except StoreItem.DoesNotExist:
        pass

    return HttpResponseRedirect(reverse('manage-store:items'))
<commit_msg>Uh nope.<commit_after># -*- coding: utf-8 -*-

from django.urls import reverse
from django.http import HttpResponseRedirect, HttpResponse
from django.shortcuts import get_object_or_404
from django.template import loader
from django.forms import inlineformset_factory
from django.db.models import Count

from Instanssi.common.http import Http403
from Instanssi.common.auth import staff_access_required
from Instanssi.admin_base.misc.custom_render import admin_render
from Instanssi.admin_store.forms import StoreItemForm, TaItemExportForm, StoreItemVariantForm
from Instanssi.store.models import StoreItem, StoreItemVariant, StoreTransaction, TransactionItem
from Instanssi.kompomaatti.models import Event

# Logging related
import logging
logger = logging.getLogger(__name__)

@staff_access_required
def index(request):
    return admin_render(request, "admin_store/index.html", {})

@staff_access_required
def export(request):
    if request.method == 'POST':
        form = TaItemExportForm(request.POST)
        if form.is_valid():
            return HttpResponseRedirect(reverse('manage-store:transactions_csv', args=(form.cleaned_data['event'],)))
    else:
        form = TaItemExportForm()

    return admin_render(request, "admin_store/export.html", {'form': form})

@staff_access_required
def amounts(request):
    item_tree = []

    # TODO: This is a really quickly made thing; needs optimizing.

    for event in Event.objects.iterator():
        counts = TransactionItem.objects\
            .filter(item__event=event)\
            .exclude(transaction__time_paid=None)\
            .values('item')\
            .annotate(Count('item'))
        if not counts:
            continue

        item_list = []
        for c in counts:
            if not c['item']:
                continue

            # Find item description
            item = StoreItem.objects.get(pk=c['item'])

            # Find available variants (if any) and count them
            variants = TransactionItem.objects\
                .filter(item=c['item']) \
                .exclude(transaction__time_paid=None)\
                .values('variant')\
                .annotate(Count('variant'))
            variant_list = []
            for v in variants:
                if not v['variant']:
                    continue
                variant = StoreItemVariant.objects.get(pk=v['variant'])
                variant_list.append({
                    'sold_variant': variant,
                    'count': v['variant__count']
                })

            # Add everything to a list for template
            item_list.append({
                'sold_item': item,
                'count': c['item__count'],
                'variants': variant_list,
            })

        # Add the event & item list to outgoing template data
        item_tree.append({
            'event': event,
            'items': item_list
        })

    # Render response
    return admin_render(request, "admin_store/amounts.html", {
        'item_tree': item_tree
    })

@staff_access_required
def items(request):
    StoreItemFormSet = inlineformset_factory(
        parent_model=StoreItem, model=StoreItemVariant, form=StoreItemVariantForm, extra=5)

    # Handle form data
    if request.method == 'POST':
        if not request.user.has_perm('store.add_storeitem'):
            raise Http403

        item_form = StoreItemForm(request.POST, request.FILES)
        variant_formset = StoreItemFormSet(request.POST, prefix="nested", instance=item_form.instance)
        if item_form.is_valid() and variant_formset.is_valid():
            item = item_form.save()
            variant_formset.save()
            logger.info('Store Item "{}" added.'.format(item.name), extra={'user': request.user})
            return HttpResponseRedirect(reverse('manage-store:items'))
    else:
        item_form = StoreItemForm()
        variant_formset = StoreItemFormSet(prefix="nested", instance=item_form.instance)

    # Get items
    m_items = StoreItem.objects.all()

    # Render response
    return admin_render(request, "admin_store/items.html", {
        'items': m_items,
        'item_form': item_form,
        'variant_formset': variant_formset
    })

@staff_access_required
def status(request):
    if not request.user.has_perm('store.view_storetransaction'):
        raise Http403

    transactions = StoreTransaction.objects.all()

    # Render response
    return admin_render(request, "admin_store/status.html", {
        'transactions': transactions,
    })

@staff_access_required
def tis_csv(request, event_id):
    if not request.user.has_perm('store.view_storetransaction'):
        raise Http403

    response = HttpResponse(content_type='text/csv')
    response['Content-Disposition'] = 'attachment; filename="instanssi_store.csv"'
    t = loader.get_template('admin_store/tis_csv.txt')
    c = {
        'data': TransactionItem.objects.filter(item__event=event_id, transaction__time_paid__isnull=False),
    }
    response.write(t.render(c))
    return response

@staff_access_required
def tis(request):
    if not request.user.has_perm('store.view_storetransaction'):
        raise Http403

    items = TransactionItem.objects.all()

    # Render response
    return admin_render(request, "admin_store/tis.html", {
        'items': items,
    })

@staff_access_required
def transaction_status(request, transaction_id):
    if not request.user.has_perm('store.view_storetransaction'):
        raise Http403

    # Get transaction
    transaction = get_object_or_404(StoreTransaction, pk=transaction_id)

    # Get items
    items = transaction.get_transaction_items()

    # Render response
    return admin_render(request, "admin_store/transactionstatus.html", {
        'transaction_id': int(transaction_id),
        'transaction': transaction,
        'items': items,
    })

@staff_access_required
def edit_item(request, item_id):
    if not request.user.has_perm('store.change_storeitem'):
        raise Http403

    StoreItemFormSet = inlineformset_factory(
        parent_model=StoreItem, model=StoreItemVariant, form=StoreItemVariantForm, extra=3)

    # Get Item
    item = get_object_or_404(StoreItem, pk=item_id)

    # Handle form data
    if request.method == 'POST':
        variant_formset = StoreItemFormSet(request.POST, instance=item)
        item_form = StoreItemForm(request.POST, request.FILES, instance=item)
        if item_form.is_valid() and variant_formset.is_valid():
            item_form.save()
            variant_formset.save()
            logger.info('Store Item "{}" edited.'.format(item.name), extra={'user': request.user})
            return HttpResponseRedirect(reverse('manage-store:edit_item', args=(item.id,)))
    else:
        item_form = StoreItemForm(instance=item)
        variant_formset = StoreItemFormSet(instance=item)

    # Render response
    return admin_render(request, "admin_store/itemedit.html", {
        'item_form': item_form,
        'variant_formset': variant_formset
    })

@staff_access_required
def delete_item(request, item_id):
    # Check for permissions
    if not request.user.has_perm('store.delete_storeitem'):
        raise Http403

    # Delete entry
    try:
        item = StoreItem.objects.get(id=item_id)
        if item.num_sold() == 0:
            item.delete()
            logger.info('Store Item "{}" deleted.'.format(item.name), extra={'user': request.user})
    except StoreItem.DoesNotExist:
        pass

    return HttpResponseRedirect(reverse('manage-store:items'))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from django.db import models
from django.contrib import admin
from Instanssi.kompomaatti.models import Event

class Sponsor(models.Model):
    event = models.ForeignKey(Event, verbose_name=u'Tapahtuma')
    name = models.CharField(u'Nimi', max_length=64, help_text=u'Sponsorin nimi')
    logo = models.ImageField(u'Kuva', upload_to='screen/sponsorlogos/', help_text=u"Sponsorin logo", blank=True)

    def __unicode__(self):
        return self.name

    class Meta:
        verbose_name=u"sponsori"
        verbose_name_plural=u"sponsorit"

    def save(self, *args, **kwargs):
        try:
            this = Sponsor.objects.get(id=self.id)
            if this.logo != self.logo:
                this.logo.delete(save=False)
        except:
            pass

        # Continue with normal save
        super(Sponsor, self).save(*args, **kwargs)

class Message(models.Model):
    event = models.ForeignKey(Event, verbose_name=u'Tapahtuma')
    show_start = models.DateTimeField(u'Alkuaika', help_text=u'Viestin näyttäminen alkaa')
    show_end = models.DateTimeField(u'Loppuaika', help_text=u'Viestin näyttäminen päättyy')
    text = models.TextField(u'Viesti', help_text=u'Viestin leipäteksti. Katso ettei tästä tule liian pitkä.')

    def __unicode__(self):
        return self.text[:32]+u' ...'

    class Meta:
        verbose_name=u"viesti"
        verbose_name_plural=u"viestit"

class IRCMessage(models.Model):
    event = models.ForeignKey(Event, verbose_name=u'Tapahtuma')
    date = models.DateTimeField(u'Aika')
    nick = models.CharField(u'Nimimerkki', max_length=64)
    message = models.TextField(u'Viesti')

    def __unicode__(self):
        return self.message[:32]+u' ...'

    class Meta:
        verbose_name=u"irc-viesti"
        verbose_name_plural=u"irc-viestit"

# Register to admin-panel
try:
    admin.site.register(IRCMessage)
    admin.site.register(Message)
    admin.site.register(Sponsor)
except:
    pass<commit_msg>screenshow: Field for image scaling<commit_after># -*- coding: utf-8 -*-

from django.db import models
from django.contrib import admin
from Instanssi.kompomaatti.models import Event
from imagekit.models import ImageSpecField
from imagekit.processors import ResizeToFit

class Sponsor(models.Model):
    event = models.ForeignKey(Event, verbose_name=u'Tapahtuma')
    name = models.CharField(u'Nimi', max_length=64, help_text=u'Sponsorin nimi')
    logo = models.ImageField(u'Kuva', upload_to='screen/sponsorlogos/', help_text=u"Sponsorin logo", blank=True)
    logo_scaled = ImageSpecField([ResizeToFit(800, 800, True)], image_field='logo', format='JPEG', options={'quality': 95})

    def __unicode__(self):
        return self.name

    class Meta:
        verbose_name=u"sponsori"
        verbose_name_plural=u"sponsorit"

    def save(self, *args, **kwargs):
        try:
            this = Sponsor.objects.get(id=self.id)
            if this.logo != self.logo:
                this.logo.delete(save=False)
        except:
            pass

        # Continue with normal save
        super(Sponsor, self).save(*args, **kwargs)

class Message(models.Model):
    event = models.ForeignKey(Event, verbose_name=u'Tapahtuma')
    show_start = models.DateTimeField(u'Alkuaika', help_text=u'Viestin näyttäminen alkaa')
    show_end = models.DateTimeField(u'Loppuaika', help_text=u'Viestin näyttäminen päättyy')
    text = models.TextField(u'Viesti', help_text=u'Viestin leipäteksti. Katso ettei tästä tule liian pitkä.')

    def __unicode__(self):
        return self.text[:32]+u' ...'

    class Meta:
        verbose_name=u"viesti"
        verbose_name_plural=u"viestit"

class IRCMessage(models.Model):
    event = models.ForeignKey(Event, verbose_name=u'Tapahtuma')
    date = models.DateTimeField(u'Aika')
    nick = models.CharField(u'Nimimerkki', max_length=64)
    message = models.TextField(u'Viesti')

    def __unicode__(self):
        return self.message[:32]+u' ...'

    class Meta:
        verbose_name=u"irc-viesti"
        verbose_name_plural=u"irc-viestit"

# Register to admin-panel
try:
    admin.site.register(IRCMessage)
    admin.site.register(Message)
    admin.site.register(Sponsor)
except:
    pass<|endoftext|>


<commit_before># -*-coding:utf-8 -*-
from email_parser.email_parser import parser
from pymongo import MongoClient
import traceback
import os

client = MongoClient('mongodb://127.0.0.1:27017/email')
db = client.email
tests = db.bulktests

def batch_parse():
    count = 1
    for i in os.listdir('../../samples/samples'):
        print count, i
        try:
            mail_par = parser('../../samples/samples/' + i, '../../files/')
            result = mail_par.parse_email()
            tests.insert_one(result)
        except:
            if not os.path.exists('../error_log/'):
                os.makedirs('../error_log/')
            log = open('../error_log/' + i + '.txt', 'w')
            log.write(i)
            traceback.print_exc(file=log)
            log.flush()
            log.close()
        count +=1

if __name__ == '__main__':
    batch_parse()
<commit_msg>small change<commit_after># -*-coding:utf-8 -*-
from email_parser import parser
from pymongo import MongoClient
import traceback
import sys
import os

client = MongoClient('mongodb://192.168.10.107:27017/email')
db = client.email
tests = db.bulktests

def batch_parse(email_path, save_path, error_path = '../error_log/'):
    count = 1
    for i in os.listdir(email_path):
        print count, i
        try:
            mail_par = parser(email_path + i, save_path)
            result = mail_par.parse_email()
            tests.insert_one(result)
        except:
            if not os.path.exists(error_path):
                os.makedirs(error_path)
            log = open(error_path + i + '.txt', 'w')
            log.write(i)
            traceback.print_exc(file=log)
            log.flush()
            log.close()
        count +=1

if __name__ == '__main__':
    batch_parse(sys.argv[1], sys.argv[2])
<|endoftext|>


<commit_before>from __future__ import print_function
import time
import socket
import sys
import os
import platform
import re
from pymavlink import mavutil
from Queue import Empty

if platform.system() == 'Windows':
    from errno import WSAECONNRESET as ECONNABORTED
else:
    from errno import ECONNABORTED

# Clean impl of mp dependencies for droneapi

import droneapi.module.api as api

def errprinter(*args):
    print(*args, file=sys.stderr)

class FakeAPI:
    def __init__(self, module):
        self.__vehicle = api.MPVehicle(module)
        self.exit = False

    def get_vehicles(self, query=None):
        return [ self.__vehicle ]

# def mav_thread(conn, state):

#     return (in_queue, out_queue)

class MavWriter():
    def __init__(self, queue):
        self.queue = queue

    def write(self, pkt):
        self.queue.put(pkt)

    def read(self):
        errprinter('writer should not have had a read request')
        os._exit(43)

def send_heartbeat(master):
    master.mav.heartbeat_send(mavutil.mavlink.MAV_TYPE_GCS, mavutil.mavlink.MAV_AUTOPILOT_INVALID, 0, 0, 0)

def request_data_stream_send(master, rate=1):
    master.mav.request_data_stream_send(master.target_system, master.target_component,
                                        mavutil.mavlink.MAV_DATA_STREAM_ALL, rate, 1)

from Queue import Queue
from threading import Thread

class MPFakeState:
    def __init__(self, master, status_printer=None):
        self.master = master
        out_queue = Queue()
        # self.mav_thread = mav_thread(master, self)
        # self.mav = master.mav

        self.api = None

        # TODO get rid of "master" object as exposed,
        # keep it private, expose something smaller for droneapi
        self.out_queue = out_queue
        self.master.mav = mavutil.mavlink.MAVLink(MavWriter(self.out_queue), srcSystem=self.master.source_system, use_native=False)

        self.command_map = {}
        self.completions = {}

        self.target_system = 0
        self.target_component = 0

        self.lat = None
        self.lon = None
        self.alt = None

        self.vx = None
        self.vy = None
        self.vz = None

        self.airspeed = None
        self.groundspeed = None

        self.pitch = None
        self.yaw = None
        self.roll = None
        self.pitchspeed = None
        self.yawspeed = None
        self.rollspeed = None

        self.mount_pitch = None
        self.mount_yaw = None
        self.mount_roll = None

        self.voltage = -1
        self.current = -1
        self.level = -1

        self.rc_readback = {}

        self.last_waypoint = 0

        self.eph = None
        self.epv = None
        self.satellites_visible = None
        self.fix_type = None  # FIXME support multiple GPSs per vehicle - possibly by using componentId

        self.rngfnd_distance = None
        self.rngfnd_voltage = None

        self.status = type('MPStatus',(object,),{
            'flightmode': 'AUTO',
            'armed': False,
        })()

        self.mav_param = {}

        # Weird
        self.mpstate = self
        self.functions = self
        self.mpstate.settings = self

        self.status_printer = status_printer

    def fix_targets(self, message):
        pass
        # """Set correct target IDs for our vehicle"""
        # settings = self.mpstate.settings
        # if hasattr(message, 'target_system'):
        #     message.target_system = settings.target_system
        # if hasattr(message, 'target_component'):
        #     message.target_component = settings.target_component

    def module(self, which):
        # psyche
        return self

    def param_set(self, name, value, retries=3):
        # TODO dumbly reimplement this using timeout loops
        # because we should actually be awaiting an ACK of PARAM_VALUE
        # changed, but we don't have a proper ack structure, we'll
        # instead just wait until the value itself was changed

        name = name.upper()
        value = float(value)
        success = False
        while retries > 0:
            retries -= 1
            self.master.param_set_send(name.upper(), value)
            tstart = time.time()
            while time.time() - tstart < 1:
                if self.mav_param[name] == value:
                    return True
                time.sleep(0.1)

        errprinter("timeout setting parameter %s to %f" % (name, value))
        return False

    def __on_change(self, *args):
        for a in args:
            for v in self.api.get_vehicles():
                v.notify_observers(a)

    def mavlink_packet(self, m):
        typ = m.get_type()
        if typ == 'STATUSTEXT':
            print(re.sub(r'(^|\n)', '>>> \1', m.text.rstrip()))
        elif typ == 'GLOBAL_POSITION_INT':
            (self.lat, self.lon) = (m.lat / 1.0e7, m.lon / 1.0e7)
            (self.vx, self.vy, self.vz) = (m.vx / 100.0, m.vy / 100.0, m.vz / 100.0)
            self.__on_change('location', 'velocity')
        elif typ == 'GPS_RAW':
            pass # better to just use global position int
            # (self.lat, self.lon) = (m.lat, m.lon)
            # self.__on_change('location')
        elif typ == 'GPS_RAW_INT':
            # (self.lat, self.lon) = (m.lat / 1.0e7, m.lon / 1.0e7)
            self.eph = m.eph
            self.epv = m.epv
            self.satellites_visible = m.satellites_visible
            self.fix_type = m.fix_type
            self.__on_change('gps_0')
        elif typ == "VFR_HUD":
            self.heading = m.heading
            self.alt = m.alt
            self.airspeed = m.airspeed
            self.groundspeed = m.groundspeed
            self.__on_change('location', 'airspeed', 'groundspeed')
        elif typ == "ATTITUDE":
            self.pitch = m.pitch
            self.yaw = m.yaw
            self.roll = m.roll
            self.pitchspeed = m.pitchspeed
            self.yawspeed = m.yawspeed
            self.rollspeed = m.rollspeed
            self.__on_change('attitude')
        elif typ == "SYS_STATUS":
            self.voltage = m.voltage_battery
            self.current = m.current_battery
            self.level = m.battery_remaining
            self.__on_change('battery')
        elif typ == "HEARTBEAT":
            self.__on_change('mode', 'armed')
        elif typ in ["WAYPOINT_CURRENT", "MISSION_CURRENT"]:
            self.last_waypoint = m.seq
        elif typ == "RC_CHANNELS_RAW":
            def set(chnum, v):
                '''Private utility for handling rc channel messages'''
                # use port to allow ch nums greater than 8
                self.rc_readback[str(m.port * 8 + chnum)] = v

            set(1, m.chan1_raw)
            set(2, m.chan2_raw)
            set(3, m.chan3_raw)
            set(4, m.chan4_raw)
            set(5, m.chan5_raw)
            set(6, m.chan6_raw)
            set(7, m.chan7_raw)
            set(8, m.chan8_raw)
        elif typ == "MOUNT_STATUS":
            self.mount_pitch = m.pointing_a / 100
            self.mount_roll = m.pointing_b / 100
            self.mount_yaw = m.pointing_c / 100
            self.__on_change('mount')
        elif typ == "RANGEFINDER":
            self.rngfnd_distance = m.distance
            self.rngfnd_voltage = m.voltage
            self.__on_change('rangefinder')

        if self.api:
            for v in self.api.get_vehicles():
                if v.mavrx_callback:
                    v.mavrx_callback(m)

    def prepare(self, await_params=False):
        # errprinter('Await heartbeat.')
        # TODO this should be more rigious. How to avoid
        #   invalid MAVLink prefix '73'
        #   invalid MAVLink prefix '13'

        params = type('PState',(object,),{
            "mav_param_count": -1,
            "mav_param_set": [],
            "loaded": False,
            "start": False,
        })()
        self.mav_param = {}
        self.pstate = params
        self.api = FakeAPI(self)

        import atexit
        self.exiting = False
        def onexit():
            self.exiting = True
        atexit.register(onexit)

        def mavlink_thread():
            # Huge try catch in case we see http://bugs.python.org/issue1856
            try:
                # Record the time we received the last "new" param.
                last_new_param = time.time()

                start_duration = 0.2
                repeat_duration = 1
                duration = start_duration

                while True:
                    send_heartbeat(self.master)
                    time.sleep(0.05)

                    # Check the time duration for last "new" params exceeds watchdog.
                    if params.start:
                        if None not in params.mav_param_set:
                            params.loaded = True

                        if not params.loaded and time.time() - last_new_param > duration:
                            c = 0
                            for i, v in enumerate(params.mav_param_set):
                                if v == None:
                                    self.master.mav.param_request_read_send(self.master.target_system, self.master.target_component, "", i)
                                    c += 1
                                    if c > 50:
                                        break
                            duration = repeat_duration
                            last_new_param = time.time()

                    while True:
                        try:
                            msg = self.out_queue.get_nowait()
                            self.master.write(msg)
                        except socket.error as error:
                            if error.errno == ECONNABORTED:
                                errprinter('reestablishing connection after read timeout')
                                try:
                                    self.master.close()
                                except:
                                    pass
                                self.master = mavutil.mavlink_connection(self.master.address)
                                continue

                            # If connection reset (closed), stop polling.
                            return
                        except Empty:
                            break
                        except Exception as e:
                            errprinter('mav send error:', e)
                            break

                    while True:
                        try:
                            msg = self.master.recv_msg()
                        except socket.error as error:
                            if error.errno == ECONNABORTED:
                                errprinter('reestablishing connection after send timeout')
                                try:
                                    self.master.close()
                                except:
                                    pass
                                self.master = mavutil.mavlink_connection(self.master.address)
                                continue

                            # If connection reset (closed), stop polling.
                            return
                        except Exception as e:
                            # TODO debug these.
                            # errprinter('mav recv error:', e)
                            msg = None
                        if not msg:
                            break

                        if msg.get_type() == 'PARAM_VALUE':
                            # If we discover a new param count, assume we
                            # are receiving a new param set.
                            if params.mav_param_count != msg.param_count:
                                params.loaded = False
                                params.start = True
                                params.mav_param_count = msg.param_count
                                params.mav_param_set = [None]*msg.param_count

                            # Attempt to set the params. We throw an error
                            # if the index is out of range of the count or
                            # we lack a param_id.
                            try:
                                if msg.param_index < msg.param_count and msg:
                                    if params.mav_param_set[msg.param_index] == None:
                                        last_new_param = time.time()
                                        duration = start_duration
                                    params.mav_param_set[msg.param_index] = msg
                                self.mav_param[msg.param_id] = msg.param_value
                            except:
                                import traceback
                                traceback.print_exc()

                        elif msg.get_type() == 'HEARTBEAT':
                            self.status.armed = (msg.base_mode & mavutil.mavlink.MAV_MODE_FLAG_SAFETY_ARMED) != 0
                            self.status.flightmode = {v: k for k, v in self.master.mode_mapping().items()}[msg.custom_mode]

                        if self.api:
                            self.mavlink_packet(msg)

            except Exception as e:
                # http://bugs.python.org/issue1856
                if self.exiting:
                    pass
                else:
                    raise e

        t = Thread(target=mavlink_thread)
        t.daemon = True
        t.start()

        # Wait for first heartbeat.
        while True:
            try:
                self.master.wait_heartbeat()
                break
            except mavutil.mavlink.MAVError:
                continue

        # Request a list of all parameters.
        request_data_stream_send(self.master)
        while True:
            # This fn actually rate limits itself to every 2s.
            # Just retry with persistence to get our first param stream.
            self.master.param_fetch_all()
            time.sleep(0.1)
            if params.mav_param_count > -1:
                break

        # We now should get parameters streaming in.
        # We may not get the full set; we leave the logic to mavlink_thread
        # to determine what params we yet need. Wait if await_params is True.
        if await_params:
            while not params.loaded:
                time.sleep(0.1)

        return self.api

def connect(ip, await_params=False, status_printer=errprinter):
    import droneapi.module.api as api
    state = MPFakeState(mavutil.mavlink_connection(ip))
    state.status_printer = status_printer
    # api.init(state)
    return state.prepare(await_params=await_params).get_vehicles()[0]
<commit_msg>Adds heartbeat timeouts. Not on main thread yet.<commit_after>from __future__ import print_function
import time
import socket
import sys
import os
import platform
import re
from pymavlink import mavutil
from Queue import Empty

if platform.system() == 'Windows':
    from errno import WSAECONNRESET as ECONNABORTED
else:
    from errno import ECONNABORTED

# Clean impl of mp dependencies for droneapi

import droneapi.module.api as api

def errprinter(*args):
    print(*args, file=sys.stderr)

class FakeAPI:
    def __init__(self, module):
        self.__vehicle = api.MPVehicle(module)
        self.exit = False

    def get_vehicles(self, query=None):
        return [ self.__vehicle ]

# def mav_thread(conn, state):

#     return (in_queue, out_queue)

class MavWriter():
    def __init__(self, queue):
        self.queue = queue

    def write(self, pkt):
        self.queue.put(pkt)

    def read(self):
        errprinter('writer should not have had a read request')
        os._exit(43)

def send_heartbeat(master):
    master.mav.heartbeat_send(mavutil.mavlink.MAV_TYPE_GCS, mavutil.mavlink.MAV_AUTOPILOT_INVALID, 0, 0, 0)

def request_data_stream_send(master, rate=1):
    master.mav.request_data_stream_send(master.target_system, master.target_component,
                                        mavutil.mavlink.MAV_DATA_STREAM_ALL, rate, 1)

from Queue import Queue
from threading import Thread

class MPFakeState:
    def __init__(self, master, status_printer=None):
        self.master = master
        out_queue = Queue()
        # self.mav_thread = mav_thread(master, self)
        # self.mav = master.mav

        self.api = None

        # TODO get rid of "master" object as exposed,
        # keep it private, expose something smaller for droneapi
        self.out_queue = out_queue
        self.master.mav = mavutil.mavlink.MAVLink(MavWriter(self.out_queue), srcSystem=self.master.source_system, use_native=False)

        self.command_map = {}
        self.completions = {}

        self.target_system = 0
        self.target_component = 0

        self.lat = None
        self.lon = None
        self.alt = None

        self.vx = None
        self.vy = None
        self.vz = None

        self.airspeed = None
        self.groundspeed = None

        self.pitch = None
        self.yaw = None
        self.roll = None
        self.pitchspeed = None
        self.yawspeed = None
        self.rollspeed = None

        self.mount_pitch = None
        self.mount_yaw = None
        self.mount_roll = None

        self.voltage = -1
        self.current = -1
        self.level = -1

        self.rc_readback = {}

        self.last_waypoint = 0

        self.eph = None
        self.epv = None
        self.satellites_visible = None
        self.fix_type = None  # FIXME support multiple GPSs per vehicle - possibly by using componentId

        self.rngfnd_distance = None
        self.rngfnd_voltage = None

        self.status = type('MPStatus',(object,),{
            'flightmode': 'AUTO',
            'armed': False,
        })()

        self.mav_param = {}

        # Weird
        self.mpstate = self
        self.functions = self
        self.mpstate.settings = self

        self.status_printer = status_printer

    def fix_targets(self, message):
        pass
        # """Set correct target IDs for our vehicle"""
        # settings = self.mpstate.settings
        # if hasattr(message, 'target_system'):
        #     message.target_system = settings.target_system
        # if hasattr(message, 'target_component'):
        #     message.target_component = settings.target_component

    def module(self, which):
        # psyche
        return self

    def param_set(self, name, value, retries=3):
        # TODO dumbly reimplement this using timeout loops
        # because we should actually be awaiting an ACK of PARAM_VALUE
        # changed, but we don't have a proper ack structure, we'll
        # instead just wait until the value itself was changed

        name = name.upper()
        value = float(value)
        success = False
        while retries > 0:
            retries -= 1
            self.master.param_set_send(name.upper(), value)
            tstart = time.time()
            while time.time() - tstart < 1:
                if self.mav_param[name] == value:
                    return True
                time.sleep(0.1)

        errprinter("timeout setting parameter %s to %f" % (name, value))
        return False

    def __on_change(self, *args):
        for a in args:
            for v in self.api.get_vehicles():
                v.notify_observers(a)

    def mavlink_packet(self, m):
        typ = m.get_type()
        if typ == 'STATUSTEXT':
            print(re.sub(r'(^|\n)', '>>> \1', m.text.rstrip()))
        elif typ == 'GLOBAL_POSITION_INT':
            (self.lat, self.lon) = (m.lat / 1.0e7, m.lon / 1.0e7)
            (self.vx, self.vy, self.vz) = (m.vx / 100.0, m.vy / 100.0, m.vz / 100.0)
            self.__on_change('location', 'velocity')
        elif typ == 'GPS_RAW':
            pass # better to just use global position int
            # (self.lat, self.lon) = (m.lat, m.lon)
            # self.__on_change('location')
        elif typ == 'GPS_RAW_INT':
            # (self.lat, self.lon) = (m.lat / 1.0e7, m.lon / 1.0e7)
            self.eph = m.eph
            self.epv = m.epv
            self.satellites_visible = m.satellites_visible
            self.fix_type = m.fix_type
            self.__on_change('gps_0')
        elif typ == "VFR_HUD":
            self.heading = m.heading
            self.alt = m.alt
            self.airspeed = m.airspeed
            self.groundspeed = m.groundspeed
            self.__on_change('location', 'airspeed', 'groundspeed')
        elif typ == "ATTITUDE":
            self.pitch = m.pitch
            self.yaw = m.yaw
            self.roll = m.roll
            self.pitchspeed = m.pitchspeed
            self.yawspeed = m.yawspeed
            self.rollspeed = m.rollspeed
            self.__on_change('attitude')
        elif typ == "SYS_STATUS":
            self.voltage = m.voltage_battery
            self.current = m.current_battery
            self.level = m.battery_remaining
            self.__on_change('battery')
        elif typ == "HEARTBEAT":
            self.__on_change('mode', 'armed')
        elif typ in ["WAYPOINT_CURRENT", "MISSION_CURRENT"]:
            self.last_waypoint = m.seq
        elif typ == "RC_CHANNELS_RAW":
            def set(chnum, v):
                '''Private utility for handling rc channel messages'''
                # use port to allow ch nums greater than 8
                self.rc_readback[str(m.port * 8 + chnum)] = v

            set(1, m.chan1_raw)
            set(2, m.chan2_raw)
            set(3, m.chan3_raw)
            set(4, m.chan4_raw)
            set(5, m.chan5_raw)
            set(6, m.chan6_raw)
            set(7, m.chan7_raw)
            set(8, m.chan8_raw)
        elif typ == "MOUNT_STATUS":
            self.mount_pitch = m.pointing_a / 100
            self.mount_roll = m.pointing_b / 100
            self.mount_yaw = m.pointing_c / 100
            self.__on_change('mount')
        elif typ == "RANGEFINDER":
            self.rngfnd_distance = m.distance
            self.rngfnd_voltage = m.voltage
            self.__on_change('rangefinder')

        if self.api:
            for v in self.api.get_vehicles():
                if v.mavrx_callback:
                    v.mavrx_callback(m)

    def prepare(self, await_params=False):
        # errprinter('Await heartbeat.')
        # TODO this should be more rigious. How to avoid
        #   invalid MAVLink prefix '73'
        #   invalid MAVLink prefix '13'

        params = type('PState',(object,),{
            "mav_param_count": -1,
            "mav_param_set": [],
            "loaded": False,
            "start": False,
        })()
        self.mav_param = {}
        self.pstate = params
        self.api = FakeAPI(self)

        import atexit
        self.exiting = False
        def onexit():
            self.exiting = True
        atexit.register(onexit)

        heartbeat_started = False

        def mavlink_thread():
            # Huge try catch in case we see http://bugs.python.org/issue1856
            try:
                # Record the time we received the last "new" param.
                last_new_param = time.time()
                last_heartbeat_sent = 0
                last_heartbeat_received = 0

                start_duration = 0.2
                repeat_duration = 1
                duration = start_duration

                while True:
                    # Downtime
                    time.sleep(0.05)

                    # Check the time duration for last "new" params exceeds watchdog.
                    if params.start:
                        if None not in params.mav_param_set:
                            params.loaded = True

                        if not params.loaded and time.time() - last_new_param > duration:
                            c = 0
                            for i, v in enumerate(params.mav_param_set):
                                if v == None:
                                    self.master.mav.param_request_read_send(self.master.target_system, self.master.target_component, "", i)
                                    c += 1
                                    if c > 50:
                                        break
                            duration = repeat_duration
                            last_new_param = time.time()

                    # Send 1 heartbeat per second
                    if time.time() - last_heartbeat_sent < 1:
                        send_heartbeat(self.master)
                        last_heartbeat_sent = time.time()
                    # Timeout after 5
                    if heartbeat_started:
                        if last_heartbeat_received == 0:
                            last_heartbeat_received = time.time()
                        elif time.time() - last_heartbeat_received > 5:
                            raise Exception('Link timeout, no heartbeat in last 5 seconds')

                    while True:
                        try:
                            msg = self.out_queue.get_nowait()
                            self.master.write(msg)
                        except socket.error as error:
                            if error.errno == ECONNABORTED:
                                errprinter('reestablishing connection after read timeout')
                                try:
                                    self.master.close()
                                except:
                                    pass
                                self.master = mavutil.mavlink_connection(self.master.address)
                                continue

                            # If connection reset (closed), stop polling.
                            return
                        except Empty:
                            break
                        except Exception as e:
                            errprinter('mav send error:', e)
                            break

                    while True:
                        try:
                            msg = self.master.recv_msg()
                        except socket.error as error:
                            if error.errno == ECONNABORTED:
                                errprinter('reestablishing connection after send timeout')
                                try:
                                    self.master.close()
                                except:
                                    pass
                                self.master = mavutil.mavlink_connection(self.master.address)
                                continue

                            # If connection reset (closed), stop polling.
                            return
                        except Exception as e:
                            # TODO debug these.
                            # errprinter('mav recv error:', e)
                            msg = None
                        if not msg:
                            break

                        if msg.get_type() == 'PARAM_VALUE':
                            # If we discover a new param count, assume we
                            # are receiving a new param set.
                            if params.mav_param_count != msg.param_count:
                                params.loaded = False
                                params.start = True
                                params.mav_param_count = msg.param_count
                                params.mav_param_set = [None]*msg.param_count

                            # Attempt to set the params. We throw an error
                            # if the index is out of range of the count or
                            # we lack a param_id.
                            try:
                                if msg.param_index < msg.param_count and msg:
                                    if params.mav_param_set[msg.param_index] == None:
                                        last_new_param = time.time()
                                        duration = start_duration
                                    params.mav_param_set[msg.param_index] = msg
                                self.mav_param[msg.param_id] = msg.param_value
                            except:
                                import traceback
                                traceback.print_exc()

                        elif msg.get_type() == 'HEARTBEAT':
                            self.status.armed = (msg.base_mode & mavutil.mavlink.MAV_MODE_FLAG_SAFETY_ARMED) != 0
                            self.status.flightmode = {v: k for k, v in self.master.mode_mapping().items()}[msg.custom_mode]
                            last_heartbeat_received = time.time()

                        if self.api:
                            self.mavlink_packet(msg)

            except Exception as e:
                # http://bugs.python.org/issue1856
                if self.exiting:
                    pass
                else:
                    raise e

        t = Thread(target=mavlink_thread)
        t.daemon = True
        t.start()

        # Wait for first heartbeat.
        while True:
            try:
                self.master.wait_heartbeat()
                break
            except mavutil.mavlink.MAVError:
                continue
        heartbeat_started = True

        # Request a list of all parameters.
        request_data_stream_send(self.master)
        while True:
            # This fn actually rate limits itself to every 2s.
            # Just retry with persistence to get our first param stream.
            self.master.param_fetch_all()
            time.sleep(0.1)
            if params.mav_param_count > -1:
                break

        # We now should get parameters streaming in.
        # We may not get the full set; we leave the logic to mavlink_thread
        # to determine what params we yet need. Wait if await_params is True.
        if await_params:
            while not params.loaded:
                time.sleep(0.1)

        return self.api

def connect(ip, await_params=False, status_printer=errprinter):
    import droneapi.module.api as api
    state = MPFakeState(mavutil.mavlink_connection(ip))
    state.status_printer = status_printer
    # api.init(state)
    return state.prepare(await_params=await_params).get_vehicles()[0]
<|endoftext|>


<commit_before>"""A set of views every cart needs.

On success, each view returns a JSON-response with the cart
representation. For the details on the format of the return value,
see the :meth:`~easycart.cart.BaseCart.encode` method of the
:class:`~easycart.cart.BaseCart` class.

Note
----
All of the views in this module accept only POST requests.

"""
from importlib import import_module

from django.conf import settings
from django.views.generic import View

__all__ = [
    'AddItem',
    'RemoveItem',
    'ChangeItemQuantity',
    'EmptyCart',
]

cart_module, cart_class = settings.EASYCART_CART_CLASS.rsplit('.', 1)
Cart = getattr(import_module(cart_module), cart_class)

class AddItem(View):
    """Add an item to the cart.

    This view expects `request.POST` to contain:

    +------------+----------------------------------------------------+
    | key        | value                                              |
    +============+====================================================+
    | `pk`       | the primary key of an item to add                  |
    +------+-----+----------------------------------------------------+
    | `quantity` | a quantity that should be associated with the item |
    +------------+----------------------------------------------------+

    """

    def post(self, request):
        cart = Cart(request)
        pk = request.POST['pk']
        quantity = request.POST['quantity']
        cart.add(pk, quantity)
        return cart.encode()

class ChangeItemQuantity(View):
    """Change the quantity of an item.

    This view expects `request.POST` to contain:

    +------------+----------------------------------------------------+
    | key        | value                                              |
    +============+====================================================+
    | `pk`       | the primary key of an item                         |
    +------+-----+----------------------------------------------------+
    | `quantity` | a new quantity to associate with the item          |
    +------------+----------------------------------------------------+

    """

    def post(self, request):
        cart = Cart(request)
        pk = request.POST['pk']
        quantity = request.POST['quantity']
        cart.change_quantity(pk, quantity)
        return cart.encode()

class RemoveItem(View):
    """Remove an item from the cart.

    Expects `request.POST` to contain key *pk*. The associated value
    should be the primary key of an item you wish to remove.

    """

    def post(self, request):
        cart = Cart(request)
        pk = request.POST['pk']
        cart.remove(pk)
        return cart.encode()

class EmptyCart(View):
    """Remove all items from the cart."""

    def post(self, request):
        cart = Cart(request)
        cart.empty()
        return cart.encode()
<commit_msg>easycart.views: Rewrite module<commit_after>"""A set of views every cart needs.

On success, each view returns a JSON-response with the cart
representation. For the details on the format of the return value,
see the :meth:`~easycart.cart.BaseCart.encode` method of the
:class:`~easycart.cart.BaseCart` class.

If a parameter required by a view is not present in the request's POST
data, then the JSON-response will have the format::

   {'error': 'MissingRequestParam', 'param': parameter_name}

Almost the same thing happens, if a parameter is invalid and results in
an exception, which is a subclass of :class:`~easycart.cart.CartException`.
In this case, the error value will be the name of the concrete exception
class (e.g. ``'ItemNotInCart'`` or ``'NegativeItemQuantity'``).
And instead of ``param`` there may be one or more items providing
additional info on the error, for example, the primary key of an item
you was trying to change or an invalid quantity passed in the request.

Note
----
All of the views in this module accept only POST requests.

"""
from importlib import import_module

from django.conf import settings
from django.http import JsonResponse
from django.views.generic import View

from easycart.cart import CartException

__all__ = [
    'AddItem',
    'RemoveItem',
    'ChangeItemQuantity',
    'EmptyCart',
]

cart_module, cart_class = settings.EASYCART_CART_CLASS.rsplit('.', 1)
Cart = getattr(import_module(cart_module), cart_class)

class CartView(View):
    """Base class for views operating the cart."""
    action = None
    """Attribute of the cart object, which will be called to perform
    some action on the cart.
    """
    required_params = ()
    """Iterable of parameters, which MUST be present in the post data."""
    optional_params = {}
    """Dictionary of parameters, which MAY be present in the post data.

    Parameters serve as keys. Associated values will be used as fallbacks
    in case the parameter is not in the post data.
    """

    def post(self, request):
        # Extract parameters from the post data
        params = {}
        for param in self.required_params:
            try:
                params[param] = request.POST[param]
            except KeyError:
                return JsonResponse({
                    'error': 'MissingRequestParam',
                    'param': param,
                })
        for param, fallback in self.optional_params.items():
            params[param] = request.POST.get(param, fallback)
        # Perform an action on the cart using these parameters
        cart = Cart(request)
        action = getattr(cart, self.action)
        try:
            action(**params)
        except CartException as exc:
            return JsonResponse(dict({'error': exc.__class__.__name__},
                                     **exc.kwargs))
        return cart.encode()

class AddItem(CartView):
    """Add an item to the cart.

    This view expects `request.POST` to contain:

    +------------+----------------------------------------------------+
    | key        | value                                              |
    +============+====================================================+
    | `pk`       | the primary key of an item to add                  |
    +------------+----------------------------------------------------+
    | `quantity` | a quantity that should be associated with the item |
    +------------+----------------------------------------------------+

    The `quantity` parameter is optional (defaults to 1).

    """
    action = 'add'
    required_params = ('pk',)
    optional_params = {'quantity': 1}

class ChangeItemQuantity(CartView):
    """Change the quantity associated with an item.

    This view expects `request.POST` to contain:

    +------------+----------------------------------------------------+
    | key        | value                                              |
    +============+====================================================+
    | `pk`       | the primary key of an item                         |
    +------------+----------------------------------------------------+
    | `quantity` | a new quantity to associate with the item          |
    +------------+----------------------------------------------------+

    """
    action = 'change_quantity'
    required_params = ('pk', 'quantity')

class RemoveItem(CartView):
    """Remove an item from the cart.

    Expects `request.POST` to contain key *pk*. The associated value
    should be the primary key of an item you wish to remove.

    """
    action = 'remove'
    required_params = ('pk',)

class EmptyCart(CartView):
    """Remove all items from the cart."""
    action = 'empty'
<|endoftext|>


<commit_before># Create your views here.
from django.shortcuts import render_to_response

def start(request):
    return render_to_response('start.html')

def dropbox_dropin_chooser(request):
	return render_to_response('dropbox-dropin-chooser.html')

def dropbox_dropin_saver(request):
	return render_to_response('dropbox-dropin-saver.html')

def dropbox_datastore(request):
	return render_to_response('dropbox-datastore.html')<commit_msg>use render() instead of render_to_response()<commit_after># Create your views here.
from django.shortcuts import render

def start(request):
    return render(request, 'start.html')

def dropbox_dropin_chooser(request):
	return render(request, 'dropbox-dropin-chooser.html')

def dropbox_dropin_saver(request):
	return render(request, 'dropbox-dropin-saver.html')

def dropbox_datastore(request):
	return render(request, 'dropbox-datastore.html')<|endoftext|>


<commit_before>"""
 * Copyright (c) 2016. Mingyu Gao
 * All rights reserved.
 *
"""
import matplotlib.backends.backend_pdf
from .format import paper_plot

def plot_setup(name, dims, fontsize=9):
    """ Setup a PDF page for plot.

    name: PDF file name. If not ending with .pdf, will automatically append.
    dims: dimension of the plot in inches, should be an array of length two.
    fontsize: fontsize for legends and labels.
    """
    paper_plot(fontsize)
    if not name.endswith('.pdf'):
        name += '.pdf'
    pdfpage = matplotlib.backends.backend_pdf.PdfPages(name)
    fig = matplotlib.pyplot.figure(figsize=dims)
    return pdfpage, fig

def plot_teardown(pdfpage):
    """ Tear down a PDF page after plotting.

    pdfpage: PDF page.
    """
    pdfpage.savefig()
    pdfpage.close()

<commit_msg>Add a context manager for PDF plot.<commit_after>"""
 * Copyright (c) 2016. Mingyu Gao
 * All rights reserved.
 *
"""
from contextlib import contextmanager
import matplotlib.backends.backend_pdf

from .format import paper_plot

def plot_setup(name, figsize=None, fontsize=9):
    """ Setup a PDF page for plot.

    name: PDF file name. If not ending with .pdf, will automatically append.
    figsize: dimension of the plot in inches, should be an array of length two.
    fontsize: fontsize for legends and labels.
    """
    paper_plot(fontsize)
    if not name.endswith('.pdf'):
        name += '.pdf'
    pdfpage = matplotlib.backends.backend_pdf.PdfPages(name)
    fig = matplotlib.pyplot.figure(figsize=figsize)
    return pdfpage, fig

def plot_teardown(pdfpage):
    """ Tear down a PDF page after plotting.

    pdfpage: PDF page.
    """
    pdfpage.savefig()
    pdfpage.close()

@contextmanager
def plot_open(name, figsize=None, fontsize=9):
    """ Open a context of PDF page for plot, used for the `with` statement.

    name: PDF file name. If not ending with .pdf, will automatically append.
    figsize: dimension of the plot in inches, should be an array of length two.
    fontsize: fontsize for legends and labels.
    """
    pdfpage, fig = plot_setup(name, figsize=figsize, fontsize=fontsize)
    yield fig
    plot_teardown(pdfpage)

<|endoftext|>


<commit_before>'''
Created on Nov 13, 2014

@author: Daniel Nam dwn28
'''

import math
from Datapoint import Datapoint

def nb(traindata, testdata):

    trainlabel = []
    testlabel = []
    trainfeature = []
    testfeature = []

    for d in traindata:
        featuremat = []
        for feature in d:
            featuremat.append(feature)
        trainfeature.append(featuremat)
#         TODO:
#         trainlabel
        trainlabel.append(d.label)

    for d in testdata:
#         TODO: add test label
        featuremat = []
        for feature in d:
            featuremat.append(feature)
        testfeature.append(featuremat)
        testlabel.append(d.label)

    n = len(trainlabel)
    npos = 0
    nneg = 0
    for label in trainlabel:
        if label == 1: npos = npos+1
        else: nneg = nneg+1
    posindex = find(trainlabel,1)
    negindex = find(trainlabel,-1)
    lpos = 0
    lneg = 0
    v = 0
    wpos = []
    wneg = []
    for idx in posindex:
        featurespace = trainfeature(idx)
        lpos = sum(featurespace) + lpos
        v = max([v,len(featurespace)])
        for k in range(len(featurespace)):
            if k+1 <= len(wpos):
                wpos[k] = wpos[k] + featurespace[k]
            else:
                wpos[k] = featurespace[k]

    for idx in negindex:
        featurespace = trainfeature(idx)
        lneg = sum(featurespace) + lneg
        v = max([v,len(featurespace)])
        for k in range(len(featurespace)):
            if k+1 <= len(wneg):
                wneg[k] = wneg[k] + featurespace[k]
            else:
                wneg[k] = featurespace[k]

    PYpos = npos/n
    PYneg = nneg/n
    PWYpos = matdiv([w+1 for w in wpos],[l+v for l in lpos])
    PWYneg = matdiv([w+1 for w in wneg],[l+v for l in lneg])
    classification = [0]*len(testfeature)
    for i in range(len(testfeature)):
        featurespace = testfeature[i]
        probpos = nbprob(PWYpos,PYpos,featurespace)
        probneg = nbprob(PWYneg,PYneg,featurespace)
        if probpos >= probneg:
            classification[i] = 1
        else: classification[i] = -1

    testAccuracy = accuracy(classification,testlabel)
    print "Accuracy: ", testAccuracy

def accuracy(result, data):
    match = 0
    for i in range(len(result)):
        if result[i]==data[i]:
            match = match+1
    return match/len(result)

def nbprob(pwy,py,feature):
    problist = []
    for i in range(len(feature)):
        problist.append(math.log(math.pow(pwy[i],feature[i])))
    return sum(problist + math.log(py))

def matdiv(mat1,mat2):
    result = []
    for i in range(len(mat1)):
        result.append(mat1[i]/mat2[i])
    return result

def find(mat,element):
    idx = []
    for i in range(len(mat)):
        if mat(i)==element:
            idx.append(i)
    return idx

# if __name__ == '__main__':
#     nb(data)
#     pass
<commit_msg>NB modified<commit_after>'''
Created on Nov 13, 2014

@author: Daniel Nam dwn28
'''

import math
from Datapoint import Datapoint
import numpy as np

def nb(traindata, testdata):

    trainlabel = []
    testlabel = []
    trainfeature = []
    testfeature = []

    for d in traindata:
        featuremat = []
        for feature in d:
            featuremat.append(feature)
        trainfeature.append(featuremat)
#         TODO:
#         trainlabel
        trainlabel.append(d.label)

    for d in testdata:
#         TODO: add test label
        featuremat = []
        for feature in d:
            featuremat.append(feature)
        testfeature.append(featuremat)
        testlabel.append(d.label)

    labels = list(set(trainlabel))
    labelindex = []
    for label in labels:
        labelindex.append(find(trainlabel,label))

    results = []
    for testcase in testfeature:
        probs = []
        for label in labels:
            nlabel = len(labelindex[labels.index(label)])
            PY = nlabel/len(trainlabel)
            index = labelindex(labels.index(label))
            nbprob = PY
            newfeatures = [[]*len(testcase)]
            for idx in index:
                testdata = trainfeature[idx]
                for i in range(len(testdata)):
                    newfeatures[i].append(testdata[i])
            for i in range(len(testcase)):
                feature = testcase[i]
                featurepool = newfeatures[i]
                PXY = (featurepool.count(feature)+1)/(len(featurepool)+len(set(featurepool)))
                nbprob = nbprob * PXY
            probs.append(nbprob)
        maxprob = max(probs)
        results.append(labels(probs.index(maxprob)))
    return results

def accuracy(result, data):
    match = 0
    for i in range(len(result)):
        if result[i]==data[i]:
            match = match+1
    return match/len(result)

def nbprob(pwy,py,feature):
    problist = []
    for i in range(len(feature)):
        problist.append(math.log(math.pow(pwy[i],feature[i])))
    return sum(problist + math.log(py))

def matdiv(mat1,mat2):
    result = []
    for i in range(len(mat1)):
        result.append(mat1[i]/mat2[i])
    return result

def find(mat,element):
    idx = []
    for i in range(len(mat)):
        if mat(i)==element:
            idx.append(i)
    return idx

# if __name__ == '__main__':
#     nb(data)
#     pass
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
File format converter.
The script converts csv file to netcdf format.

"""

import sys
import os.path
import csv
import pupynere

def update_name(old_name):
    new = old_name.split('.')
    new[-1] = 'nc'
    return '.'.join(new)

def read_file(fl):
    with open(fl, 'r') as f:
        for line in f:
            if not line.startswith('#'):
                yield line.strip().split(',')

def csv_to_nc(csvfl):
    """
    Transforms csv file into netcdf format.

    :Args:
      - csvfl cvs data file, example: data.csv

    :Returns:
      - netcdf file, example: data.nc
    """

    vv = zip(*[map(float, l) for l in read_file(csvfl)])

    transformed_file = update_name(csvfl)
    nc = pupynere.netcdf_file(transformed_file, 'w')

    nc.createDimension('dim', None)

    for i, item in enumerate(vv):
       nc.createVariable('var_%02d' % i, 'd', ('dim',))[:] = vv[i]
    return 0

if __name__ == "__main__":
    import sys
    if len(sys.argv) < 2:
        sys.stderr.write("Usage: nc.py csvfile\n")
        raise SystemExit(1)

    if not os.path.exists(sys.argv[1]):
        sys.stderr.write("Error: csvdata file {0} not found\n".format(sys.argv[1]))
        raise SystemExit(1)

    csv_to_nc(sys.argv[1])

<commit_msg>Add docstrings<commit_after>#!/usr/bin/env python
"""
File format converter.
The script converts csv file to netcdf format.

"""

import sys
import os.path
import csv
import pupynere

def update_name(old_name):
    '''Return filename with new ext - .nc'''
    new = old_name.split('.')
    new[-1] = 'nc'
    return '.'.join(new)

def read_file(fl):
    '''Yields line by line

    :Args:
      - fl, csv file for transformation
    '''
    with open(fl, 'r') as f:
        for line in f:
            if not line.startswith('#'):
                yield line.strip().split(',')

def csv_to_nc(csvfl):
    """
    Transforms csv file into netcdf format.

    :Args:
      - csvfl cvs data file, example: data.csv

    :Returns:
      - netcdf file, example: data.nc
    """

    vv = zip(*[map(float, l) for l in read_file(csvfl)])

    transformed_file = update_name(csvfl)
    nc = pupynere.netcdf_file(transformed_file, 'w')

    nc.createDimension('dim', None)

    for i, item in enumerate(vv):
       nc.createVariable('var_%02d' % i, 'd', ('dim',))[:] = vv[i]
    return 0

if __name__ == "__main__":
    import sys
    if len(sys.argv) < 2:
        sys.stderr.write("Usage: nc.py csvfile\n")
        raise SystemExit(1)

    if not os.path.exists(sys.argv[1]):
        sys.stderr.write("Error: csvdata file {0} not found\n".format(sys.argv[1]))
        raise SystemExit(1)

    csv_to_nc(sys.argv[1])

<|endoftext|>


<commit_before>import logging
import os
import shutil
from pathlib import Path
from collections import defaultdict
from edalize.edatool import Edatool

logger = logging.getLogger(__name__)

class Libero(Edatool):
    @classmethod
    def get_doc(cls, api_ver):
        if api_ver == 0:
            return {'description': "The Libero backend supports Microsemi Libero to build systems and program the FPGA",
                    'members': [
                        {'name': 'family',
                         'type': 'String',
                         'desc': 'FPGA family (e.g. PolarFire)'},
                        {'name': 'die',
                         'type': 'String',
                         'desc': 'FPGA device (e.g. MPF300TS)'},
                        {'name': 'package',
                         'type': 'String',
                         'desc': 'FPGA package type (e.g. FCG1152)'},
                        {'name': 'speed',
                         'type': 'String',
                         'desc': 'FPGA speed rating (e.g. -1)'},
                        {'name': 'dievoltage',
                         'type': 'String',
                         'desc': 'FPGA die voltage (e.g. 1.0)'},
                        {'name': 'range',
                         'type': 'String',
                         'desc': 'FPGA temperature range (e.g. IND)'},
                        {'name': 'defiostd',
                         'type': 'String',
                         'desc': 'FPGA default IO std (e.g. "LVCMOS 1.8V"'},
                        {'name': 'hdl',
                         'type': 'String',
                         'desc': 'Default HDL (e.g. "VERILOG"'},
                    ]
                    }

    argtypes = ['vlogdefine', 'vlogparam', 'generic']
    mandatory_options = ['family', 'die', 'package', 'range']

    tool_options_defaults = {
        'range': 'IND',
    }

    def _set_tool_options_defaults(self):
        for key, default_value in self.tool_options_defaults.items():
            if not key in self.tool_options:
                logger.info("Set Libero tool option %s to default value %s"
                            % (key, str(default_value)))
                self.tool_options[key] = default_value

    def _check_mandatory_options(self):
        shouldExit = 0
        for key in self.mandatory_options:
            if not key in self.tool_options:
                logger.error("Libero option \"%s\" must be defined", key)
                shouldExit = 1
        if shouldExit:
            raise RuntimeError("Missing required tool options")

    def configure_main(self):
        """
        Configuration is the first phase of the build.

        This writes the project TCL file. It first collects all
        sources, IPs and constraints and then writes them to the TCL file along
        with the build steps.
        """
        self._set_tool_options_defaults()
        self._check_mandatory_options()
        (src_files, incdirs) = self._get_fileset_files(force_slash=True)
        self.jinja_env.filters['src_file_filter'] = self.src_file_filter
        self.jinja_env.filters['constraint_file_filter'] = self.constraint_file_filter
        self.jinja_env.filters['tcl_file_filter'] = self.tcl_file_filter

        escaped_name = self.name.replace(".", "_")

        # Add Edalize working dir to Synthesys includes removing duplicates
        incdirs = list(set(incdirs.__add__(["."])))

        # Build source files that are part of libraries
        library_files = defaultdict(list)
        for f in src_files:
            if f.logical_name:
                library_files[f.logical_name].append(f.name)

        template_vars = {
            'name': escaped_name,
            'src_files': src_files,
            'library_files': library_files,
            'incdirs': incdirs,
            'vlogparam': self.vlogparam,
            'vlogdefine': self.vlogdefine,
            'generic': self.generic,
            'tool_options': self.tool_options,
            'toplevel': self.toplevel,
            'generic': self.generic,
            'prj_root': "./prj",
            'op': "{",
            'cl': "}",
            'sp': " "
        }

        # Set preferred HDL language based on file type amount if not user defined.
        # According to docs, projects can be mixed but one language must be
        # defined as preferred
        if not "hdl" in self.tool_options:
            verilogFiles = 0
            VHDLFiles = 0
            for f in src_files:
                t = f.file_type.split('-')[0]
                if t == "verilogSource" or t == "systemVerilogSource":
                    verilogFiles += 1
                elif t == "vhdlSource":
                    VHDLFiles += 1
            if verilogFiles >= VHDLFiles:
                self.tool_options["hdl"] = "VERILOG"
            else:
                self.tool_options["hdl"] = "VHDL"

        # Render the TCL project file
        self.render_template('libero-project.tcl.j2',
                             escaped_name + '-project.tcl',
                             template_vars)

        # Render the TCL run file
        self.render_template('libero-run.tcl.j2',
                             escaped_name + '-run.tcl',
                             template_vars)

        # Render the Synthesize TCL file
        self.render_template('libero-syn-user.tcl.j2',
                             escaped_name + '-syn-user.tcl',
                             template_vars)

        logger.info("Cores and Libero TCL Scripts generated.")

    def src_file_filter(self, f):
        file_types = {
            'verilogSource': '-hdl_source {',
            'systemVerilogSource': '-hdl_source {',
            'vhdlSource': "-hdl_source {",
            'PDC': '-io_pdc {',
            'SDC': '-sdc {',
        }
        _file_type = f.file_type.split('-')[0]
        if _file_type in file_types:
            # Do not return library files here
            if f.logical_name:
                return ''
            return file_types[_file_type] + f.name
        return ''

    def tcl_file_filter(self, f):
        file_types = {
            'tclSource': 'source ',
        }
        _file_type = f.file_type.split('-')[0]
        if _file_type in file_types:
            return file_types[_file_type] + f.name
        return ''

    def constraint_file_filter(self, f, type="ALL"):
        file_types = {
            'PDC': 'constraint/io/',
            'SDC': 'constraint/',
        }
        _file_type = f.file_type.split('-')[0]
        if _file_type in file_types:
            filename = f.name.split("/")[-1]
            if type == "ALL":
                return file_types[_file_type] + filename
            elif _file_type == type:
                return file_types[_file_type] + filename
        return ''

    def build_main(self):
        logger.info("Executing Libero TCL Scripts.")
        escaped_name = self.name.replace(".", "_")
        if shutil.which("libero"):
            self._run_tool('libero', ['SCRIPT:' + escaped_name + '-run.tcl'])
        else:
            filePath = os.path.join(Path(self.work_root).relative_to(
                os.getcwd()), escaped_name + '-run.tcl')
            logger.warn(
                "Libero not found on path, execute manually the script \"" + filePath + "\"")

    def run_main(self):
        pass
<commit_msg>Added a src file filter for floor planning PDC files to the libero target. Libero requires that these constraints are put in a different file from the IO constraints.<commit_after>import logging
import os
import shutil
from pathlib import Path
from collections import defaultdict
from edalize.edatool import Edatool

logger = logging.getLogger(__name__)

class Libero(Edatool):
    @classmethod
    def get_doc(cls, api_ver):
        if api_ver == 0:
            return {'description': "The Libero backend supports Microsemi Libero to build systems and program the FPGA",
                    'members': [
                        {'name': 'family',
                         'type': 'String',
                         'desc': 'FPGA family (e.g. PolarFire)'},
                        {'name': 'die',
                         'type': 'String',
                         'desc': 'FPGA device (e.g. MPF300TS)'},
                        {'name': 'package',
                         'type': 'String',
                         'desc': 'FPGA package type (e.g. FCG1152)'},
                        {'name': 'speed',
                         'type': 'String',
                         'desc': 'FPGA speed rating (e.g. -1)'},
                        {'name': 'dievoltage',
                         'type': 'String',
                         'desc': 'FPGA die voltage (e.g. 1.0)'},
                        {'name': 'range',
                         'type': 'String',
                         'desc': 'FPGA temperature range (e.g. IND)'},
                        {'name': 'defiostd',
                         'type': 'String',
                         'desc': 'FPGA default IO std (e.g. "LVCMOS 1.8V"'},
                        {'name': 'hdl',
                         'type': 'String',
                         'desc': 'Default HDL (e.g. "VERILOG"'},
                    ]
                    }

    argtypes = ['vlogdefine', 'vlogparam', 'generic']
    mandatory_options = ['family', 'die', 'package', 'range']

    tool_options_defaults = {
        'range': 'IND',
    }

    def _set_tool_options_defaults(self):
        for key, default_value in self.tool_options_defaults.items():
            if not key in self.tool_options:
                logger.info("Set Libero tool option %s to default value %s"
                            % (key, str(default_value)))
                self.tool_options[key] = default_value

    def _check_mandatory_options(self):
        shouldExit = 0
        for key in self.mandatory_options:
            if not key in self.tool_options:
                logger.error("Libero option \"%s\" must be defined", key)
                shouldExit = 1
        if shouldExit:
            raise RuntimeError("Missing required tool options")

    def configure_main(self):
        """
        Configuration is the first phase of the build.

        This writes the project TCL file. It first collects all
        sources, IPs and constraints and then writes them to the TCL file along
        with the build steps.
        """
        self._set_tool_options_defaults()
        self._check_mandatory_options()
        (src_files, incdirs) = self._get_fileset_files(force_slash=True)
        self.jinja_env.filters['src_file_filter'] = self.src_file_filter
        self.jinja_env.filters['constraint_file_filter'] = self.constraint_file_filter
        self.jinja_env.filters['tcl_file_filter'] = self.tcl_file_filter

        escaped_name = self.name.replace(".", "_")

        # Add Edalize working dir to Synthesys includes removing duplicates
        incdirs = list(set(incdirs.__add__(["."])))

        # Build source files that are part of libraries
        library_files = defaultdict(list)
        for f in src_files:
            if f.logical_name:
                library_files[f.logical_name].append(f.name)

        template_vars = {
            'name': escaped_name,
            'src_files': src_files,
            'library_files': library_files,
            'incdirs': incdirs,
            'vlogparam': self.vlogparam,
            'vlogdefine': self.vlogdefine,
            'generic': self.generic,
            'tool_options': self.tool_options,
            'toplevel': self.toplevel,
            'generic': self.generic,
            'prj_root': "./prj",
            'op': "{",
            'cl': "}",
            'sp': " "
        }

        # Set preferred HDL language based on file type amount if not user defined.
        # According to docs, projects can be mixed but one language must be
        # defined as preferred
        if not "hdl" in self.tool_options:
            verilogFiles = 0
            VHDLFiles = 0
            for f in src_files:
                t = f.file_type.split('-')[0]
                if t == "verilogSource" or t == "systemVerilogSource":
                    verilogFiles += 1
                elif t == "vhdlSource":
                    VHDLFiles += 1
            if verilogFiles >= VHDLFiles:
                self.tool_options["hdl"] = "VERILOG"
            else:
                self.tool_options["hdl"] = "VHDL"

        # Render the TCL project file
        self.render_template('libero-project.tcl.j2',
                             escaped_name + '-project.tcl',
                             template_vars)

        # Render the TCL run file
        self.render_template('libero-run.tcl.j2',
                             escaped_name + '-run.tcl',
                             template_vars)

        # Render the Synthesize TCL file
        self.render_template('libero-syn-user.tcl.j2',
                             escaped_name + '-syn-user.tcl',
                             template_vars)

        logger.info("Cores and Libero TCL Scripts generated.")

    def src_file_filter(self, f):
        file_types = {
            'verilogSource': '-hdl_source {',
            'systemVerilogSource': '-hdl_source {',
            'vhdlSource': "-hdl_source {",
            'PDC': '-io_pdc {',
            'SDC': '-sdc {',
            'FPPDC': '-fp_pdc {',
        }
        _file_type = f.file_type.split('-')[0]
        if _file_type in file_types:
            # Do not return library files here
            if f.logical_name:
                return ''
            return file_types[_file_type] + f.name
        return ''

    def tcl_file_filter(self, f):
        file_types = {
            'tclSource': 'source ',
        }
        _file_type = f.file_type.split('-')[0]
        if _file_type in file_types:
            return file_types[_file_type] + f.name
        return ''

    def constraint_file_filter(self, f, type="ALL"):
        file_types = {
            'PDC': 'constraint/io/',
            'SDC': 'constraint/',
            'FPPDC': 'constraint/fp/',
        }
        _file_type = f.file_type.split('-')[0]
        if _file_type in file_types:
            filename = f.name.split("/")[-1]
            if type == "ALL":
                return file_types[_file_type] + filename
            elif _file_type == type:
                return file_types[_file_type] + filename
        return ''

    def build_main(self):
        logger.info("Executing Libero TCL Scripts.")
        escaped_name = self.name.replace(".", "_")
        if shutil.which("libero"):
            self._run_tool('libero', ['SCRIPT:' + escaped_name + '-run.tcl'])
        else:
            filePath = os.path.join(Path(self.work_root).relative_to(
                os.getcwd()), escaped_name + '-run.tcl')
            logger.warn(
                "Libero not found on path, execute manually the script \"" + filePath + "\"")

    def run_main(self):
        pass
<|endoftext|>