<commit_before>from django.db import models
from django.db.models import Q
import operator

from msgvis.apps.base.models import MappedValuesQuerySet
from msgvis.apps.corpus import models as corpus_models
from msgvis.apps.dimensions import registry

MAX_CATEGORICAL_LEVELS = 10

def find_messages(queryset):
    """If the given queryset is actually a :class:`.Dataset` model, get its messages queryset."""
    if isinstance(queryset, corpus_models.Dataset):
        queryset = queryset.message_set.all()
    return queryset

def levels_or(field_name, domain):
    filter_ors = []
    for level in domain:
        if level is None or level == "":
            filter_ors.append((field_name + "__isnull", True))
        else:
            filter_ors.append((field_name, level))

    return reduce(operator.or_, [Q(x) for x in filter_ors])

class DataTable(object):
    """
    This class knows how to calculate appropriate visualization data
    for a given pair of dimensions.
    """

    def __init__(self, primary_dimension, secondary_dimension=None):
        """
        Construct a DataTable for one or two dimensions.

        Dimensions may be string dimension keys or
        :class:`msgvis.apps.dimensions.models.CategoricalDimension` objects.

        :type primary_dimension: registry.models.CategoricalDimension
        :type secondary_dimension: registry.models.CategoricalDimension

        :return:
        """

        # Look up the dimensions if needed
        if isinstance(primary_dimension, basestring):
            primary_dimension = registry.get_dimension(primary_dimension)

        if secondary_dimension is not None and isinstance(secondary_dimension, basestring):
            secondary_dimension = registry.get_dimension(secondary_dimension)

        self.primary_dimension = primary_dimension
        self.secondary_dimension = secondary_dimension
        self.mode = "default"

    def set_mode(self, mode):
        self.mode = mode

    def render(self, queryset, desired_primary_bins=None, desired_secondary_bins=None):
        """
        Given a set of messages (already filtered as necessary),
        calculate the data table.

        Optionally, a number of primary and secondary bins may be given.

        The result is a list of dictionaries. Each
        dictionary contains a key for each dimension
        and a value key for the count.
        """

        # Type checking
        queryset = find_messages(queryset)

        # Filter out null time
        queryset = queryset.exclude(time__isnull=True)

        if not self.secondary_dimension:
            # If there is only one dimension, we should be able to fall back
            # on that dimension's group_by() implementation.
            queryset = self.primary_dimension.group_by(queryset,
                                                       grouping_key=self.primary_dimension.key,
                                                       bins=desired_primary_bins)

            return queryset.annotate(value=models.Count('id'))

        else:
            # Now it gets nasty...
            primary_group = self.primary_dimension.get_grouping_expression(queryset,
                                                                           bins=desired_primary_bins)

            secondary_group = self.secondary_dimension.get_grouping_expression(queryset,
                                                                               bins=desired_secondary_bins)

            if primary_group is None or secondary_group is None:
                # There is no data to group
                return queryset.values()

            queryset, internal_primary_key = self.primary_dimension.select_grouping_expression(
                queryset,
                primary_group)

            queryset, internal_secondary_key = self.secondary_dimension.select_grouping_expression(
                queryset,
                secondary_group)

            # Group the data
            queryset = queryset.values(internal_primary_key,
                                       internal_secondary_key)

            # Count the messages
            queryset = queryset.annotate(value=models.Count('id'))

            # We may need to remap some fields
            mapping = {}
            if internal_primary_key != self.primary_dimension.key:
                mapping[internal_primary_key] = self.primary_dimension.key
            if internal_secondary_key != self.secondary_dimension.key:
                mapping[internal_secondary_key] = self.secondary_dimension.key

            if len(mapping) > 0:
                return MappedValuesQuerySet.create_from(queryset, mapping)
            else:
                return queryset

    def render_others(self, queryset_for_others, domains):
        """
        Given a set of messages (already filtered as necessary),
        calculate the data table.

        Optionally, a number of primary and secondary bins may be given.

        The result is a list of dictionaries. Each
        dictionary contains a key for each dimension
        and a value key for the count.
        """

        # Type checking
        queryset_for_others = find_messages(queryset_for_others)

        # Filter out null time
        queryset_for_others = queryset_for_others.exclude(time__isnull=True)

        if not self.secondary_dimension and self.primary_dimension.is_categorical():
            # If there is only one dimension, we should be able to fall back
            # on that dimension's group_by() implementation.

            queryset_for_others = queryset_for_others.exclude(levels_or(self.primary_dimension.field_name, domains[self.primary_dimension.key]))
            domains[self.primary_dimension.key].append(u'others')

            return [{self.primary_dimension.key: u'others', 'value': queryset_for_others.count()}]
    """
        else:
            if self.primary_dimension.is_categorical() and self.secondary_dimension.is_categorical():

            # Now it gets nasty...
            primary_group = self.primary_dimension.get_grouping_expression(queryset,
                                                                           bins=desired_primary_bins)

            secondary_group = self.secondary_dimension.get_grouping_expression(queryset,
                                                                               bins=desired_secondary_bins)

            if primary_group is None or secondary_group is None:
                # There is no data to group
                return queryset.values()

            queryset, internal_primary_key = self.primary_dimension.select_grouping_expression(
                queryset,
                primary_group)

            queryset, internal_secondary_key = self.secondary_dimension.select_grouping_expression(
                queryset,
                secondary_group)

            # Group the data
            queryset = queryset.values(internal_primary_key,
                                       internal_secondary_key)

            # Count the messages
            queryset = queryset.annotate(value=models.Count('id'))

            # We may need to remap some fields
            mapping = {}
            if internal_primary_key != self.primary_dimension.key:
                mapping[internal_primary_key] = self.primary_dimension.key
            if internal_secondary_key != self.secondary_dimension.key:
                mapping[internal_secondary_key] = self.secondary_dimension.key

            if len(mapping) > 0:
                return MappedValuesQuerySet.create_from(queryset, mapping)
            else:
                return queryset
    """

    def domain(self, dimension, queryset, filter=None, exclude=None, desired_bins=None):
        """Return the sorted levels in this dimension"""
        if filter is not None:
            queryset = dimension.filter(queryset, **filter)

        if exclude is not None:
            queryset = dimension.exclude(queryset, **exclude)

        queryset = queryset.exclude(time__isnull=True)
        domain = dimension.get_domain(queryset, bins=desired_bins)
        labels = dimension.get_domain_labels(domain)

        return domain, labels

    def filter_search_key(self, domain, labels, search_key):
        match_domain = []
        match_labels = []
        for i in range(len(domain)):
            level = domain[i]
            if level is not None and level.lower().find(search_key.lower()) != -1 :
                match_domain.append(level)

                if labels is not None:
                    match_labels.append(labels[i])

        return match_domain, match_labels

    def generate(self, dataset, filters=None, exclude=None, page_size=30, page=None, search_key=None):
        """
        Generate a complete data table response.

        This includes 'table', which provides the non-zero
        message frequency for each combination of primary and secondary dimension values,
        respecting the filters.

        It also includes 'domains', which provides, for both
        primary and secondary dimensions, the levels of the
        dimension irrespective of filters (except on those actual dimensions).
        """

        queryset = dataset.message_set.all()

        # Filter the data (look for filters on the primary/secondary dimensions at the same time
        primary_filter = None
        secondary_filter = None
        if filters is not None:
            for filter in filters:
                dimension = filter['dimension']
                queryset = dimension.filter(queryset, **filter)

                if dimension == self.primary_dimension:
                    primary_filter = filter
                if dimension == self.secondary_dimension:
                    secondary_filter = filter

        primary_exclude = None
        secondary_exclude = None
        if exclude is not None:
            for exclude_filter in exclude:
                dimension = exclude_filter['dimension']
                queryset = dimension.exclude(queryset, **exclude_filter)

                if dimension == self.primary_dimension:
                    primary_exclude = exclude_filter
                if dimension == self.secondary_dimension:
                    secondary_exclude = exclude_filter

        table = None
        domains = {}
        domain_labels = {}
        max_page = None
        queryset_for_others = None

        # Include the domains for primary and (secondary) dimensions
        domain, labels = self.domain(self.primary_dimension,
                                     dataset.message_set.all(),
                                     primary_filter, primary_exclude)

        # paging the first dimension, this is for the filter distribution
        if primary_filter is None and self.secondary_dimension is None and page is not None:

            if search_key is not None:
                domain, labels = self.filter_search_key(domain, labels, search_key)
            start = (page - 1) * page_size
            end = min(start + page_size, len(domain))
            max_page = (len(domain) / page_size) + 1

            # no level left
            if len(domain) == 0 or start > len(domain):
                return None

            domain = domain[start:end]
            if labels is not None:
                labels = labels[start:end]

            queryset = queryset.filter(levels_or(self.primary_dimension.field_name, domain))
        else:
            if self.mode == 'enable_others' and self.primary_dimension.is_categorical() and len(domain) > MAX_CATEGORICAL_LEVELS:
                domain = domain[:MAX_CATEGORICAL_LEVELS]

                queryset_for_others = queryset
                queryset = queryset.filter(levels_or(self.primary_dimension.field_name, domain))

                if labels is not None:
                    labels = labels[:MAX_CATEGORICAL_LEVELS]

        domains[self.primary_dimension.key] = domain
        if labels is not None:
            domain_labels[self.primary_dimension.key] = labels

        if self.secondary_dimension:
            domain, labels = self.domain(self.secondary_dimension,
                                         dataset.message_set.all(),
                                         secondary_filter, secondary_exclude)

            if self.mode == 'enable_others' and self.secondary_dimension.is_categorical() and len(domain) > MAX_CATEGORICAL_LEVELS:
                domain = domain[:MAX_CATEGORICAL_LEVELS]

                if queryset_for_others is None:
                    queryset_for_others = queryset
                queryset = queryset.filter(levels_or(self.secondary_dimension.field_name, domain))

                if labels is not None:
                    labels = labels[:MAX_CATEGORICAL_LEVELS]

            domains[self.secondary_dimension.key] = domain
            if labels is not None:
                domain_labels[self.secondary_dimension.key] = labels

        # Render a table
        table = self.render(queryset)

        if self.mode == "enable_others":
            # adding others to the results
            table_for_others = self.render_others(queryset_for_others, domains)
            table = list(table)
            table.extend(table_for_others)

        results = {
            'table': table,
            'domains': domains,
            'domain_labels': domain_labels
        }
        if max_page is not None:
            results['max_page'] = max_page

        return results
<commit_msg>got others work for both categorical dimensions<commit_after>from django.db import models
from django.db.models import Q
import operator

from msgvis.apps.base.models import MappedValuesQuerySet
from msgvis.apps.corpus import models as corpus_models
from msgvis.apps.dimensions import registry

MAX_CATEGORICAL_LEVELS = 10

def find_messages(queryset):
    """If the given queryset is actually a :class:`.Dataset` model, get its messages queryset."""
    if isinstance(queryset, corpus_models.Dataset):
        queryset = queryset.message_set.all()
    return queryset

def levels_or(field_name, domain):
    filter_ors = []
    for level in domain:
        if level is None or level == "":
            filter_ors.append((field_name + "__isnull", True))
        else:
            filter_ors.append((field_name, level))

    return reduce(operator.or_, [Q(x) for x in filter_ors])

class DataTable(object):
    """
    This class knows how to calculate appropriate visualization data
    for a given pair of dimensions.
    """

    def __init__(self, primary_dimension, secondary_dimension=None):
        """
        Construct a DataTable for one or two dimensions.

        Dimensions may be string dimension keys or
        :class:`msgvis.apps.dimensions.models.CategoricalDimension` objects.

        :type primary_dimension: registry.models.CategoricalDimension
        :type secondary_dimension: registry.models.CategoricalDimension

        :return:
        """

        # Look up the dimensions if needed
        if isinstance(primary_dimension, basestring):
            primary_dimension = registry.get_dimension(primary_dimension)

        if secondary_dimension is not None and isinstance(secondary_dimension, basestring):
            secondary_dimension = registry.get_dimension(secondary_dimension)

        self.primary_dimension = primary_dimension
        self.secondary_dimension = secondary_dimension
        self.mode = "default"

    def set_mode(self, mode):
        self.mode = mode

    def render(self, queryset, desired_primary_bins=None, desired_secondary_bins=None):
        """
        Given a set of messages (already filtered as necessary),
        calculate the data table.

        Optionally, a number of primary and secondary bins may be given.

        The result is a list of dictionaries. Each
        dictionary contains a key for each dimension
        and a value key for the count.
        """

        # Type checking
        queryset = find_messages(queryset)

        # Filter out null time
        queryset = queryset.exclude(time__isnull=True)

        if not self.secondary_dimension:
            # If there is only one dimension, we should be able to fall back
            # on that dimension's group_by() implementation.
            queryset = self.primary_dimension.group_by(queryset,
                                                       grouping_key=self.primary_dimension.key,
                                                       bins=desired_primary_bins)

            return queryset.annotate(value=models.Count('id'))

        else:
            # Now it gets nasty...
            primary_group = self.primary_dimension.get_grouping_expression(queryset,
                                                                           bins=desired_primary_bins)

            secondary_group = self.secondary_dimension.get_grouping_expression(queryset,
                                                                               bins=desired_secondary_bins)

            if primary_group is None or secondary_group is None:
                # There is no data to group
                return queryset.values()

            queryset, internal_primary_key = self.primary_dimension.select_grouping_expression(
                queryset,
                primary_group)

            queryset, internal_secondary_key = self.secondary_dimension.select_grouping_expression(
                queryset,
                secondary_group)

            # Group the data
            queryset = queryset.values(internal_primary_key,
                                       internal_secondary_key)

            # Count the messages
            queryset = queryset.annotate(value=models.Count('id'))

            # We may need to remap some fields
            mapping = {}
            if internal_primary_key != self.primary_dimension.key:
                mapping[internal_primary_key] = self.primary_dimension.key
            if internal_secondary_key != self.secondary_dimension.key:
                mapping[internal_secondary_key] = self.secondary_dimension.key

            if len(mapping) > 0:
                return MappedValuesQuerySet.create_from(queryset, mapping)
            else:
                return queryset

    def render_others(self, queryset_for_others, domains):
        """
        Given a set of messages (already filtered as necessary),
        calculate the data table.

        Optionally, a number of primary and secondary bins may be given.

        The result is a list of dictionaries. Each
        dictionary contains a key for each dimension
        and a value key for the count.
        """

        # Type checking
        queryset_for_others = find_messages(queryset_for_others)

        # Filter out null time
        queryset_for_others = queryset_for_others.exclude(time__isnull=True)

        if not self.secondary_dimension and self.primary_dimension.is_categorical():
            # If there is only one dimension, we should be able to fall back
            # on that dimension's group_by() implementation.

            queryset_for_others = queryset_for_others.exclude(levels_or(self.primary_dimension.field_name, domains[self.primary_dimension.key]))
            domains[self.primary_dimension.key].append(u'others')

            return [{self.primary_dimension.key: u'others', 'value': queryset_for_others.count()}]

        elif self.secondary_dimension:
            if self.primary_dimension.is_categorical() and self.secondary_dimension.is_categorical():
                original_queryset = queryset_for_others
                others_results = []
                domains[self.primary_dimension.key].append(u'others')
                domains[self.secondary_dimension.key].append(u'others')

                # primary others x secondary others
                queryset_for_others = queryset_for_others.exclude(levels_or(self.primary_dimension.field_name, domains[self.primary_dimension.key]))
                queryset_for_others = queryset_for_others.exclude(levels_or(self.secondary_dimension.field_name, domains[self.secondary_dimension.key]))

                others_results.append({self.primary_dimension.key: u'others',
                                       self.secondary_dimension.key: u'others',
                                       'value': queryset_for_others.count()})

                # primary top ones x secondary others
                queryset_for_others = original_queryset
                queryset_for_others = queryset_for_others.exclude(levels_or(self.secondary_dimension.field_name, domains[self.secondary_dimension.key]))

                queryset_for_others = self.primary_dimension.group_by(queryset_for_others,
                                                                      grouping_key=self.primary_dimension.key)

                queryset_for_others = queryset_for_others.annotate(value=models.Count('id'))
                primary_top_results = list(queryset_for_others)
                for r in primary_top_results:
                    r[self.secondary_dimension.key] = u'others'
                others_results.extend(primary_top_results)

                # primary others x secondary top ones
                queryset_for_others = original_queryset
                queryset_for_others = queryset_for_others.exclude(levels_or(self.primary_dimension.field_name, domains[self.primary_dimension.key]))

                queryset_for_others = self.secondary_dimension.group_by(queryset_for_others,
                                                                        grouping_key=self.secondary_dimension.key)

                queryset_for_others = queryset_for_others.annotate(value=models.Count('id'))
                secondary_top_results = list(queryset_for_others)
                for r in secondary_top_results:
                    r[self.primary_dimension.key] = u'others'
                others_results.extend(secondary_top_results)

                return others_results

        """

            # Now it gets nasty...
            primary_group = self.primary_dimension.get_grouping_expression(queryset,
                                                                           bins=desired_primary_bins)

            secondary_group = self.secondary_dimension.get_grouping_expression(queryset,
                                                                               bins=desired_secondary_bins)

            if primary_group is None or secondary_group is None:
                # There is no data to group
                return queryset.values()

            queryset, internal_primary_key = self.primary_dimension.select_grouping_expression(
                queryset,
                primary_group)

            queryset, internal_secondary_key = self.secondary_dimension.select_grouping_expression(
                queryset,
                secondary_group)

            # Group the data
            queryset = queryset.values(internal_primary_key,
                                       internal_secondary_key)

            # Count the messages
            queryset = queryset.annotate(value=models.Count('id'))

            # We may need to remap some fields
            mapping = {}
            if internal_primary_key != self.primary_dimension.key:
                mapping[internal_primary_key] = self.primary_dimension.key
            if internal_secondary_key != self.secondary_dimension.key:
                mapping[internal_secondary_key] = self.secondary_dimension.key

            if len(mapping) > 0:
                return MappedValuesQuerySet.create_from(queryset, mapping)
            else:
                return queryset
    """

    def domain(self, dimension, queryset, filter=None, exclude=None, desired_bins=None):
        """Return the sorted levels in this dimension"""
        if filter is not None:
            queryset = dimension.filter(queryset, **filter)

        if exclude is not None:
            queryset = dimension.exclude(queryset, **exclude)

        queryset = queryset.exclude(time__isnull=True)
        domain = dimension.get_domain(queryset, bins=desired_bins)
        labels = dimension.get_domain_labels(domain)

        return domain, labels

    def filter_search_key(self, domain, labels, search_key):
        match_domain = []
        match_labels = []
        for i in range(len(domain)):
            level = domain[i]
            if level is not None and level.lower().find(search_key.lower()) != -1 :
                match_domain.append(level)

                if labels is not None:
                    match_labels.append(labels[i])

        return match_domain, match_labels

    def generate(self, dataset, filters=None, exclude=None, page_size=30, page=None, search_key=None):
        """
        Generate a complete data table response.

        This includes 'table', which provides the non-zero
        message frequency for each combination of primary and secondary dimension values,
        respecting the filters.

        It also includes 'domains', which provides, for both
        primary and secondary dimensions, the levels of the
        dimension irrespective of filters (except on those actual dimensions).
        """

        queryset = dataset.message_set.all()

        # Filter the data (look for filters on the primary/secondary dimensions at the same time
        primary_filter = None
        secondary_filter = None
        if filters is not None:
            for filter in filters:
                dimension = filter['dimension']
                queryset = dimension.filter(queryset, **filter)

                if dimension == self.primary_dimension:
                    primary_filter = filter
                if dimension == self.secondary_dimension:
                    secondary_filter = filter

        primary_exclude = None
        secondary_exclude = None
        if exclude is not None:
            for exclude_filter in exclude:
                dimension = exclude_filter['dimension']
                queryset = dimension.exclude(queryset, **exclude_filter)

                if dimension == self.primary_dimension:
                    primary_exclude = exclude_filter
                if dimension == self.secondary_dimension:
                    secondary_exclude = exclude_filter

        table = None
        domains = {}
        domain_labels = {}
        max_page = None
        queryset_for_others = None

        # Include the domains for primary and (secondary) dimensions
        domain, labels = self.domain(self.primary_dimension,
                                     dataset.message_set.all(),
                                     primary_filter, primary_exclude)

        # paging the first dimension, this is for the filter distribution
        if primary_filter is None and self.secondary_dimension is None and page is not None:

            if search_key is not None:
                domain, labels = self.filter_search_key(domain, labels, search_key)
            start = (page - 1) * page_size
            end = min(start + page_size, len(domain))
            max_page = (len(domain) / page_size) + 1

            # no level left
            if len(domain) == 0 or start > len(domain):
                return None

            domain = domain[start:end]
            if labels is not None:
                labels = labels[start:end]

            queryset = queryset.filter(levels_or(self.primary_dimension.field_name, domain))
        else:
            if self.mode == 'enable_others' and self.primary_dimension.is_categorical() and len(domain) > MAX_CATEGORICAL_LEVELS:
                domain = domain[:MAX_CATEGORICAL_LEVELS]

                queryset_for_others = queryset
                queryset = queryset.filter(levels_or(self.primary_dimension.field_name, domain))

                if labels is not None:
                    labels = labels[:MAX_CATEGORICAL_LEVELS]

        domains[self.primary_dimension.key] = domain
        if labels is not None:
            domain_labels[self.primary_dimension.key] = labels

        if self.secondary_dimension:
            domain, labels = self.domain(self.secondary_dimension,
                                         dataset.message_set.all(),
                                         secondary_filter, secondary_exclude)

            if self.mode == 'enable_others' and self.secondary_dimension.is_categorical() and len(domain) > MAX_CATEGORICAL_LEVELS:
                domain = domain[:MAX_CATEGORICAL_LEVELS]

                if queryset_for_others is None:
                    queryset_for_others = queryset
                queryset = queryset.filter(levels_or(self.secondary_dimension.field_name, domain))

                if labels is not None:
                    labels = labels[:MAX_CATEGORICAL_LEVELS]

            domains[self.secondary_dimension.key] = domain
            if labels is not None:
                domain_labels[self.secondary_dimension.key] = labels

        # Render a table
        table = self.render(queryset)

        if self.mode == "enable_others":
            # adding others to the results
            table_for_others = self.render_others(queryset_for_others, domains)
            table = list(table)
            table.extend(table_for_others)

        results = {
            'table': table,
            'domains': domains,
            'domain_labels': domain_labels
        }
        if max_page is not None:
            results['max_page'] = max_page

        return results
<|endoftext|>


<commit_before># Copyright (c) 2006-2007 The Regents of The University of Michigan
# Copyright (c) 2009 Advanced Micro Devices, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Brad Beckmann

import m5
from m5.objects import *
from m5.defines import buildEnv
from m5.util import addToPath
from Ruby import create_topology

#
# Note: the cache latency is only used by the sequencer on fast path hits
#
class Cache(RubyCache):
    latency = 3

def define_options(parser):
    return

def create_system(options, system, dma_ports, ruby_system):

    if buildEnv['PROTOCOL'] != 'Network_test':
        panic("This script requires the Network_test protocol to be built.")

    cpu_sequencers = []

    #
    # The Garnet tester protocol does not support fs nor dma
    #
    assert(dma_ports == [])

    #
    # The ruby network creation expects the list of nodes in the system to be
    # consistent with the NetDest list.  Therefore the l1 controller nodes must be
    # listed before the directory nodes and directory nodes before dma nodes, etc.
    #
    l1_cntrl_nodes = []
    dir_cntrl_nodes = []

    #
    # Must create the individual controllers before the network to ensure the
    # controller constructors are called before the network constructor
    #

    for i in xrange(options.num_cpus):
        #
        # First create the Ruby objects associated with this cpu
        # Only one cache exists for this protocol, so by default use the L1D
        # config parameters.
        #
        cache = Cache(size = options.l1d_size,
                      assoc = options.l1d_assoc)

        #
        # Only one unified L1 cache exists.  Can cache instructions and data.
        #
        l1_cntrl = L1Cache_Controller(version = i,
                                      cacheMemory = cache,
                                      ruby_system = ruby_system)

        cpu_seq = RubySequencer(icache = cache,
                                dcache = cache,
                                using_network_tester = True,
                                ruby_system = ruby_system)

        l1_cntrl.sequencer = cpu_seq
        exec("ruby_system.l1_cntrl%d = l1_cntrl" % i)

        #
        # Add controllers and sequencers to the appropriate lists
        #
        cpu_sequencers.append(cpu_seq)
        l1_cntrl_nodes.append(l1_cntrl)

    phys_mem_size = sum(map(lambda r: r.size(), system.mem_ranges))
    assert(phys_mem_size % options.num_dirs == 0)
    mem_module_size = phys_mem_size / options.num_dirs

    # Run each of the ruby memory controllers at a ratio of the frequency of
    # the ruby system.
    # clk_divider value is a fix to pass regression.
    ruby_system.memctrl_clk_domain = DerivedClockDomain(
                                          clk_domain=ruby_system.clk_domain,
                                          clk_divider=3)

    for i in xrange(options.num_dirs):
        #
        # Create the Ruby objects associated with the directory controller
        #

        mem_cntrl = RubyMemoryControl(
                              clk_domain = ruby_system.memctrl_clk_domain,
                              version = i,
                              ruby_system = ruby_system)

        dir_size = MemorySize('0B')
        dir_size.value = mem_module_size

        dir_cntrl = Directory_Controller(version = i,
                                         directory = \
                                         RubyDirectoryMemory(version = i,
                                                             size = dir_size),
                                         memBuffer = mem_cntrl,
                                         ruby_system = ruby_system)

        exec("ruby_system.dir_cntrl%d = dir_cntrl" % i)
        dir_cntrl_nodes.append(dir_cntrl)

    all_cntrls = l1_cntrl_nodes + dir_cntrl_nodes
    topology = create_topology(all_cntrls, options)
    return (cpu_sequencers, dir_cntrl_nodes, topology)
<commit_msg>config: ruby: remove memory controller from network test It is not in use and not required as such. (transplanted from 43bc3665c52baec3821a8e310eece545f29d3415)<commit_after># Copyright (c) 2006-2007 The Regents of The University of Michigan
# Copyright (c) 2009 Advanced Micro Devices, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Brad Beckmann

import m5
from m5.objects import *
from m5.defines import buildEnv
from m5.util import addToPath
from Ruby import create_topology

#
# Note: the cache latency is only used by the sequencer on fast path hits
#
class Cache(RubyCache):
    latency = 3

def define_options(parser):
    return

def create_system(options, system, dma_ports, ruby_system):

    if buildEnv['PROTOCOL'] != 'Network_test':
        panic("This script requires the Network_test protocol to be built.")

    cpu_sequencers = []

    #
    # The Garnet tester protocol does not support fs nor dma
    #
    assert(dma_ports == [])

    #
    # The ruby network creation expects the list of nodes in the system to be
    # consistent with the NetDest list.  Therefore the l1 controller nodes must be
    # listed before the directory nodes and directory nodes before dma nodes, etc.
    #
    l1_cntrl_nodes = []
    dir_cntrl_nodes = []

    #
    # Must create the individual controllers before the network to ensure the
    # controller constructors are called before the network constructor
    #

    for i in xrange(options.num_cpus):
        #
        # First create the Ruby objects associated with this cpu
        # Only one cache exists for this protocol, so by default use the L1D
        # config parameters.
        #
        cache = Cache(size = options.l1d_size,
                      assoc = options.l1d_assoc)

        #
        # Only one unified L1 cache exists.  Can cache instructions and data.
        #
        l1_cntrl = L1Cache_Controller(version = i,
                                      cacheMemory = cache,
                                      ruby_system = ruby_system)

        cpu_seq = RubySequencer(icache = cache,
                                dcache = cache,
                                using_network_tester = True,
                                ruby_system = ruby_system)

        l1_cntrl.sequencer = cpu_seq
        exec("ruby_system.l1_cntrl%d = l1_cntrl" % i)

        #
        # Add controllers and sequencers to the appropriate lists
        #
        cpu_sequencers.append(cpu_seq)
        l1_cntrl_nodes.append(l1_cntrl)

    phys_mem_size = sum(map(lambda r: r.size(), system.mem_ranges))
    assert(phys_mem_size % options.num_dirs == 0)
    mem_module_size = phys_mem_size / options.num_dirs

    for i in xrange(options.num_dirs):
        dir_size = MemorySize('0B')
        dir_size.value = mem_module_size

        dir_cntrl = Directory_Controller(version = i,
                                         directory = \
                                         RubyDirectoryMemory(version = i,
                                                             size = dir_size),
                                         ruby_system = ruby_system)

        exec("ruby_system.dir_cntrl%d = dir_cntrl" % i)
        dir_cntrl_nodes.append(dir_cntrl)

    all_cntrls = l1_cntrl_nodes + dir_cntrl_nodes
    topology = create_topology(all_cntrls, options)
    return (cpu_sequencers, dir_cntrl_nodes, topology)
<|endoftext|>


<commit_before>"""
Models for the organization application
"""

from django.contrib.auth.models import User
from django.core.mail import EmailMessage
from django.db import models
from django.template.loader import render_to_string

from muckrock.settings import MONTHLY_REQUESTS

from datetime import datetime
import logging
import stripe

logger = logging.getLogger(__name__)

class Organization(models.Model):
    """Orginization to allow pooled requests and collaboration"""

    name = models.CharField(max_length=255, unique=True)
    slug = models.SlugField(max_length=255, unique=True)
    owner = models.ForeignKey(User)
    date_update = models.DateField()
    num_requests = models.IntegerField(default=0)
    max_users = models.IntegerField(default=50)
    monthly_cost = models.IntegerField(default=45000)
    monthly_requests = models.IntegerField(default=MONTHLY_REQUESTS.get('org', 0))
    stripe_id = models.CharField(max_length=255, blank=True)
    active = models.BooleanField(default=False)

    def __unicode__(self):
        return self.name

    @models.permalink
    def get_absolute_url(self):
        """The url for this object"""
        return ('org-detail', [], {'slug': self.slug})

    def save(self, *args, **kwargs):
        """Save the Organization"""
        # TODO: add custom save code so that the Stripe plan associated with this
        # organization is updated with the cost, just in case the cost is changed
        # from the backend.
        super(Organization, self).save(*args, **kwargs)

    def get_requests(self):
        """Get the number of requests left for this month"""
        not_this_month = self.date_update.month != datetime.now().month
        not_this_year = self.date_update.year != datetime.now().year
        if not_this_month or not_this_year and self.active:
            # update requests if they have not yet been updated this month
            self.date_update = datetime.now()
            self.num_requests = self.monthly_requests
            self.save()
        return self.num_requests

    def is_owned_by(self, user):
        """Answers whether the passed user owns the org"""
        return self.owner == user

    def is_active(self):
        """Is this organization active?"""
        return self.active

    def add_member(self, user):
        """Add a user to this organization"""
        profile = user.get_profile()
        if not profile.is_member_of(self): # doesn't update if already a member
            profile.organization = self
            profile.save()
            # send an email notifying the user
            msg = render_to_string('text/organization/add_member.txt', {
                'member_name': user.first_name,
                'organization_name': self.name,
                'organization_owner': self.owner.get_full_name(),
                'organization_link': self.get_absolute_url()
            })
            email = EmailMessage(
                subject='[MuckRock] You were added to an organization',
                body=msg,
                from_email='info@muckrock.com',
                to=[user.email],
                bcc=['diagnostics@muckrock.com']
            )
            email.send(fail_silently=False)
            logger.info('%s was added as a member of the %s organization', user.username, self.name)
        return

    def remove_member(self, user):
        """Remove a user (who isn't the owner) from this organization"""
        if not self.is_owned_by(user):
            profile = user.get_profile()
            profile.organization = None
            profile.save()
            # send an email notifying the user
            msg = render_to_string('text/organization/remove_member.txt', {
                'member_name': user.first_name,
                'organization_name': self.name,
                'organization_owner': self.owner.get_full_name(),
            })
            email = EmailMessage(
                subject='[MuckRock] You were removed from an organization',
                body=msg,
                from_email='info@muckrock.com',
                to=[user.email],
                bcc=['diagnostics@muckrock.com']
            )
            email.send(fail_silently=False)
            logger.info('%s was removed as a member of the %s organization',
                user.username, self.name)
        return

    def create_plan(self):
        """Creates an organization-specific Stripe plan"""
        if not self.stripe_id:
            plan_name = self.name + ' Plan'
            plan_id = self.slug + '-org-plan'
            plan = stripe.Plan.create(
                amount=self.monthly_cost,
                interval='month',
                name=plan_name,
                currency='usd',
                id=plan_id)
            self.stripe_id = plan.id
            self.save()
        else:
            raise ValueError('This organization already has an associated plan.')
        return

    def delete_plan(self):
        """Deletes this organization's specific Stripe plan"""
        if self.stripe_id:
            plan = stripe.Plan.retrieve(self.stripe_id)
            plan.delete()
            self.stripe_id = ''
            self.save()
        else:
            raise ValueError('This organization has no associated plan to cancel.')
        return

    def update_plan(self):
        """
        Deletes and recreates an organization's plan.
        Plans must be deleted and recreated because Stripe prohibits plans
        from updating any information except their name.
        """
        if self.stripe_id:
            self.delete_plan()
            self.create_plan()
            new_plan = stripe.Plan.retrieve(self.stripe_id)
            customer = stripe.Customer.retrieve(self.owner.get_profile().stripe_id)
            customer.update_subscription(plan=new_plan.id)
            customer.save()
        else:
            raise ValueError('This organization has no associated plan to update.')
        return

    def start_subscription(self):
        """Subscribes the owner to this org's plan"""
        profile = self.owner.get_profile()
        org_plan = stripe.Plan.retrieve(self.stripe_id)
        customer = stripe.Customer.retrieve(profile.stripe_id)
        customer.update_subscription(plan=org_plan.name)
        customer.save()
        # if the owner has a pro account, downgrade him to a community account
        if profile.acct_type == 'pro':
            profile.acct_type = 'community'
            profile.save()
        self.active = True
        self.save()
        return

    def pause_subscription(self):
        """Cancels the owner's subscription to this org's plan"""
        customer = stripe.Customer.retrieve(self.stripe_id)
        customer.cancel_subscription()
        customer.save()
        self.active = False
        self.save()
        return
<commit_msg>updating plan on save caused infinite loop, so i took it out<commit_after>"""
Models for the organization application
"""

from django.contrib.auth.models import User
from django.core.mail import EmailMessage
from django.db import models
from django.template.loader import render_to_string

from muckrock.settings import MONTHLY_REQUESTS

from datetime import datetime
import logging
import stripe

logger = logging.getLogger(__name__)

class Organization(models.Model):
    """Orginization to allow pooled requests and collaboration"""

    name = models.CharField(max_length=255, unique=True)
    slug = models.SlugField(max_length=255, unique=True)
    owner = models.ForeignKey(User)
    date_update = models.DateField()
    num_requests = models.IntegerField(default=0)
    max_users = models.IntegerField(default=50)
    monthly_cost = models.IntegerField(default=45000)
    monthly_requests = models.IntegerField(default=MONTHLY_REQUESTS.get('org', 0))
    stripe_id = models.CharField(max_length=255, blank=True)
    active = models.BooleanField(default=False)

    def __unicode__(self):
        return self.name

    @models.permalink
    def get_absolute_url(self):
        """The url for this object"""
        return ('org-detail', [], {'slug': self.slug})

    def get_requests(self):
        """Get the number of requests left for this month"""
        not_this_month = self.date_update.month != datetime.now().month
        not_this_year = self.date_update.year != datetime.now().year
        if not_this_month or not_this_year and self.active:
            # update requests if they have not yet been updated this month
            self.date_update = datetime.now()
            self.num_requests = self.monthly_requests
            self.save()
        return self.num_requests

    def is_owned_by(self, user):
        """Answers whether the passed user owns the org"""
        return self.owner == user

    def is_active(self):
        """Is this organization active?"""
        return self.active

    def add_member(self, user):
        """Add a user to this organization"""
        profile = user.get_profile()
        if not profile.is_member_of(self): # doesn't update if already a member
            profile.organization = self
            profile.save()
            # send an email notifying the user
            msg = render_to_string('text/organization/add_member.txt', {
                'member_name': user.first_name,
                'organization_name': self.name,
                'organization_owner': self.owner.get_full_name(),
                'organization_link': self.get_absolute_url()
            })
            email = EmailMessage(
                subject='[MuckRock] You were added to an organization',
                body=msg,
                from_email='info@muckrock.com',
                to=[user.email],
                bcc=['diagnostics@muckrock.com']
            )
            email.send(fail_silently=False)
            logger.info('%s was added as a member of the %s organization', user.username, self.name)
        return

    def remove_member(self, user):
        """Remove a user (who isn't the owner) from this organization"""
        if not self.is_owned_by(user):
            profile = user.get_profile()
            profile.organization = None
            profile.save()
            # send an email notifying the user
            msg = render_to_string('text/organization/remove_member.txt', {
                'member_name': user.first_name,
                'organization_name': self.name,
                'organization_owner': self.owner.get_full_name(),
            })
            email = EmailMessage(
                subject='[MuckRock] You were removed from an organization',
                body=msg,
                from_email='info@muckrock.com',
                to=[user.email],
                bcc=['diagnostics@muckrock.com']
            )
            email.send(fail_silently=False)
            logger.info('%s was removed as a member of the %s organization',
                user.username, self.name)
        return

    def create_plan(self):
        """Creates an organization-specific Stripe plan"""
        if not self.stripe_id:
            plan_name = self.name + ' Plan'
            plan_id = self.slug + '-org-plan'
            plan = stripe.Plan.create(
                amount=self.monthly_cost,
                interval='month',
                name=plan_name,
                currency='usd',
                id=plan_id)
            self.stripe_id = plan.id
            self.save()
        else:
            raise ValueError('This organization already has an associated plan.')
        return

    def delete_plan(self):
        """Deletes this organization's specific Stripe plan"""
        if self.stripe_id:
            plan = stripe.Plan.retrieve(self.stripe_id)
            plan.delete()
            self.stripe_id = ''
            self.save()
        else:
            raise ValueError('This organization has no associated plan to cancel.')
        return

    def update_plan(self):
        """
        Deletes and recreates an organization's plan.
        Plans must be deleted and recreated because Stripe prohibits plans
        from updating any information except their name.
        """
        if self.stripe_id:
            self.delete_plan()
            self.create_plan()
            new_plan = stripe.Plan.retrieve(self.stripe_id)
            customer = stripe.Customer.retrieve(self.owner.get_profile().stripe_id)
            customer.update_subscription(plan=new_plan.id)
            customer.save()
        else:
            raise ValueError('This organization has no associated plan to update.')
        return

    def start_subscription(self):
        """Subscribes the owner to this org's plan"""
        profile = self.owner.get_profile()
        org_plan = stripe.Plan.retrieve(self.stripe_id)
        customer = stripe.Customer.retrieve(profile.stripe_id)
        customer.update_subscription(plan=org_plan.name)
        customer.save()
        # if the owner has a pro account, downgrade him to a community account
        if profile.acct_type == 'pro':
            profile.acct_type = 'community'
            profile.save()
        self.active = True
        self.save()
        return

    def pause_subscription(self):
        """Cancels the owner's subscription to this org's plan"""
        customer = stripe.Customer.retrieve(self.stripe_id)
        customer.cancel_subscription()
        customer.save()
        self.active = False
        self.save()
        return
<|endoftext|>


<commit_before>import os
from time import time
from datetime import datetime, timedelta

import numpy as np
import sys
from keras import Input
from keras.callbacks import TensorBoard, EarlyStopping
from keras.engine import Model
from keras.layers import LSTM, TimeDistributed, Dense, Flatten
from keras.optimizers import Adam
from keras.regularizers import L1L2
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split

from topoml_util import geom_scaler
from topoml_util.slack_send import notify

SCRIPT_VERSION = '1.0.5'
SCRIPT_NAME = os.path.basename(__file__)
TIMESTAMP = str(datetime.now()).replace(':', '.')
SIGNATURE = SCRIPT_NAME + ' ' + TIMESTAMP
TRAINING_DATA_FILE = '../files/neighborhoods/neighborhoods_train.npz'
SCRIPT_START = time()

# Hyperparameters
BATCH_SIZE = int(os.getenv('BATCH_SIZE', 1024))
TRAIN_VALIDATE_SPLIT = float(os.getenv('TRAIN_VALIDATE_SPLIT', 0.1))
REPEAT_DEEP_ARCH = int(os.getenv('REPEAT_DEEP_ARCH', 0))
LSTM_SIZE = int(os.getenv('LSTM_SIZE', 128))
DENSE_SIZE = int(os.getenv('DENSE_SIZE', 64))
EPOCHS = int(os.getenv('EPOCHS', 100))
LEARNING_RATE = float(os.getenv('LEARNING_RATE', 1e-4))
PATIENCE = int(os.getenv('PATIENCE', 16))
RECURRENT_DROPOUT = float(os.getenv('RECURRENT_DROPOUT', 0.5))
GEOM_SCALE = float(os.getenv('GEOM_SCALE', 0))  # If no default or 0: overridden when data is known
OPTIMIZER = Adam(lr=LEARNING_RATE, clipnorm=1.)

# Load data
train_loaded = np.load(TRAINING_DATA_FILE)
train_geoms = train_loaded['input_geoms']
train_labels = train_loaded['above_or_below_median']

# Determine final test mode or standard
if len(sys.argv) > 1 and sys.argv[1] in ['-t', '--test']:
    print('Training in final test mode')
    TEST_DATA_FILE = '../files/neighborhoods/neighborhoods_test.npz'
    test_loaded = np.load(TEST_DATA_FILE)
    test_geoms = test_loaded['input_geoms']
    test_labels = test_loaded['above_or_below_median']
else:
    print('Training in standard training mode')
    # Split the training data in random seen/unseen sets
    train_geoms, test_geoms, train_labels, test_labels = train_test_split(train_geoms, train_labels, test_size=0.1)

# Normalize
geom_scale = GEOM_SCALE or geom_scaler.scale(train_geoms)
train_geoms = geom_scaler.transform(train_geoms, geom_scale)
test_geoms = geom_scaler.transform(test_geoms, geom_scale)  # re-use variance from training

message = '''
running {} with
version: {}                batch size: {}
train/validate split: {}   repeat deep: {}
lstm size: {}              dense size: {}
epochs: {}                 learning rate: {}
geometry scale: {:.3E}     recurrent dropout: {}
patience {}
'''.format(
    SIGNATURE,
    SCRIPT_VERSION,         BATCH_SIZE,
    TRAIN_VALIDATE_SPLIT,   REPEAT_DEEP_ARCH,
    LSTM_SIZE,              DENSE_SIZE,
    EPOCHS,                 LEARNING_RATE,
    geom_scale,             RECURRENT_DROPOUT,
    PATIENCE,
)
print(message)

# Shape determination
geom_max_points, geom_vector_len = train_geoms.shape[1:]
output_size = train_labels.shape[-1]

# Build model
inputs = Input(shape=(geom_max_points, geom_vector_len))
model = LSTM(LSTM_SIZE,
             return_sequences=True,
             # kernel_regularizer=L1L2(l1=0.01, l2=0.01),
             recurrent_dropout=RECURRENT_DROPOUT)(inputs)
model = TimeDistributed(Dense(DENSE_SIZE, activation='relu'))(model)

for layer in range(REPEAT_DEEP_ARCH):
    model = LSTM(LSTM_SIZE, return_sequences=True, recurrent_dropout=RECURRENT_DROPOUT)(model)

model = Dense(DENSE_SIZE, activation='relu')(model)
model = Flatten()(model)
model = Dense(output_size, activation='softmax')(model)

model = Model(inputs=inputs, outputs=model)
model.compile(
    loss='categorical_crossentropy',
    metrics=['accuracy'],
    optimizer=OPTIMIZER),
model.summary()

# Callbacks
callbacks = [
    TensorBoard(log_dir='./tensorboard_log/' + SIGNATURE, write_graph=False),
    # EarlyStopping(patience=PATIENCE, min_delta=0.001)
]

history = model.fit(
    x=train_geoms,
    y=train_labels,
    epochs=EPOCHS,
    batch_size=BATCH_SIZE,
    validation_split=TRAIN_VALIDATE_SPLIT,
    callbacks=callbacks).history

# Run on unseen test data
test_pred = [np.argmax(prediction) for prediction in model.predict(test_geoms)]
test_labels = [np.argmax(label) for label in test_labels]
accuracy = accuracy_score(test_labels, test_pred)

runtime = time() - SCRIPT_START
message = '''
test accuracy of {:f} in {} with
version: {}                batch size {}
train/validate split {}    repeat deep arch {}
lstm size {}               dense size {}
epochs {}                  learning rate {}
geometry scale {:.3E}        recurrent dropout {}
patience {}
'''.format(
    accuracy, timedelta(seconds=runtime),
    SCRIPT_VERSION, BATCH_SIZE,
    TRAIN_VALIDATE_SPLIT, REPEAT_DEEP_ARCH,
    LSTM_SIZE, DENSE_SIZE,
    len(history['val_loss']), LEARNING_RATE,
    geom_scale, RECURRENT_DROPOUT,
    PATIENCE,
)

notify(SIGNATURE, message)
print(SCRIPT_NAME, 'finished successfully with', message)
<commit_msg>smaller hidden dense, lr decay<commit_after>import os
from time import time
from datetime import datetime, timedelta

import numpy as np
import sys
from keras import Input
from keras.callbacks import TensorBoard, EarlyStopping
from keras.engine import Model
from keras.layers import LSTM, TimeDistributed, Dense, Flatten
from keras.optimizers import Adam
from keras.regularizers import L1L2
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split

from topoml_util import geom_scaler
from topoml_util.slack_send import notify

SCRIPT_VERSION = '1.0.6'
SCRIPT_NAME = os.path.basename(__file__)
TIMESTAMP = str(datetime.now()).replace(':', '.')
SIGNATURE = SCRIPT_NAME + ' ' + TIMESTAMP
TRAINING_DATA_FILE = '../files/neighborhoods/neighborhoods_train.npz'
SCRIPT_START = time()

# Hyperparameters
BATCH_SIZE = int(os.getenv('BATCH_SIZE', 1024))
TRAIN_VALIDATE_SPLIT = float(os.getenv('TRAIN_VALIDATE_SPLIT', 0.1))
REPEAT_DEEP_ARCH = int(os.getenv('REPEAT_DEEP_ARCH', 0))
LSTM_SIZE = int(os.getenv('LSTM_SIZE', 128))
DENSE_SIZE = int(os.getenv('DENSE_SIZE', 32))
EPOCHS = int(os.getenv('EPOCHS', 200))
LEARNING_RATE = float(os.getenv('LEARNING_RATE', 1e-4))
PATIENCE = int(os.getenv('PATIENCE', 16))
RECURRENT_DROPOUT = float(os.getenv('RECURRENT_DROPOUT', 0.5))
GEOM_SCALE = float(os.getenv('GEOM_SCALE', 0))  # If no default or 0: overridden when data is known
OPTIMIZER = Adam(lr=LEARNING_RATE, clipnorm=1., decay=1e-6)

# Load data
train_loaded = np.load(TRAINING_DATA_FILE)
train_geoms = train_loaded['input_geoms']
train_labels = train_loaded['above_or_below_median']

# Determine final test mode or standard
if len(sys.argv) > 1 and sys.argv[1] in ['-t', '--test']:
    print('Training in final test mode')
    TEST_DATA_FILE = '../files/neighborhoods/neighborhoods_test.npz'
    test_loaded = np.load(TEST_DATA_FILE)
    test_geoms = test_loaded['input_geoms']
    test_labels = test_loaded['above_or_below_median']
else:
    print('Training in standard training mode')
    # Split the training data in random seen/unseen sets
    train_geoms, test_geoms, train_labels, test_labels = train_test_split(train_geoms, train_labels, test_size=0.1)

# Normalize
geom_scale = GEOM_SCALE or geom_scaler.scale(train_geoms)
train_geoms = geom_scaler.transform(train_geoms, geom_scale)
test_geoms = geom_scaler.transform(test_geoms, geom_scale)  # re-use variance from training

message = '''
running {} with
version: {}                batch size: {}
train/validate split: {}   repeat deep: {}
lstm size: {}              dense size: {}
epochs: {}                 learning rate: {}
geometry scale: {:.3E}     recurrent dropout: {}
patience {}
'''.format(
    SIGNATURE,
    SCRIPT_VERSION,         BATCH_SIZE,
    TRAIN_VALIDATE_SPLIT,   REPEAT_DEEP_ARCH,
    LSTM_SIZE,              DENSE_SIZE,
    EPOCHS,                 LEARNING_RATE,
    geom_scale,             RECURRENT_DROPOUT,
    PATIENCE,
)
print(message)

# Shape determination
geom_max_points, geom_vector_len = train_geoms.shape[1:]
output_size = train_labels.shape[-1]

# Build model
inputs = Input(shape=(geom_max_points, geom_vector_len))
model = LSTM(LSTM_SIZE,
             return_sequences=True,
             # kernel_regularizer=L1L2(l1=0.01, l2=0.01),
             recurrent_dropout=RECURRENT_DROPOUT)(inputs)
model = TimeDistributed(Dense(DENSE_SIZE, activation='relu'))(model)

for layer in range(REPEAT_DEEP_ARCH):
    model = LSTM(LSTM_SIZE, return_sequences=True, recurrent_dropout=RECURRENT_DROPOUT)(model)

model = Dense(DENSE_SIZE, activation='relu')(model)
model = Flatten()(model)
model = Dense(output_size, activation='softmax')(model)

model = Model(inputs=inputs, outputs=model)
model.compile(
    loss='categorical_crossentropy',
    metrics=['accuracy'],
    optimizer=OPTIMIZER),
model.summary()

# Callbacks
callbacks = [
    TensorBoard(log_dir='./tensorboard_log/' + SIGNATURE, write_graph=False),
    # EarlyStopping(patience=PATIENCE, min_delta=0.001)
]

history = model.fit(
    x=train_geoms,
    y=train_labels,
    epochs=EPOCHS,
    batch_size=BATCH_SIZE,
    validation_split=TRAIN_VALIDATE_SPLIT,
    callbacks=callbacks).history

# Run on unseen test data
test_pred = [np.argmax(prediction) for prediction in model.predict(test_geoms)]
test_labels = [np.argmax(label) for label in test_labels]
accuracy = accuracy_score(test_labels, test_pred)

runtime = time() - SCRIPT_START
message = '''
test accuracy of {:f} in {} with
version: {}                batch size {}
train/validate split {}    repeat deep arch {}
lstm size {}               dense size {}
epochs {}                  learning rate {}
geometry scale {:.3E}        recurrent dropout {}
patience {}
'''.format(
    accuracy, timedelta(seconds=runtime),
    SCRIPT_VERSION, BATCH_SIZE,
    TRAIN_VALIDATE_SPLIT, REPEAT_DEEP_ARCH,
    LSTM_SIZE, DENSE_SIZE,
    len(history['val_loss']), LEARNING_RATE,
    geom_scale, RECURRENT_DROPOUT,
    PATIENCE,
)

notify(SIGNATURE, message)
print(SCRIPT_NAME, 'finished successfully with', message)
<|endoftext|>


<commit_before># Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import abc
import os
import shutil
import sys
import tempfile
import uuid

from muranoclient.common import exceptions as muranoclient_exc
import six

from murano.common import config
from murano.common.i18n import _LE
from murano.dsl import exceptions
from murano.engine import yaql_yaml_loader
from murano.openstack.common import log as logging
from murano.packages import exceptions as pkg_exc
from murano.packages import load_utils

LOG = logging.getLogger(__name__)

class PackageLoader(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractmethod
    def get_package(self, name):
        pass

    @abc.abstractmethod
    def get_package_by_class(self, name):
        pass

class ApiPackageLoader(PackageLoader):
    def __init__(self, murano_client_factory, tenant_id):
        self._cache_directory = self._get_cache_directory()
        self._murano_client_factory = murano_client_factory
        self.tenant_id = tenant_id

    def get_package_by_class(self, name):
        filter_opts = {'class_name': name}
        try:
            package_definition = self._get_definition(filter_opts)
        except LookupError:
            exc_info = sys.exc_info()
            raise exceptions.NoPackageForClassFound(name), None, exc_info[2]
        return self._get_package_by_definition(package_definition)

    def get_package(self, name):
        filter_opts = {'fqn': name}
        try:
            package_definition = self._get_definition(filter_opts)
        except LookupError:
            exc_info = sys.exc_info()
            raise exceptions.NoPackageFound(name), None, exc_info[2]
        return self._get_package_by_definition(package_definition)

    @staticmethod
    def _get_cache_directory():
        base_directory = (
            config.CONF.packages_opts.packages_cache or
            os.path.join(tempfile.gettempdir(), 'murano-packages-cache')
        )
        directory = os.path.abspath(os.path.join(base_directory,
                                                 str(uuid.uuid4())))
        os.makedirs(directory)

        LOG.debug('Cache for package loader is located at: %s' % directory)
        return directory

    def _get_definition(self, filter_opts):
        try:
            packages = list(self._murano_client_factory().packages.filter(
                **filter_opts))
            if len(packages) > 1:
                LOG.debug('Ambiguous package resolution: '
                          'more then 1 package found for query "{0}", '
                          'will resolve based on the ownership'.
                          format(filter_opts))
                return get_best_package_match(packages, self.tenant_id)
            elif len(packages) == 1:
                return packages[0]
            else:
                LOG.debug('There are no packages matching filter '
                          '{0}'.format(filter_opts))
                raise LookupError()
        except muranoclient_exc.HTTPException:
            LOG.debug('Failed to get package definition from repository')
            raise LookupError()

    def _get_package_by_definition(self, package_def):
        package_id = package_def.id
        package_name = package_def.fully_qualified_name
        package_directory = os.path.join(self._cache_directory, package_name)

        if os.path.exists(package_directory):
            try:
                return load_utils.load_from_dir(
                    package_directory, preload=True,
                    loader=yaql_yaml_loader.YaqlYamlLoader)
            except pkg_exc.PackageLoadError:
                LOG.exception(_LE(
                    'Unable to load package from cache. Clean-up...'))
                shutil.rmtree(package_directory, ignore_errors=True)
        try:
            package_data = self._murano_client_factory().packages.download(
                package_id)
        except muranoclient_exc.HTTPException as e:
            msg = 'Error loading package id {0}: {1}'.format(
                package_id, str(e)
            )
            exc_info = sys.exc_info()
            raise pkg_exc.PackageLoadError(msg), None, exc_info[2]
        package_file = None
        try:
            with tempfile.NamedTemporaryFile(delete=False) as package_file:
                package_file.write(package_data)

            return load_utils.load_from_file(
                package_file.name,
                target_dir=package_directory,
                drop_dir=False,
                loader=yaql_yaml_loader.YaqlYamlLoader
            )
        except IOError:
            msg = 'Unable to extract package data for %s' % package_id
            exc_info = sys.exc_info()
            raise pkg_exc.PackageLoadError(msg), None, exc_info[2]
        finally:
            try:
                if package_file:
                    os.remove(package_file.name)
            except OSError:
                pass

    def cleanup(self):
        shutil.rmtree(self._cache_directory, ignore_errors=True)

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.cleanup()
        return False

class DirectoryPackageLoader(PackageLoader):
    def __init__(self, base_path):
        self._base_path = base_path
        self._processed_entries = set()
        self._packages_by_class = {}
        self._packages_by_name = {}

        self._build_index()

    def get_package(self, name):
        return self._packages_by_name.get(name)

    def get_package_by_class(self, name):
        return self._packages_by_class.get(name)

    def _build_index(self):
        for entry in os.listdir(self._base_path):
            folder = os.path.join(self._base_path, entry)
            if not os.path.isdir(folder) or entry in self._processed_entries:
                continue

            try:
                package = load_utils.load_from_dir(
                    folder, preload=True,
                    loader=yaql_yaml_loader.YaqlYamlLoader)
            except pkg_exc.PackageLoadError:
                LOG.exception(_LE('Unable to load package from path: '
                                  '{0}').format(entry))
                continue

            for c in package.classes:
                self._packages_by_class[c] = package
            self._packages_by_name[package.full_name] = package

            self._processed_entries.add(entry)

def get_best_package_match(packages, tenant_id):
    public = None
    other = []
    for package in packages:
        if package.owner_id == tenant_id:
            return package
        elif package.is_public:
            public = package
        else:
            other.append(package)
    if public is not None:
        return public
    elif other:
        return other[0]
<commit_msg>Use catalog=True for package_loader queries<commit_after># Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import abc
import os
import shutil
import sys
import tempfile
import uuid

from muranoclient.common import exceptions as muranoclient_exc
import six

from murano.common import config
from murano.common.i18n import _LE
from murano.dsl import exceptions
from murano.engine import yaql_yaml_loader
from murano.openstack.common import log as logging
from murano.packages import exceptions as pkg_exc
from murano.packages import load_utils

LOG = logging.getLogger(__name__)

class PackageLoader(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractmethod
    def get_package(self, name):
        pass

    @abc.abstractmethod
    def get_package_by_class(self, name):
        pass

class ApiPackageLoader(PackageLoader):
    def __init__(self, murano_client_factory, tenant_id):
        self._cache_directory = self._get_cache_directory()
        self._murano_client_factory = murano_client_factory
        self.tenant_id = tenant_id

    def get_package_by_class(self, name):
        filter_opts = {'class_name': name}
        try:
            package_definition = self._get_definition(filter_opts)
        except LookupError:
            exc_info = sys.exc_info()
            raise exceptions.NoPackageForClassFound(name), None, exc_info[2]
        return self._get_package_by_definition(package_definition)

    def get_package(self, name):
        filter_opts = {'fqn': name}
        try:
            package_definition = self._get_definition(filter_opts)
        except LookupError:
            exc_info = sys.exc_info()
            raise exceptions.NoPackageFound(name), None, exc_info[2]
        return self._get_package_by_definition(package_definition)

    @staticmethod
    def _get_cache_directory():
        base_directory = (
            config.CONF.packages_opts.packages_cache or
            os.path.join(tempfile.gettempdir(), 'murano-packages-cache')
        )
        directory = os.path.abspath(os.path.join(base_directory,
                                                 str(uuid.uuid4())))
        os.makedirs(directory)

        LOG.debug('Cache for package loader is located at: %s' % directory)
        return directory

    def _get_definition(self, filter_opts):
        filter_opts['catalog'] = True
        try:
            packages = list(self._murano_client_factory().packages.filter(
                **filter_opts))
            if len(packages) > 1:
                LOG.debug('Ambiguous package resolution: '
                          'more then 1 package found for query "{0}", '
                          'will resolve based on the ownership'.
                          format(filter_opts))
                return get_best_package_match(packages, self.tenant_id)
            elif len(packages) == 1:
                return packages[0]
            else:
                LOG.debug('There are no packages matching filter '
                          '{0}'.format(filter_opts))
                raise LookupError()
        except muranoclient_exc.HTTPException:
            LOG.debug('Failed to get package definition from repository')
            raise LookupError()

    def _get_package_by_definition(self, package_def):
        package_id = package_def.id
        package_name = package_def.fully_qualified_name
        package_directory = os.path.join(self._cache_directory, package_name)

        if os.path.exists(package_directory):
            try:
                return load_utils.load_from_dir(
                    package_directory, preload=True,
                    loader=yaql_yaml_loader.YaqlYamlLoader)
            except pkg_exc.PackageLoadError:
                LOG.exception(_LE(
                    'Unable to load package from cache. Clean-up...'))
                shutil.rmtree(package_directory, ignore_errors=True)
        try:
            package_data = self._murano_client_factory().packages.download(
                package_id)
        except muranoclient_exc.HTTPException as e:
            msg = 'Error loading package id {0}: {1}'.format(
                package_id, str(e)
            )
            exc_info = sys.exc_info()
            raise pkg_exc.PackageLoadError(msg), None, exc_info[2]
        package_file = None
        try:
            with tempfile.NamedTemporaryFile(delete=False) as package_file:
                package_file.write(package_data)

            return load_utils.load_from_file(
                package_file.name,
                target_dir=package_directory,
                drop_dir=False,
                loader=yaql_yaml_loader.YaqlYamlLoader
            )
        except IOError:
            msg = 'Unable to extract package data for %s' % package_id
            exc_info = sys.exc_info()
            raise pkg_exc.PackageLoadError(msg), None, exc_info[2]
        finally:
            try:
                if package_file:
                    os.remove(package_file.name)
            except OSError:
                pass

    def cleanup(self):
        shutil.rmtree(self._cache_directory, ignore_errors=True)

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.cleanup()
        return False

class DirectoryPackageLoader(PackageLoader):
    def __init__(self, base_path):
        self._base_path = base_path
        self._processed_entries = set()
        self._packages_by_class = {}
        self._packages_by_name = {}

        self._build_index()

    def get_package(self, name):
        return self._packages_by_name.get(name)

    def get_package_by_class(self, name):
        return self._packages_by_class.get(name)

    def _build_index(self):
        for entry in os.listdir(self._base_path):
            folder = os.path.join(self._base_path, entry)
            if not os.path.isdir(folder) or entry in self._processed_entries:
                continue

            try:
                package = load_utils.load_from_dir(
                    folder, preload=True,
                    loader=yaql_yaml_loader.YaqlYamlLoader)
            except pkg_exc.PackageLoadError:
                LOG.exception(_LE('Unable to load package from path: '
                                  '{0}').format(entry))
                continue

            for c in package.classes:
                self._packages_by_class[c] = package
            self._packages_by_name[package.full_name] = package

            self._processed_entries.add(entry)

def get_best_package_match(packages, tenant_id):
    public = None
    other = []
    for package in packages:
        if package.owner_id == tenant_id:
            return package
        elif package.is_public:
            public = package
        else:
            other.append(package)
    if public is not None:
        return public
    elif other:
        return other[0]
<|endoftext|>


<commit_before>#
# This file is part of my.gpodder.org.
#
# my.gpodder.org is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# my.gpodder.org is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
# License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with my.gpodder.org. If not, see <http://www.gnu.org/licenses/>.
#

from mygpo.api.basic_auth import require_valid_user, check_username
from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseForbidden, Http404, HttpResponseNotAllowed
from mygpo.api.httpresponse import JsonResponse
from mygpo.exceptions import ParameterMissing
from django.shortcuts import get_object_or_404
from mygpo.api.sanitizing import sanitize_url
from mygpo.api.models import Device, Podcast, Episode, ToplistEntry
from mygpo.api.models.episodes import Chapter
from mygpo.data.models import PodcastTag
from django.utils.translation import ugettext as _
from datetime import datetime, timedelta
from mygpo.log import log
from mygpo.utils import parse_time
import dateutil.parser
from django.contrib.sites.models import Site
from django.views.decorators.csrf import csrf_exempt

try:
    #try to import the JSON module (if we are on Python 2.6)
    import json

    # Python 2.5 seems to have a different json module
    if not 'dumps' in dir(json):
        raise ImportError

except ImportError:
    # No JSON module available - fallback to simplejson (Python < 2.6)
    print "No JSON module available - fallback to simplejson (Python < 2.6)"
    import simplejson as json

@csrf_exempt
def top_tags(request, count):
    tags = DirectoryEntry.objects.top_tags(int(count))
    resp = []
    for t in tags:
        resp.append( {
            'tag': t.tag,
            'usage': t.entries
            } )
    return JsonResponse(resp)

@csrf_exempt
def tag_podcasts(request, tag, count):
    resp = []
    for p in DirectoryEntry.objects.podcasts_for_tag(tag)[:int(count)]:
        resp.append( podcast_data(p.get_podcast()) )

    return JsonResponse(resp)

def podcast_info(request):
    url = sanitize_url(request.GET.get('url', ''))
    podcast = get_object_or_404(Podcast, url=url)
    resp = podcast_data(podcast)

    return JsonResponse(resp)

def episode_info(request):
    podcast_url = sanitize_url(request.GET.get('podcast', ''))
    episode_url = sanitize_url(request.GET.get('url', ''), podcast=False, episode=True)
    print podcast_url
    print episode_url
    episode = get_object_or_404(Episode, url=episode_url, podcast__url=podcast_url)

    resp = episode_data(episode)
    return JsonResponse(resp)

def podcast_data(podcast):
    site = Site.objects.get_current()
    if podcast.group:
        e = ToplistEntry.objects.get(podcast_group=podcast.group)
    else:
        e = ToplistEntry.objects.get(podcast=podcast)
    return {
        "url": podcast.url,
        "title": podcast.title,
        "description": podcast.description,
        "subscribers": e.subscriptions,
        "logo_url": podcast.logo_url,
        "website": podcast.link,
        "mygpo_link": 'http://%s/podcast/%s' % (site.domain, podcast.id),
        }

def episode_data(episode):
    site = Site.objects.get_current()

    return {
        "title": episode.title,
        "url": episode.url,
        "podcast_title": episode.podcast.title,
        "podcast_url": episode.podcast.url,
        "description": episode.description,
        "website": episode.link,
        "mygpo_link": 'http://%s/episode/%s' % (site.domain, episode.id),
        }
<commit_msg>fix bug in podcast_data<commit_after>#
# This file is part of my.gpodder.org.
#
# my.gpodder.org is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# my.gpodder.org is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
# License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with my.gpodder.org. If not, see <http://www.gnu.org/licenses/>.
#

from mygpo.api.basic_auth import require_valid_user, check_username
from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseForbidden, Http404, HttpResponseNotAllowed
from mygpo.api.httpresponse import JsonResponse
from mygpo.exceptions import ParameterMissing
from django.shortcuts import get_object_or_404
from mygpo.api.sanitizing import sanitize_url
from mygpo.api.models import Device, Podcast, Episode, ToplistEntry
from mygpo.api.models.episodes import Chapter
from mygpo.data.models import PodcastTag
from django.utils.translation import ugettext as _
from datetime import datetime, timedelta
from mygpo.log import log
from mygpo.utils import parse_time
import dateutil.parser
from django.contrib.sites.models import Site
from django.views.decorators.csrf import csrf_exempt

try:
    #try to import the JSON module (if we are on Python 2.6)
    import json

    # Python 2.5 seems to have a different json module
    if not 'dumps' in dir(json):
        raise ImportError

except ImportError:
    # No JSON module available - fallback to simplejson (Python < 2.6)
    print "No JSON module available - fallback to simplejson (Python < 2.6)"
    import simplejson as json

@csrf_exempt
def top_tags(request, count):
    tags = DirectoryEntry.objects.top_tags(int(count))
    resp = []
    for t in tags:
        resp.append( {
            'tag': t.tag,
            'usage': t.entries
            } )
    return JsonResponse(resp)

@csrf_exempt
def tag_podcasts(request, tag, count):
    resp = []
    for p in DirectoryEntry.objects.podcasts_for_tag(tag)[:int(count)]:
        resp.append( podcast_data(p.get_podcast()) )

    return JsonResponse(resp)

def podcast_info(request):
    url = sanitize_url(request.GET.get('url', ''))
    podcast = get_object_or_404(Podcast, url=url)
    resp = podcast_data(podcast)

    return JsonResponse(resp)

def episode_info(request):
    podcast_url = sanitize_url(request.GET.get('podcast', ''))
    episode_url = sanitize_url(request.GET.get('url', ''), podcast=False, episode=True)
    episode = get_object_or_404(Episode, url=episode_url, podcast__url=podcast_url)

    resp = episode_data(episode)
    return JsonResponse(resp)

def podcast_data(podcast):
    site = Site.objects.get_current()
    if podcast.group:
        try:
            e = ToplistEntry.objects.get(podcast_group=podcast.group)

        # no toplist entry has been created for the group yet
        except ToplistEntry.DoesNotExist:
            e = ToplistEntry.objects.get(podcast=podcast)
    else:
        e = ToplistEntry.objects.get(podcast=podcast)

    return {
        "url": podcast.url,
        "title": podcast.title,
        "description": podcast.description,
        "subscribers": e.subscriptions,
        "logo_url": podcast.logo_url,
        "website": podcast.link,
        "mygpo_link": 'http://%s/podcast/%s' % (site.domain, podcast.id),
        }

def episode_data(episode):
    site = Site.objects.get_current()

    return {
        "title": episode.title,
        "url": episode.url,
        "podcast_title": episode.podcast.title,
        "podcast_url": episode.podcast.url,
        "description": episode.description,
        "website": episode.link,
        "mygpo_link": 'http://%s/episode/%s' % (site.domain, episode.id),
        }
<|endoftext|>


<commit_before>from datetime import timedelta
from mysite.search.models import Project
from mysite.customs.models import RoundupBugTracker
from celery.task import PeriodicTask
from celery.registry import tasks
from mysite.search.launchpad_crawl import grab_lp_bugs, lpproj2ohproj
import mysite.customs.miro
import mysite.customs.bugtrackers.gnome_love

class GrabLaunchpadBugs(PeriodicTask):
    run_every = timedelta(days=1)
    def run(self, **kwargs):
        logger = self.get_logger(**kwargs)
        for lp_project in lpproj2ohproj:
            openhatch_proj = lpproj2ohproj[lp_project]
            logger.info("Started to grab lp.net bugs for %s into %s" % (
                    lp_project, openhatch_proj))
            grab_lp_bugs(lp_project=lp_project,
                         openhatch_project=openhatch_proj)

class GrabMiroBugs(PeriodicTask):
    run_every = timedelta(days=1)
    def run(self, **kwargs):
        logger = self.get_logger(**kwargs)
        logger.info("Started to grab Miro bitesized bugs")
        mysite.customs.miro.grab_miro_bugs()

class GrabGnomeLoveBugs(PeriodicTask):
    run_every = timedelta(days=1)
    def run(self, **kwargs):
        logger = self.get_logger(**kwargs)
        logger.info("Started to grab GNOME Love bugs")
        mysite.customs.bugtrackers.gnome_love.grab()

class GrabPythonBugs(PeriodicTask):
    run_every = timedelta(days=1)
    def run(self, **kwargs):
        logger = self.get_logger(**kwargs)
        logger.info("Started to grab Python 'easy' bugs")
        bug_tracker_name = self.__class__.__name__
        python_core, _ = Project.objects.get_or_create(name='Python', language='Python')
        # FIXME: Do we need this line?
        RoundupBugTracker.objects.filter(name=bug_tracker_name).delete()
        p, _ = RoundupBugTracker.objects.get_or_create(name=bug_tracker_name, project=python_core)
        p.include_these_roundup_bug_statuses = '1,3'
        p.roundup_root_url = 'http://bugs.python.org'
        p.csv_keyword = '6' # Only grab bugs marked "easy"
        p.my_bugs_are_always_good_for_newcomers = True
        p.save()
        p.grab()

class GrabPythonBugsInDocumentation(PeriodicTask):
    run_every = timedelta(days=1)
    def run(self, **kwargs):
        logger = self.get_logger(**kwargs)
        logger.info("Started to grab Python documentation bugs")
        bug_tracker_name = self.__class__.__name__
        python_core, _ = Project.objects.get_or_create(name='Python', language='Python')
        # FIXME: Do we need this line?
        RoundupBugTracker.objects.filter(name=bug_tracker_name).delete()
        p, _ = RoundupBugTracker.objects.get_or_create(name=bug_tracker_name, project=python_core)
        p.include_these_roundup_bug_statuses = '1,3'
        p.roundup_root_url = 'http://bugs.python.org'
        p.my_bugs_concern_just_documentation = True
        p.components = '4'
        p.save()
        p.grab()
        """e.g.,
        http://bugs.python.org/issue?@action=export_csv&@columns=title,id,activity,status
        &@sort=activity&@group=priority&@filter=components,status
        &@pagesize=50&@startwith=0&status=1&components=4"""

tasks.register(GrabMiroBugs)
tasks.register(GrabGnomeLoveBugs)
tasks.register(GrabLaunchpadBugs)
tasks.register(GrabPythonBugs)
tasks.register(GrabPythonBugsInDocumentation)
<commit_msg>Fix Python bug grabber to call it "Python (project)"<commit_after>from datetime import timedelta
from mysite.search.models import Project
from mysite.customs.models import RoundupBugTracker
from celery.task import PeriodicTask
from celery.registry import tasks
from mysite.search.launchpad_crawl import grab_lp_bugs, lpproj2ohproj
import mysite.customs.miro
import mysite.customs.bugtrackers.gnome_love

class GrabLaunchpadBugs(PeriodicTask):
    run_every = timedelta(days=1)
    def run(self, **kwargs):
        logger = self.get_logger(**kwargs)
        for lp_project in lpproj2ohproj:
            openhatch_proj = lpproj2ohproj[lp_project]
            logger.info("Started to grab lp.net bugs for %s into %s" % (
                    lp_project, openhatch_proj))
            grab_lp_bugs(lp_project=lp_project,
                         openhatch_project=openhatch_proj)

class GrabMiroBugs(PeriodicTask):
    run_every = timedelta(days=1)
    def run(self, **kwargs):
        logger = self.get_logger(**kwargs)
        logger.info("Started to grab Miro bitesized bugs")
        mysite.customs.miro.grab_miro_bugs()

class GrabGnomeLoveBugs(PeriodicTask):
    run_every = timedelta(days=1)
    def run(self, **kwargs):
        logger = self.get_logger(**kwargs)
        logger.info("Started to grab GNOME Love bugs")
        mysite.customs.bugtrackers.gnome_love.grab()

class GrabPythonBugs(PeriodicTask):
    run_every = timedelta(days=1)
    def run(self, **kwargs):
        logger = self.get_logger(**kwargs)
        logger.info("Started to grab Python 'easy' bugs")
        bug_tracker_name = self.__class__.__name__
        python_core, _ = Project.objects.get_or_create(name='Python (project)', language='Python')
        # FIXME: Do we need this line?
        RoundupBugTracker.objects.filter(name=bug_tracker_name).delete()
        p, _ = RoundupBugTracker.objects.get_or_create(name=bug_tracker_name, project=python_core)
        p.include_these_roundup_bug_statuses = '1,3'
        p.roundup_root_url = 'http://bugs.python.org'
        p.csv_keyword = '6' # Only grab bugs marked "easy"
        p.my_bugs_are_always_good_for_newcomers = True
        p.save()
        p.grab()

class GrabPythonBugsInDocumentation(PeriodicTask):
    run_every = timedelta(days=1)
    def run(self, **kwargs):
        logger = self.get_logger(**kwargs)
        logger.info("Started to grab Python documentation bugs")
        bug_tracker_name = self.__class__.__name__
        python_core, _ = Project.objects.get_or_create(name='Python (project)', language='Python')
        # FIXME: Do we need this line?
        RoundupBugTracker.objects.filter(name=bug_tracker_name).delete()
        p, _ = RoundupBugTracker.objects.get_or_create(name=bug_tracker_name, project=python_core)
        p.include_these_roundup_bug_statuses = '1,3'
        p.roundup_root_url = 'http://bugs.python.org'
        p.my_bugs_concern_just_documentation = True
        p.components = '4'
        p.save()
        p.grab()
        """e.g.,
        http://bugs.python.org/issue?@action=export_csv&@columns=title,id,activity,status
        &@sort=activity&@group=priority&@filter=components,status
        &@pagesize=50&@startwith=0&status=1&components=4"""

tasks.register(GrabMiroBugs)
tasks.register(GrabGnomeLoveBugs)
tasks.register(GrabLaunchpadBugs)
tasks.register(GrabPythonBugs)
tasks.register(GrabPythonBugsInDocumentation)
<|endoftext|>