<commit_before>"""
.. module:: appliance
   :platform: Unix
   :synopsis: Contains classes for representing appliances and appliance
      traces.

.. moduleauthor:: Phil Ngo <ngo.phil@gmail.com>
.. moduleauthor:: Miguel Perez <miguel@invalid.com>
.. moduleauthor:: Stephen Suffian <steve@invalid.com>
.. moduleauthor:: Sabina Tomkins <sabina.tomkins@gmail.com>

"""

import numpy as np
import pandas as pd
import pprint
from utils import order_traces
import utils
import decimal
import json
import warnings

class ApplianceTrace(object):
    """This class represents appliance traces.

    Traces are power used by a single (or single set of) appliances sampled
    at a consistent rate.

    """

    def __init__(self, series, metadata):
        '''
        Initializes a trace object from a series and a metadata dictionary.
        Series must be sampled at a particular sample rate
        '''
        self.series = series
        self.metadata = metadata

    def get_sampling_rate(self):
        '''
        Returns a string representing the rate at which the series was sampled.
        '''
        return self.series.index.freq

    def get_time_of_day(self, start_time, end_time):
        '''
        Given a start and end datetime.time, it returns a trace
        within that time period.
        '''
        new_series = self.series.ix[start_time:end_time]
        return ApplianceTrace(new_series,self.metadata)

    def get_windows(self, window_length, window_step):
        """
        Returns a numpy array with stacked sliding windows of data.
        """
        total_length = self.series.size
        n_steps = int((total_length - window_length) / window_step)
        windows = []
        for step in range(n_steps):
            start = step * window_step
            window = self.series[start:start + window_length].tolist()
            windows.append(window)
        return np.array(windows,dtype=np.float)

    def get_total_usage(self):
        '''
        Returns the total usage of this trace
        '''
        return self.series.sum()

    def get_daily_usage(self):
        '''
        Returns the total daily usage of this trace
        '''
        return series.resample('D', how='mean')

    def print_trace(self):
        '''
        Prints a summary of the trace
        '''
        print 'Series size: {}'.format(self.series.size)
        print 'Sampling rate: {}'.format(self.get_sampling_rate)
        print 'Metadata: '
        pprint.pprint(self.metadata)

    def resample(self,sample_rate, method='mean'):
        '''
        Returns a new trace resampled to a given sample rate, defined by the
        offset aliases described in panda time series.
        http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
        '''

        try:
            new_series = self.series.astype(float)
            new_series = new_series.resample(sample_rate,how=method)
            new_series = new_series.fillna(0)
            new_series = new_series.map(decimal.Decimal)
            new_series.name = self.series.name
        except ValueError:
            raise utils.SampleError(sample_rate)
        return ApplianceTrace(new_series,self.metadata)

    def split_by(self,rate):
        '''
        Returns a list of traces formed by splitting this trace by day ('D')
        or week ('W')
        '''
        # set rate to group by
        if rate == 'D' or rate == '1D':
            groupby_rate = self.series.index.date
        elif rate == 'W' or rate == '1W':
            groupby_rate = self.series.index.week

        elif rate == 'M' or rate== '1M':
            groupby_rate = self.series.index.month

        else:
            raise NotImplementedError('Looking for "week" or "day"')

        traces=[]
        for i, group in enumerate(self.series.groupby(groupby_rate)):
            metadata = dict.copy(self.metadata)
            metadata['trace_num'] = i
            traces.append(ApplianceTrace(group[1],metadata))
        return traces

    def to_daily_usage_json(self,method='utc_dict'):
        '''
        Returns the daily usage sum trace in a json format for calendar view
        '''
        d_sum = self.resample('D', 'sum')
        if method == 'utc_dict':
            data = {}
            for i, v in d_sum.series.iteritems():
                kwh = v/1000
                unixtime = str(i.strftime("%s"))
                data.update({unixtime:float(kwh)})
        elif method == 'date_list':
            data = []
            for i, v in d_sum.series.iteritems():
                kwh = v/1000
                data.append({'date':i.strftime('%Y-%m-%d %H:%M'),
                             'reading': float(kwh)})
        else:
            raise NotImplementedError
        json_string = json.dumps(data, ensure_ascii=False)
        return json_string

    def to_json(self):
        '''
        Returns the trace in a json format amenable to d3 visualization.
        '''
        data = []
        for i, v in self.series.iteritems():
            kwh = v/1000
            data.append({'date':i.strftime('%Y-%m-%d %H:%M'),
                         'reading': float(kwh)})
        json_string = json.dumps(data, ensure_ascii=False,
                                 indent=4, separators=(',', ': '))
        return json_string

class ApplianceInstance(object):
    """
    This class represents appliance instances, which may have multiple
    appliance traces.

    Instances hold traces from a single instance of an appliance sampled
    at a consistent rate. Traces may or may not be consecutive.

    """

    def __init__(self,traces,metadata):
        '''Initialize an appliance trace with a list of ApplianceTraces'''
        self.traces = order_traces(traces)
        self.metadata=metadata

    def concatenate_traces(self, how="strict"):
        '''
        Takes its own list of traces and attempts to concatenate them.
        '''
        if how == "strict":
            self.traces = [pd.concat(traces)]
        else:
            raise NotImplementedError

    def get_time_of_day(self,start_time,end_time):
        '''
        Given a start and end datetime.time, it returns an instance with traces
        within that time period.
        '''
        new_traces = []
        for trace in self.traces:
            new_traces.append(trace.get_time_of_day(start_time,end_time))
        return ApplianceInstance(new_traces,self.metadata)

    def resample(self,sample_rate):
        '''
        Returns an instance with resampled traces.
        '''
        new_traces = []
        for trace in self.traces:
            new_traces.append(trace.resample(sample_rate))
        return ApplianceInstance(new_traces,self.metadata)

    def split_by(self,rate):
        '''
        Return a new ApplianceTrace in which each instance split into multiple
        traces split by day ('D') or week ('W') - (Sun-Sat?)
        '''
        traces=[]
        for trace in self.traces:
            traces.extend(trace.split_by(rate))
        return ApplianceInstance(traces,self.metadata)

class ApplianceSet(object):
    """
    This class represents appliance sets, which contain a set of temporally
    aligned appliance instances.

    Appliance sets are most frequently used as ground-truth for various
    algorithms, representing a particular home, building, or metered unit.

    """
    def __init__(self,instances,metadata):
        '''
        Initializes an appliance set given a list of instances.
        '''
        if not utils.instances_aligned(instances):
            self.instances = utils.align_instances(instances)
            warnings.warn("Aligning misaligned traces using default procedure")
        else:
            self.instances = instances
        self.metadata = metadata

    def generate_top_k_set(self,k):
        '''
        Get top k energy-consuming appliances
        '''
        # TODO compare speeds of individual instance summing vs dataframe building and summing
        # TODO more intelligently create the metadata
        df = get_dataframe()
        total_usages = df.sum(axis=0)
        usage_order = np.argsort(total_usages)[::-1] # assumes correctly ordered columns
        top_k_instances = [self.instances[i] for i in usage_order[:k]]
        return ApplianceSet(top_k_instances,
                            {"name":"top_{}".format(k)})

    def generate_non_zero_set(self):
        '''
        Get all energy-consuming appliances (and drop instances with traces
        with all zeros)
        '''
        # TODO compare speeds of individual instance summing vs dataframe building and summing
        # TODO intelligently create the metadata
        df = get_dataframe()
        total_usages = df.sum(axis=0)
        usage_order = np.argsort(total_usages)[::-1] # assumes correctly ordered columns
        non_zero_instances = [self.instances[i] for i in usage_order if total_usages[i] > 0 ]
        return ApplianceSet(non_zero_instances,
                            {"name":"non_zero"})

    def get_dataframe(self):
        '''
        Makes a new dataframe of the appliance instances. Throws an exception if
        if the appliance instances have traces that don't align.
        '''
        # TODO concatenate all traces into a single trace
        # TODO change this to ordered dict
        # TODO actually throw an exception
        series_dict = {instance.traces[0].series.name:instance.traces[0].series
                       for instance in self.instances}
        return pd.DataFrame.from_dict(series_dict)

    def get_time_of_day(self,start_time,end_time):
        '''
        Given a start and end datetime.time, it returns an ApplianceType with
        traces within that time period.
        '''
        new_instances=[]
        for instance in self.instances:
            new_instances.append(instance.get_time_of_day(start_time,end_time))
        return ApplianceType(new_instances,self.metadata)
    def resample(self, sample_rate):
        '''
        Returns a new ApplianceSet instance with resampled traces.
        '''
        new_instances=[]
        for instance in self.instances:
            new_instances.append(instance.resample(sample_rate))
        return ApplianceSet(new_instances,self.metadata)

    def split_by(self, rate):
        '''
        Returns a new ApplianceSet object for which each trace in each instance
        is split into multiple traces from unique days ('D') or weeks ('W')
        '''
        instances = []
        for instance in self.instances:
            new_instance = instance.split_by(rate)
            instances.append(new_instance)
        return ApplianceSet(instances,self.metadata)

class ApplianceType(object):
    """This class represents appliance types, which contain a set of
    ApplianceInstances which share particular attributes, but which are not
    necessarily temporally aligned.

    Appliance types are most frequently used to generate models of particular
    types of appliances.
    """

    def __init__(self, instances, metadata):
        '''
        Initialize a type object with a list of instances. (Check for
        uniqueness?)
        '''
        # TODO Check for uniqueness?
        self.instances = instances
        self.metadata = metadata
        try:
            self.instance_id_index={instance.metadata['dataid']:i for i,instance in enumerate(instances)}
        except KeyError:
            print 'Warning: no "dataid" key found in metadata.'

    def get_instance_by_id(self,instance_id):
        '''
        Gets the instance of the type with a specified instance id
        '''
        index = self.instance_id_index[instance_id]
        return ApplianceType.instances[index]

    def get_time_of_day(self,start_time,end_time):
        '''
        Given a start and end datetime.time, it returns an ApplianceSet with
        traces within that time period.
        '''
        new_instances=[]
        for instance in self.instances:
            new_instances.append(instance.get_time_of_day(start_time,end_time))
        return ApplianceSet(new_instances,self.metadata)

    def resample(self,sample_rate):
        '''
        Returns a new ApplianceType with resampled traces.
        '''
        new_instances=[]
        for instance in self.instances:
            new_instances.append(instance.resample(sample_rate))
        return ApplianceType(new_instances,self.metadata)

    def split_by(self, rate):
        '''
        Returns a new ApplianceType object for which each trace in each instance
        is split into multiple traces from unique days ('D') or weeks ('W')
        '''
        instances=[]
        for instance in self.instances:
            new_instance= instance.split_by(rate)
            instances.append(new_instance)
        return ApplianceType(instances,self.metadata)

class AlignmentError(Exception):
    """
    Exception raised for errors in the alignment of the data.
    """
    def __str__(self):
        return '''Improperly aligned traces'''
<commit_msg>added resampling by month<commit_after>"""
.. module:: appliance
   :platform: Unix
   :synopsis: Contains classes for representing appliances and appliance
      traces.

.. moduleauthor:: Phil Ngo <ngo.phil@gmail.com>
.. moduleauthor:: Miguel Perez <miguel@invalid.com>
.. moduleauthor:: Stephen Suffian <steve@invalid.com>
.. moduleauthor:: Sabina Tomkins <sabina.tomkins@gmail.com>

"""

import numpy as np
import pandas as pd
import pprint
from utils import order_traces
import utils
import decimal
import json
import warnings

class ApplianceTrace(object):
    """This class represents appliance traces.

    Traces are power used by a single (or single set of) appliances sampled
    at a consistent rate.

    """

    def __init__(self, series, metadata):
        '''
        Initializes a trace object from a series and a metadata dictionary.
        Series must be sampled at a particular sample rate
        '''
        self.series = series
        self.metadata = metadata

    def get_sampling_rate(self):
        '''
        Returns a string representing the rate at which the series was sampled.
        '''
        return self.series.index.freq

    def get_time_of_day(self, start_time, end_time):
        '''
        Given a start and end datetime.time, it returns a trace
        within that time period.
        '''
        new_series = self.series.ix[start_time:end_time]
        return ApplianceTrace(new_series,self.metadata)

    def get_windows(self, window_length, window_step):
        """
        Returns a numpy array with stacked sliding windows of data.
        """
        total_length = self.series.size
        n_steps = int((total_length - window_length) / window_step)
        windows = []
        for step in range(n_steps):
            start = step * window_step
            window = self.series[start:start + window_length].tolist()
            windows.append(window)
        return np.array(windows,dtype=np.float)

    def get_total_usage(self):
        '''
        Returns the total usage of this trace
        '''
        return self.series.sum()

    def get_daily_usage(self):
        '''
        Returns the total daily usage of this trace
        '''
        return series.resample('D', how='mean')

    def print_trace(self):
        '''
        Prints a summary of the trace
        '''
        print 'Series size: {}'.format(self.series.size)
        print 'Sampling rate: {}'.format(self.get_sampling_rate)
        print 'Metadata: '
        pprint.pprint(self.metadata)

    def resample(self,sample_rate, method='mean'):
        '''
        Returns a new trace resampled to a given sample rate, defined by the
        offset aliases described in panda time series.
        http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
        '''

        try:
            new_series = self.series.astype(float)
            new_series = new_series.resample(sample_rate,how=method)
            new_series = new_series.fillna(0)
            new_series = new_series.map(decimal.Decimal)
            new_series.name = self.series.name
        except ValueError:
            raise utils.SampleError(sample_rate)
        return ApplianceTrace(new_series,self.metadata)

    def split_by(self,rate):
        '''
        Returns a list of traces formed by splitting this trace by day ('D')
        or week ('W')
        '''
        # set rate to group by
        if rate == 'D' or rate == '1D':
            groupby_rate = self.series.index.date
        elif rate == 'W' or rate == '1W':
            groupby_rate = self.series.index.week

        elif rate == 'M' or rate== '1M':
            groupby_rate = self.series.index.month

        else:
            raise NotImplementedError('Looking for "week" or "day"')

        traces=[]
        for i, group in enumerate(self.series.groupby(groupby_rate)):
            metadata = dict.copy(self.metadata)
            metadata['trace_num'] = i
            traces.append(ApplianceTrace(group[1],metadata))
        return traces

    def to_daily_usage_json(self,method='utc_dict'):
        '''
        Returns the daily usage sum trace in a json format for calendar view
        '''
        d_sum = self.resample('D', 'sum')
        m_sum = self.resample('M', 'sum')
        if method == 'month_sums':
            data = []
            for i, v in m_sum.series.iteritems():
                kwh = v/1000
                data.append({'date':i.strftime('%Y-%m-%d'),
                             'reading': float(kwh)})
        elif method == 'utc_dict':
            data = {}
            for i, v in d_sum.series.iteritems():
                kwh = v/1000
                unixtime = str(i.strftime("%s"))
                data.update({unixtime:float(kwh)})
        elif method == 'date_list':
            data = []
            for i, v in d_sum.series.iteritems():
                kwh = v/1000
                data.append({'date':i.strftime('%Y-%m-%d'),
                             'reading': float(kwh)})
        else:
            raise NotImplementedError
        json_string = json.dumps(data, ensure_ascii=False)
        return json_string

    def to_json(self):
        '''
        Returns the trace in a json format amenable to d3 visualization.
        '''
        data = []
        for i, v in self.series.iteritems():
            kwh = v/1000
            data.append({'date':i.strftime('%Y-%m-%d %H:%M'),
                         'reading': float(kwh)})
        json_string = json.dumps(data, ensure_ascii=False,
                                 indent=4, separators=(',', ': '))
        return json_string

class ApplianceInstance(object):
    """
    This class represents appliance instances, which may have multiple
    appliance traces.

    Instances hold traces from a single instance of an appliance sampled
    at a consistent rate. Traces may or may not be consecutive.

    """

    def __init__(self,traces,metadata):
        '''Initialize an appliance trace with a list of ApplianceTraces'''
        self.traces = order_traces(traces)
        self.metadata=metadata

    def concatenate_traces(self, how="strict"):
        '''
        Takes its own list of traces and attempts to concatenate them.
        '''
        if how == "strict":
            self.traces = [pd.concat(traces)]
        else:
            raise NotImplementedError

    def get_time_of_day(self,start_time,end_time):
        '''
        Given a start and end datetime.time, it returns an instance with traces
        within that time period.
        '''
        new_traces = []
        for trace in self.traces:
            new_traces.append(trace.get_time_of_day(start_time,end_time))
        return ApplianceInstance(new_traces,self.metadata)

    def resample(self,sample_rate):
        '''
        Returns an instance with resampled traces.
        '''
        new_traces = []
        for trace in self.traces:
            new_traces.append(trace.resample(sample_rate))
        return ApplianceInstance(new_traces,self.metadata)

    def split_by(self,rate):
        '''
        Return a new ApplianceTrace in which each instance split into multiple
        traces split by day ('D') or week ('W') - (Sun-Sat?)
        '''
        traces=[]
        for trace in self.traces:
            traces.extend(trace.split_by(rate))
        return ApplianceInstance(traces,self.metadata)

class ApplianceSet(object):
    """
    This class represents appliance sets, which contain a set of temporally
    aligned appliance instances.

    Appliance sets are most frequently used as ground-truth for various
    algorithms, representing a particular home, building, or metered unit.

    """
    def __init__(self,instances,metadata):
        '''
        Initializes an appliance set given a list of instances.
        '''
        if not utils.instances_aligned(instances):
            self.instances = utils.align_instances(instances)
            warnings.warn("Aligning misaligned traces using default procedure")
        else:
            self.instances = instances
        self.metadata = metadata

    def generate_top_k_set(self,k):
        '''
        Get top k energy-consuming appliances
        '''
        # TODO compare speeds of individual instance summing vs dataframe building and summing
        # TODO more intelligently create the metadata
        df = get_dataframe()
        total_usages = df.sum(axis=0)
        usage_order = np.argsort(total_usages)[::-1] # assumes correctly ordered columns
        top_k_instances = [self.instances[i] for i in usage_order[:k]]
        return ApplianceSet(top_k_instances,
                            {"name":"top_{}".format(k)})

    def generate_non_zero_set(self):
        '''
        Get all energy-consuming appliances (and drop instances with traces
        with all zeros)
        '''
        # TODO compare speeds of individual instance summing vs dataframe building and summing
        # TODO intelligently create the metadata
        df = get_dataframe()
        total_usages = df.sum(axis=0)
        usage_order = np.argsort(total_usages)[::-1] # assumes correctly ordered columns
        non_zero_instances = [self.instances[i] for i in usage_order if total_usages[i] > 0 ]
        return ApplianceSet(non_zero_instances,
                            {"name":"non_zero"})

    def get_dataframe(self):
        '''
        Makes a new dataframe of the appliance instances. Throws an exception if
        if the appliance instances have traces that don't align.
        '''
        # TODO concatenate all traces into a single trace
        # TODO change this to ordered dict
        # TODO actually throw an exception
        series_dict = {instance.traces[0].series.name:instance.traces[0].series
                       for instance in self.instances}
        return pd.DataFrame.from_dict(series_dict)

    def get_time_of_day(self,start_time,end_time):
        '''
        Given a start and end datetime.time, it returns an ApplianceType with
        traces within that time period.
        '''
        new_instances=[]
        for instance in self.instances:
            new_instances.append(instance.get_time_of_day(start_time,end_time))
        return ApplianceType(new_instances,self.metadata)
    def resample(self, sample_rate):
        '''
        Returns a new ApplianceSet instance with resampled traces.
        '''
        new_instances=[]
        for instance in self.instances:
            new_instances.append(instance.resample(sample_rate))
        return ApplianceSet(new_instances,self.metadata)

    def split_by(self, rate):
        '''
        Returns a new ApplianceSet object for which each trace in each instance
        is split into multiple traces from unique days ('D') or weeks ('W')
        '''
        instances = []
        for instance in self.instances:
            new_instance = instance.split_by(rate)
            instances.append(new_instance)
        return ApplianceSet(instances,self.metadata)

class ApplianceType(object):
    """This class represents appliance types, which contain a set of
    ApplianceInstances which share particular attributes, but which are not
    necessarily temporally aligned.

    Appliance types are most frequently used to generate models of particular
    types of appliances.
    """

    def __init__(self, instances, metadata):
        '''
        Initialize a type object with a list of instances. (Check for
        uniqueness?)
        '''
        # TODO Check for uniqueness?
        self.instances = instances
        self.metadata = metadata
        try:
            self.instance_id_index={instance.metadata['dataid']:i for i,instance in enumerate(instances)}
        except KeyError:
            print 'Warning: no "dataid" key found in metadata.'

    def get_instance_by_id(self,instance_id):
        '''
        Gets the instance of the type with a specified instance id
        '''
        index = self.instance_id_index[instance_id]
        return ApplianceType.instances[index]

    def get_time_of_day(self,start_time,end_time):
        '''
        Given a start and end datetime.time, it returns an ApplianceSet with
        traces within that time period.
        '''
        new_instances=[]
        for instance in self.instances:
            new_instances.append(instance.get_time_of_day(start_time,end_time))
        return ApplianceSet(new_instances,self.metadata)

    def resample(self,sample_rate):
        '''
        Returns a new ApplianceType with resampled traces.
        '''
        new_instances=[]
        for instance in self.instances:
            new_instances.append(instance.resample(sample_rate))
        return ApplianceType(new_instances,self.metadata)

    def split_by(self, rate):
        '''
        Returns a new ApplianceType object for which each trace in each instance
        is split into multiple traces from unique days ('D') or weeks ('W')
        '''
        instances=[]
        for instance in self.instances:
            new_instance= instance.split_by(rate)
            instances.append(new_instance)
        return ApplianceType(instances,self.metadata)

class AlignmentError(Exception):
    """
    Exception raised for errors in the alignment of the data.
    """
    def __str__(self):
        return '''Improperly aligned traces'''
<|endoftext|>


<commit_before>"""
Couple djangorestframework and cities_light.

It defines a urlpatterns variables, with the following urls:

- cities-light-api-city-list
- cities-light-api-city-detail
- cities-light-api-region-list
- cities-light-api-region-detail
- cities-light-api-country-list
- cities-light-api-country-detail

If rest_framework (v3) is installed, all you have to do is add this url
include::

    url(r'^cities_light/api/', include('cities_light.contrib.restframework3')),

And that's all !
"""
from rest_framework import viewsets, relations
from rest_framework.serializers import HyperlinkedModelSerializer
from rest_framework import routers

from django.conf.urls import url, include

from ..loading import get_cities_models

Country, Region, City = get_cities_models()

class CitySerializer(HyperlinkedModelSerializer):
    """
    HyperlinkedModelSerializer for City.
    """
    url = relations.HyperlinkedIdentityField(
        view_name='cities-light-api-city-detail')
    country = relations.HyperlinkedRelatedField(
        view_name='cities-light-api-country-detail', read_only=True)
    region = relations.HyperlinkedRelatedField(
        view_name='cities-light-api-region-detail', read_only=True)

    class Meta:
        model = City
        exclude = ('slug',)

class RegionSerializer(HyperlinkedModelSerializer):
    """
    HyperlinkedModelSerializer for Region.
    """
    url = relations.HyperlinkedIdentityField(
        view_name='cities-light-api-region-detail')
    country = relations.HyperlinkedRelatedField(
        view_name='cities-light-api-country-detail', read_only=True)

    class Meta:
        model = Region
        exclude = ('slug',)

class CountrySerializer(HyperlinkedModelSerializer):
    """
    HyperlinkedModelSerializer for Country.
    """
    url = relations.HyperlinkedIdentityField(
        view_name='cities-light-api-country-detail')

    class Meta:
        model = Country
        fields = '__all__'

class CitiesLightListModelViewSet(viewsets.ReadOnlyModelViewSet):
    def get_queryset(self):
        """
        Allows a GET param, 'q', to be used against name_ascii.
        """
        queryset = super(CitiesLightListModelViewSet, self).get_queryset()

        if self.request.GET.get('q', None):
            return queryset.filter(name_ascii__icontains=self.request.GET['q'])

        return queryset

class CountryModelViewSet(CitiesLightListModelViewSet):
    serializer_class = CountrySerializer
    queryset = Country.objects.all()

class RegionModelViewSet(CitiesLightListModelViewSet):
    serializer_class = RegionSerializer
    queryset = Region.objects.all()

class CityModelViewSet(CitiesLightListModelViewSet):
    """
    ListRetrieveView for City.
    """
    serializer_class = CitySerializer
    queryset = City.objects.all()

    def get_queryset(self):
        """
        Allows a GET param, 'q', to be used against search_names.
        """
        queryset = super(CitiesLightListModelViewSet, self).get_queryset()

        if self.request.GET.get('q', None):
            return queryset.filter(
                search_names__icontains=self.request.GET['q'])

        return queryset

router = routers.SimpleRouter()
router.register(r'cities', CityModelViewSet, basename='cities-light-api-city')
router.register(r'countries', CountryModelViewSet,
                basename='cities-light-api-country')
router.register(r'regions', RegionModelViewSet,
                basename='cities-light-api-region')

urlpatterns = [
    url(r'^', include(router.urls)),
]
<commit_msg>add subregion to contrib drf<commit_after>"""
Couple djangorestframework and cities_light.

It defines a urlpatterns variables, with the following urls:

- cities-light-api-city-list
- cities-light-api-city-detail
- cities-light-api-region-list
- cities-light-api-region-detail
- cities-light-api-country-list
- cities-light-api-country-detail

If rest_framework (v3) is installed, all you have to do is add this url
include::

    url(r'^cities_light/api/', include('cities_light.contrib.restframework3')),

And that's all !
"""
from rest_framework import viewsets, relations
from rest_framework.serializers import HyperlinkedModelSerializer
from rest_framework import routers

from django.conf.urls import url, include

from ..loading import get_cities_models

Country, Region, Subregion, City = get_cities_models()

class CitySerializer(HyperlinkedModelSerializer):
    """
    HyperlinkedModelSerializer for City.
    """
    url = relations.HyperlinkedIdentityField(
        view_name='cities-light-api-city-detail')
    country = relations.HyperlinkedRelatedField(
        view_name='cities-light-api-country-detail', read_only=True)
    region = relations.HyperlinkedRelatedField(
        view_name='cities-light-api-region-detail', read_only=True)
    subregion = relations.HyperlinkedRelatedField(
        view_name='cities-light-api-subregion-detail', read_only=True)

    class Meta:
        model = City
        exclude = ('slug',)

class SubRegionSerializer(HyperlinkedModelSerializer):
    """
    HyperlinkedModelSerializer for SubRegion.
    """
    url = relations.HyperlinkedIdentityField(
        view_name='cities-light-api-city-detail')
    country = relations.HyperlinkedRelatedField(
        view_name='cities-light-api-country-detail', read_only=True)
    region = relations.HyperlinkedRelatedField(
        view_name='cities-light-api-region-detail', read_only=True)

    class Meta:
        model = SubRegion
        exclude = ('slug',)

class RegionSerializer(HyperlinkedModelSerializer):
    """
    HyperlinkedModelSerializer for Region.
    """
    url = relations.HyperlinkedIdentityField(
        view_name='cities-light-api-region-detail')
    country = relations.HyperlinkedRelatedField(
        view_name='cities-light-api-country-detail', read_only=True)

    class Meta:
        model = Region
        exclude = ('slug',)

class CountrySerializer(HyperlinkedModelSerializer):
    """
    HyperlinkedModelSerializer for Country.
    """
    url = relations.HyperlinkedIdentityField(
        view_name='cities-light-api-country-detail')

    class Meta:
        model = Country
        fields = '__all__'

class CitiesLightListModelViewSet(viewsets.ReadOnlyModelViewSet):
    def get_queryset(self):
        """
        Allows a GET param, 'q', to be used against name_ascii.
        """
        queryset = super(CitiesLightListModelViewSet, self).get_queryset()

        if self.request.GET.get('q', None):
            return queryset.filter(name_ascii__icontains=self.request.GET['q'])

        return queryset

class CountryModelViewSet(CitiesLightListModelViewSet):
    serializer_class = CountrySerializer
    queryset = Country.objects.all()

class RegionModelViewSet(CitiesLightListModelViewSet):
    serializer_class = RegionSerializer
    queryset = Region.objects.all()

class SubRegionModelViewSet(CitiesLightListModelViewSet):
    serializer_class = SubRegionSerializer
    queryset = SubRegion.objects.all()

class CityModelViewSet(CitiesLightListModelViewSet):
    """
    ListRetrieveView for City.
    """
    serializer_class = CitySerializer
    queryset = City.objects.all()

    def get_queryset(self):
        """
        Allows a GET param, 'q', to be used against search_names.
        """
        queryset = super(CitiesLightListModelViewSet, self).get_queryset()

        if self.request.GET.get('q', None):
            return queryset.filter(
                search_names__icontains=self.request.GET['q'])

        return queryset

router = routers.SimpleRouter()
router.register(r'cities', CityModelViewSet, basename='cities-light-api-city')
router.register(r'countries', CountryModelViewSet,
                basename='cities-light-api-country')
router.register(r'regions', RegionModelViewSet,
                basename='cities-light-api-region')
router.register(r'subregions', SubRegionModelViewSet,
                basename='cities-light-api-subregion')

urlpatterns = [
    url(r'^', include(router.urls)),
]
<|endoftext|>


<commit_before>import numpy as np
import cudarray as ca
from .base import Op, Output, SplitMixin, Unary

class Flatten(Unary):
    def setup(self):
        shape = self.x.shape
        self.shape = (shape[0], np.prod(shape[1:]))
        self.array = ca.zeros(self.shape)
        self.grad_array = ca.zeros(self.shape)

    def fprop(self):
        self.array = ca.reshape(self.x.array, self.shape)

    def bprop(self):
        self.x.grad_array = ca.reshape(self.grad_array, self.x.shape)

class Reshape(Unary):
    def __init__(self, newshape):
        if not isinstance(newshape, tuple):
            if hasattr(newshape, '__iter__'):
                newshape = tuple(newshape)
            else:
                newshape = (newshape,)
        if not 0 <= list(newshape).count(-1) <= 1:
            raise ValueError('invalid newshape: %s' % newshape)
        self.newshape = newshape

    def setup(self):
        size = self.x.array.size
        newshape_size = np.prod(self.newshape)
        self.shape = tuple(d if d != -1 else -size//newshape_size
                           for d in self.newshape)
        self.array = ca.zeros(self.shape)
        self.grad_array = ca.zeros(self.shape)

    def fprop(self):
        self.array = ca.reshape(self.x.array, self.shape)

    def bprop(self):
        self.x.grad_array = ca.reshape(self.grad_array, self.x.shape)

class Slices(Op, SplitMixin):
    def __init__(self, splits):
        self.splits = splits

    def __call__(self, x):
        self.x = x
        self.inputs = [x]
        self.outputs = [Output()(self) for _ in range(len(self.splits)+1)]
        self.bpropable = x.bpropable
        return self.outputs

    def setup(self):
        splits = [0] + self.splits + [self.x.shape[0]]
        self.slices = []
        for i in range(len(splits) - 1):
            self.slices.append((splits[i], splits[i+1]))
        for i, (start, end) in enumerate(self.slices):
            shape = (end-start,) + self.x.shape[1:]
            self.outputs[i].shape = shape
            self.outputs[i].array = self.x.array[start:end, :]
            self.outputs[i].bpropable = self.bpropable
            if self.bpropable:
                self.outputs[i].grad_array = ca.zeros(shape)

    def fprop(self):
        for i, (start, end) in enumerate(self.slices):
            self.outputs[i].array = self.x.array[start:end, :]

    def bprop(self):
        for i, (start, end) in enumerate(self.slices):
            ca.copyto(self.x.grad_array[start:end, :],
                      self.outputs[i].grad_array)

class Transpose(Unary):
    def __init__(self, contiguous=False):
        self.contiguous = contiguous

    def setup(self):
        if len(self.x.shape) == 1:
            self.shape = (1,) + self.x.shape
        elif len(self.x.shape) == 2:
            self.shape = tuple(reversed(self.x.shape))
        else:
            raise ValueError('invalid shape for transpose: %s'
                             % str(self.x.shape))
        self.array = ca.zeros(self.shape)
        self.grad_array = ca.zeros(self.shape)

    def fprop(self):
        self.array = ca.transpose(self.x.array)
        if self.contiguous:
            self.array = ca.ascontiguousarray(self.array)

    def bprop(self):
        self.x.grad_array = ca.transpose(self.grad_array)
        if self.contiguous:
            self.array = ca.ascontiguousarray(self.x.grad_array)

class VSplit(Op, SplitMixin):
    def __init__(self, len_x):
        self.n_splits = len_x

    def __call__(self, x):
        self.x = x
        self.inputs = [x]
        self.bpropable = x.bpropable
        self.outputs = [Output()(self) for i in range(self.n_splits)]
        return self.outputs

    def setup(self):
        shape = self.x.shape[1:]
        for i in range(self.n_splits):
            self.outputs[i].shape = shape
            self.outputs[i].array = self.x.array[i]
            self.outputs[i].grad_array = ca.zeros(shape)

    def fprop(self):
        for i in range(self.n_splits):
            self.outputs[i].array = self.x.array[i]

    def bprop(self):
        for i in range(self.n_splits):
            ca.copyto(self.x.grad_array[i], self.outputs[i].grad_array)

class VStack(Op):
    def __call__(self, *xs):
        self.n_sources = len(xs)
        self.inputs = xs
        return self

    def setup(self):
        shape = self.inputs[0].shape
        for expr in self.inputs:
            if shape != expr.shape:
                raise ValueError('shape mismatch: %s and %s'
                                 % (shape, expr.shape))
        self.shape = (self.n_sources,) + shape
        self.array = ca.zeros(self.shape)
        self.grad_array = ca.zeros(self.shape)

    def fprop(self):
        for i in range(self.n_sources):
            ca.copyto(self.array[i], self.inputs[i].array)

    def bprop(self):
        for i in range(self.n_sources):
            ca.copyto(self.inputs[i].grad_array, self.grad_array[i])

class Concatenate(Op):
    def __init__(self, axis):
        self.axis = axis
        self.a_size = -1

    def __call__(self, a, b):
        self.a = a
        self.b = b
        self.bpropable = a.bpropable or b.bpropable
        self.inputs = [a, b]
        return self

    def setup(self):
        a_shp = self.a.shape
        b_shp = self.b.shape
        concat_size = a_shp[self.axis] + b_shp[self.axis]
        self.a_size = a_shp[self.axis]
        self.shape = (a_shp[:self.axis] + (concat_size,) + a_shp[self.axis+1:])
        self.array = ca.zeros(self.shape)
        self.grad_array = ca.zeros(self.shape)

    def fprop(self):
        ca.extra.concatenate(self.a.array, self.b.array, axis=self.axis,
                             out=self.array)

    def bprop(self):
        ca.extra.split(self.grad_array, a_size=self.a_size, axis=self.axis,
                       out_a=self.a.grad_array, out_b=self.b.grad_array)

def transpose(x):
    return Transpose()(x)
<commit_msg>small fixes<commit_after>import numpy as np
import cudarray as ca
from .base import Op, Output, SplitMixin, Unary

class Flatten(Unary):
    def setup(self):
        shape = self.x.shape
        self.shape = (shape[0], np.prod(shape[1:]))
        self.array = ca.zeros(self.shape)
        self.grad_array = ca.zeros(self.shape)

    def fprop(self):
        self.array = ca.reshape(self.x.array, self.shape)

    def bprop(self):
        self.x.grad_array = ca.reshape(self.grad_array, self.x.shape)

class Reshape(Unary):
    def __init__(self, newshape):
        if not isinstance(newshape, tuple):
            if hasattr(newshape, '__iter__'):
                newshape = tuple(newshape)
            else:
                newshape = (newshape,)
        if not 0 <= list(newshape).count(-1) <= 1:
            raise ValueError('invalid newshape: %s' % newshape)
        self.newshape = newshape

    def setup(self):
        size = self.x.array.size
        newshape_size = np.prod(self.newshape)
        self.shape = tuple(d if d != -1 else -size//newshape_size
                           for d in self.newshape)
        self.array = ca.zeros(self.shape)
        self.grad_array = ca.zeros(self.shape)

    def fprop(self):
        self.array = ca.reshape(self.x.array, self.shape)

    def bprop(self):
        self.x.grad_array = ca.reshape(self.grad_array, self.x.shape)

class Slices(Op, SplitMixin):
    def __init__(self, splits):
        self.splits = splits
        self.outputs = [Output()(self) for _ in range(len(self.splits)+1)]

    def __call__(self, x):
        self.x = x
        self.inputs = [x]
        self.bpropable = x.bpropable
        for output in self.outputs:
            output.bpropable = self.bpropable
        return self.outputs

    def setup(self):
        splits = [0] + self.splits + [self.x.shape[0]]
        self.slices = []
        for i in range(len(splits) - 1):
            self.slices.append((splits[i], splits[i+1]))
        for i, (start, end) in enumerate(self.slices):
            shape = (end-start,) + self.x.shape[1:]
            self.outputs[i].shape = shape
            self.outputs[i].array = self.x.array[start:end]
            if self.bpropable:
                self.outputs[i].grad_array = ca.zeros(shape)

    def fprop(self):
        for i, (start, end) in enumerate(self.slices):
            self.outputs[i].array = self.x.array[start:end]

    def bprop(self):
        for i, (start, end) in enumerate(self.slices):
            ca.copyto(self.x.grad_array[start:end], self.outputs[i].grad_array)

class Transpose(Unary):
    def __init__(self, contiguous=False):
        self.contiguous = contiguous

    def setup(self):
        if len(self.x.shape) == 1:
            self.shape = (1,) + self.x.shape
        elif len(self.x.shape) == 2:
            self.shape = tuple(reversed(self.x.shape))
        else:
            raise ValueError('invalid shape for transpose: %s'
                             % str(self.x.shape))
        self.array = ca.zeros(self.shape)
        self.grad_array = ca.zeros(self.shape)

    def fprop(self):
        self.array = ca.transpose(self.x.array)
        if self.contiguous:
            self.array = ca.ascontiguousarray(self.array)

    def bprop(self):
        self.x.grad_array = ca.transpose(self.grad_array)
        if self.contiguous:
            self.array = ca.ascontiguousarray(self.x.grad_array)

class VSplit(Op, SplitMixin):
    def __init__(self, len_x):
        self.n_splits = len_x

    def __call__(self, x):
        self.x = x
        self.inputs = [x]
        self.bpropable = x.bpropable
        self.outputs = [Output()(self) for i in range(self.n_splits)]
        return self.outputs

    def setup(self):
        shape = self.x.shape[1:]
        for i in range(self.n_splits):
            self.outputs[i].shape = shape
            self.outputs[i].array = self.x.array[i]
            self.outputs[i].grad_array = ca.zeros(shape)

    def fprop(self):
        for i in range(self.n_splits):
            self.outputs[i].array = self.x.array[i]

    def bprop(self):
        for i in range(self.n_splits):
            ca.copyto(self.x.grad_array[i], self.outputs[i].grad_array)

class VStack(Op):
    def __call__(self, *xs):
        self.n_sources = len(xs)
        self.inputs = xs
        return self

    def setup(self):
        shape = self.inputs[0].shape
        for expr in self.inputs:
            if shape != expr.shape:
                raise ValueError('shape mismatch: %s and %s'
                                 % (shape, expr.shape))
        self.shape = (self.n_sources,) + shape
        self.array = ca.zeros(self.shape)
        self.grad_array = ca.zeros(self.shape)

    def fprop(self):
        for i in range(self.n_sources):
            ca.copyto(self.array[i], self.inputs[i].array)

    def bprop(self):
        for i in range(self.n_sources):
            ca.copyto(self.inputs[i].grad_array, self.grad_array[i])

class Concatenate(Op):
    def __init__(self, axis):
        self.axis = axis
        self.a_size = -1

    def __call__(self, a, b):
        self.a = a
        self.b = b
        self.bpropable = a.bpropable or b.bpropable
        self.inputs = [a, b]
        return self

    def setup(self):
        a_shp = self.a.shape
        b_shp = self.b.shape
        concat_size = a_shp[self.axis] + b_shp[self.axis]
        self.a_size = a_shp[self.axis]
        self.shape = (a_shp[:self.axis] + (concat_size,) + a_shp[self.axis+1:])
        self.array = ca.zeros(self.shape)
        self.grad_array = ca.zeros(self.shape)

    def fprop(self):
        ca.extra.concatenate(self.a.array, self.b.array, axis=self.axis,
                             out=self.array)

    def bprop(self):
        ca.extra.split(self.grad_array, a_size=self.a_size, axis=self.axis,
                       out_a=self.a.grad_array, out_b=self.b.grad_array)

def transpose(x):
    return Transpose()(x)
<|endoftext|>


<commit_before>import re
import networkx as nx

from .ExpressionNode import ExpressionNode
from .modules import modules
from .util import hdl_syntax, hdl_entity, hdl_arch, hdl_inst
from .util import parse_net, sub_brackets

class ExpressionGraph(nx.DiGraph):
    """Defines a di-graph of arithmetic expressions

    The graph is defined as follows:
    - Each node is a module that computes an arithmetic expression
    - Each edge is a net connecting two modules

    Attributes:
        name (string): The name of the graph
        next_net (int): The next net name to be used
        next_block (int): The next block name to be used
        blocks (list): The list of blocks in the graph
        in_ports (list of ((string, int), string)): The list of input ports
        out_ports (list of ((string, int), string)): The list of output ports
    """

    def __init__(self, name="graph", in_ports=None, out_ports=None):
        """Initializes the graph

        Specifying in_ports and out_ports is optional.
        Doing so implies that the graph is already connected inside of a larger
        structure.
        If they are not specified, the graph will finds its own.

        Args:
            name (string): The name of the graph
            in_ports (list of tuples): The list of input ports
            out_ports (list of tuples): The list of output ports
        """

        # Make sure that in_ports and out_ports are both None or both lists
        if any(x is not None for x in [in_ports, out_ports]) and \
                not all(x is not None for x in [in_ports, out_ports]):
            raise ValueError(("in_ports and out_ports"
                "must both be None or both be lists"))

        self.name = name

        # Save the input and output ports
        self.in_ports = in_ports
        self.out_ports = out_ports

        # Procedurally-generated net names start with "n1"
        self.next_net = 1

        # Procedurally-generated block names start with "block1"
        self.next_block = 0
        self.blocks = [None]

        super().__init__()

    def add_node(self, node, **attr):
        """Adds a node to the graph

        Args:
            node (ExpressionNode): The name of the node to add
            attr (dict): The attributes of the node to add
        """

        if not isinstance(node, ExpressionNode):
            raise TypeError("Node must be an ExpressionNode")

        # Add GraphViz attributes
        attr = modules[node.value]
        attr["shape"] = attr.get("shape", "square")
        attr["fillcolor"] = attr.get("fillcolor", "white")
        attr["label"] = attr.get("label", "")
        attr["style"] = attr.get("style", "filled")

        # Add the node to the graph
        super().add_node(node, **attr)
        return node

    def remove_node(self, node):
        """Removes a node from the graph

        Args:
            node (ExpressionNode): The name of the node to remove
        """

        if not isinstance(node, ExpressionNode):
            raise TypeError("Node must be an ExpressionNode")

        # Remove the node from the graph
        super().remove_node(node)
        return node

    def add_edge(self, node1, pin1, node2, pin2):
        """Adds a directed edge to the graph, from node1 to node2

        Args:
            node1 (ExpressionNode): The name of the first node
            pin1 (tuple): (name, index) of the pin on the first node
            node2 (ExpressionNode): The name of the second node
            pin2 (tuple): (name, index) of the pin on the second node
        """

        if not isinstance(node1, ExpressionNode):
            raise TypeError("Node1 must be an ExpressionNode")
        if not isinstance(node2, ExpressionNode):
            raise TypeError("Node2 must be an ExpressionNode")

        # Connect the nodes
        proposed_net_name = "$n{0}_{1}".format(self.next_net, self.name)
        actual_net_name = node1.add_child(node2, pin1, pin2)
        if proposed_net_name == actual_net_name:
            self.next_net += 1

        # Styles the edge for GraphViz visualization
        attr = {
            "arrowhead": "none",
            "headport": "ne",
            "tailport": "sw",
            "ins": [pin1],
            "outs": [pin2],
            "edge_nets": [actual_net_name]
        }

        # Initialize weight to 1
        # This is later modified by logical effort
        attr["weight"] = 1

        # If the two nodes are already connnected, simply update the pins
        if self.has_edge(node1, node2):
            edge_data = self.get_edge_data(node1, node2, default = attr)
            edge_data["ins"].append(pin1)
            edge_data["outs"].append(pin2)
            edge_data["edge_nets"].append(actual_net_name)
        else:
        # Add the edge to the graph
            super().add_edge(node1, node2, **attr)

        return self.get_edge_data(node1, node2)

    def add_block(self, *nodes):
        """Creates a new module block of nodes

        This is a sub-graph of nodes that will be flattened together into a
        single monolithic module.

        Args:
            nodes (list): The list of nodes to add to the block
        """
        if not all([isinstance(node, ExpressionNode) for node in nodes]):
            raise TypeError("Nodes must be ExpressionNodes")
        if not all([node.block is None for node in nodes]):
            raise ValueError("Nodes must not already be in a block")

        # If the block is empty, don't create it
        if not nodes:
            return None

        # Get the name of the block
        block_id = self.next_block

        # Assign the block to the nodes
        for node in nodes:
            node.block = block_id

        # Create the block
        new_block = self.subgraph(nodes)
        self.blocks[block_id] = new_block
        if block_id == len(self.blocks) - 1:
            self.blocks.append(None)

        # Increment the next block name
        self.next_block = next(
                x for x in range(len(self.blocks)) if self.blocks[x] is None
        )

        return block_id

    def remove_block(self, block_id):
        """Removes a block from the graph

        Args:
            block_id (int): The ID of the block to remove
        """
        if block_id not in range(len(self.blocks)) or self.blocks[block_id] is None:
            raise ValueError("Invalid block ID")

        # Remove the block from the graph
        block = self.blocks[block_id]
        for node in block:
            node.block = None
        self.blocks[block_id] = None

    def reset_blocks(self):
        """Removes all blocks from the graph"""

        # Remove all blocks from the graph
        for block_id in [x for x in self.blocks if x is not None]:
            self.remove_block(block_id)

        # Reset the next block name
        self.next_block = 0
        self.blocks = [None]

    def _get_internal_nets(self):
        """Returns the internal nets of the graph"""

        # Get all nets in the graph
        in_nets = set()
        out_nets = set()

        # Get the nets from nodes
        for node in self.nodes:
            for net in node.in_nets.values():
                in_nets.update([parse_net(net) for x in net])
            for net in node.out_nets.values():
                out_nets.update([parse_net(net) for x in net])

        # Get the nets from blocks
        for block in [x for x in self.blocks if x is not None]:
            block_in_nets, block_out_nets = block._get_internal_nets()
            in_nets.update(block_in_nets)
            out_nets.update(block_out_nets)

        # If the graph has pre-defined ports, filter them out
        if self.in_ports is not None:
            in_ports = [x[0] for x in in_ports[0]]
            in_nets = [x for x in in_nets if x.split("[")[0] not in in_ports]
            out_ports = [x[0] for x in out_ports[0]]
            out_nets = [x for x in out_nets if x.split("[")[0] not in out_ports]

        return (set(in_nets), set(out_nets))

    def _get_ports(self):
        """Returns the ports of the graph"""

        # If the ports are defined, return them
        if self.in_ports is not None:
            return (self.in_ports, self.out_ports)

        # Otherwise, find them from nodes and blocks
        in_ports, out_ports = self._get_internal_nets()

        # Signals generated inside this graph are not inputs
        in_ports = in_ports - out_ports

        # Form the ports
        # Assume that all these retrieved nets are 1-bit
        in_internal = [(sub_brackets(x),1) for x in in_ports]
        in_external = list(in_ports)
        in_ports = (in_internal, in_external)

        out_internal = [(sub_brackets(x),1) for x in out_ports]
        out_external = list(out_ports)
        out_ports = (out_internal, out_external)

        return (in_ports, out_ports)

    def hdl(
        self,
        language="verilog",
        flat=False,
        full_flat=False,
        module_name=None,
        description_string="start of unnamed graph"
    ):
        """Creates a HDL description of the graph

        Args:
            language (str): The language in which to generate the HDL
            flat (bool): If True, flatten the graph's HDL
            full_flat (bool): If True, flatten all modules in the graph's HDL
            module_name (str): The name of the module to generate
            description_string (str): String commend to prepend to the HDL

        Returns:
            str: HDL module definition representing the graph
            list: Set of HDL module definitions used in the graph

        """
        # Check that the language is supported
        if language not in ["verilog", "vhdl"]:
            raise ValueError("Unsupported hardware-descriptive language")

        # If module name is not defined, set it to graph's name
        if module_name is None:
            module_name = self.name

        # Set language-specific syntax
        syntax = hdl_syntax[language]

        # Create the HDL

        hdl = ""
        module_defs = set()

        # Pull in the HDL description of blocks
        for block_id in range(len(self.blocks)):
            if self.blocks[block_id] is None:
                continue
            block = self.blocks[block_id]

            block_hdl, block_defs = block.hdl(
                    language=language,
                    flat=full_flat,
                    full_flat=True,
                    module_name=module_name + "_block_" + str(block_id),
                    description_string="block {0}".format(block_id)
            )

            hdl += block_hdl
            module_defs.update(block_defs)

        # Pull in the HDL description of nodes outside of blocks
        for node in self:
            if node.block is not None:
                continue
            node_hdl, node_defs = node.hdl(
                    language=language,
                    flat=full_flat
            )

            hdl += node_hdl
            module_defs.update(node_defs)

        # This HDL description will have multiple instances in it
        # By default, util.hdl_inst names all instances "U0"
        # These names need to be made unique
        U_count = 0
        for U in re.finditer(r"U\d+", hdl):
            hdl = hdl[:U.start()] + "U" + str(U_count) + hdl[U.end():]
            U_count += 1

        # Add wire definitions
        in_wires, out_wires = self._get_internal_nets()
        wires = in_wires | out_wires

        wire_hdl = syntax["wire_def"].format(",".join(wires))

        # Assemble the HDL
        hdl = syntax["comment_string"] + description_string + "\n" + \
                wire_hdl + "\n" + hdl

        # If flat HDL is desired, it can returned here
        if flat:
            return hdl, module_defs

        # If flat HDL is not desired, wrap the graph in a module
        ## First get in_ports and out_ports
        (in_ports, out_ports) = self._get_ports()
        ## Then create the entity
        entity = hdl_entity(module_name, in_ports[0], out_ports[0], language)
        ## Then create the architecture
        arch = hdl_arch(module_name, hdl, language)
        ## Add the entity and architecture to the module_defs
        module_defs.add(entity+arch)
        ## Create an instance of the module
        inst_ports = [[x[0][0],x[1]] for x in ins + outs]
        inst = hdl_inst(module_name, inst_ports, language)

        return hdl, module_defs

if __name__ == "__main__":
    raise RuntimeError("This module is not intended to be run directly")
<commit_msg>Reformat ExpressionGraph add_edge<commit_after>import re
import networkx as nx

from .ExpressionNode import ExpressionNode
from .modules import modules
from .util import hdl_syntax, hdl_entity, hdl_arch, hdl_inst
from .util import parse_net, sub_brackets

class ExpressionGraph(nx.DiGraph):
    """Defines a di-graph of arithmetic expressions

    The graph is defined as follows:
    - Each node is a module that computes an arithmetic expression
    - Each edge is a net connecting two modules

    Attributes:
        name (string): The name of the graph
        next_net (int): The next net name to be used
        next_block (int): The next block name to be used
        blocks (list): The list of blocks in the graph
        in_ports (list of ((string, int), string)): The list of input ports
        out_ports (list of ((string, int), string)): The list of output ports
    """

    def __init__(self, name="graph", in_ports=None, out_ports=None):
        """Initializes the graph

        Specifying in_ports and out_ports is optional.
        Doing so implies that the graph is already connected inside of a larger
        structure.
        If they are not specified, the graph will finds its own.

        Args:
            name (string): The name of the graph
            in_ports (list of tuples): The list of input ports
            out_ports (list of tuples): The list of output ports
        """

        # Make sure that in_ports and out_ports are both None or both lists
        if any(x is not None for x in [in_ports, out_ports]) and \
                not all(x is not None for x in [in_ports, out_ports]):
            raise ValueError(("in_ports and out_ports"
                "must both be None or both be lists"))

        self.name = name

        # Save the input and output ports
        self.in_ports = in_ports
        self.out_ports = out_ports

        # Procedurally-generated net names start with "n1"
        self.next_net = 1

        # Procedurally-generated block names start with "block1"
        self.next_block = 0
        self.blocks = [None]

        super().__init__()

    def add_node(self, node, **attr):
        """Adds a node to the graph

        Args:
            node (ExpressionNode): The name of the node to add
            attr (dict): The attributes of the node to add
        """

        if not isinstance(node, ExpressionNode):
            raise TypeError("Node must be an ExpressionNode")

        # Add GraphViz attributes
        attr = modules[node.value]
        attr["shape"] = attr.get("shape", "square")
        attr["fillcolor"] = attr.get("fillcolor", "white")
        attr["label"] = attr.get("label", "")
        attr["style"] = attr.get("style", "filled")

        # Add the node to the graph
        super().add_node(node, **attr)
        return node

    def remove_node(self, node):
        """Removes a node from the graph

        Args:
            node (ExpressionNode): The name of the node to remove
        """

        if not isinstance(node, ExpressionNode):
            raise TypeError("Node must be an ExpressionNode")

        # Remove the node from the graph
        super().remove_node(node)
        return node

    def add_edge(self, parent, pin1, child, pin2):
        """Adds a directed edge to the graph, from parent to child

        Args:
            parent (ExpressionNode): The name of the first node
            pin1 (tuple): (name, index) of the pin on the first node
            child (ExpressionNode): The name of the second node
            pin2 (tuple): (name, index) of the pin on the second node
        """

        if not isinstance(parent, ExpressionNode):
            raise TypeError("Node1 must be an ExpressionNode")
        if not isinstance(child, ExpressionNode):
            raise TypeError("Node2 must be an ExpressionNode")

        # Connect the nodes
        proposed_net_name = "$n{0}_{1}".format(self.next_net, self.name)
        actual_net_name = parent.add_child(child, pin1, pin2)
        if proposed_net_name == actual_net_name:
            self.next_net += 1

        # Styles the edge for GraphViz visualization
        attr = {
            "arrowhead": "none",
            "headport": "ne",
            "tailport": "sw",
            "ins": [pin1],
            "outs": [pin2],
            "edge_nets": [actual_net_name]
        }

        # Initialize weight to 1
        # This is later modified by logical effort
        attr["weight"] = 1

        # If the two nodes are already connnected, simply update the pins
        if self.has_edge(parent, child):
            edge_data = self.get_edge_data(parent, child, default = attr)
            edge_data["ins"].append(pin1)
            edge_data["outs"].append(pin2)
            edge_data["edge_nets"].append(actual_net_name)
        else:
        # Add the edge to the graph
            super().add_edge(parent, child, **attr)

        return self.get_edge_data(parent, child)

    def add_block(self, *nodes):
        """Creates a new module block of nodes

        This is a sub-graph of nodes that will be flattened together into a
        single monolithic module.

        Args:
            nodes (list): The list of nodes to add to the block
        """
        if not all([isinstance(node, ExpressionNode) for node in nodes]):
            raise TypeError("Nodes must be ExpressionNodes")
        if not all([node.block is None for node in nodes]):
            raise ValueError("Nodes must not already be in a block")

        # If the block is empty, don't create it
        if not nodes:
            return None

        # Get the name of the block
        block_id = self.next_block

        # Assign the block to the nodes
        for node in nodes:
            node.block = block_id

        # Create the block
        new_block = self.subgraph(nodes)
        self.blocks[block_id] = new_block
        if block_id == len(self.blocks) - 1:
            self.blocks.append(None)

        # Increment the next block name
        self.next_block = next(
                x for x in range(len(self.blocks)) if self.blocks[x] is None
        )

        return block_id

    def remove_block(self, block_id):
        """Removes a block from the graph

        Args:
            block_id (int): The ID of the block to remove
        """
        if block_id not in range(len(self.blocks)) or self.blocks[block_id] is None:
            raise ValueError("Invalid block ID")

        # Remove the block from the graph
        block = self.blocks[block_id]
        for node in block:
            node.block = None
        self.blocks[block_id] = None

    def reset_blocks(self):
        """Removes all blocks from the graph"""

        # Remove all blocks from the graph
        for block_id in [x for x in self.blocks if x is not None]:
            self.remove_block(block_id)

        # Reset the next block name
        self.next_block = 0
        self.blocks = [None]

    def _get_internal_nets(self):
        """Returns the internal nets of the graph"""

        # Get all nets in the graph
        in_nets = set()
        out_nets = set()

        # Get the nets from nodes
        for node in self.nodes:
            for net in node.in_nets.values():
                in_nets.update([parse_net(net) for x in net])
            for net in node.out_nets.values():
                out_nets.update([parse_net(net) for x in net])

        # Get the nets from blocks
        for block in [x for x in self.blocks if x is not None]:
            block_in_nets, block_out_nets = block._get_internal_nets()
            in_nets.update(block_in_nets)
            out_nets.update(block_out_nets)

        # If the graph has pre-defined ports, filter them out
        if self.in_ports is not None:
            in_ports = [x[0] for x in in_ports[0]]
            in_nets = [x for x in in_nets if x.split("[")[0] not in in_ports]
            out_ports = [x[0] for x in out_ports[0]]
            out_nets = [x for x in out_nets if x.split("[")[0] not in out_ports]

        return (set(in_nets), set(out_nets))

    def _get_ports(self):
        """Returns the ports of the graph"""

        # If the ports are defined, return them
        if self.in_ports is not None:
            return (self.in_ports, self.out_ports)

        # Otherwise, find them from nodes and blocks
        in_ports, out_ports = self._get_internal_nets()

        # Signals generated inside this graph are not inputs
        in_ports = in_ports - out_ports

        # Form the ports
        # Assume that all these retrieved nets are 1-bit
        in_internal = [(sub_brackets(x),1) for x in in_ports]
        in_external = list(in_ports)
        in_ports = (in_internal, in_external)

        out_internal = [(sub_brackets(x),1) for x in out_ports]
        out_external = list(out_ports)
        out_ports = (out_internal, out_external)

        return (in_ports, out_ports)

    def hdl(
        self,
        language="verilog",
        flat=False,
        full_flat=False,
        module_name=None,
        description_string="start of unnamed graph"
    ):
        """Creates a HDL description of the graph

        Args:
            language (str): The language in which to generate the HDL
            flat (bool): If True, flatten the graph's HDL
            full_flat (bool): If True, flatten all modules in the graph's HDL
            module_name (str): The name of the module to generate
            description_string (str): String commend to prepend to the HDL

        Returns:
            str: HDL module definition representing the graph
            list: Set of HDL module definitions used in the graph

        """
        # Check that the language is supported
        if language not in ["verilog", "vhdl"]:
            raise ValueError("Unsupported hardware-descriptive language")

        # If module name is not defined, set it to graph's name
        if module_name is None:
            module_name = self.name

        # Set language-specific syntax
        syntax = hdl_syntax[language]

        # Create the HDL

        hdl = ""
        module_defs = set()

        # Pull in the HDL description of blocks
        for block_id in range(len(self.blocks)):
            if self.blocks[block_id] is None:
                continue
            block = self.blocks[block_id]

            block_hdl, block_defs = block.hdl(
                    language=language,
                    flat=full_flat,
                    full_flat=True,
                    module_name=module_name + "_block_" + str(block_id),
                    description_string="block {0}".format(block_id)
            )

            hdl += block_hdl
            module_defs.update(block_defs)

        # Pull in the HDL description of nodes outside of blocks
        for node in self:
            if node.block is not None:
                continue
            node_hdl, node_defs = node.hdl(
                    language=language,
                    flat=full_flat
            )

            hdl += node_hdl
            module_defs.update(node_defs)

        # This HDL description will have multiple instances in it
        # By default, util.hdl_inst names all instances "U0"
        # These names need to be made unique
        U_count = 0
        for U in re.finditer(r"U\d+", hdl):
            hdl = hdl[:U.start()] + "U" + str(U_count) + hdl[U.end():]
            U_count += 1

        # Add wire definitions
        in_wires, out_wires = self._get_internal_nets()
        wires = in_wires | out_wires

        wire_hdl = syntax["wire_def"].format(",".join(wires))

        # Assemble the HDL
        hdl = syntax["comment_string"] + description_string + "\n" + \
                wire_hdl + "\n" + hdl

        # If flat HDL is desired, it can returned here
        if flat:
            return hdl, module_defs

        # If flat HDL is not desired, wrap the graph in a module
        ## First get in_ports and out_ports
        (in_ports, out_ports) = self._get_ports()
        ## Then create the entity
        entity = hdl_entity(module_name, in_ports[0], out_ports[0], language)
        ## Then create the architecture
        arch = hdl_arch(module_name, hdl, language)
        ## Add the entity and architecture to the module_defs
        module_defs.add(entity+arch)
        ## Create an instance of the module
        inst_ports = [[x[0][0],x[1]] for x in ins + outs]
        inst = hdl_inst(module_name, inst_ports, language)

        return hdl, module_defs

if __name__ == "__main__":
    raise RuntimeError("This module is not intended to be run directly")
<|endoftext|>


<commit_before>"""
pbs_bullet

A script to keep an eye on HPC jobs using the PBS system, and
send pushbullet notifications on a few events.

I have not tested this. Use with extreme caution.

Jono Gray
j.gray@soton.ac.uk
"""

import sys
import argparse
import json
import urllib2, urllib
import logging
from time import sleep

logging.basicConfig()
logger = logging.getLogger(__name__)

try:
    from subprocess import check_output, call
except:
    logger.error("pbs_bullet uses the check_output command, added in python 2.7.")
    sys.exit(1)

def arguments():
    parser = argparse.ArgumentParser(
        description="Watch a PBS job. Can optionally send notifications by pushbullet, and kill the job if low on memory.")
    parser.add_argument('jobid', help='Job id to monitor.')
    parser.add_argument('--pushbullet-token', type=str, dest="pb_token", default=None)
    parser.add_argument('--notify-on', type=str, nargs='*', dest="notify_on",
        choices=["start", "finish", "killed", "fail"], default=["start", "finish", "killed", "fail"],
        help="Events to send a notification on.")
    parser.add_argument('--low-mem', type=float, dest="kill_threshold", default=0.,
        help="Kill the job using qdel if free memory on any one node drops below this \%.")
    parser.add_argument('--poll-interval', type=int, dest="poll_interval", default=300,
        help="Number of seconds to wait between each check. Defaults to 5 minutes.")
    parser.add_argument('--log-level', dest='log_level', type=str, choices=['debug',
        'info', 'warning', 'error'], default='info', nargs="?")
    args, extras = parser.parse_known_args()
    return args

def parse_job(jobdetails):
    """
    Turn the output of qstat -f into a dictionary.
    """
    lines = jobdetails.replace("\n\t", "").splitlines()[1:-1]
    return dict(map(lambda line: tuple(line.strip().split(" = ")), lines))

def get_nodes(job_dict):
    """
    Return a list of the nodes in use from a job dictionary.
    """

    nodes = job_dict['exec_host']
    return list(set(map(lambda x: x.split('/')[0], nodes.split('+'))))

def kill_job(jobid):
    """
    Attempt to kill a job using qdel.
    """

    return call(['qdel', jobid])

def check_free(node):
    """
    Use rsh and free to get the percentage of free memory on a node.
    """

    return check_output(["rsh", node, "free", "|",  "awk",  "'FNR == 3 {print $4/($3+$4)*100}'"])

def start_notify(jobid, jobdetails, nodes, pb_token):
    """
    Send a notification that the job has started.
    """
    title = "%s, id: %s, started." % (jobdetails['Job_Name'], str(jobid))
    body = "Running on nodes %s, and due to finish %s." % (", ".join(nodes), jobdetails['etime'])
    send_notification(title, body, pb_token)

def finish_notify(jobid, jobdetails, pb_token):
    """
    Send a notification that the job has completed.
    """
    title = "%s, id: %s, finished." % (jobdetails['Job_Name'], str(jobid))
    body = ""
    send_notification(title, body, pb_token)

def kill_notify(jobid, jobdetails, nodes, freemem, pb_token):
    """
    Send a notification that the job is being killed.
    """
    title = "Attempting to kill job %s, id: %s." % (jobdetails['Job_Name'], str(jobid))
    body = make_free_str(nodes, freemem)
    send_notification(title, "\n".join(body), pb_token)

def send_notification(title, body, token, target=None):
    """
    Send a pushbullet notification.
    """
    data = {"type":"note", "title": title, "body": body}
    if target is not None:
        data['device_iden'] = target
    note = json.dumps(data)
    logger.debug("Sending %s to pushbullet." % note)
    request = urllib2.Request('https://api.pushbullet.com/v2/pushes', note, headers={
        'Authorization':"Bearer %s" % token,
        'Content-Type':'application/json',
        'Accept':'*/*'
    })
    try:
        resp = urllib2.urlopen(request)
    except urllib2.HTTPError as e:
        logger.error("Pushbullet notify error.")
        logger.error(e.read())

def create_listener(name, token):
    """
    Register this device to receive pushbullet notifications,
    and if successful return the identifier.
    """
    data = urllib.urlencode({'nickname':name, 'type':'streaming'})
    logger.debug("Adding %s to pushbullet." % name)
    request = urllib2.Request('https://api.pushbullet.com/v2/devices', data, headers={
        'Authorization':"Bearer %s" % token,
        'Accept':'*/*'
    })
    try:
        resp = urllib2.urlopen(request)
        return json.load(resp)['iden']
    except urllib2.HTTPError as e:
        logger.error("Pushbullet register error.")
        logger.error(e.read())

def delete_listener(iden, token):
    """
    Unregister this device as a listener.
    """
    logger.debug("Deleting %s from pushbullet." % iden)
    request = urllib2.Request('https://api.pushbullet.com/v2/devices/%s' % iden, headers={
        'Authorization':"Bearer %s" % token,
        'Accept':'*/*'
    })
    request.get_method = lambda: 'DELETE'
    try:
        resp = urllib2.urlopen(request)
        return json.load(resp)
    except urllib2.HTTPError as e:
        logger.error("Pushbullet delete error.")
        logger.error(e.read())

def delete_push(push, token):
    """
    Delete a push.
    """
    logger.debug("Deleting push id %s from pushbullet." % push['iden'])
    request = urllib2.Request('https://api.pushbullet.com/v2/pushes/%s' % push['iden'], headers={
        'Authorization':"Bearer %s" % token,
        'Accept':'*/*'
    })
    request.get_method = lambda: 'DELETE'
    try:
        resp = urllib2.urlopen(request)
        return json.load(resp)
    except urllib2.HTTPError as e:
        logger.error("Pushbullet delete error.")
        logger.error(e.read())

def check_pushes(iden, token):
    """
    Return any undismissed pushes for this device, and dismiss
    them.
    """
    data = urllib.urlencode({'active':'0'})
    logger.debug("Checking pushes for %s." % iden)
    request = urllib2.Request('https://api.pushbullet.com/v2/pushes?%s' % data, headers={
        'Authorization':"Bearer %s" % token,
        'Accept':'*/*'
    })
    try:
        resp = urllib2.urlopen(request)
        pushes = json.load(resp)['pushes']
        pushes = filter(lambda push: 'target_device_iden' in push.keys() and push['target_device_iden'] == iden, pushes)
        logger.debug("Got %d pushes." % len(pushes))
        #Delete them from the server
        map(lambda push: delete_push(push, token), pushes)
        pushes.reverse()
        return pushes

    except urllib2.HTTPError as e:
        logger.error("Pushbullet check error.")
        logger.error(e.read())

def parse_push(push, token, jobid, jobdetails):
    """
    Take a push, and execute some commands.
    This is very primitive - we look for preset strings
    in the body of the push.
    """
    logger.debug("Parsing push msg.")

    try:
        cmd = push['body'].lower()
        logger.debug(cmd)
        target = push['source_device_iden']
        commands = []
        if 'showstart' in cmd:
            # Return the starttime for this job.
            try:
                body = check_output(['showstart', jobid])
                title = "Job %s (%s) Start Time" % (jobdetails['Job_Name'], jobid)
            except Exception as e:
                body = str(e)
                title = "Showstart failed."
            send_notification(title, body, token, target=target)
            commands.append('showstart')
        if 'cancel' in cmd:
            # Cancel the job
            try:
                kill_job(jobid)
                title = "Attempting to kill job %s, id: %s." % (jobdetails['Job_Name'], str(jobid))
                body = ""
            except Exception as e:
                body = str(e)
                title = "qdel failed."
            send_notification(title, body, token, target=target)
            commands.append('showstart')
            commands.append('cancel')
        if 'freemem' in cmd:
            # Get the free memory for nodes
            try:
                nodes = get_nodes(jobdetails)
                freemem = map(check_free, nodes)
                body = make_free_str(nodes, freemem)
                title = "Job %s (%s) Free Memory" % (jobdetails['Job_Name'], jobid)
            except Exception as e:
                body = str(e)
                title = "Freemem check failed."
            send_notification(title, body, token, target=target)
            commands.append('freemem')
        assert commands
    except KeyError:
        logger.debug("No body in this push.")
    except AssertionError:
        logger.debug("No commands in this push.")

def make_free_str(nodes, freemem):
    return "Free memory - %s" % ", ".join(map(lambda (node, free): "%s: %s%%" % (node, free.strip()), zip(nodes, freemem)))

def main():
    args = arguments()
    numeric_level = getattr(logging, args.log_level.upper(), None)
    if not isinstance(numeric_level, int):
        raise ValueError('Invalid log level: %s' % log_level)

    logger.setLevel(numeric_level)
    jobid = args.jobid
    pb_token = args.pb_token
    sleep_time = args.poll_interval
    notify_on = args.notify_on
    lowmem = args.kill_threshold

    started = False
    finished = False

    try:
        if pb_token is not None:
            try:
                logger.debug("Checking status for job %s" % jobid)
                jobdetails = parse_job(check_output(['qstat', '-f', jobid]))
            except Exception as e:
                logger.error('qstat command failed. Bailing out.')
                logger.error('Error was:')
                logger.error(e)
                raise
            name = "%s - %s" % (jobdetails['Job_Name'], jobid)
            iden = create_listener(name, pb_token)

        while not finished:
            try:
                logger.debug("Checking status for job %s" % jobid)
                jobdetails = parse_job(check_output(['qstat', '-f', jobid]))
            except Exception as e:
                logger.error('qstat command failed. Bailing out.')
                logger.error('Error was:')
                logger.error(e)
                if pb_token is not None:
                    delete_listener(iden, pb_token)
                break
            logger.debug("Job state is %s" % jobdetails['job_state'])
            if jobdetails['job_state'] == 'R':
                logger.debug("Job %s is running." % jobid)
                nodes = get_nodes(jobdetails)
                if not started:
                    started = True

                    if pb_token is not None and "start" in notify_on:
                        start_notify(jobid, jobdetails, nodes, pb_token)

                #Check memory use
                try:
                    logger.debug("Checking memory on %s" % ", ".join(nodes))
                    freemem = map(check_free, nodes)
                    logger.debug(str(freemem))
                    logger.debug(str(nodes))
                    logger.debug(make_free_str(nodes, freemem))
                    if filter(lambda x: float(x) < lowmem, freemem):
                        logger.debug("Free memory below threshold. Killing the job.")
                        try:
                            kill_job(jobid)
                        except Exception as e:
                            logger.error("qdel command failed.")
                            logger.error('Error was:')
                            logger.error(e)
                        if pb_token is not None and "kill" in notify_on:
                            kill_notify(jobid, jobdetails, nodes, freemem, pb_token)
                except Exception as e:
                    logger.error("Freemem check failed.")
                    logger.error(e)

            elif jobdetails['job_state'] != 'R' and started:
                #Job finished. Notify if appropriate
                finished = True
                if pb_token is not None and "finish" in notify_on:
                    finish_notify(jobid, jobdetails, pb_token)
                break

            #Check for pushed commands
            if pb_token is not None:
                map(lambda push: parse_push(push, pb_token, jobid, jobdetails), check_pushes(iden, pb_token))
            logger.debug("Sleeping for %ds" % sleep_time)
            sleep(sleep_time)
    except Exception:
        raise
    finally:
        if pb_token is not None:
            delete_listener(iden, pb_token)

if __name__ == "__main__":
    main()<commit_msg>debugging<commit_after>"""
pbs_bullet

A script to keep an eye on HPC jobs using the PBS system, and
send pushbullet notifications on a few events.

I have not tested this. Use with extreme caution.

Jono Gray
j.gray@soton.ac.uk
"""

import sys
import argparse
import json
import urllib2, urllib
import logging
from time import sleep

logging.basicConfig()
logger = logging.getLogger(__name__)

try:
    from subprocess import check_output, call
except:
    logger.error("pbs_bullet uses the check_output command, added in python 2.7.")
    sys.exit(1)

def arguments():
    parser = argparse.ArgumentParser(
        description="Watch a PBS job. Can optionally send notifications by pushbullet, and kill the job if low on memory.")
    parser.add_argument('jobid', help='Job id to monitor.')
    parser.add_argument('--pushbullet-token', type=str, dest="pb_token", default=None)
    parser.add_argument('--notify-on', type=str, nargs='*', dest="notify_on",
        choices=["start", "finish", "killed", "fail"], default=["start", "finish", "killed", "fail"],
        help="Events to send a notification on.")
    parser.add_argument('--low-mem', type=float, dest="kill_threshold", default=0.,
        help="Kill the job using qdel if free memory on any one node drops below this \%.")
    parser.add_argument('--poll-interval', type=int, dest="poll_interval", default=300,
        help="Number of seconds to wait between each check. Defaults to 5 minutes.")
    parser.add_argument('--log-level', dest='log_level', type=str, choices=['debug',
        'info', 'warning', 'error'], default='info', nargs="?")
    args, extras = parser.parse_known_args()
    return args

def parse_job(jobdetails):
    """
    Turn the output of qstat -f into a dictionary.
    """
    lines = jobdetails.replace("\n\t", "").splitlines()[1:-1]
    return dict(map(lambda line: tuple(line.strip().split(" = ")), lines))

def get_nodes(job_dict):
    """
    Return a list of the nodes in use from a job dictionary.
    """

    nodes = job_dict['exec_host']
    return list(set(map(lambda x: x.split('/')[0], nodes.split('+'))))

def kill_job(jobid):
    """
    Attempt to kill a job using qdel.
    """

    return call(['qdel', jobid])

def check_free(node):
    """
    Use rsh and free to get the percentage of free memory on a node.
    """

    return check_output(["rsh", node, "free", "|",  "awk",  "'FNR == 3 {print $4/($3+$4)*100}'"])

def start_notify(jobid, jobdetails, nodes, pb_token):
    """
    Send a notification that the job has started.
    """
    title = "%s, id: %s, started." % (jobdetails['Job_Name'], str(jobid))
    body = "Running on nodes %s, and started %s." % (", ".join(nodes), jobdetails['etime'])
    send_notification(title, body, pb_token)

def finish_notify(jobid, jobdetails, pb_token):
    """
    Send a notification that the job has completed.
    """
    title = "%s, id: %s, finished." % (jobdetails['Job_Name'], str(jobid))
    body = ""
    send_notification(title, body, pb_token)

def kill_notify(jobid, jobdetails, nodes, freemem, pb_token):
    """
    Send a notification that the job is being killed.
    """
    title = "Attempting to kill job %s, id: %s." % (jobdetails['Job_Name'], str(jobid))
    body = make_free_str(nodes, freemem)
    send_notification(title, "\n".join(body), pb_token)

def send_notification(title, body, token, target=None):
    """
    Send a pushbullet notification.
    """
    data = {"type":"note", "title": title, "body": body}
    if target is not None:
        data['device_iden'] = target
    note = json.dumps(data)
    logger.debug("Sending %s to pushbullet." % note)
    request = urllib2.Request('https://api.pushbullet.com/v2/pushes', note, headers={
        'Authorization':"Bearer %s" % token,
        'Content-Type':'application/json',
        'Accept':'*/*'
    })
    try:
        resp = urllib2.urlopen(request)
    except urllib2.HTTPError as e:
        logger.error("Pushbullet notify error.")
        logger.error(e.read())

def create_listener(name, token):
    """
    Register this device to receive pushbullet notifications,
    and if successful return the identifier.
    """
    data = urllib.urlencode({'nickname':name, 'type':'streaming'})
    logger.debug("Adding %s to pushbullet." % name)
    request = urllib2.Request('https://api.pushbullet.com/v2/devices', data, headers={
        'Authorization':"Bearer %s" % token,
        'Accept':'*/*'
    })
    try:
        resp = urllib2.urlopen(request)
        return json.load(resp)['iden']
    except urllib2.HTTPError as e:
        logger.error("Pushbullet register error.")
        logger.error(e.read())

def delete_listener(iden, token):
    """
    Unregister this device as a listener.
    """
    logger.debug("Deleting %s from pushbullet." % iden)
    request = urllib2.Request('https://api.pushbullet.com/v2/devices/%s' % iden, headers={
        'Authorization':"Bearer %s" % token,
        'Accept':'*/*'
    })
    request.get_method = lambda: 'DELETE'
    try:
        resp = urllib2.urlopen(request)
        return json.load(resp)
    except urllib2.HTTPError as e:
        logger.error("Pushbullet delete error.")
        logger.error(e.read())

def delete_push(push, token):
    """
    Delete a push.
    """
    logger.debug("Deleting push id %s from pushbullet." % push['iden'])
    request = urllib2.Request('https://api.pushbullet.com/v2/pushes/%s' % push['iden'], headers={
        'Authorization':"Bearer %s" % token,
        'Accept':'*/*'
    })
    request.get_method = lambda: 'DELETE'
    try:
        resp = urllib2.urlopen(request)
        return json.load(resp)
    except urllib2.HTTPError as e:
        logger.error("Pushbullet delete error.")
        logger.error(e.read())

def check_pushes(iden, token):
    """
    Return any undismissed pushes for this device, and dismiss
    them.
    """
    data = urllib.urlencode({'active':'0'})
    logger.debug("Checking pushes for %s." % iden)
    request = urllib2.Request('https://api.pushbullet.com/v2/pushes?%s' % data, headers={
        'Authorization':"Bearer %s" % token,
        'Accept':'*/*'
    })
    try:
        resp = urllib2.urlopen(request)
        pushes = json.load(resp)['pushes']
        pushes = filter(lambda push: 'target_device_iden' in push.keys() and push['target_device_iden'] == iden, pushes)
        logger.debug("Got %d pushes." % len(pushes))
        #Delete them from the server
        map(lambda push: delete_push(push, token), pushes)
        pushes.reverse()
        return pushes

    except urllib2.HTTPError as e:
        logger.error("Pushbullet check error.")
        logger.error(e.read())

def parse_push(push, token, jobid, jobdetails):
    """
    Take a push, and execute some commands.
    This is very primitive - we look for preset strings
    in the body of the push.
    """
    logger.debug("Parsing push msg.")

    try:
        cmd = push['body'].lower()
        logger.debug(cmd)
        logger.debug(push)
        target = push['source_device_iden']
        commands = []
        if 'showstart' in cmd:
            # Return the starttime for this job.
            try:
                body = check_output(['showstart', jobid])
                title = "Job %s (%s) Start Time" % (jobdetails['Job_Name'], jobid)
            except Exception as e:
                body = str(e)
                title = "Showstart failed."
            send_notification(title, body, token, target=target)
            commands.append('showstart')
        if 'cancel' in cmd:
            # Cancel the job
            try:
                kill_job(jobid)
                title = "Attempting to kill job %s, id: %s." % (jobdetails['Job_Name'], str(jobid))
                body = ""
            except Exception as e:
                body = str(e)
                title = "qdel failed."
            send_notification(title, body, token, target=target)
            commands.append('showstart')
            commands.append('cancel')
        if 'freemem' in cmd:
            # Get the free memory for nodes
            try:
                nodes = get_nodes(jobdetails)
                freemem = map(check_free, nodes)
                body = make_free_str(nodes, freemem)
                title = "Job %s (%s) Free Memory" % (jobdetails['Job_Name'], jobid)
            except Exception as e:
                body = str(e)
                title = "Freemem check failed."
            send_notification(title, body, token, target=target)
            commands.append('freemem')
        assert commands
    except KeyError as e:
        logger.debug(e)
        logger.debug("No body in this push.")
    except AssertionError:
        logger.debug("No commands in this push.")

def make_free_str(nodes, freemem):
    return "Free memory - %s" % ", ".join(map(lambda (node, free): "%s: %s%%" % (node, free.strip()), zip(nodes, freemem)))

def main():
    args = arguments()
    numeric_level = getattr(logging, args.log_level.upper(), None)
    if not isinstance(numeric_level, int):
        raise ValueError('Invalid log level: %s' % log_level)

    logger.setLevel(numeric_level)
    jobid = args.jobid
    pb_token = args.pb_token
    sleep_time = args.poll_interval
    notify_on = args.notify_on
    lowmem = args.kill_threshold

    started = False
    finished = False

    try:
        if pb_token is not None:
            try:
                logger.debug("Checking status for job %s" % jobid)
                jobdetails = parse_job(check_output(['qstat', '-f', jobid]))
            except Exception as e:
                logger.error('qstat command failed. Bailing out.')
                logger.error('Error was:')
                logger.error(e)
                raise
            name = "%s - %s" % (jobdetails['Job_Name'], jobid)
            iden = create_listener(name, pb_token)

        while not finished:
            try:
                logger.debug("Checking status for job %s" % jobid)
                jobdetails = parse_job(check_output(['qstat', '-f', jobid]))
            except Exception as e:
                logger.error('qstat command failed. Bailing out.')
                logger.error('Error was:')
                logger.error(e)
                raise
            logger.debug("Job state is %s" % jobdetails['job_state'])
            if jobdetails['job_state'] == 'R':
                logger.debug("Job %s is running." % jobid)
                nodes = get_nodes(jobdetails)
                if not started:
                    started = True

                    if pb_token is not None and "start" in notify_on:
                        start_notify(jobid, jobdetails, nodes, pb_token)

                #Check memory use
                try:
                    logger.debug("Checking memory on %s" % ", ".join(nodes))
                    freemem = map(check_free, nodes)
                    logger.debug(str(freemem))
                    logger.debug(str(nodes))
                    logger.debug(make_free_str(nodes, freemem))
                    if filter(lambda x: float(x) < lowmem, freemem):
                        logger.debug("Free memory below threshold. Killing the job.")
                        try:
                            kill_job(jobid)
                        except Exception as e:
                            logger.error("qdel command failed.")
                            logger.error('Error was:')
                            logger.error(e)
                        if pb_token is not None and "kill" in notify_on:
                            kill_notify(jobid, jobdetails, nodes, freemem, pb_token)
                except Exception as e:
                    logger.error("Freemem check failed.")
                    logger.error(e)

            elif jobdetails['job_state'] != 'R' and started:
                #Job finished. Notify if appropriate
                finished = True
                if pb_token is not None and "finish" in notify_on:
                    finish_notify(jobid, jobdetails, pb_token)
                break

            #Check for pushed commands
            if pb_token is not None:
                map(lambda push: parse_push(push, pb_token, jobid, jobdetails), check_pushes(iden, pb_token))
            logger.debug("Sleeping for %ds" % sleep_time)
            sleep(sleep_time)
    except Exception:
        raise
    finally:
        if pb_token is not None:
            delete_listener(iden, pb_token)

if __name__ == "__main__":
    main()<|endoftext|>


<commit_before># TSP Exact Undirected Solver
#
#
#
# For algorithms class, Spring 2015
# By Marshall Ehlinger

import math

def lexPermute(p):
	# If permutation array "a" (p in the code below) has two consecutive elements in increasing order,
    # Find the largest index i so that a[i]< a[i+1]
	for i in range (len(p) - 2, -1, -1):
		if p[i] < p[i+1]:

			# Find the largest index j so that a[i]< a[j]
			for k in range (len(p) - 1, i, -1):
				if p[k] > p[i]:
					j = k
					break

			# Swap (a[i] and a[j])
			temp = p[i]
			p[i] = p[j]
			p[j] = temp

			# Reverse the order of  ai+1 all the way up to a[n]
			for g in range(i+1, int(math.ceil((len(p) + (i+1)) / 2))):
				temp = p[len(p) - (g-i)]
				p[len(p) - (g-i)] = p[g]
				p[g] = temp

			print p # Recursive print
			lexPermute(p)

	return p

def getArrayFromCSV(filename):
	# Returns a two-dimensional array of data from comma-seperated text file,
	# formatted as a 2-dimensional adjacency matrix, to a 2D list of lists
	with open("6.txt","r") as csv:
		dataList = csv.readlines()
	dataList = dataList[0].split(",")
	for i in range(0, len(dataList)):
		dataList[i] = int(dataList[i])
	return dataList

########
# TEST #
########
perm = getArrayFromCSV("6.txt")
print perm
lexPermute(perm)<commit_msg>TSP: comma seperated text values now read and written into 2D list<commit_after># TSP Exact Undirected Solver
#
#
#
# For algorithms class, Spring 2015
# By Marshall Ehlinger

import math

def lexPermute(p):
	# If permutation array "a" (p in the code below) has two consecutive elements in increasing order,
    # Find the largest index i so that a[i]< a[i+1]
	for i in range (len(p) - 2, -1, -1):
		if p[i] < p[i+1]:

			# Find the largest index j so that a[i]< a[j]
			for k in range (len(p) - 1, i, -1):
				if p[k] > p[i]:
					j = k
					break

			# Swap (a[i] and a[j])
			temp = p[i]
			p[i] = p[j]
			p[j] = temp

			# Reverse the order of  ai+1 all the way up to a[n]
			for g in range(i+1, int(math.ceil((len(p) + (i+1)) / 2))):
				temp = p[len(p) - (g-i)]
				p[len(p) - (g-i)] = p[g]
				p[g] = temp

			print p # Recursive print
			lexPermute(p)

	return p

def getArrayFromCSV(filename):
	# Returns a two-dimensional array of data from comma-seperated text file,
	# formatted as a 2-dimensional adjacency matrix, to a 2D list of lists
	with open(filename,"r") as csv:
		dataList = csv.read().splitlines()
	for i in range(0, len(dataList)):
		dataList[i] = dataList[i].split(",")
		for j in range(0, len(dataList[i])):
			dataList[i][j].rstrip()
			dataList[i][j] = int(dataList[i][j])
	return dataList

########
# TEST #
########
perm = getArrayFromCSV("tests/6.txt")
print perm
lexPermute(perm)<|endoftext|>