<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import itertools
import warnings
import numpy
import collections
import time

import dedupe.backport as backport
import dedupe.lr as lr

def randomPairsWithReplacement(n_records, sample_size) :
    # If the population is very large relative to the sample
    # size than we'll get very few duplicates by chance
    warnings.warn("There may be duplicates in the sample")

    try :
        random_indices = numpy.random.randint(n_records,
                                              size=sample_size*2)
    except OverflowError:
        max_int = numpy.iinfo('int').max
        warnings.warn("Asked to sample pairs from %d records, will only sample pairs from first %d records" % (n_records, max_int))
        random_indices = numpy.random.randint(max_int,
                                              size=sample_size*2)

    random_indices = random_indices.reshape((-1, 2))
    random_indices.sort(axis=1)

    return random_indices

def randomPairs(n_records, sample_size):
    """
    Return random combinations of indices for a square matrix of size
    n records
    """

    if n_records < 2 :
        raise ValueError("Needs at least two records")
    n = n_records * (n_records - 1) / 2

    # numpy doesn't always throw an overflow error so we need to
    # check to make sure that the largest number we'll use is smaller
    # than the numpy's maximum unsigned integer
    if 8 * n > numpy.iinfo('uint').max :
        return randomPairsWithReplacement(n_records, sample_size)

    if sample_size >= n:
        if sample_size > n :
            warnings.warn("Requested sample of size %d, only returning %d possible pairs" % (sample_size, n))

        random_indices = numpy.arange(n)
    else :
        random_indices = numpy.random.randint(n, size=sample_size)

    random_indices.dtype = 'uint'

    b = 1 - 2 * n_records

    x = numpy.trunc((-b - numpy.sqrt(b ** 2 - 8 * random_indices)) / 2)
    y = random_indices + x * (b + x + 2) / 2 + 1

    return numpy.column_stack((x, y)).astype(int)

def randomPairsMatch(n_records_A, n_records_B, sample_size):
    """
    Return random combinations of indices for record list A and B
    """
    if not n_records_A or not n_records_B :
        raise ValueError("There must be at least one record in A and in B")

    if sample_size >= n_records_A * n_records_B :

        if sample_size > n_records_A * n_records_B :
            warnings.warn("Requested sample of size %d, only returning %d possible pairs" % (sample_size, n_records_A * n_records_B))

        return backport.cartesian((numpy.arange(n_records_A),
                                   numpy.arange(n_records_B)))

    A_samples = numpy.random.randint(n_records_A, size=sample_size)
    B_samples = numpy.random.randint(n_records_B, size=sample_size)
    pairs = zip(A_samples,B_samples)
    set_pairs = set(pairs)

    while len(set_pairs) < sample_size:
        set_pairs.update(randomPairsMatch(n_records_A,
                                          n_records_B,
                                          (sample_size-len(set_pairs))))
    else:
        return set_pairs

def trainModel(training_data, data_model, alpha=.001):
    """
    Use logistic regression to train weights for all fields in the data model
    """

    labels = numpy.array(training_data['label'] == 'match', dtype='i4')
    examples = training_data['distances']

    (weight, bias) = lr.lr(labels, examples, alpha)

    for i, name in enumerate(data_model['fields']) :
        data_model['fields'][name].weight = float(weight[i])

    data_model['bias'] = bias

    return data_model

def derivedDistances(primary_distances, data_model) :
    distances = primary_distances

    current_column = len(data_model.field_comparators)

    for cat_index, length in data_model.categorical_indices :
        start = current_column
        end = start + length

        distances[:,start:end] =\
                (distances[:, cat_index][:,None]
                 == numpy.arange(2, 2 + length)[None,:])

        distances[:,cat_index][distances[:,cat_index] > 1] = 0

        current_column = end

    for interaction in data_model.interactions :
        distances[:,current_column] =\
                numpy.prod(distances[:,interaction], axis=1)

        current_column += 1

    missing_data = numpy.isnan(distances[:,:current_column])

    distances[:,:current_column][missing_data] = 0

    distances[:,current_column:] =\
            1 - missing_data[:,data_model.missing_field_indices]

    return distances

def fieldDistances(record_pairs, data_model):
    num_records = len(record_pairs)

    distances = numpy.empty((num_records, data_model.total_fields))

    field_comparators = data_model.field_comparators.items()

    for i, (record_1, record_2) in enumerate(record_pairs) :
        for j, (field, compare) in enumerate(field_comparators) :
            distances[i,j] = compare(record_1[field],
                                     record_2[field])

    distances = derivedDistances(distances, data_model)

    return distances

def scorePairs(field_distances, data_model):
    fields = data_model['fields']

    field_weights = [fields[name].weight for name in fields]
    bias = data_model['bias']

    scores = numpy.dot(field_distances, field_weights)

    scores = numpy.exp(scores + bias) / (1 + numpy.exp(scores + bias))

    return scores

class ScoreRecords(object) :
    def __init__(self, data_model, threshold) :
        self.data_model = data_model
        self.threshold = threshold
        self.score_queue = None

    def __call__(self, records_queue, score_queue) :
        self.score_queue = score_queue
        while True :
            record_pairs = records_queue.get()
            if record_pairs is None :
                break

            self.fieldDistance(record_pairs)

        score_queue.put(None)

    def fieldDistance(self, record_pairs) :
        ids = []
        records = []

        for record_pair in record_pairs :
            ((id_1, record_1, smaller_ids_1),
             (id_2, record_2, smaller_ids_2)) = record_pair

            if set.isdisjoint(smaller_ids_1, smaller_ids_2) :

                ids.append((id_1, id_2))
                records.append((record_1, record_2))

        if records :
            distances = fieldDistances(records, self.data_model)
            scores = scorePairs(distances, self.data_model)

            scored_pairs = numpy.rec.fromarrays((ids, scores),
                                                dtype= [('pairs', 'object', 2),
                                                        ('score', 'f4', 1)])

            filtered_pairs = scored_pairs[scores > self.threshold]

            self.score_queue.put(filtered_pairs)

def mergeScores(score_queue, result_queue, stop_signals) :
    scored_pairs = numpy.empty(0, dtype= [('pairs', 'object', 2),
                                          ('score', 'f4', 1)])

    seen_signals = 0
    while seen_signals < stop_signals  :
        score_chunk = score_queue.get()
        if score_chunk is not None :
            scored_pairs = numpy.concatenate((scored_pairs, score_chunk))
        else :
            seen_signals += 1

    print "transferring array from mergeScores"

    result_queue.put(scored_pairs)

def scoreDuplicates(records, data_model, num_processes=1, threshold=0) :
    record_pairs_queue = backport.SimpleQueue()
    score_queue =  backport.SimpleQueue()
    result_queue = backport.SimpleQueue()

    n_map_processes = max(num_processes-1, 1)
    score_records = ScoreRecords(data_model, threshold)
    map_processes = [backport.Process(target=score_records,
                                      args=(record_pairs_queue,
                                            score_queue))
                     for _ in xrange(n_map_processes)]
    [process.start() for process in map_processes]

    reduce_process = backport.Process(target=mergeScores,
                                      args=(score_queue,
                                            result_queue,
                                            n_map_processes))
    reduce_process.start()

    fillQueue(record_pairs_queue, records, n_map_processes)

    scored_pairs = result_queue.get()

    print "transfer complete"

    return scored_pairs

def fillQueue(queue, iterable, stop_signals) :
    iterable = iter(iterable)
    chunk_size = 100000
    multiplier = 1.1

    # initial values
    i = 0
    n_records = 0
    t0 = time.time()
    last_rate = 10000

    while True :
        chunk = list(itertools.islice(iterable, chunk_size))
        if chunk :
            queue.put(chunk)

            n_records += chunk_size
            i += 1

            if i % 10 :
                time_delta = time.time() - t0

                current_rate = n_records/time_delta

                # chunk_size is always either growing or shrinking, if
                # the shrinking led to a faster rate, keep
                # shrinking. Same with growing. If the rate decreased,
                # reverse directions
                if current_rate < last_rate :
                    multiplier = 1/multiplier

                chunk_size = max(chunk_size * multiplier, 1)

                last_rate = current_rate
                n_records = 0
                t0 = time.time()

        else :
            # put poison pills in queue to tell scorers that they are
            # done
            [queue.put(None) for _ in xrange(stop_signals)]
            break

def peek(records) :
    try :
        record = records.next()
    except AttributeError as e:
        if "has no attribute 'next'" not in str(e) :
            raise
        try :
            records = iter(records)
            record = records.next()
        except StopIteration :
            return None, records

    return record, itertools.chain([record], records)

def freezeData(data) : # pragma : no cover
    return [(frozendict(record_1),
             frozendict(record_2))
            for record_1, record_2 in data]

class frozendict(collections.Mapping):
    """Don't forget the docstrings!!"""

    def __init__(self, *args, **kwargs): # pragma : no cover
        self._d = dict(*args, **kwargs)

    def __iter__(self):                  # pragma : no cover
        return iter(self._d)

    def __len__(self):                   # pragma : no cover
        return len(self._d)

    def __getitem__(self, key):          # pragma : no cover
        return self._d[key]

    def __repr__(self) :
        return '<frozendict %s>' % repr(self._d)

    def __hash__(self):
        try:
            return self._cached_hash
        except AttributeError:
            h = self._cached_hash = hash(frozenset(self._d.iteritems()))
            return h
<commit_msg>testing speeds if we don't have dtype object<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import itertools
import warnings
import numpy
import collections
import time
import tempfile

import dedupe.backport as backport
import dedupe.lr as lr

def randomPairsWithReplacement(n_records, sample_size) :
    # If the population is very large relative to the sample
    # size than we'll get very few duplicates by chance
    warnings.warn("There may be duplicates in the sample")

    try :
        random_indices = numpy.random.randint(n_records,
                                              size=sample_size*2)
    except OverflowError:
        max_int = numpy.iinfo('int').max
        warnings.warn("Asked to sample pairs from %d records, will only sample pairs from first %d records" % (n_records, max_int))
        random_indices = numpy.random.randint(max_int,
                                              size=sample_size*2)

    random_indices = random_indices.reshape((-1, 2))
    random_indices.sort(axis=1)

    return random_indices

def randomPairs(n_records, sample_size):
    """
    Return random combinations of indices for a square matrix of size
    n records
    """

    if n_records < 2 :
        raise ValueError("Needs at least two records")
    n = n_records * (n_records - 1) / 2

    # numpy doesn't always throw an overflow error so we need to
    # check to make sure that the largest number we'll use is smaller
    # than the numpy's maximum unsigned integer
    if 8 * n > numpy.iinfo('uint').max :
        return randomPairsWithReplacement(n_records, sample_size)

    if sample_size >= n:
        if sample_size > n :
            warnings.warn("Requested sample of size %d, only returning %d possible pairs" % (sample_size, n))

        random_indices = numpy.arange(n)
    else :
        random_indices = numpy.random.randint(n, size=sample_size)

    random_indices.dtype = 'uint'

    b = 1 - 2 * n_records

    x = numpy.trunc((-b - numpy.sqrt(b ** 2 - 8 * random_indices)) / 2)
    y = random_indices + x * (b + x + 2) / 2 + 1

    return numpy.column_stack((x, y)).astype(int)

def randomPairsMatch(n_records_A, n_records_B, sample_size):
    """
    Return random combinations of indices for record list A and B
    """
    if not n_records_A or not n_records_B :
        raise ValueError("There must be at least one record in A and in B")

    if sample_size >= n_records_A * n_records_B :

        if sample_size > n_records_A * n_records_B :
            warnings.warn("Requested sample of size %d, only returning %d possible pairs" % (sample_size, n_records_A * n_records_B))

        return backport.cartesian((numpy.arange(n_records_A),
                                   numpy.arange(n_records_B)))

    A_samples = numpy.random.randint(n_records_A, size=sample_size)
    B_samples = numpy.random.randint(n_records_B, size=sample_size)
    pairs = zip(A_samples,B_samples)
    set_pairs = set(pairs)

    while len(set_pairs) < sample_size:
        set_pairs.update(randomPairsMatch(n_records_A,
                                          n_records_B,
                                          (sample_size-len(set_pairs))))
    else:
        return set_pairs

def trainModel(training_data, data_model, alpha=.001):
    """
    Use logistic regression to train weights for all fields in the data model
    """

    labels = numpy.array(training_data['label'] == 'match', dtype='i4')
    examples = training_data['distances']

    (weight, bias) = lr.lr(labels, examples, alpha)

    for i, name in enumerate(data_model['fields']) :
        data_model['fields'][name].weight = float(weight[i])

    data_model['bias'] = bias

    return data_model

def derivedDistances(primary_distances, data_model) :
    distances = primary_distances

    current_column = len(data_model.field_comparators)

    for cat_index, length in data_model.categorical_indices :
        start = current_column
        end = start + length

        distances[:,start:end] =\
                (distances[:, cat_index][:,None]
                 == numpy.arange(2, 2 + length)[None,:])

        distances[:,cat_index][distances[:,cat_index] > 1] = 0

        current_column = end

    for interaction in data_model.interactions :
        distances[:,current_column] =\
                numpy.prod(distances[:,interaction], axis=1)

        current_column += 1

    missing_data = numpy.isnan(distances[:,:current_column])

    distances[:,:current_column][missing_data] = 0

    distances[:,current_column:] =\
            1 - missing_data[:,data_model.missing_field_indices]

    return distances

def fieldDistances(record_pairs, data_model):
    num_records = len(record_pairs)

    distances = numpy.empty((num_records, data_model.total_fields))

    field_comparators = data_model.field_comparators.items()

    for i, (record_1, record_2) in enumerate(record_pairs) :
        for j, (field, compare) in enumerate(field_comparators) :
            distances[i,j] = compare(record_1[field],
                                     record_2[field])

    distances = derivedDistances(distances, data_model)

    return distances

def scorePairs(field_distances, data_model):
    fields = data_model['fields']

    field_weights = [fields[name].weight for name in fields]
    bias = data_model['bias']

    scores = numpy.dot(field_distances, field_weights)

    scores = numpy.exp(scores + bias) / (1 + numpy.exp(scores + bias))

    return scores

class ScoreRecords(object) :
    def __init__(self, data_model, threshold) :
        self.data_model = data_model
        self.threshold = threshold
        self.score_queue = None

    def __call__(self, records_queue, score_queue) :
        self.score_queue = score_queue
        while True :
            record_pairs = records_queue.get()
            if record_pairs is None :
                break

            self.fieldDistance(record_pairs)

        score_queue.put(None)

    def fieldDistance(self, record_pairs) :
        ids = []
        records = []

        for record_pair in record_pairs :
            ((id_1, record_1, smaller_ids_1),
             (id_2, record_2, smaller_ids_2)) = record_pair

            if set.isdisjoint(smaller_ids_1, smaller_ids_2) :

                ids.append((id_1, id_2))
                records.append((record_1, record_2))

        if records :
            distances = fieldDistances(records, self.data_model)
            scores = scorePairs(distances, self.data_model)

            scored_pairs = numpy.rec.fromarrays((ids, scores),
                                                dtype= [('pairs', 'i4', 2),
                                                        ('score', 'f4', 1)])

            filtered_pairs = scored_pairs[scores > self.threshold]

            self.score_queue.put(filtered_pairs)

def mergeScores(score_queue, result_queue, stop_signals) :
    scored_pairs = numpy.empty(0, dtype= [('pairs', 'i4', 2),
                                          ('score', 'f4', 1)])

    seen_signals = 0
    while seen_signals < stop_signals  :
        score_chunk = score_queue.get()
        if score_chunk is not None :
            scored_pairs = numpy.concatenate((scored_pairs, score_chunk))
        else :
            seen_signals += 1

    print "transferring array from mergeScores"
    print time.time()
    scored_pairs_file, file_path = tempfile.mkstemp()

    fp = numpy.memmap(file_path,
                      dtype=scored_pairs.dtype,
                      shape=scored_pairs.shape)
    fp[:] = scored_pairs[:]

    result_queue.put(file_path)

def scoreDuplicates(records, data_model, num_processes=1, threshold=0) :
    record_pairs_queue = backport.SimpleQueue()
    score_queue =  backport.SimpleQueue()
    result_queue = backport.SimpleQueue()

    n_map_processes = max(num_processes-1, 1)
    score_records = ScoreRecords(data_model, threshold)
    map_processes = [backport.Process(target=score_records,
                                      args=(record_pairs_queue,
                                            score_queue))
                     for _ in xrange(n_map_processes)]
    [process.start() for process in map_processes]

    reduce_process = backport.Process(target=mergeScores,
                                      args=(score_queue,
                                            result_queue,
                                            n_map_processes))
    reduce_process.start()

    fillQueue(record_pairs_queue, records, n_map_processes)

    scored_pairs_file = result_queue.get()
    scored_pairs = numpy.memmap(scored_pairs_file,
                                dtype=[('pairs', 'i4', 2),
                                       ('score', 'f4', 1)])
    print "transfer complete"
    print time.time()

    return scored_pairs

def fillQueue(queue, iterable, stop_signals) :
    iterable = iter(iterable)
    chunk_size = 100000
    multiplier = 1.1

    # initial values
    i = 0
    n_records = 0
    t0 = time.time()
    last_rate = 10000

    while True :
        chunk = list(itertools.islice(iterable, chunk_size))
        if chunk :
            queue.put(chunk)

            n_records += chunk_size
            i += 1

            if i % 10 :
                time_delta = time.time() - t0

                current_rate = n_records/time_delta

                # chunk_size is always either growing or shrinking, if
                # the shrinking led to a faster rate, keep
                # shrinking. Same with growing. If the rate decreased,
                # reverse directions
                if current_rate < last_rate :
                    multiplier = 1/multiplier

                chunk_size = max(chunk_size * multiplier, 1)

                last_rate = current_rate
                n_records = 0
                t0 = time.time()

        else :
            # put poison pills in queue to tell scorers that they are
            # done
            [queue.put(None) for _ in xrange(stop_signals)]
            break

def peek(records) :
    try :
        record = records.next()
    except AttributeError as e:
        if "has no attribute 'next'" not in str(e) :
            raise
        try :
            records = iter(records)
            record = records.next()
        except StopIteration :
            return None, records

    return record, itertools.chain([record], records)

def freezeData(data) : # pragma : no cover
    return [(frozendict(record_1),
             frozendict(record_2))
            for record_1, record_2 in data]

class frozendict(collections.Mapping):
    """Don't forget the docstrings!!"""

    def __init__(self, *args, **kwargs): # pragma : no cover
        self._d = dict(*args, **kwargs)

    def __iter__(self):                  # pragma : no cover
        return iter(self._d)

    def __len__(self):                   # pragma : no cover
        return len(self._d)

    def __getitem__(self, key):          # pragma : no cover
        return self._d[key]

    def __repr__(self) :
        return '<frozendict %s>' % repr(self._d)

    def __hash__(self):
        try:
            return self._cached_hash
        except AttributeError:
            h = self._cached_hash = hash(frozenset(self._d.iteritems()))
            return h
<|endoftext|>


<commit_before>from django.shortcuts import render, get_object_or_404
from .models import MealOption, Restaurant
from django.http import HttpResponseRedirect
from django.core.urlresolvers import reverse

def index(request):
	# get a random restaurant
	restaurant = Restaurant.objects.order_by('?').first()
	# try to get a random meal, otherwise we'll just say "something" from <restaurant>
	if (restaurant):
		meal = restaurant.mealoption_set.order_by('?').first()
	else:
		meal = None

	return render(request, 'lunch/index.html', {
    	'meal_option': meal,
    	'restaurant': restaurant
    	})

def restaurant_detail(request, restaurant_id):
	restaurant = get_object_or_404(Restaurant, pk=restaurant_id)

	# save meal option
	if ('meal_option' in request.POST):
		meal = MealOption(restaurant=restaurant, name=request.POST['meal_option'])
		meal.save()

	return render(request, 'lunch/restaurant.html', {
		'restaurant': restaurant,
		'suggestion': MealOption.get_random()
		})

def add_restaurant(request):
	# save restaurant and redirect to index
	if ('restaurant' in request.POST and 'facebook_id' in request.POST):
		restaurant = Restaurant(name=request.POST['restaurant'], facebook_id=request.POST['facebook_id'])
		restaurant.save()
		return HttpResponseRedirect(reverse('lunch:index'))
	else:
		return render(request, 'lunch/add_restaurant.html')<commit_msg>turned around creation of new meal object t's a bit fancier this way<commit_after>from django.shortcuts import render, get_object_or_404
from .models import MealOption, Restaurant
from django.http import HttpResponseRedirect
from django.core.urlresolvers import reverse

def index(request):
	# get a random restaurant
	restaurant = Restaurant.objects.order_by('?').first()
	# try to get a random meal, otherwise we'll just say "something" from <restaurant>
	if (restaurant):
		meal = restaurant.mealoption_set.order_by('?').first()
	else:
		meal = None

	return render(request, 'lunch/index.html', {
    	'meal_option': meal,
    	'restaurant': restaurant
    	})

def restaurant_detail(request, restaurant_id):
	restaurant = get_object_or_404(Restaurant, pk=restaurant_id)

	# save meal option
	if ('meal_option' in request.POST):
		restaurant.mealoption_set.create(name=request.POST['meal_option'])

	return render(request, 'lunch/restaurant.html', {
		'restaurant': restaurant,
		'suggestion': MealOption.get_random()
		})

# todo: check for facebook_id or set to null
def add_restaurant(request):
	# save restaurant and redirect to index
	if ('restaurant' in request.POST and 'facebook_id' in request.POST):
		restaurant = Restaurant(name=request.POST['restaurant'], facebook_id=request.POST['facebook_id'])
		restaurant.save()
		return HttpResponseRedirect(reverse('lunch:index'))
	else:
		return render(request, 'lunch/add_restaurant.html')<|endoftext|>


<commit_before>#Dresses a cobra.Model with arrays and vectors so that linear algebra operations
#can be carried out on
from sys import maxint
from copy import deepcopy
from warnings import warn

from numpy import array, hstack, ndarray
from scipy.sparse import lil_matrix, dok_matrix

from .Model import Model

class ArrayBasedModel(Model):
    """ArrayBasedModel is a class that adds arrays and vectors to
    a cobra.Model to make it easier to perform linear algebra operations.

    S : :class:`scipy.sparse.lil_matrix` (in CPython)
        Stoichiometric matrix of the model

    lower_bounds

    upper_bounds

    objective_coefficients

    b

    constraint_sense

    """
    def __setstate__(self, state):
        """Make sure all cobra.Objects in the model point to the model

        TODO: Make sure that the genes and metabolites referenced by
        the reactions.(genes|metabolites) point to the model's genes
        and metabolites.

        """
        self.__dict__.update(state)
        [[setattr(x, '_model', self)
          for x in self.__dict__[y]]
         for y in ['reactions', 'genes', 'metabolites']]

    def __init__(self, description=None, deepcopy_model=False, matrix_type='scipy.lil_matrix'):
        """
        description: None | String | cobra.Model

        deepcopy_model: Boolean.  If True and description is
        a cobra.Model then make a deepcopy of the Model before
        creating the ArrayBasedModel.

        matrix_type: String. Specifies which type of backend matrix to use for self.S.
        Currently, 'scipy.lil_matrix' or 'scipy.dok_matrix'

        """
        if deepcopy_model and isinstance(description, Model):
            description = description.copy()
        Model.__init__(self, description)
        self.S = None
        self.matrix_type = matrix_type
        self.b = None
        self.constraint_sense = None
        self.update()

    @property
    def lower_bounds(self):
        return self._lower_bounds

    @property
    def upper_bounds(self):
        return self._upper_bounds

    @property
    def objective_coefficients(self):
        return self._objective_coefficients

    def copy(self, print_time=False):
        """Provides a partial 'deepcopy' of the Model.  All of the Metabolite, Gene,
        and Reaction objects are created anew but in a faster fashion than deepcopy

        print_time: Boolean used for debugging

        """
        the_copy = Model.copy(self, print_time)
        the_copy.update()
        return the_copy

    def add_metabolites(self, metabolite_list,
                        expand_stoichiometric_matrix=True):
        """Will add a list of metabolites to the the object, if they do not
        exist and then expand the stochiometric matrix

        metabolite_list: A list of :class:`~cobra.core.Metabolite` objects

        expand_stoichimetric_matrix: Boolean.  If True and self.S is
        not None then it will add rows to self.S.  self.S must be
        created after adding reactions and metabolites to self before
        it can be expanded.  Trying to expand self.S when self only
        contains metabolites is ludacris.

        """
        Model.add_metabolites(metabolite_list)
        self.constraint_sense = [x._constraint_sense for x in self.metabolites]
        if self.S is not None and expand_stoichiometric_matrix:
            s_expansion = len(self.metabolites) - self.S.shape[0]
            if s_expansion > 0:
                self.S = self.S.todok()
                self.S.resize((self.S.shape[0] + s_expansion,
                               self.S.shape[1]))
                self.S = self.S.tolil()

    def _update_reaction(self, reaction):
        """Updates everything associated with the reaction.id of reaction.

        reaction: A cobra.Reaction object, or a list of these objects.

        WARNING: This function is only used after the Model has been
        converted to matrices.  It is typically faster to access the objects
        in the Model directly.  This function will eventually moved to another
        module for advanced users due to the potential for mistakes.

        """
        if not hasattr(reaction, '__iter__'):
            reaction = [reaction]
        Model._update_reaction(reaction)
        for the_reaction in reaction:
            try:
                reaction_index = self.reactions.index(the_reaction.id)
            except KeyError:
                print the_reaction.id + ' is not in the model'
                continue

            #zero reaction stoichiometry column
            the_column = self.S[:, reaction_index]
            for nonzero_index in the_column.nonzero()[0]:
                the_column[nonzero_index, 0] = 0
            self._lower_bounds[reaction_index] = the_reaction.lower_bound
            self._upper_bounds[reaction_index] = the_reaction.upper_bound
            self.objective_coefficients[reaction_index] = the_reaction.objective_coefficient
            self.add_metabolites(the_reaction._metabolites)
            #Make sure that the metabolites are the ones contained in the model
            the_reaction._metabolites = [self.metabolite.get_by_id(x.id)
                                        for x in the_reaction._metabolites]
            #Update the stoichiometric matrix
            metabolite_indices = map(self.metabolites.index, the_reaction._metabolites)
            for (index, metabolite_index) in enumerate(metabolite_indices):
                self.S[metabolite_index, reaction_index] = the_reaction.stoichiometric_coefficients[index]

    def add_reactions(self, reaction_list, update_matrices=False):
        """Will add a cobra.Reaction object to the model, if
        reaction.id is not in self.reactions.

        reaction_list: A :class:`~cobra.core.Reaction` object or a list of them

        update_matrices:  Boolean.  If true populate / update matrices
        S, lower_bounds, upper_bounds, .... Note this is slow to run
        for very large models and using this option with repeated calls
        will degrade performance.  Better to call self.update() after
        adding all reactions.

         If the stoichiometric matrix is initially empty then initialize a 1x1
         sparse matrix and add more rows as needed in the self.add_metabolites
         function

        """
        Model.add_reactions(reaction_list)
        if update_matrices:
            self._update_matrices(reaction_list)

    def _construct_matrices(self):
        """Large sparse matrices take time to construct and to read / write.
        This function allows one to let the model exists without cobra_model.S
        and then generate it at needed.

        """
        self._update_matrices() #This does basic construction as well.

    def _update_reaction_vectors(self):
        """regenerates the lower_bounds, upper_bounds,
        and objective_coefficients vectors.

        WARNING: This function is only used after the Model has been
        converted to matrices.  It is typically faster to access the objects
        in the Model directly.  This function will eventually moved to another
        module for advanced users due to the potential for mistakes.

        """
        self._lower_bounds = LinkedArray(self.reactions, "lower_bound")
        self._upper_bounds = LinkedArray(self.reactions, "upper_bound")
        self._objective_coefficients = LinkedArray(self.reactions,
                                                   "objective_coefficient")

    # TODO deprecate and use @property for _b and constraint sense
    def _update_metabolite_vectors(self):
        """regenerates _b and _constraint_sense

        WARNING: This function is only used after the Model has been
        converted to matrices.  It is typically faster to access the objects
        in the Model directly.  This function will eventually moved to another
        module for advanced users due to the potential for mistakes.

        """
        _b, _constraint_sense = [], []
        [(_b.append(x._bound),
          _constraint_sense.append(x._constraint_sense))
         for x in self.metabolites]
        self.b = array(_b)
        self.constraint_sense = _constraint_sense

    # TODO deprecate and use @property
    def _update_matrices(self, reaction_list=None):
        """
        reaction_list: None or a list of cobra.Reaction objects that are in
        self.reactions.  If None then reconstruct the whole matrix.

        NOTE: reaction_list is assumed to be at the end of self.reactions.

        In the future, we'll be able to use reactions from anywhere in the
        list

        WARNING: This function is only used after the Model has been
        converted to matrices.  It is typically faster to access the objects
        in the Model directly.  This function will eventually moved to another
        module for advanced users due to the potential for mistakes.

        """
        #Pretty much all of these things are unnecessary to use the objects and
        #interact with the optimization solvers.  It might be best to move them
        #to linear algebra modules.
        #If no reactions are present in the Model, initialize the arrays
        if not self.S or reaction_list is None:
            reaction_list = self.reactions
            SMatrix = SMatrix_classes[self.matrix_type]
            self.S = SMatrix((len(self.metabolites),
                              len(self.reactions)), model=self)
            self._update_reaction_vectors()
        else: #Expand the arrays to accomodate the new reaction
            self.S.resize((len(self.metabolites),
                           len(self.reactions)))
            lower_bounds = array([x.lower_bound
                                  for x in reaction_list])
            upper_bounds = array([x.upper_bound
                                  for x in reaction_list])
            objective_coefficients = array([x.objective_coefficient
                                            for x in reaction_list])
            self._lower_bounds = hstack((self._lower_bounds, lower_bounds)).view(LinkedArray)
            self._lower_bounds.__init__(self.reactions, "lower_bound")
            self._upper_bounds = hstack((self._upper_bounds, upper_bounds))
            self._upper_bounds.__init__(self.reactions, "upper_bound")
            self._objective_coefficients = hstack((self._objective_coefficients,
                                                  objective_coefficients))
            self._objective_coefficients.__init__(self.reactions,
                                                  "objective_coefficient")

        #Use dok format to speed up additions.
        coefficient_dictionary = {}
        #SPEED this up. This is the slow part.  Can probably use a dict to index.
        for the_reaction in reaction_list:
            reaction_index = self.reactions.index(the_reaction.id)
            for the_key, the_value in the_reaction._metabolites.items():
                coefficient_dictionary[(self.metabolites.index(the_key.id),
                                        reaction_index)] = the_value

        if not self.S.getformat() == 'dok':
            self.S = self.S.todok()
        self.S.update(coefficient_dictionary)
        if self.matrix_type == 'scipy.lil_matrix':
            try:
                self.S = self.S.tolil()
            except Exception, e:
                warn('Unable to convert S to lil_matrix maintaining in dok_matrix format: %s'%e)

    def update(self):
        """Regenerates the stoichiometric matrix and vectors

        """
        self._update_matrices()
        self._update_metabolite_vectors()

class LinkedArray(ndarray):
    """A ndarray which updates an attribute from a list"""
    def __new__(cls, list, attribute):
        # construct a new ndarray with the values from the list
        # For example, if the list if model.reactions and the attribute is
        # "lower_bound" create an array of [reaction.lower_bound for ... ]
        return array([getattr(i, attribute) for i in list]).view(cls)
    def __init__(self, list, attribute):
        self._list = list
        self._attr = attribute

    def __setitem__(self, index, value):
        numpy.ndarray.__setitem__(self, index, value)
        setattr(self._list[index], self._attr, value)
    def __setslice__(self, i, j, value):
        numpy.ndarray.__setslice__(self, i, j, value)
        if j == maxint:
            j = len(self)
        if hasattr(value, "__getitem__"):  # setting to a list
            for index in range(i, j):
                setattr(self._list[index], self._attr, value[index])
        else:
            for index in range(i, j):
                setattr(self._list[index], self._attr, value)

class SMatrix_dok(dok_matrix):
    """A 2D sparse dok matrix which maintains links to a cobra Model"""
    def __init__(self, *args, **kwargs):
        dok_matrix.__init__(self, *args)
        self.format = "dok"
        self._model = kwargs["model"] if "model" in kwargs else None

    def __setitem__(self, index, value):
        dok_matrix.__setitem__(self, index, value)
        if type(index[0]) is int and type(index[1]) is int:
            reaction = self._model.reactions[index[1]]
            if value != 0:
                reaction.add_metabolites({self._model.metabolites[index[0]]: value}, combine=False)
            else:  # setting 0 means metabolites should be removed
                metabolite = self._model.metabolites[index[0]]
                if metabolite in reaction._metabolites:
                    reaction.pop(metabolite)

    def tolil(self):
        new = SMatrix_lil(dok_matrix.tolil(self), model=self._model)
        return new

class SMatrix_lil(lil_matrix):
    """A 2D sparse lil matrix which maintains links to a cobra Model"""
    def __init__(self, *args, **kwargs):
        lil_matrix.__init__(self, *args)
        self.format = "lil"
        self._model = kwargs["model"] if "model" in kwargs else None

    def __setitem__(self, index, value):
        lil_matrix.__setitem__(self, index, value)
        if isinstance(index[0], int):
            metabolites = [self._model.metabolites[index[0]]]
        else:
            metabolites = self._model.metabolites[index[0]]

        if isinstance(index[1], int):
            reactions = [self._model.reactions[index[1]]]
        else:
            reactions = self._model.reactions[index[1]]

        if value == 0:  # remove_metabolites
            met_set = set(metabolites)
            for reaction in reactions:
                to_remove = met_set.intersection(reaction._metabolites)
                for i in to_remove:
                    reaction.pop(i)
        else:  # add metabolites
            met_dict = {met: value for met in metabolites}
            for reaction in reactions:
                reaction.add_metabolites(met_dict, combine=False)

    def todok(self):
        new = SMatrix_dok(lil_matrix.todok(self), model=self._model)
        return new

SMatrix_classes = {"scipy.dok_matrix": SMatrix_dok,
                   "scipy.lil_matrix": SMatrix_lil}
<commit_msg>Linked array for metabolite bounds<commit_after>#Dresses a cobra.Model with arrays and vectors so that linear algebra operations
#can be carried out on
from sys import maxint
from copy import deepcopy
from warnings import warn

from numpy import array, hstack, ndarray
from scipy.sparse import lil_matrix, dok_matrix

from .Model import Model

class ArrayBasedModel(Model):
    """ArrayBasedModel is a class that adds arrays and vectors to
    a cobra.Model to make it easier to perform linear algebra operations.

    S : :class:`scipy.sparse.lil_matrix` (in CPython)
        Stoichiometric matrix of the model

    lower_bounds

    upper_bounds

    objective_coefficients

    b

    constraint_sense

    """
    def __setstate__(self, state):
        """Make sure all cobra.Objects in the model point to the model

        TODO: Make sure that the genes and metabolites referenced by
        the reactions.(genes|metabolites) point to the model's genes
        and metabolites.

        """
        self.__dict__.update(state)
        [[setattr(x, '_model', self)
          for x in self.__dict__[y]]
         for y in ['reactions', 'genes', 'metabolites']]

    def __init__(self, description=None, deepcopy_model=False, matrix_type='scipy.lil_matrix'):
        """
        description: None | String | cobra.Model

        deepcopy_model: Boolean.  If True and description is
        a cobra.Model then make a deepcopy of the Model before
        creating the ArrayBasedModel.

        matrix_type: String. Specifies which type of backend matrix to use for self.S.
        Currently, 'scipy.lil_matrix' or 'scipy.dok_matrix'

        """
        if deepcopy_model and isinstance(description, Model):
            description = description.copy()
        Model.__init__(self, description)
        self.S = None
        self.matrix_type = matrix_type
        self.b = None
        self.constraint_sense = None
        self.update()

    @property
    def lower_bounds(self):
        return self._lower_bounds

    @property
    def upper_bounds(self):
        return self._upper_bounds

    @property
    def objective_coefficients(self):
        return self._objective_coefficients

    @property
    def b(self):
        return self._b

    def copy(self):
        """Provides a partial 'deepcopy' of the Model.  All of the Metabolite, Gene,
        and Reaction objects are created anew but in a faster fashion than deepcopy

        """
        the_copy = Model.copy(self)
        the_copy.update()
        return the_copy

    def add_metabolites(self, metabolite_list,
                        expand_stoichiometric_matrix=True):
        """Will add a list of metabolites to the the object, if they do not
        exist and then expand the stochiometric matrix

        metabolite_list: A list of :class:`~cobra.core.Metabolite` objects

        expand_stoichimetric_matrix: Boolean.  If True and self.S is
        not None then it will add rows to self.S.  self.S must be
        created after adding reactions and metabolites to self before
        it can be expanded.  Trying to expand self.S when self only
        contains metabolites is ludacris.

        """
        Model.add_metabolites(metabolite_list)
        self.constraint_sense = [x._constraint_sense for x in self.metabolites]
        if self.S is not None and expand_stoichiometric_matrix:
            s_expansion = len(self.metabolites) - self.S.shape[0]
            if s_expansion > 0:
                self.S = self.S.todok()
                self.S.resize((self.S.shape[0] + s_expansion,
                               self.S.shape[1]))
                self.S = self.S.tolil()

    def _update_reaction(self, reaction):
        """Updates everything associated with the reaction.id of reaction.

        reaction: A cobra.Reaction object, or a list of these objects.

        WARNING: This function is only used after the Model has been
        converted to matrices.  It is typically faster to access the objects
        in the Model directly.  This function will eventually moved to another
        module for advanced users due to the potential for mistakes.

        """
        if not hasattr(reaction, '__iter__'):
            reaction = [reaction]
        Model._update_reaction(reaction)
        for the_reaction in reaction:
            try:
                reaction_index = self.reactions.index(the_reaction.id)
            except KeyError:
                print the_reaction.id + ' is not in the model'
                continue

            #zero reaction stoichiometry column
            the_column = self.S[:, reaction_index]
            for nonzero_index in the_column.nonzero()[0]:
                the_column[nonzero_index, 0] = 0
            self._lower_bounds[reaction_index] = the_reaction.lower_bound
            self._upper_bounds[reaction_index] = the_reaction.upper_bound
            self.objective_coefficients[reaction_index] = the_reaction.objective_coefficient
            self.add_metabolites(the_reaction._metabolites)
            #Make sure that the metabolites are the ones contained in the model
            the_reaction._metabolites = [self.metabolite.get_by_id(x.id)
                                        for x in the_reaction._metabolites]
            #Update the stoichiometric matrix
            metabolite_indices = map(self.metabolites.index, the_reaction._metabolites)
            for (index, metabolite_index) in enumerate(metabolite_indices):
                self.S[metabolite_index, reaction_index] = the_reaction.stoichiometric_coefficients[index]

    def add_reactions(self, reaction_list, update_matrices=False):
        """Will add a cobra.Reaction object to the model, if
        reaction.id is not in self.reactions.

        reaction_list: A :class:`~cobra.core.Reaction` object or a list of them

        update_matrices:  Boolean.  If true populate / update matrices
        S, lower_bounds, upper_bounds, .... Note this is slow to run
        for very large models and using this option with repeated calls
        will degrade performance.  Better to call self.update() after
        adding all reactions.

         If the stoichiometric matrix is initially empty then initialize a 1x1
         sparse matrix and add more rows as needed in the self.add_metabolites
         function

        """
        Model.add_reactions(reaction_list)
        if update_matrices:
            self._update_matrices(reaction_list)

    def _construct_matrices(self):
        """Large sparse matrices take time to construct and to read / write.
        This function allows one to let the model exists without cobra_model.S
        and then generate it at needed.

        """
        self._update_matrices() #This does basic construction as well.

    def _update_reaction_vectors(self):
        """regenerates the lower_bounds, upper_bounds,
        and objective_coefficients vectors.

        WARNING: This function is only used after the Model has been
        converted to matrices.  It is typically faster to access the objects
        in the Model directly.  This function will eventually moved to another
        module for advanced users due to the potential for mistakes.

        """
        self._lower_bounds = LinkedArray(self.reactions, "lower_bound")
        self._upper_bounds = LinkedArray(self.reactions, "upper_bound")
        self._objective_coefficients = LinkedArray(self.reactions,
                                                   "objective_coefficient")

    # TODO deprecate and use @property for _b and constraint sense
    def _update_metabolite_vectors(self):
        """regenerates _b and _constraint_sense

        WARNING: This function is only used after the Model has been
        converted to matrices.  It is typically faster to access the objects
        in the Model directly.  This function will eventually moved to another
        module for advanced users due to the potential for mistakes.

        """
        self._b = LinkedArray(self.metabolites, "_b")
        self.constraint_sense = [x._constraint_sensense for x in self.metabolites]

    # TODO deprecate and use @property
    def _update_matrices(self, reaction_list=None):
        """
        reaction_list: None or a list of cobra.Reaction objects that are in
        self.reactions.  If None then reconstruct the whole matrix.

        NOTE: reaction_list is assumed to be at the end of self.reactions.

        In the future, we'll be able to use reactions from anywhere in the
        list

        WARNING: This function is only used after the Model has been
        converted to matrices.  It is typically faster to access the objects
        in the Model directly.  This function will eventually moved to another
        module for advanced users due to the potential for mistakes.

        """
        #Pretty much all of these things are unnecessary to use the objects and
        #interact with the optimization solvers.  It might be best to move them
        #to linear algebra modules.
        #If no reactions are present in the Model, initialize the arrays
        if not self.S or reaction_list is None:
            reaction_list = self.reactions
            SMatrix = SMatrix_classes[self.matrix_type]
            self.S = SMatrix((len(self.metabolites),
                              len(self.reactions)), model=self)
            self._update_reaction_vectors()
        else: #Expand the arrays to accomodate the new reaction
            self.S.resize((len(self.metabolites),
                           len(self.reactions)))
            lower_bounds = array([x.lower_bound
                                  for x in reaction_list])
            upper_bounds = array([x.upper_bound
                                  for x in reaction_list])
            objective_coefficients = array([x.objective_coefficient
                                            for x in reaction_list])
            self._lower_bounds = hstack((self._lower_bounds, lower_bounds)).view(LinkedArray)
            self._lower_bounds.__init__(self.reactions, "lower_bound")
            self._upper_bounds = hstack((self._upper_bounds, upper_bounds))
            self._upper_bounds.__init__(self.reactions, "upper_bound")
            self._objective_coefficients = hstack((self._objective_coefficients,
                                                  objective_coefficients))
            self._objective_coefficients.__init__(self.reactions,
                                                  "objective_coefficient")

        #Use dok format to speed up additions.
        coefficient_dictionary = {}
        #SPEED this up. This is the slow part.  Can probably use a dict to index.
        for the_reaction in reaction_list:
            reaction_index = self.reactions.index(the_reaction.id)
            for the_key, the_value in the_reaction._metabolites.items():
                coefficient_dictionary[(self.metabolites.index(the_key.id),
                                        reaction_index)] = the_value

        if not self.S.getformat() == 'dok':
            self.S = self.S.todok()
        self.S.update(coefficient_dictionary)
        if self.matrix_type == 'scipy.lil_matrix':
            try:
                self.S = self.S.tolil()
            except Exception, e:
                warn('Unable to convert S to lil_matrix maintaining in dok_matrix format: %s'%e)

    def update(self):
        """Regenerates the stoichiometric matrix and vectors

        """
        self._update_matrices()
        self._update_metabolite_vectors()

class LinkedArray(ndarray):
    """A ndarray which updates an attribute from a list"""
    def __new__(cls, list, attribute):
        # construct a new ndarray with the values from the list
        # For example, if the list if model.reactions and the attribute is
        # "lower_bound" create an array of [reaction.lower_bound for ... ]
        return array([getattr(i, attribute) for i in list]).view(cls)
    def __init__(self, list, attribute):
        self._list = list
        self._attr = attribute

    def __setitem__(self, index, value):
        numpy.ndarray.__setitem__(self, index, value)
        setattr(self._list[index], self._attr, value)

    def __setslice__(self, i, j, value):
        numpy.ndarray.__setslice__(self, i, j, value)
        if j == maxint:
            j = len(self)
        if hasattr(value, "__getitem__"):  # setting to a list
            for index in range(i, j):
                setattr(self._list[index], self._attr, value[index])
        else:
            for index in range(i, j):
                setattr(self._list[index], self._attr, value)

    def _extend(self, other):
        extended = hstack(self, other).view(LinkedArray)
        extended.__init__(self._list, self._attr)
        self = extended

class SMatrix_dok(dok_matrix):
    """A 2D sparse dok matrix which maintains links to a cobra Model"""
    def __init__(self, *args, **kwargs):
        dok_matrix.__init__(self, *args)
        self.format = "dok"
        self._model = kwargs["model"] if "model" in kwargs else None

    def __setitem__(self, index, value):
        dok_matrix.__setitem__(self, index, value)
        if type(index[0]) is int and type(index[1]) is int:
            reaction = self._model.reactions[index[1]]
            if value != 0:
                reaction.add_metabolites({self._model.metabolites[index[0]]: value}, combine=False)
            else:  # setting 0 means metabolites should be removed
                metabolite = self._model.metabolites[index[0]]
                if metabolite in reaction._metabolites:
                    reaction.pop(metabolite)

    def tolil(self):
        new = SMatrix_lil(dok_matrix.tolil(self), model=self._model)
        return new

class SMatrix_lil(lil_matrix):
    """A 2D sparse lil matrix which maintains links to a cobra Model"""
    def __init__(self, *args, **kwargs):
        lil_matrix.__init__(self, *args)
        self.format = "lil"
        self._model = kwargs["model"] if "model" in kwargs else None

    def __setitem__(self, index, value):
        lil_matrix.__setitem__(self, index, value)
        if isinstance(index[0], int):
            metabolites = [self._model.metabolites[index[0]]]
        else:
            metabolites = self._model.metabolites[index[0]]

        if isinstance(index[1], int):
            reactions = [self._model.reactions[index[1]]]
        else:
            reactions = self._model.reactions[index[1]]

        if value == 0:  # remove_metabolites
            met_set = set(metabolites)
            for reaction in reactions:
                to_remove = met_set.intersection(reaction._metabolites)
                for i in to_remove:
                    reaction.pop(i)
        else:  # add metabolites
            met_dict = {met: value for met in metabolites}
            for reaction in reactions:
                reaction.add_metabolites(met_dict, combine=False)

    def todok(self):
        new = SMatrix_dok(lil_matrix.todok(self), model=self._model)
        return new

SMatrix_classes = {"scipy.dok_matrix": SMatrix_dok,
                   "scipy.lil_matrix": SMatrix_lil}
<|endoftext|>


<commit_before># stdlib
import os
from pathlib import Path
import shutil
from typing import Dict
from typing import Optional
from urllib.request import urlretrieve

# third party
from jinja2 import Environment
from jinja2 import FileSystemLoader
from jinja2 import Template
import yaml

# relative
from .lib import manifest_template_path
from .lib import repo_src_path

HAGRID_TEMPLATE_PATH = str(manifest_template_path())

def read_yml_file(filename: str) -> Optional[Dict]:

    template = None

    with open(filename) as fp:
        try:
            template = yaml.safe_load(fp)
        except yaml.YAMLError as exc:
            raise exc

    return template

def git_url_for_file(file_path: str, base_url: str, hash: str) -> str:
    return os.path.join(base_url, hash, file_path)

def get_local_abs_path(target_dir: str, file_path: str) -> str:
    local_path = os.path.join(target_dir, file_path)
    return os.path.expanduser(local_path)

def setup_from_manifest_template(release_type: str) -> None:
    template = read_yml_file(HAGRID_TEMPLATE_PATH)

    if template is None:
        raise ValueError(
            f"Failed to read {HAGRID_TEMPLATE_PATH}. Please check the file name or path is correct."
        )

    git_hash = template["hash"]
    git_base_url = template["baseUrl"]
    target_dir = template["target_dir"]

    files_to_download = template["files"]

    for files in files_to_download:
        for trg_file_path, src_file_path in files.items():
            if release_type == "development":
                copy_files_from_repo(
                    src_file_path=src_file_path, trg_file_path=trg_file_path
                )
            else:
                local_destination = get_local_abs_path(target_dir, trg_file_path)
                link_to_file = git_url_for_file(src_file_path, git_base_url, git_hash)
                download_file(
                    link_to_file=link_to_file, local_destination=local_destination
                )

def render_templates(env_vars: dict, release_type: str) -> None:
    template = read_yml_file(HAGRID_TEMPLATE_PATH)

    if template is None:
        raise ValueError("Failed to read hagrid template.")

    print("Rendering ...............")

    files_to_download = template["files"]
    target_dir = (
        repo_src_path() if release_type == "development" else template["target_dir"]
    )

    jinja_template = JinjaTemplate(target_dir)

    for files in files_to_download:
        for trg_file_path, _ in files.items():
            jinja_template.substitute_vars(trg_file_path, env_vars)

    print("Rendering ............... DOne...........")

class JinjaTemplate(object):
    def __init__(self, template_dir: str) -> None:
        self.directory = os.path.expanduser(template_dir)
        self.environ = Environment(loader=FileSystemLoader(self.directory))

    def read_template_from_path(self, filepath: str) -> Template:
        print(filepath)
        return self.environ.get_template(name=filepath)

    def substitute_vars(self, template_path: str, vars_to_substitute: dict) -> None:

        template = self.read_template_from_path(template_path)
        rendered_template = template.render(vars_to_substitute)
        self.save_to(rendered_template, template.name)

    def save_to(self, message: str, filename: str) -> None:
        base_dir = self.directory
        with open(os.path.join(base_dir, filename), "w") as fp:
            fp.write(message)

def download_file(link_to_file: str, local_destination: str) -> str:

    file_dir = os.path.dirname(local_destination)
    print("File Dir: ", file_dir)
    os.makedirs(file_dir, exist_ok=True)

    try:
        print("Link to file", link_to_file)
        resultFilePath, _ = urlretrieve(link_to_file, local_destination)
    except Exception as e:
        raise e
    return resultFilePath

def copy_files_from_repo(src_file_path: str, trg_file_path: str) -> None:

    repo_directory = Path(repo_src_path())

    local_src_path = repo_directory / src_file_path
    local_destination = repo_directory / trg_file_path

    file_dir = os.path.dirname(local_destination)
    os.makedirs(file_dir, exist_ok=True)
    try:
        shutil.copyfile(local_src_path, local_destination)
    except shutil.SameFileError:
        print(f"{src_file_path}: Source and destination represents the same file.")

    # If destination is a directory.
    except IsADirectoryError:
        print("Destination is a directory.")

    # If there is any permission issue
    except PermissionError:
        print("Permission denied.")

    except Exception as e:
        print(
            f"While copying file from src: {src_file_path} to dest: {trg_file_path}, following exception occurred: {e}"
        )
<commit_msg>replace urlretrieve with requests.get due to security issue [B310:blacklist]<commit_after># stdlib
import os
from pathlib import Path
import shutil
from typing import Dict
from typing import Optional
from typing import Union

# third party
from jinja2 import Environment
from jinja2 import FileSystemLoader
from jinja2 import Template
import requests
import yaml

# relative
from .lib import manifest_template_path
from .lib import repo_src_path

HAGRID_TEMPLATE_PATH = str(manifest_template_path())

def read_yml_file(filename: str) -> Optional[Dict]:

    template = None

    with open(filename) as fp:
        try:
            template = yaml.safe_load(fp)
        except yaml.YAMLError as exc:
            raise exc

    return template

def git_url_for_file(file_path: str, base_url: str, hash: str) -> str:
    return os.path.join(base_url, hash, file_path)

def get_local_abs_path(target_dir: str, file_path: str) -> str:
    local_path = os.path.join(target_dir, file_path)
    return os.path.expanduser(local_path)

def setup_from_manifest_template(release_type: str) -> None:
    template = read_yml_file(HAGRID_TEMPLATE_PATH)

    if template is None:
        raise ValueError(
            f"Failed to read {HAGRID_TEMPLATE_PATH}. Please check the file name or path is correct."
        )

    git_hash = template["hash"]
    git_base_url = template["baseUrl"]
    target_dir = template["target_dir"]

    files_to_download = template["files"]

    for files in files_to_download:
        for trg_file_path, src_file_path in files.items():
            if release_type == "development":
                copy_files_from_repo(
                    src_file_path=src_file_path, trg_file_path=trg_file_path
                )
            else:
                local_destination = get_local_abs_path(target_dir, trg_file_path)
                link_to_file = git_url_for_file(src_file_path, git_base_url, git_hash)
                download_file(
                    link_to_file=link_to_file, local_destination=local_destination
                )

def render_templates(env_vars: dict, release_type: str) -> None:
    template = read_yml_file(HAGRID_TEMPLATE_PATH)

    if template is None:
        raise ValueError("Failed to read hagrid template.")

    files_to_download = template["files"]
    target_dir = (
        repo_src_path() if release_type == "development" else template["target_dir"]
    )

    jinja_template = JinjaTemplate(target_dir)

    for files in files_to_download:
        for trg_file_path, _ in files.items():
            jinja_template.substitute_vars(trg_file_path, env_vars)

class JinjaTemplate(object):
    def __init__(self, template_dir: Union[str, os.PathLike]) -> None:
        self.directory = os.path.expanduser(template_dir)
        self.environ = Environment(
            loader=FileSystemLoader(self.directory), autoescape=True
        )

    def read_template_from_path(self, filepath: str) -> Template:
        print(filepath)
        return self.environ.get_template(name=filepath)

    def substitute_vars(self, template_path: str, vars_to_substitute: dict) -> None:

        template = self.read_template_from_path(template_path)
        rendered_template = template.render(vars_to_substitute)
        self.save_to(rendered_template, template.name)

    def save_to(self, message: str, filename: str) -> None:
        base_dir = self.directory
        with open(os.path.join(base_dir, filename), "w") as fp:
            fp.write(message)

def download_file(link_to_file: str, local_destination: str) -> None:

    file_dir = os.path.dirname(local_destination)
    print("File Dir: ", file_dir)
    os.makedirs(file_dir, exist_ok=True)

    try:
        print("Link to file", link_to_file)

        # download file
        response = requests.get(link_to_file)
        if response.status_code != 200:
            raise Exception(f"Failed to download: {link_to_file}")

        # Save file to the local destination
        open(local_destination, "wb").write(response.content)

    except Exception as e:
        raise e

def copy_files_from_repo(src_file_path: str, trg_file_path: str) -> None:

    repo_directory = Path(repo_src_path())

    local_src_path = repo_directory / src_file_path
    local_destination = repo_directory / trg_file_path

    file_dir = os.path.dirname(local_destination)
    os.makedirs(file_dir, exist_ok=True)
    try:
        shutil.copyfile(local_src_path, local_destination)
    except shutil.SameFileError:
        print(f"{src_file_path}: Source and destination represents the same file.")

    # If destination is a directory.
    except IsADirectoryError:
        print("Destination is a directory.")

    # If there is any permission issue
    except PermissionError:
        print("Permission denied.")

    except Exception as e:
        print(
            f"While copying file from src: {src_file_path} to dest: {trg_file_path}, following exception occurred: {e}"
        )
<|endoftext|>


<commit_before># region Description
"""
test_apple_arp_dos.py: Unit tests for Raw-packet script: apple_arp_dos.py
Author: Vladimir Ivanov
License: MIT
Copyright 2019, Raw-packet Project
"""
# endregion

# region Import
from sys import path
from os.path import dirname, abspath
from os import system, kill
from signal import SIGTERM
from time import sleep, time
from subprocess import Popen, run, PIPE
import unittest

root_path = dirname(dirname(dirname(dirname(dirname(dirname(abspath(__file__)))))))
# endregion

# region Authorship information
__author__ = 'Vladimir Ivanov'
__copyright__ = 'Copyright 2019, Raw-packet Project'
__credits__ = ['']
__license__ = 'MIT'
__version__ = '0.2.1'
__maintainer__ = 'Vladimir Ivanov'
__email__ = 'ivanov.vladimir.mail@gmail.com'
__status__ = 'Development'
# endregion

# region Main class - ScriptAppleArpDosTest
class ScriptAppleArpDosTest(unittest.TestCase):

    # region Properties
    path.append(root_path)
    from raw_packet.Utils.base import Base
    base: Base = Base()
    network_interface: str = 'wlan0'
    target_ipv4_address: str = '192.168.0.53'
    bad_network_interface: str = 'wlan0123'
    bad_ipv4_address: str = '192.168.0.1234'
    bad_mac_address: str = '12:34:56:78:90:abc'
    # endregion

    def test01_main_arp_scan(self):
        apple_arp_dos = Popen(['python3 ' + root_path + '/Scripts/Apple/apple_arp_dos.py -i ' +
                               self.network_interface], shell=True, stdout=PIPE)
        find_target: bool = False
        start_time = time()
        for output_line in apple_arp_dos.stdout:
            output_line: str = output_line.decode('utf-8')
            print(output_line[:-1])
            if self.target_ipv4_address in output_line:
                find_target = True
                break
            else:
                if int(time() - start_time) > 10:
                    kill(apple_arp_dos.pid, SIGTERM)
                    while self.base.get_process_pid('/apple_arp_dos.py') != -1:
                        kill(self.base.get_process_pid('/apple_arp_dos.py'), SIGTERM)
                        sleep(0.5)
                    break
        self.assertTrue(find_target)

    def test02_main_nmap_scan(self):
        apple_arp_dos = Popen(['python3 ' + root_path + '/Scripts/Apple/apple_arp_dos.py -i ' +
                               self.network_interface + ' -n'], shell=True, stdout=PIPE)
        find_target: bool = False
        start_time = time()
        for output_line in apple_arp_dos.stdout:
            output_line: str = output_line.decode('utf-8')
            print(output_line[:-1])
            if self.target_ipv4_address in output_line:
                find_target = True
                break
            else:
                if int(time() - start_time) > 120:
                    kill(apple_arp_dos.pid, SIGTERM)
                    while self.base.get_process_pid('/apple_arp_dos.py') != -1:
                        kill(self.base.get_process_pid('/apple_arp_dos.py'), SIGTERM)
                        sleep(0.5)
                    break
        self.assertTrue(find_target)

    def test03_main_bad_interface(self):
        apple_arp_dos = run(['python3 ' + root_path + '/Scripts/Apple/apple_arp_dos.py -i ' +
                             self.bad_network_interface], shell=True, stdout=PIPE)
        apple_arp_dos_output: bytes = apple_arp_dos.stdout
        apple_arp_dos_output: str = apple_arp_dos_output.decode('utf-8')
        print(apple_arp_dos_output)
        self.assertIn(self.bad_network_interface, apple_arp_dos_output)

    def test04_main_bad_target_ip(self):
        apple_arp_dos = run(['python3 ' + root_path + '/Scripts/Apple/apple_arp_dos.py -i ' +
                             self.network_interface + ' -t ' + self.bad_ipv4_address], shell=True, stdout=PIPE)
        apple_arp_dos_output: bytes = apple_arp_dos.stdout
        apple_arp_dos_output: str = apple_arp_dos_output.decode('utf-8')
        print(apple_arp_dos_output)
        self.assertIn(self.bad_ipv4_address, apple_arp_dos_output)

    def test05_main_bad_target_mac(self):
        apple_arp_dos = run(['python3 ' + root_path + '/Scripts/Apple/apple_arp_dos.py -i ' +
                             self.network_interface + ' -t ' + self.target_ipv4_address +
                             ' -m ' + self.bad_mac_address], shell=True, stdout=PIPE)
        apple_arp_dos_output: bytes = apple_arp_dos.stdout
        apple_arp_dos_output: str = apple_arp_dos_output.decode('utf-8')
        print(apple_arp_dos_output)
        self.assertIn(self.bad_mac_address, apple_arp_dos_output)

    def test06_main(self):
        command: str = 'python3 ' + root_path + '/Scripts/Apple/apple_arp_dos.py -i ' + self.network_interface + \
                       ' -t ' + self.target_ipv4_address
        process = Popen(command, shell=True)
        sleep(10)
        response = system("ping -c 1 " + self.target_ipv4_address)
        kill(process.pid, SIGTERM)
        while self.base.get_process_pid('/apple_arp_dos.py') != -1:
            kill(self.base.get_process_pid('/apple_arp_dos.py'), SIGTERM)
            sleep(0.5)
        self.assertNotEqual(response, 0)

# endregion
<commit_msg>Update test_apple_arp_dos.py<commit_after># region Description
"""
test_apple_arp_dos.py: Unit tests for Raw-packet script: apple_arp_dos.py
Author: Vladimir Ivanov
License: MIT
Copyright 2019, Raw-packet Project
"""
# endregion

# region Import
from sys import path
from os.path import dirname, abspath
from os import system, kill
from signal import SIGTERM
from time import sleep, time
from subprocess import Popen, run, PIPE
import unittest

root_path = dirname(dirname(dirname(dirname(dirname(dirname(abspath(__file__)))))))
# endregion

# region Authorship information
__author__ = 'Vladimir Ivanov'
__copyright__ = 'Copyright 2019, Raw-packet Project'
__credits__ = ['']
__license__ = 'MIT'
__version__ = '0.2.1'
__maintainer__ = 'Vladimir Ivanov'
__email__ = 'ivanov.vladimir.mail@gmail.com'
__status__ = 'Development'
# endregion

# region Main class - ScriptAppleArpDosTest
class ScriptAppleArpDosTest(unittest.TestCase):

    # region Properties
    path.append(root_path)
    from raw_packet.Utils.base import Base
    base: Base = Base()
    network_interface: str = 'eth1'
    target_ipv4_address: str = '192.168.1.5'
    bad_network_interface: str = 'wlan0123'
    bad_ipv4_address: str = '192.168.0.1234'
    bad_mac_address: str = '12:34:56:78:90:abc'
    # endregion

    def test01_main_arp_scan(self):
        apple_arp_dos = Popen(['python3 ' + root_path + '/Scripts/Apple/apple_arp_dos.py -i ' +
                               self.network_interface], shell=True, stdout=PIPE)
        find_target: bool = False
        start_time = time()
        for output_line in apple_arp_dos.stdout:
            output_line: str = output_line.decode('utf-8')
            print(output_line[:-1])
            if self.target_ipv4_address in output_line:
                find_target = True
                break
            else:
                if int(time() - start_time) > 10:
                    kill(apple_arp_dos.pid, SIGTERM)
                    while self.base.get_process_pid('/apple_arp_dos.py') != -1:
                        kill(self.base.get_process_pid('/apple_arp_dos.py'), SIGTERM)
                        sleep(0.5)
                    break
        self.assertTrue(find_target)

    def test02_main_nmap_scan(self):
        apple_arp_dos = Popen(['python3 ' + root_path + '/Scripts/Apple/apple_arp_dos.py -i ' +
                               self.network_interface + ' -n'], shell=True, stdout=PIPE)
        find_target: bool = False
        start_time = time()
        for output_line in apple_arp_dos.stdout:
            output_line: str = output_line.decode('utf-8')
            print(output_line[:-1])
            if self.target_ipv4_address in output_line:
                find_target = True
                break
            else:
                if int(time() - start_time) > 120:
                    kill(apple_arp_dos.pid, SIGTERM)
                    while self.base.get_process_pid('/apple_arp_dos.py') != -1:
                        kill(self.base.get_process_pid('/apple_arp_dos.py'), SIGTERM)
                        sleep(0.5)
                    break
        self.assertTrue(find_target)

    def test03_main_bad_interface(self):
        apple_arp_dos = run(['python3 ' + root_path + '/Scripts/Apple/apple_arp_dos.py -i ' +
                             self.bad_network_interface], shell=True, stdout=PIPE)
        apple_arp_dos_output: bytes = apple_arp_dos.stdout
        apple_arp_dos_output: str = apple_arp_dos_output.decode('utf-8')
        print(apple_arp_dos_output)
        self.assertIn(self.bad_network_interface, apple_arp_dos_output)

    def test04_main_bad_target_ip(self):
        apple_arp_dos = run(['python3 ' + root_path + '/Scripts/Apple/apple_arp_dos.py -i ' +
                             self.network_interface + ' -t ' + self.bad_ipv4_address], shell=True, stdout=PIPE)
        apple_arp_dos_output: bytes = apple_arp_dos.stdout
        apple_arp_dos_output: str = apple_arp_dos_output.decode('utf-8')
        print(apple_arp_dos_output)
        self.assertIn(self.bad_ipv4_address, apple_arp_dos_output)

    def test05_main_bad_target_mac(self):
        apple_arp_dos = run(['python3 ' + root_path + '/Scripts/Apple/apple_arp_dos.py -i ' +
                             self.network_interface + ' -t ' + self.target_ipv4_address +
                             ' -m ' + self.bad_mac_address], shell=True, stdout=PIPE)
        apple_arp_dos_output: bytes = apple_arp_dos.stdout
        apple_arp_dos_output: str = apple_arp_dos_output.decode('utf-8')
        print(apple_arp_dos_output)
        self.assertIn(self.bad_mac_address, apple_arp_dos_output)

    def test06_main(self):
        command: str = 'python3 ' + root_path + '/Scripts/Apple/apple_arp_dos.py -i ' + self.network_interface + \
                       ' -t ' + self.target_ipv4_address
        process = Popen(command, shell=True)
        sleep(10)
        response = system("ping -c 1 " + self.target_ipv4_address)
        kill(process.pid, SIGTERM)
        while self.base.get_process_pid('/apple_arp_dos.py') != -1:
            kill(self.base.get_process_pid('/apple_arp_dos.py'), SIGTERM)
            sleep(0.5)
        self.assertNotEqual(response, 0)

# endregion
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Stack endpoint for Heat v1 ReST API.
"""

import httplib
import itertools
import json
import socket
import urlparse
from webob import exc

from heat.api.openstack.v1 import util
from heat.common import wsgi
from heat.engine import api as engine_api
from heat.engine import rpcapi as engine_rpcapi

import heat.openstack.common.rpc.common as rpc_common
from heat.openstack.common import log as logging

logger = logging.getLogger('heat.api.openstack.v1.stacks')

class InstantiationData(object):
    """
    The data accompanying a PUT or POST request to create or update a stack.
    """

    PARAMS = (
        PARAM_STACK_NAME,
        PARAM_TEMPLATE,
        PARAM_TEMPLATE_URL,
        PARAM_USER_PARAMS,
    ) = (
        'stack_name',
        'template',
        'template_url',
        'parameters',
    )

    def __init__(self, data):
        """Initialise from the request object."""
        self.data = data

    @staticmethod
    def json_parse(data, data_type):
        """
        Parse the supplied data as JSON, raising the appropriate exception
        if it is in the wrong format.
        """

        try:
            return json.loads(data)
        except ValueError:
            err_reason = "%s not in valid JSON format" % data_type
            raise exc.HTTPBadRequest(explanation=err_reason)

    def stack_name(self):
        """
        Return the stack name.
        """
        if self.PARAM_STACK_NAME not in self.data:
            raise exc.HTTPBadRequest(explanation=_("No stack name specified"))
        return self.data[self.PARAM_STACK_NAME]

    def _load_template(self, template_url):
        """
        Retrieve a template from a URL, in JSON format.
        """
        logger.debug('Template URL %s' % template_url)
        url = urlparse.urlparse(template_url)
        err_reason = _("Could not retrieve template")

        try:
            ConnType = (url.scheme == 'https' and httplib.HTTPSConnection
                                               or httplib.HTTPConnection)
            conn = ConnType(url.netloc)

            try:
                conn.request("GET", url.path)
                resp = conn.getresponse()
                logger.info('status %d' % r1.status)

                if resp.status != 200:
                    raise exc.HTTPBadRequest(explanation=err_reason)

                return self.json_parse(resp.read(), 'Template')
            finally:
                conn.close()
        except socket.gaierror:
            raise exc.HTTPBadRequest(explanation=err_reason)

    def template(self):
        """
        Get template file contents, either inline or from a URL, in JSON
        format.
        """
        if self.PARAM_TEMPLATE in self.data:
            return self.data[self.PARAM_TEMPLATE]
        elif self.PARAM_TEMPLATE_URL in self.data:
            return self._load_template(self.data[self.PARAM_TEMPLATE_URL])

        raise exc.HTTPBadRequest(explanation=_("No template specified"))

    def user_params(self):
        """
        Get the user-supplied parameters for the stack in JSON format.
        """
        return self.data.get(self.PARAM_USER_PARAMS, {})

    def args(self):
        """
        Get any additional arguments supplied by the user.
        """
        params = self.data.items()
        return dict((k, v) for k, v in params if k not in self.PARAMS)

def format_stack(req, stack, keys=[]):
    include_key = lambda k: k in keys if keys else True

    def transform(key, value):
        if not include_key(key):
            return

        if key == engine_api.STACK_ID:
            yield ('id', value['stack_id'])
            yield ('links', [util.make_link(req, value)])
        else:
            # TODO(zaneb): ensure parameters can be formatted for XML
            #elif key == engine_api.STACK_PARAMETERS:
            #    return key, json.dumps(value)
            yield (key, value)

    return dict(itertools.chain.from_iterable(
        transform(k, v) for k, v in stack.items()))

class StackController(object):
    """
    WSGI controller for stacks resource in Heat v1 API
    Implements the API actions
    """

    def __init__(self, options):
        self.options = options
        self.engine_rpcapi = engine_rpcapi.EngineAPI()

    def default(self, req, **args):
        raise exc.HTTPNotFound()

    @util.tenant_local
    def index(self, req):
        """
        Lists summary information for all stacks
        """

        try:
            stack_list = self.engine_rpcapi.list_stacks(req.context)
        except rpc_common.RemoteError as ex:
            return util.remote_error(ex, True)

        summary_keys = (engine_api.STACK_ID,
                        engine_api.STACK_NAME,
                        engine_api.STACK_DESCRIPTION,
                        engine_api.STACK_STATUS,
                        engine_api.STACK_STATUS_DATA,
                        engine_api.STACK_CREATION_TIME,
                        engine_api.STACK_DELETION_TIME,
                        engine_api.STACK_UPDATED_TIME)

        stacks = stack_list['stacks']

        return {'stacks': [format_stack(req, s, summary_keys) for s in stacks]}

    @util.tenant_local
    def create(self, req, body):
        """
        Create a new stack
        """

        data = InstantiationData(body)

        try:
            result = self.engine_rpcapi.create_stack(req.context,
                                                     data.stack_name(),
                                                     data.template(),
                                                     data.user_params(),
                                                     data.args())
        except rpc_common.RemoteError as ex:
            return util.remote_error(ex, True)

        if 'Description' in result:
            raise exc.HTTPBadRequest(explanation=result['Description'])

        raise exc.HTTPCreated(location=util.make_url(req, result))

    @util.tenant_local
    def lookup(self, req, stack_name, path='', body=None):
        """
        Redirect to the canonical URL for a stack
        """

        try:
            identity = self.engine_rpcapi.identify_stack(req.context,
                                                         stack_name)
        except rpc_common.RemoteError as ex:
            return util.remote_error(ex)

        location = util.make_url(req, identity)
        if path:
            location = '/'.join([location, path])

        raise exc.HTTPFound(location=location)

    @util.identified_stack
    def show(self, req, identity):
        """
        Gets detailed information for a stack
        """

        try:
            stack_list = self.engine_rpcapi.show_stack(req.context,
                                                       identity)
        except rpc_common.RemoteError as ex:
            return util.remote_error(ex)

        if not stack_list['stacks']:
            raise exc.HTTPInternalServerError()

        stack = stack_list['stacks'][0]

        return {'stack': format_stack(req, stack)}

    @util.identified_stack
    def template(self, req, identity):
        """
        Get the template body for an existing stack
        """

        try:
            templ = self.engine_rpcapi.get_template(req.context,
                                                    identity)
        except rpc_common.RemoteError as ex:
            return util.remote_error(ex)

        if templ is None:
            raise exc.HTTPNotFound()

        # TODO(zaneb): always set Content-type to application/json
        return templ

    @util.identified_stack
    def update(self, req, identity, body):
        """
        Update an existing stack with a new template and/or parameters
        """
        data = InstantiationData(body)

        try:
            res = self.engine_rpcapi.update_stack(req.context,
                                                  identity,
                                                  data.template(),
                                                  data.user_params(),
                                                  data.args())
        except rpc_common.RemoteError as ex:
            return util.remote_error(ex)

        if 'Description' in res:
            raise exc.HTTPBadRequest(explanation=res['Description'])

        raise exc.HTTPAccepted()

    @util.identified_stack
    def delete(self, req, identity):
        """
        Delete the specified stack
        """

        try:
            res = self.engine_rpcapi.delete_stack(req.context,
                                                  identity,
                                                  cast=False)

        except rpc_common.RemoteError as ex:
            return util.remote_error(ex)

        if res is not None:
            raise exc.HTTPBadRequest(explanation=res['Error'])

        raise exc.HTTPNoContent()

    @util.tenant_local
    def validate_template(self, req, body):
        """
        Implements the ValidateTemplate API action
        Validates the specified template
        """

        data = InstantiationData(body)

        try:
            result = self.engine_rpcapi.validate_template(req.context,
                                                          data.template())
        except rpc_common.RemoteError as ex:
            return util.remote_error(ex, True)

        if 'Error' in result:
            raise exc.HTTPBadRequest(explanation=result['Error'])

        return result

def create_resource(options):
    """
    Stacks resource factory method.
    """
    # TODO(zaneb) handle XML based on Content-type/Accepts
    deserializer = wsgi.JSONRequestDeserializer()
    serializer = wsgi.JSONResponseSerializer()
    return wsgi.Resource(StackController(options), deserializer, serializer)
<commit_msg>r1 not defined<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Stack endpoint for Heat v1 ReST API.
"""

import httplib
import itertools
import json
import socket
import urlparse
from webob import exc

from heat.api.openstack.v1 import util
from heat.common import wsgi
from heat.engine import api as engine_api
from heat.engine import rpcapi as engine_rpcapi

import heat.openstack.common.rpc.common as rpc_common
from heat.openstack.common import log as logging

logger = logging.getLogger('heat.api.openstack.v1.stacks')

class InstantiationData(object):
    """
    The data accompanying a PUT or POST request to create or update a stack.
    """

    PARAMS = (
        PARAM_STACK_NAME,
        PARAM_TEMPLATE,
        PARAM_TEMPLATE_URL,
        PARAM_USER_PARAMS,
    ) = (
        'stack_name',
        'template',
        'template_url',
        'parameters',
    )

    def __init__(self, data):
        """Initialise from the request object."""
        self.data = data

    @staticmethod
    def json_parse(data, data_type):
        """
        Parse the supplied data as JSON, raising the appropriate exception
        if it is in the wrong format.
        """

        try:
            return json.loads(data)
        except ValueError:
            err_reason = "%s not in valid JSON format" % data_type
            raise exc.HTTPBadRequest(explanation=err_reason)

    def stack_name(self):
        """
        Return the stack name.
        """
        if self.PARAM_STACK_NAME not in self.data:
            raise exc.HTTPBadRequest(explanation=_("No stack name specified"))
        return self.data[self.PARAM_STACK_NAME]

    def _load_template(self, template_url):
        """
        Retrieve a template from a URL, in JSON format.
        """
        logger.debug('Template URL %s' % template_url)
        url = urlparse.urlparse(template_url)
        err_reason = _("Could not retrieve template")

        try:
            ConnType = (url.scheme == 'https' and httplib.HTTPSConnection
                                               or httplib.HTTPConnection)
            conn = ConnType(url.netloc)

            try:
                conn.request("GET", url.path)
                resp = conn.getresponse()
                logger.info('status %d' % resp.status)

                if resp.status != 200:
                    raise exc.HTTPBadRequest(explanation=err_reason)

                return self.json_parse(resp.read(), 'Template')
            finally:
                conn.close()
        except socket.gaierror:
            raise exc.HTTPBadRequest(explanation=err_reason)

    def template(self):
        """
        Get template file contents, either inline or from a URL, in JSON
        format.
        """
        if self.PARAM_TEMPLATE in self.data:
            return self.data[self.PARAM_TEMPLATE]
        elif self.PARAM_TEMPLATE_URL in self.data:
            return self._load_template(self.data[self.PARAM_TEMPLATE_URL])

        raise exc.HTTPBadRequest(explanation=_("No template specified"))

    def user_params(self):
        """
        Get the user-supplied parameters for the stack in JSON format.
        """
        return self.data.get(self.PARAM_USER_PARAMS, {})

    def args(self):
        """
        Get any additional arguments supplied by the user.
        """
        params = self.data.items()
        return dict((k, v) for k, v in params if k not in self.PARAMS)

def format_stack(req, stack, keys=[]):
    include_key = lambda k: k in keys if keys else True

    def transform(key, value):
        if not include_key(key):
            return

        if key == engine_api.STACK_ID:
            yield ('id', value['stack_id'])
            yield ('links', [util.make_link(req, value)])
        else:
            # TODO(zaneb): ensure parameters can be formatted for XML
            #elif key == engine_api.STACK_PARAMETERS:
            #    return key, json.dumps(value)
            yield (key, value)

    return dict(itertools.chain.from_iterable(
        transform(k, v) for k, v in stack.items()))

class StackController(object):
    """
    WSGI controller for stacks resource in Heat v1 API
    Implements the API actions
    """

    def __init__(self, options):
        self.options = options
        self.engine_rpcapi = engine_rpcapi.EngineAPI()

    def default(self, req, **args):
        raise exc.HTTPNotFound()

    @util.tenant_local
    def index(self, req):
        """
        Lists summary information for all stacks
        """

        try:
            stack_list = self.engine_rpcapi.list_stacks(req.context)
        except rpc_common.RemoteError as ex:
            return util.remote_error(ex, True)

        summary_keys = (engine_api.STACK_ID,
                        engine_api.STACK_NAME,
                        engine_api.STACK_DESCRIPTION,
                        engine_api.STACK_STATUS,
                        engine_api.STACK_STATUS_DATA,
                        engine_api.STACK_CREATION_TIME,
                        engine_api.STACK_DELETION_TIME,
                        engine_api.STACK_UPDATED_TIME)

        stacks = stack_list['stacks']

        return {'stacks': [format_stack(req, s, summary_keys) for s in stacks]}

    @util.tenant_local
    def create(self, req, body):
        """
        Create a new stack
        """

        data = InstantiationData(body)

        try:
            result = self.engine_rpcapi.create_stack(req.context,
                                                     data.stack_name(),
                                                     data.template(),
                                                     data.user_params(),
                                                     data.args())
        except rpc_common.RemoteError as ex:
            return util.remote_error(ex, True)

        if 'Description' in result:
            raise exc.HTTPBadRequest(explanation=result['Description'])

        raise exc.HTTPCreated(location=util.make_url(req, result))

    @util.tenant_local
    def lookup(self, req, stack_name, path='', body=None):
        """
        Redirect to the canonical URL for a stack
        """

        try:
            identity = self.engine_rpcapi.identify_stack(req.context,
                                                         stack_name)
        except rpc_common.RemoteError as ex:
            return util.remote_error(ex)

        location = util.make_url(req, identity)
        if path:
            location = '/'.join([location, path])

        raise exc.HTTPFound(location=location)

    @util.identified_stack
    def show(self, req, identity):
        """
        Gets detailed information for a stack
        """

        try:
            stack_list = self.engine_rpcapi.show_stack(req.context,
                                                       identity)
        except rpc_common.RemoteError as ex:
            return util.remote_error(ex)

        if not stack_list['stacks']:
            raise exc.HTTPInternalServerError()

        stack = stack_list['stacks'][0]

        return {'stack': format_stack(req, stack)}

    @util.identified_stack
    def template(self, req, identity):
        """
        Get the template body for an existing stack
        """

        try:
            templ = self.engine_rpcapi.get_template(req.context,
                                                    identity)
        except rpc_common.RemoteError as ex:
            return util.remote_error(ex)

        if templ is None:
            raise exc.HTTPNotFound()

        # TODO(zaneb): always set Content-type to application/json
        return templ

    @util.identified_stack
    def update(self, req, identity, body):
        """
        Update an existing stack with a new template and/or parameters
        """
        data = InstantiationData(body)

        try:
            res = self.engine_rpcapi.update_stack(req.context,
                                                  identity,
                                                  data.template(),
                                                  data.user_params(),
                                                  data.args())
        except rpc_common.RemoteError as ex:
            return util.remote_error(ex)

        if 'Description' in res:
            raise exc.HTTPBadRequest(explanation=res['Description'])

        raise exc.HTTPAccepted()

    @util.identified_stack
    def delete(self, req, identity):
        """
        Delete the specified stack
        """

        try:
            res = self.engine_rpcapi.delete_stack(req.context,
                                                  identity,
                                                  cast=False)

        except rpc_common.RemoteError as ex:
            return util.remote_error(ex)

        if res is not None:
            raise exc.HTTPBadRequest(explanation=res['Error'])

        raise exc.HTTPNoContent()

    @util.tenant_local
    def validate_template(self, req, body):
        """
        Implements the ValidateTemplate API action
        Validates the specified template
        """

        data = InstantiationData(body)

        try:
            result = self.engine_rpcapi.validate_template(req.context,
                                                          data.template())
        except rpc_common.RemoteError as ex:
            return util.remote_error(ex, True)

        if 'Error' in result:
            raise exc.HTTPBadRequest(explanation=result['Error'])

        return result

def create_resource(options):
    """
    Stacks resource factory method.
    """
    # TODO(zaneb) handle XML based on Content-type/Accepts
    deserializer = wsgi.JSONRequestDeserializer()
    serializer = wsgi.JSONResponseSerializer()
    return wsgi.Resource(StackController(options), deserializer, serializer)
<|endoftext|>


<commit_before>from django.contrib import admin

from .models import Author, Book, Note, NoteTag, Section

@admin.register(Author)
class AuthorAdmin(admin.ModelAdmin):
    list_display = ['name', 'goodreads_id']

@admin.register(Section)
class SectionAdmin(admin.ModelAdmin):
    list_display = ['title', 'subtitle', 'get_page_display', 'book']
    list_filter = ['book']

@admin.register(Book)
class BookAdmin(admin.ModelAdmin):
    list_display = ['title', 'slug']

@admin.register(Note)
class NoteAdmin(admin.ModelAdmin):
    list_display = ['subject', 'section', 'book', 'get_page_display']
    search_fields = ['subject', 'quote', 'comment']

@admin.register(NoteTag)
class NoteTagAdmin(admin.ModelAdmin):
    list_display = ['slug', 'description', 'colour']
<commit_msg>Set prepopulated_fields for AuthorAdmin<commit_after>from django.contrib import admin

from .models import Author, Book, Note, NoteTag, Section

@admin.register(Author)
class AuthorAdmin(admin.ModelAdmin):
    list_display = ['name', 'goodreads_id']
    prepopulated_fields = {'slug': ('name',), }

@admin.register(Section)
class SectionAdmin(admin.ModelAdmin):
    list_display = ['title', 'subtitle', 'get_page_display', 'book']
    list_filter = ['book']

@admin.register(Book)
class BookAdmin(admin.ModelAdmin):
    list_display = ['title', 'slug']

@admin.register(Note)
class NoteAdmin(admin.ModelAdmin):
    list_display = ['subject', 'section', 'book', 'get_page_display']
    search_fields = ['subject', 'quote', 'comment']

@admin.register(NoteTag)
class NoteTagAdmin(admin.ModelAdmin):
    list_display = ['slug', 'description', 'colour']
<|endoftext|>