<commit_before># -*- coding: utf-8 -*-

# datapackage.py - Load and manage data packages defined by dataprotocols.org
# Copyright (C) 2013 Tryggvi Björgvinsson
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import urllib
import urlparse
import csv
import json
import itertools
import os
import datetime, time
import base64
import re

class DataPackage(object):
    """
    Package for loading and managing a data package as defined by:
    http://www.dataprotocols.org/en/latest/data-packages.html
    """

    FIELD_PARSERS = {
        'number': float,
        'integer': int,
        'date': lambda x: \
            datetime.datetime.strptime(x, '%Y-%m-%d').date(),
        'time': lambda x: time.strptime(x, '%H:%M'),
        'datetime': lambda x: \
            datetime.datetime.strptime(x, '%Y-%m-%dT%H:%M:%S%Z'),
        'boolean': bool,
        'binary': base64.b64decode,
        'object': json.loads,
        'json': json.loads,
        'geojson': json.loads,
        'array': list,
        }

    def _field_parser(self, field):
        """
        Return a type casting function (field parser) based on the data
        package field types. This returns the default type/functions for
        the types except when the field type is either data or datetime.
        In that case it returns the correctly formatted string (YYYY/MM/DD
        is even supported). Also if the field type is geopoint it returns
        the a function that parses the value as as
        {'lat': latitude_as_float, 'lon': longitude_as_float}
        """

        # If a format is provided with the field we need a different way
        # to cast it as datetime
        if (field['type'] == 'date' or field['type'] == 'datetime') \
                and 'format' in field:

            # Get the format into its own variable, we need to make some
            # replacements
            format_string = field['format']
            # Order of the replacements is important since month and minutes
            # can be denoted in a similar fashion
            replacement_order = [('hh','%m'), (':mm',':%M'), ('ss', '%S'),
                                 ('yyyy','%Y'), ('yy','%y'), ('mm', '%m'),
                                 ('dd','%d')]

            # For each replacement we substitute (and ignore the case)
            for (old, new) in replacement_order:
                format_string = re.sub("(?i)%s" % old, new, format_string)

            # Return the parser (here's a difference between date and datetime
            if field['type'] == 'datetime':
                return lambda x: datetime.datetime.strptime(x, format_string)
            else:
                return lambda x: \
                    datetime.datetime.strptime(x, format_string).date()

        # If type is geopoint we need to create a parser that can parse three
        # different formats into one dictionary
        if field['type'] == 'geopoint':
            def parse_geopoint(value):
                # Try to load it as json ( if it an dictionary object or array)
                try:
                    parsed = json.loads(value)
                    if type(parsed) == list:
                        # Geopoint coded as [123.4, 567.8]
                        return {'lat':parsed[0], 'lon':parsed[1]}
                    else:
                        # Geopoint coded as {'lat':123.4, 'lon':567.8}
                        return parsed
                except:
                    # Geopoint probably coded as "123.4, 567.8"
                    geotuple = value.split(',')
                    return {'lat':float(geotuple[0]), 'lon':float(geotuple[1])}

            # Return the parser
            return parse_geopoint

        # If none of the edge cases we use the default field parsers and fall
        # back on unicode type if no parser is found
        return self.FIELD_PARSERS.get(field['type'], unicode)

    def __init__(self, uri, opener=None):
        """
        Construct a DataPackage.

        :param basestring uri: URI or file path to the data package.
            ``datapackage.json`` should exist under this URI.
        :param function opener: optional function instead of ``urllib.urlopen``
            to read data; e.g. ``opener=zf.open`` where ``zf = ZipFS('a.zip')``
        :type opener: function
        """
        self.opener = opener or urllib.urlopen
        # Bind the URI to this instance
        self.uri = uri
        # Load the descriptor as a dictionary into a variable
        self.descriptor = self.get_descriptor()
        # Load the resources as a dictionary into a variable
        self.resources = self.get_resources()

    def open_resource(self, path):
        return self.opener(urlparse.urljoin(self.uri, path))

    @property
    def title(self):
        """
        The title of the dataset as described by its descriptor.
        Default is an empty string if no title is present
        """

        return self.descriptor.get('title', u'')

    @property
    def description(self):
        """
        The descriptor of the dataset as described by its descriptor.
        Default is an empty string if no description is present
        """

        return self.descriptor.get('description', u'')

    @property
    def data(self):
        """
        An iterator that returns dictionary representation of the rows in
        all resources.
        """

        # Get all of the generators for the resources
        data_generators = [self.get_data(k) for k in self.resources.keys()]
        return itertools.chain.from_iterable(data_generators)

    def _package_is_local(self):
        """
        Checks to see if the data package is located on the local file system.
        This simple check just looks if there is a scheme or netloc associated
        with the data package URI (and will therefore return False when the
        URI uses the file: scheme)
        """

        parsed_results = urlparse.urlparse(self.uri)
        return parsed_results.scheme == '' or parsed_results.netloc == ''

    def get_descriptor(self):
        """
        Get the descriptor for the data package (as defined by the standard)
        as a dictionary. This uses the URI provided by the constructor and
        performs a join with the descriptor URN. This follows the join rules
        of urlparse.urljoin which means for URLs that if the URI does
        not end with a slash the last piece of the URI will be replaced with
        the descriptor URN.
        """
        descriptor = self.open_resource('datapackage.json')

        # Load the descriptor json contents
        json_descriptor = json.load(descriptor)

        # Return the descriptor json contents (as the dict json.load returns
        return json_descriptor

    def get_resources(self):
        """
        Get the data package's resources as a dictionary. The key for each
        resource is the value of its id attribute. If no id is provided then
        the key is an empty string. This means that resources can be
        overwritten if they have the same (or no id).
        """

        # Initialise the empty dictionary
        sources = {}
        # Loop through the resources
        for resource in self.descriptor['resources']:
            # Create a resource dictionary
            source = {'location':
                          # Location is url path or None (in that order)
                          resource.get('url', resource.get('path', None)),
                      'fields':
                          # Fields are found in schema.fields
                          resource.get('schema', {}).get('fields', [])
                      }
            # Add the resource to the resource dictionary collection
            sources[resource.get('id', u'')] = source

        # Return the resource collection
        return sources

    def get_data(self, id=''):
        """
        Generator that yields the data for a given resource.
        The resource defaults to a resource identified by the empty string
        (no id attribute in descriptor)
        """

        # Check if the id can be found in the resources or throw a KeyError
        if id not in self.resources:
            raise KeyError("Source not found")

        # Get the resource dictionary representation
        resource_dict = self.resources[id]
        # Open the resource location
        resource_file = self.open_resource(resource_dict['location'])
        # We assume CSV so we create the csv file
        reader = csv.reader(resource_file)
        # Throw away the first line (headers)
        reader.next()
        # For each row we yield it as a dictionary where keys are the field
        # names and the value the value in that row
        for row in reader:
            yield dict((field['id'], self._field_parser(field)(row[idx].decode("utf-8")))
                       for idx, field in enumerate(resource_dict['fields']))
<commit_msg>read the encoding of each resource and decode the file respectively<commit_after># -*- coding: utf-8 -*-

# datapackage.py - Load and manage data packages defined by dataprotocols.org
# Copyright (C) 2013 Tryggvi Björgvinsson
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import urllib
import urlparse
import csv
import json
import itertools
import os
import datetime, time
import base64
import re

class DataPackage(object):
    """
    Package for loading and managing a data package as defined by:
    http://www.dataprotocols.org/en/latest/data-packages.html
    """

    FIELD_PARSERS = {
        'number': float,
        'integer': int,
        'date': lambda x: \
            datetime.datetime.strptime(x, '%Y-%m-%d').date(),
        'time': lambda x: time.strptime(x, '%H:%M'),
        'datetime': lambda x: \
            datetime.datetime.strptime(x, '%Y-%m-%dT%H:%M:%S%Z'),
        'boolean': bool,
        'binary': base64.b64decode,
        'object': json.loads,
        'json': json.loads,
        'geojson': json.loads,
        'array': list,
        }

    def _field_parser(self, field):
        """
        Return a type casting function (field parser) based on the data
        package field types. This returns the default type/functions for
        the types except when the field type is either data or datetime.
        In that case it returns the correctly formatted string (YYYY/MM/DD
        is even supported). Also if the field type is geopoint it returns
        the a function that parses the value as as
        {'lat': latitude_as_float, 'lon': longitude_as_float}
        """

        # If a format is provided with the field we need a different way
        # to cast it as datetime
        if (field['type'] == 'date' or field['type'] == 'datetime') \
                and 'format' in field:

            # Get the format into its own variable, we need to make some
            # replacements
            format_string = field['format']
            # Order of the replacements is important since month and minutes
            # can be denoted in a similar fashion
            replacement_order = [('hh','%m'), (':mm',':%M'), ('ss', '%S'),
                                 ('yyyy','%Y'), ('yy','%y'), ('mm', '%m'),
                                 ('dd','%d')]

            # For each replacement we substitute (and ignore the case)
            for (old, new) in replacement_order:
                format_string = re.sub("(?i)%s" % old, new, format_string)

            # Return the parser (here's a difference between date and datetime
            if field['type'] == 'datetime':
                return lambda x: datetime.datetime.strptime(x, format_string)
            else:
                return lambda x: \
                    datetime.datetime.strptime(x, format_string).date()

        # If type is geopoint we need to create a parser that can parse three
        # different formats into one dictionary
        if field['type'] == 'geopoint':
            def parse_geopoint(value):
                # Try to load it as json ( if it an dictionary object or array)
                try:
                    parsed = json.loads(value)
                    if type(parsed) == list:
                        # Geopoint coded as [123.4, 567.8]
                        return {'lat':parsed[0], 'lon':parsed[1]}
                    else:
                        # Geopoint coded as {'lat':123.4, 'lon':567.8}
                        return parsed
                except:
                    # Geopoint probably coded as "123.4, 567.8"
                    geotuple = value.split(',')
                    return {'lat':float(geotuple[0]), 'lon':float(geotuple[1])}

            # Return the parser
            return parse_geopoint

        # If none of the edge cases we use the default field parsers and fall
        # back on unicode type if no parser is found
        return self.FIELD_PARSERS.get(field['type'], unicode)

    def __init__(self, uri, opener=None):
        """
        Construct a DataPackage.

        :param basestring uri: URI or file path to the data package.
            ``datapackage.json`` should exist under this URI.
        :param function opener: optional function instead of ``urllib.urlopen``
            to read data; e.g. ``opener=zf.open`` where ``zf = ZipFS('a.zip')``
        :type opener: function
        """
        self.opener = opener or urllib.urlopen
        # Bind the URI to this instance
        self.uri = uri
        # Load the descriptor as a dictionary into a variable
        self.descriptor = self.get_descriptor()
        # Load the resources as a dictionary into a variable
        self.resources = self.get_resources()

    def open_resource(self, path):
        return self.opener(urlparse.urljoin(self.uri, path))

    @property
    def title(self):
        """
        The title of the dataset as described by its descriptor.
        Default is an empty string if no title is present
        """

        return self.descriptor.get('title', u'')

    @property
    def description(self):
        """
        The descriptor of the dataset as described by its descriptor.
        Default is an empty string if no description is present
        """

        return self.descriptor.get('description', u'')

    @property
    def data(self):
        """
        An iterator that returns dictionary representation of the rows in
        all resources.
        """

        # Get all of the generators for the resources
        data_generators = [self.get_data(k) for k in self.resources.keys()]
        return itertools.chain.from_iterable(data_generators)

    def _package_is_local(self):
        """
        Checks to see if the data package is located on the local file system.
        This simple check just looks if there is a scheme or netloc associated
        with the data package URI (and will therefore return False when the
        URI uses the file: scheme)
        """

        parsed_results = urlparse.urlparse(self.uri)
        return parsed_results.scheme == '' or parsed_results.netloc == ''

    def get_descriptor(self):
        """
        Get the descriptor for the data package (as defined by the standard)
        as a dictionary. This uses the URI provided by the constructor and
        performs a join with the descriptor URN. This follows the join rules
        of urlparse.urljoin which means for URLs that if the URI does
        not end with a slash the last piece of the URI will be replaced with
        the descriptor URN.
        """
        descriptor = self.open_resource('datapackage.json')

        # Load the descriptor json contents
        json_descriptor = json.load(descriptor)

        # Return the descriptor json contents (as the dict json.load returns
        return json_descriptor

    def get_resources(self):
        """
        Get the data package's resources as a dictionary. The key for each
        resource is the value of its id attribute. If no id is provided then
        the key is an empty string. This means that resources can be
        overwritten if they have the same (or no id).
        """

        # Initialise the empty dictionary
        sources = {}
        # Loop through the resources
        for resource in self.descriptor['resources']:
            # Create a resource dictionary
            source = {'location':
                          # Location is url path or None (in that order)
                          resource.get('url', resource.get('path', None)),
                      'encoding':
                        # The encoding of the file - defaults to utf-8
                          resource.get('encoding','utf-8'),
                      'fields':
                          # Fields are found in schema.fields
                          resource.get('schema', {}).get('fields', [])
                      }
            # Add the resource to the resource dictionary collection
            sources[resource.get('id', u'')] = source

        # Return the resource collection
        return sources

    def get_data(self, id=''):
        """
        Generator that yields the data for a given resource.
        The resource defaults to a resource identified by the empty string
        (no id attribute in descriptor)
        """

        # Check if the id can be found in the resources or throw a KeyError
        if id not in self.resources:
            raise KeyError("Source not found")

        # Get the resource dictionary representation
        resource_dict = self.resources[id]
        # Open the resource location
        resource_file = self.open_resource(resource_dict['location'])
        # We assume CSV so we create the csv file
        reader = csv.reader(resource_file)
        # Throw away the first line (headers)
        reader.next()
        # For each row we yield it as a dictionary where keys are the field
        # names and the value the value in that row
        for row in reader:
            yield dict((field['id'],
            self._field_parser(field)(row[idx].decode(resource_dict.get('encoding'))))
                       for idx, field in enumerate(resource_dict['fields']))
<|endoftext|>


<commit_before>from flask import Blueprint, request, url_for

from datawire.auth import require
from datawire.model import Service
from datawire.views.util import jsonify, obj_or_404
from datawire.views.util import get_limit, get_offset

services = Blueprint('services', __name__)

@services.route('/services')
def index():
    q = Service.all()
    result = {'count': q.count()}
    q = q.limit(get_limit())
    q = q.offset(get_offset())
    result['results'] = q
    return jsonify(result)

@services.route('/services/<key>')
def get(key):
    service = obj_or_404(Service.by_key(key))
    require.service.publish(service)
    return jsonify(service)
<commit_msg>Use shallow serialisation for services listing. <commit_after>from flask import Blueprint, request, url_for

from datawire.auth import require
from datawire.model import Service
from datawire.views.util import jsonify, obj_or_404
from datawire.views.util import get_limit, get_offset

services = Blueprint('services', __name__)

@services.route('/services')
def index():
    q = Service.all()
    result = {'count': q.count()}
    q = q.limit(get_limit())
    q = q.offset(get_offset())
    result['results'] = q
    return jsonify(result, refs=True)

@services.route('/services/<key>')
def get(key):
    service = obj_or_404(Service.by_key(key))
    require.service.view(service)
    return jsonify(service)
<|endoftext|>


<commit_before>import libcore
import matplotlib.pyplot as plt
from os.path import join
import numpy as np

"""Berechnete Bestrahlungsstärke E. Siehe Rechnung in Ordner MessungenAufgabe_1-2"""
IRRADIANCE_WATT_PER_SQUARE_METER = 0.121705797795879

"""Wellenlänge der verwendeten grünen LED"""
WAVELENGTH_METER = 0.000000525

"""Pixelgröße aus Datenblatt entnommen"""
PIXEL_AREA_METER = 0.0000045 ** 2

TIME_OF_EXPOSURE_MS = [0.02,
                       1.645,
                       3.27,
                       4.895,
                       6.52,
                       8.145,
                       9.77,
                       11.395,
                       13.02]

def plot_mean_of_photons():
    mean_of_photons_for_texp = []

    for texp in TIME_OF_EXPOSURE_MS:
        texp_sec = texp / 1000
        mean_of_photons = libcore.get_mean_of_photons(PIXEL_AREA_METER,
                                            IRRADIANCE_WATT_PER_SQUARE_METER,
                                            texp_sec,
                                            WAVELENGTH_METER)

        mean_of_photons_for_texp.append(mean_of_photons)

    plt.plot(TIME_OF_EXPOSURE_MS, mean_of_photons_for_texp, 'ro')
    plt.xlabel('Time of Exposure [ms]')
    plt.ylabel('Mean of Photons per Pixel')

    plt.show()

def get_mean_gray_value_without_dark_noise():
    # ../MessungenAufgabe_1-2/offen/*
    path_open = join(join(join("..", "MessungenAufgabe_1-2"), "offen", "*"))

    # ../MessungenAufgabe_1-2/geschlossen/*
    path_closed = join(join(join("..", "MessungenAufgabe_1-2"), "geschlossen"), "*")

    # Verarbeite images_open
    images_open = libcore.get_sorted_images(path_open)

    mean_open = libcore.get_mean_of_two_images(images_open)
    mean_open = np.matrix(mean_open)

    # Verarbeite images_closed
    images_closed = libcore.get_sorted_images(path_closed)

    mean_closed = libcore.get_mean_of_two_images(images_closed)
    mean_closed = np.matrix(mean_closed)

    mean_gray_value_without_dark_noise = mean_open - mean_closed

    return mean_gray_value_without_dark_noise

def get_variance_gray_value_without_dark_noise():
    # ../MessungenAufgabe_1-2/offen/*
    path_open = join(join(join("..", "MessungenAufgabe_1-2"), "offen", "*"))

    # ../MessungenAufgabe_1-2/geschlossen/*
    path_closed = join(join(join("..", "MessungenAufgabe_1-2"), "geschlossen"), "*")

    # Verarbeite images_open
    images_open = libcore.get_sorted_images(path_open)

    variance_open = libcore.get_time_variance_of_two_images(images_open)
    variance_open = np.matrix(variance_open)

    # Verarbeite images_closed
    images_closed = libcore.get_sorted_images(path_closed)

    variance_closed = libcore.get_time_variance_of_two_images(images_closed)
    variance_closed = np.matrix(variance_closed)

    variance_gray_value_without_dark_noise = variance_open - variance_closed

    return variance_gray_value_without_dark_noise

def plot_photo_transfer():
    mean_gray_value_without_dark_noise = get_mean_gray_value_without_dark_noise()
    variance_gray_value_without_dark_noise = get_variance_gray_value_without_dark_noise()

    plt.plot(mean_gray_value_without_dark_noise, variance_gray_value_without_dark_noise, 'ro')
    plt.xlabel('gray value - dark value')
    plt.ylabel('variance gray value')

    plt.show()

def main():
    #plot_mean_of_photons()
    #get_mean_gray_value_without_dark_noise()
    plot_photo_transfer()

if __name__ == '__main__':
    main()
<commit_msg>Phototransfer-Methode fertig gestellt mit Einzeichnen des Sättigungspunktes und der linearen Regeressionsgeraden.<commit_after>import libcore
import matplotlib.pyplot as plt
import numpy as np

from os.path import join
from scipy import stats

"""Berechnete Bestrahlungsstärke E. Siehe Rechnung in Ordner MessungenAufgabe_1-2"""
IRRADIANCE_WATT_PER_SQUARE_METER = 0.121705797795879

"""Wellenlänge der verwendeten grünen LED"""
WAVELENGTH_METER = 0.000000525

"""Pixelgröße aus Datenblatt entnommen"""
PIXEL_AREA_METER = 0.0000045 ** 2

TIME_OF_EXPOSURE_MS = [0.02,
                       1.645,
                       3.27,
                       4.895,
                       6.52,
                       8.145,
                       9.77,
                       11.395,
                       13.02]

def plot_mean_of_photons():
    mean_of_photons_for_texp = []

    for texp in TIME_OF_EXPOSURE_MS:
        texp_sec = texp / 1000
        mean_of_photons = libcore.get_mean_of_photons(PIXEL_AREA_METER,
                                            IRRADIANCE_WATT_PER_SQUARE_METER,
                                            texp_sec,
                                            WAVELENGTH_METER)

        mean_of_photons_for_texp.append(mean_of_photons)

    plt.plot(TIME_OF_EXPOSURE_MS, mean_of_photons_for_texp, 'ro')
    plt.xlabel('Time of Exposure [ms]')
    plt.ylabel('Mean of Photons per Pixel')

    plt.show()

def get_mean_gray_value_without_dark_noise():
    # ../MessungenAufgabe_1-2/offen/*
    path_open = join(join(join("..", "MessungenAufgabe_1-2"), "offen", "*"))

    # ../MessungenAufgabe_1-2/geschlossen/*
    path_closed = join(join(join("..", "MessungenAufgabe_1-2"), "geschlossen"), "*")

    # Verarbeite images_open
    images_open = libcore.get_sorted_images(path_open)

    mean_open = libcore.get_mean_of_two_images(images_open)
    mean_open = np.array(mean_open)

    # Verarbeite images_closed
    images_closed = libcore.get_sorted_images(path_closed)

    mean_closed = libcore.get_mean_of_two_images(images_closed)
    mean_closed = np.array(mean_closed)

    mean_gray_value_without_dark_noise = mean_open - mean_closed

    return mean_gray_value_without_dark_noise

def get_variance_gray_value_without_dark_noise():
    # ../MessungenAufgabe_1-2/offen/*
    path_open = join(join(join("..", "MessungenAufgabe_1-2"), "offen", "*"))

    # ../MessungenAufgabe_1-2/geschlossen/*
    path_closed = join(join(join("..", "MessungenAufgabe_1-2"), "geschlossen"), "*")

    # Verarbeite images_open
    images_open = libcore.get_sorted_images(path_open)

    variance_open = libcore.get_time_variance_of_two_images(images_open)
    variance_open = np.array(variance_open)

    # Verarbeite images_closed
    images_closed = libcore.get_sorted_images(path_closed)

    variance_closed = libcore.get_time_variance_of_two_images(images_closed)
    variance_closed = np.array(variance_closed)

    variance_gray_value_without_dark_noise = variance_open - variance_closed

    return variance_gray_value_without_dark_noise

def plot_photo_transfer():
    mean_gray_value_without_dark_noise = get_mean_gray_value_without_dark_noise()
    variance_gray_value_without_dark_noise = get_variance_gray_value_without_dark_noise()

    # Limits der Axen setzen
    x_begin = 0
    x_end = 300
    plt.xlim(x_begin, x_end)
    plt.ylim(-1, np.max(variance_gray_value_without_dark_noise) * 1.3)

    # Plotte mittlere Grauwerte und Varianz ohne Dunkelstrom
    plt.plot(mean_gray_value_without_dark_noise, variance_gray_value_without_dark_noise, 'ro')
    plt.xlabel('gray value - dark value')
    plt.ylabel('variance gray value')

    # Sättingspunkt ist an der Stelle der maximalen Varianz
    saturation_index = np.argmax(variance_gray_value_without_dark_noise)

    # Sättingungspunkt im Plot einzeichnen
    saturation_x_coord = mean_gray_value_without_dark_noise[saturation_index]
    saturation_y_coord = variance_gray_value_without_dark_noise[saturation_index]

    plt.annotate('Saturation', xy=(saturation_x_coord, saturation_y_coord),
                xycoords='data',
                xytext=(-30, +30),
                textcoords='offset points',
                fontsize=16,
                arrowprops=dict(arrowstyle="->", connectionstyle="arc3,rad=.2"))

    plt.plot([saturation_x_coord, saturation_x_coord], [-1, saturation_y_coord],
             color='red', linewidth=1.5, linestyle="--")

    # Lineare Regeression der steigenden Varianz bilden.
    # Die Sättigung soll dabei nicht miteinfließen.
    # Man geht davon aus, dass bei 70% des Sättigungspunktes die Sättigung anfängt
    mean_sat_begin = mean_gray_value_without_dark_noise[saturation_index] * 0.7

    # Ermittle die Indizes der Matrix-Einträge die einen kleineren Wert als mean_sat_begin haben
    mean_without_sat_indices = mean_gray_value_without_dark_noise < mean_sat_begin

    # Mean-Matrix von der Sättigung bereinigt
    mean_without_sat = mean_gray_value_without_dark_noise[mean_without_sat_indices]
    variance_without_sat = variance_gray_value_without_dark_noise[mean_without_sat_indices]

    # Steigung und Y-Achsenabschnitt der Geraden
    slope, intercept, _, _, _ = stats.linregress(mean_without_sat, variance_without_sat)

    # Anfang und Ende der Geraden bestimmen
    line_begin = intercept
    line_end = intercept + slope * x_end

    plt.plot([x_begin, x_end], [line_begin, line_end])

    plt.show()

def main():
    #plot_mean_of_photons()
    plot_photo_transfer()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from datetime import datetime
from dbaas.celery import app
import models
import logging
from notification.models import TaskHistory
from util import exec_remote_command_host, get_worker_name, \
    build_context_script, get_dict_lines
from registered_functions.functools import get_function
from workflow.steps.util.dns import ChangeTTLTo5Minutes, ChangeTTLTo3Hours
from workflow.steps.util.db_monitor import DisableMonitoring, EnableMonitoring
from workflow.steps.util.zabbix import DisableAlarms, EnableAlarms
from workflow.workflow import steps_for_instances

LOG = logging.getLogger(__name__)

@app.task(bind=True)
def execute_scheduled_maintenance(self, maintenance_id):
    LOG.debug("Maintenance id: {}".format(maintenance_id))
    maintenance = models.Maintenance.objects.get(id=maintenance_id)
    models.Maintenance.objects.filter(id=maintenance_id).update(
        status=maintenance.RUNNING, started_at=datetime.now()
    )
    LOG.info("Maintenance {} is RUNNING".format(maintenance,))

    worker_name = get_worker_name()
    task_history = TaskHistory.register(
        request=self.request, worker_name=worker_name
    )
    task_history.relevance = TaskHistory.RELEVANCE_CRITICAL
    LOG.info("id: {} | task: {} | kwargs: {} | args: {}".format(
        self.request.id, self.request.task,
        self.request.kwargs, str(self.request.args)
    ))
    task_history.update_details(
        persist=True, details="Executing Maintenance: {}".format(maintenance)
    )
    for hm in models.HostMaintenance.objects.filter(maintenance=maintenance):
        main_output = {}
        hm.status = hm.RUNNING
        hm.started_at = datetime.now()
        hm.save()
        if hm.host is None:
            hm.status = hm.UNAVAILABLEHOST
            hm.finished_at = datetime.now()
            hm.save()
            continue

        host = hm.host
        update_task = "\nRunning Maintenance on {}".format(host)

        if maintenance.disable_alarms:
            disable_alarms(hm.host)

        param_dict = {}
        params = models.MaintenanceParameters.objects.filter(
            maintenance=maintenance
        )
        for param in params:
            param_function = get_function(param.function_name)
            param_dict[param.parameter_name] = param_function(host.id)

        main_script = build_context_script(param_dict, maintenance.main_script)
        exit_status = exec_remote_command_host(host, main_script, main_output)

        if exit_status == 0:
            hm.status = hm.SUCCESS
        else:

            if maintenance.rollback_script:
                rollback_output = {}
                hm.status = hm.ROLLBACK
                hm.save()

                rollback_script = build_context_script(
                    param_dict, maintenance.rollback_script
                )
                exit_status = exec_remote_command_host(
                    host, rollback_script, rollback_output
                )

                if exit_status == 0:
                    hm.status = hm.ROLLBACK_SUCCESS
                else:
                    hm.status = hm.ROLLBACK_ERROR

                hm.rollback_log = get_dict_lines(rollback_output)

            else:
                hm.status = hm.ERROR

        if maintenance.disable_alarms:
            enable_alarms(hm.host)

        update_task += "...status: {}".format(hm.status)

        task_history.update_details(persist=True, details=update_task)

        hm.main_log = get_dict_lines(main_output)
        hm.finished_at = datetime.now()
        hm.save()

    models.Maintenance.objects.filter(id=maintenance_id).update(
        status=maintenance.FINISHED, finished_at=datetime.now()
    )
    task_history.update_status_for(
        TaskHistory.STATUS_SUCCESS, details='Maintenance executed succesfully'
    )
    LOG.info("Maintenance: {} has FINISHED".format(maintenance))

def disable_alarms(host):
    for instance in host.instances.all():
        DisableMonitoring(instance).do()
        DisableAlarms(instance).do()

def enable_alarms(host):
    for instance in host.instances.all():
        EnableMonitoring(instance).do()
        EnableAlarms(instance).do()

def region_migration_prepare(infra):
    instance = infra.instances.first()
    ChangeTTLTo5Minutes(instance).do()

def region_migration_finish(infra):
    instance = infra.instances.first()
    ChangeTTLTo3Hours(instance).do()

@app.task(bind=True)
def region_migration_start(self, infra, instances, since_step=None):
    steps = [{
        'Disable monitoring and alarms': (
            'workflow.steps.util.zabbix.DestroyAlarms',
            'workflow.steps.util.db_monitor.DisableMonitoring',
        )}, {
        'Stopping infra': (
            'workflow.steps.util.database.Stop',
            'workflow.steps.util.database.CheckIsDown',
        )}, {
        'Creating new virtual machine': (
            'workflow.steps.util.vm.MigrationCreateNewVM',
        )}, {
        'Creating new infra': (
            'workflow.steps.util.vm.MigrationWaitingBeReady',
            'workflow.steps.util.infra.MigrationCreateInstance',
            'workflow.steps.util.disk.MigrationCreateExport',
        )}, {
        'Configuring new infra': (
            'workflow.steps.util.volume_provider.MountDataVolume',
            'workflow.steps.util.plan.InitializationMigration',
            'workflow.steps.util.plan.ConfigureMigration',
        )}, {
        'Preparing new environment': (
            'workflow.steps.util.disk.AddDiskPermissionsOldest',
            'workflow.steps.util.disk.MountOldestExportMigration',
            'workflow.steps.util.disk.CopyDataBetweenExportsMigration',
            'workflow.steps.util.disk.FilePermissionsMigration',
            'workflow.steps.util.disk.UnmountNewerExportMigration',
            'workflow.steps.util.vm.ChangeInstanceHost',
            'workflow.steps.util.vm.UpdateOSDescription',
            'workflow.steps.util.infra.OfferingMigration',
            'workflow.steps.util.infra.UpdateMigrateEnvironment',
            'workflow.steps.util.infra.UpdateMigratePlan',
        )}, {
        'Starting new infra': (
            'workflow.steps.util.database.Start',
            'workflow.steps.util.database.CheckIsUp',
        )}, {
        'Enabling access': (
            'workflow.steps.util.dns.ChangeEndpoint',
            'workflow.steps.util.acl.ReplicateAclsMigration',
        )}, {
        'Destroying old infra': (
            'workflow.steps.util.disk.DisableOldestExportMigration',
            'workflow.steps.util.disk.DiskUpdateHost',
            'workflow.steps.util.vm.RemoveHostMigration',
        )}, {
        'Enabling monitoring and alarms': (
            'workflow.steps.util.db_monitor.EnableMonitoring',
            'workflow.steps.util.zabbix.CreateAlarms',
        )}, {
        'Restart replication': (
            'workflow.steps.util.database.SetSlavesMigration',
        )
    }]

    task = TaskHistory()
    task.task_id = self.request.id
    task.task_name = "migrating_zone"
    task.task_status = TaskHistory.STATUS_RUNNING
    task.context = {'infra': infra, 'instances': instances}
    task.arguments = {'infra': infra, 'instances': instances}
    task.user = 'admin'
    task.save()

    if steps_for_instances(steps, instances, task, since_step=since_step):
        task.set_status_success('Region migrated with success')
    else:
        task.set_status_error('Could not migrate region')

    database = infra.databases.first()
    database.environment = infra.environment
    database.save()

@app.task(bind=True)
def create_database(
    self, name, plan, environment, team, project, description, task,
    subscribe_to_email_events=True, is_protected=False, user=None,
    retry_from=None
):
    task = TaskHistory.register(
        request=self.request, task_history=task, user=user,
        worker_name=get_worker_name()
    )

    from tasks_create_database import create_database
    create_database(
        name, plan, environment, team, project, description, task,
        subscribe_to_email_events, is_protected, user, retry_from
    )

@app.task(bind=True)
def restore_database(self, database, task, snapshot, user, retry_from=None):
    task = TaskHistory.register(
        request=self.request, task_history=task, user=user,
        worker_name=get_worker_name()
    )

    from backup.models import Snapshot
    snapshot = Snapshot.objects.get(id=snapshot)

    from tasks_restore_backup import restore_snapshot
    restore_snapshot(database, snapshot.group, task, retry_from)

@app.task(bind=True)
def create_database_rollback(self, rollback_from, task, user):
    task = TaskHistory.register(
        request=self.request, task_history=task, user=user,
        worker_name=get_worker_name()
    )

    from tasks_create_database import rollback_create
    rollback_create(rollback_from, task, user)
<commit_msg>add steps metric collector<commit_after>from datetime import datetime
from dbaas.celery import app
import models
import logging
from notification.models import TaskHistory
from util import exec_remote_command_host, get_worker_name, \
    build_context_script, get_dict_lines
from registered_functions.functools import get_function
from workflow.steps.util.dns import ChangeTTLTo5Minutes, ChangeTTLTo3Hours
from workflow.steps.util.db_monitor import DisableMonitoring, EnableMonitoring
from workflow.steps.util.zabbix import DisableAlarms, EnableAlarms
from workflow.workflow import steps_for_instances

LOG = logging.getLogger(__name__)

@app.task(bind=True)
def execute_scheduled_maintenance(self, maintenance_id):
    LOG.debug("Maintenance id: {}".format(maintenance_id))
    maintenance = models.Maintenance.objects.get(id=maintenance_id)
    models.Maintenance.objects.filter(id=maintenance_id).update(
        status=maintenance.RUNNING, started_at=datetime.now()
    )
    LOG.info("Maintenance {} is RUNNING".format(maintenance,))

    worker_name = get_worker_name()
    task_history = TaskHistory.register(
        request=self.request, worker_name=worker_name
    )
    task_history.relevance = TaskHistory.RELEVANCE_CRITICAL
    LOG.info("id: {} | task: {} | kwargs: {} | args: {}".format(
        self.request.id, self.request.task,
        self.request.kwargs, str(self.request.args)
    ))
    task_history.update_details(
        persist=True, details="Executing Maintenance: {}".format(maintenance)
    )
    for hm in models.HostMaintenance.objects.filter(maintenance=maintenance):
        main_output = {}
        hm.status = hm.RUNNING
        hm.started_at = datetime.now()
        hm.save()
        if hm.host is None:
            hm.status = hm.UNAVAILABLEHOST
            hm.finished_at = datetime.now()
            hm.save()
            continue

        host = hm.host
        update_task = "\nRunning Maintenance on {}".format(host)

        if maintenance.disable_alarms:
            disable_alarms(hm.host)

        param_dict = {}
        params = models.MaintenanceParameters.objects.filter(
            maintenance=maintenance
        )
        for param in params:
            param_function = get_function(param.function_name)
            param_dict[param.parameter_name] = param_function(host.id)

        main_script = build_context_script(param_dict, maintenance.main_script)
        exit_status = exec_remote_command_host(host, main_script, main_output)

        if exit_status == 0:
            hm.status = hm.SUCCESS
        else:

            if maintenance.rollback_script:
                rollback_output = {}
                hm.status = hm.ROLLBACK
                hm.save()

                rollback_script = build_context_script(
                    param_dict, maintenance.rollback_script
                )
                exit_status = exec_remote_command_host(
                    host, rollback_script, rollback_output
                )

                if exit_status == 0:
                    hm.status = hm.ROLLBACK_SUCCESS
                else:
                    hm.status = hm.ROLLBACK_ERROR

                hm.rollback_log = get_dict_lines(rollback_output)

            else:
                hm.status = hm.ERROR

        if maintenance.disable_alarms:
            enable_alarms(hm.host)

        update_task += "...status: {}".format(hm.status)

        task_history.update_details(persist=True, details=update_task)

        hm.main_log = get_dict_lines(main_output)
        hm.finished_at = datetime.now()
        hm.save()

    models.Maintenance.objects.filter(id=maintenance_id).update(
        status=maintenance.FINISHED, finished_at=datetime.now()
    )
    task_history.update_status_for(
        TaskHistory.STATUS_SUCCESS, details='Maintenance executed succesfully'
    )
    LOG.info("Maintenance: {} has FINISHED".format(maintenance))

def disable_alarms(host):
    for instance in host.instances.all():
        DisableMonitoring(instance).do()
        DisableAlarms(instance).do()

def enable_alarms(host):
    for instance in host.instances.all():
        EnableMonitoring(instance).do()
        EnableAlarms(instance).do()

def region_migration_prepare(infra):
    instance = infra.instances.first()
    ChangeTTLTo5Minutes(instance).do()

def region_migration_finish(infra):
    instance = infra.instances.first()
    ChangeTTLTo3Hours(instance).do()

@app.task(bind=True)
def region_migration_start(self, infra, instances, since_step=None):
    steps = [{
        'Disable monitoring and alarms': (
            'workflow.steps.util.zabbix.DestroyAlarms',
            'workflow.steps.util.db_monitor.DisableMonitoring',
        )}, {
        'Stopping infra': (
            'workflow.steps.util.database.Stop',
            'workflow.steps.util.database.CheckIsDown',
        )}, {
        'Creating new virtual machine': (
            'workflow.steps.util.vm.MigrationCreateNewVM',
        )}, {
        'Creating new infra': (
            'workflow.steps.util.vm.MigrationWaitingBeReady',
            'workflow.steps.util.infra.MigrationCreateInstance',
            'workflow.steps.util.disk.MigrationCreateExport',
        )}, {
        'Configuring new infra': (
            'workflow.steps.util.volume_provider.MountDataVolume',
            'workflow.steps.util.plan.InitializationMigration',
            'workflow.steps.util.plan.ConfigureMigration',
            'workflow.steps.util.metric_collector.ConfigureTelegraf',
        )}, {
        'Preparing new environment': (
            'workflow.steps.util.disk.AddDiskPermissionsOldest',
            'workflow.steps.util.disk.MountOldestExportMigration',
            'workflow.steps.util.disk.CopyDataBetweenExportsMigration',
            'workflow.steps.util.disk.FilePermissionsMigration',
            'workflow.steps.util.disk.UnmountNewerExportMigration',
            'workflow.steps.util.vm.ChangeInstanceHost',
            'workflow.steps.util.vm.UpdateOSDescription',
            'workflow.steps.util.infra.OfferingMigration',
            'workflow.steps.util.infra.UpdateMigrateEnvironment',
            'workflow.steps.util.infra.UpdateMigratePlan',
        )}, {
        'Starting new infra': (
            'workflow.steps.util.database.Start',
            'workflow.steps.util.database.CheckIsUp',
            'workflow.steps.util.metric_collector.RestartTelegraf',
        )}, {
        'Enabling access': (
            'workflow.steps.util.dns.ChangeEndpoint',
            'workflow.steps.util.acl.ReplicateAclsMigration',
        )}, {
        'Destroying old infra': (
            'workflow.steps.util.disk.DisableOldestExportMigration',
            'workflow.steps.util.disk.DiskUpdateHost',
            'workflow.steps.util.vm.RemoveHostMigration',
        )}, {
        'Enabling monitoring and alarms': (
            'workflow.steps.util.db_monitor.EnableMonitoring',
            'workflow.steps.util.zabbix.CreateAlarms',
        )}, {
        'Restart replication': (
            'workflow.steps.util.database.SetSlavesMigration',
        )
    }]

    task = TaskHistory()
    task.task_id = self.request.id
    task.task_name = "migrating_zone"
    task.task_status = TaskHistory.STATUS_RUNNING
    task.context = {'infra': infra, 'instances': instances}
    task.arguments = {'infra': infra, 'instances': instances}
    task.user = 'admin'
    task.save()

    if steps_for_instances(steps, instances, task, since_step=since_step):
        task.set_status_success('Region migrated with success')
    else:
        task.set_status_error('Could not migrate region')

    database = infra.databases.first()
    database.environment = infra.environment
    database.save()

@app.task(bind=True)
def create_database(
    self, name, plan, environment, team, project, description, task,
    subscribe_to_email_events=True, is_protected=False, user=None,
    retry_from=None
):
    task = TaskHistory.register(
        request=self.request, task_history=task, user=user,
        worker_name=get_worker_name()
    )

    from tasks_create_database import create_database
    create_database(
        name, plan, environment, team, project, description, task,
        subscribe_to_email_events, is_protected, user, retry_from
    )

@app.task(bind=True)
def restore_database(self, database, task, snapshot, user, retry_from=None):
    task = TaskHistory.register(
        request=self.request, task_history=task, user=user,
        worker_name=get_worker_name()
    )

    from backup.models import Snapshot
    snapshot = Snapshot.objects.get(id=snapshot)

    from tasks_restore_backup import restore_snapshot
    restore_snapshot(database, snapshot.group, task, retry_from)

@app.task(bind=True)
def create_database_rollback(self, rollback_from, task, user):
    task = TaskHistory.register(
        request=self.request, task_history=task, user=user,
        worker_name=get_worker_name()
    )

    from tasks_create_database import rollback_create
    rollback_create(rollback_from, task, user)
<|endoftext|>


<commit_before>from .. import utils

class TCPCall(object):
    """Command
    Provides ways to send commands
    """
    def __init__(self, loop, service):
        self.loop = loop
        self.service = service

    async def ping(self, remote):
        """Ping

        Args:
            remote: Remote Destination
        Returns:
            Remote Node
        """
        reader, writer = await remote.connect_tcp(self.loop)
        echo = utils.get_echo_bytes()
        await self.service.protocol._do_ping(writer, echo)
        writer.close()

        await self.service.event.do_ping(remote, echo)

    async def store(self, remote, key, value):
        """Store

        Args:
            remote: Remote Destination
            key: Key
            value: Value
        Returns:
            None
        """
        reader, writer = await remote.connect_tcp(self.loop)
        echo = utils.get_echo_bytes()
        await self.service.protocol._do_store(writer, echo, key, value)
        writer.close()

        await self.service.event.do_store(remote, echo, key, value)

    async def findNode(self, remote):
        """findNode

        Args:
            remote: Remote Hash
        Returns:
            Remote Node
        """
        pass

    async def findValue(self, key):
        """findValue

        Args:
            key: Key
        Returns:
            (key, value)
        """
        pass

    async def pong_ping(self, remote, echo):
        """Pong

        Args:
            remote: Remote Destination
            echo: Echo Value
        Returns:
            None
        """
        reader, writer = await remote.connect_tcp(self.loop)
        await self.service.protocol._do_pong_ping(writer, echo)
        writer.close()
        await self.service.event.do_pong_ping(remote, echo)

    async def pong_store(self, remote, echo, key):
        """

        Args:
            remote: Remote Destination
            echo: Echo Value
            key: Key Saved
        Returns:
            None
        """
        reader, writer = await remote.connect_tcp(self.loop)
        await self.service.protocol._do_pong_store(writer, echo, key)
        writer.close()

        await self.service.event.do_pong_store(remote, echo, key)

    async def pong_findNode(self, remote, echo):
        """Pong

        Args:
            remote: Remote Destination
            echo: Echo Value
        Returns:
            None
        """
        pass

    async def pong_findValue(self, remote, echo):
        """Pong

        Args:
            remote: Remote Destination
            echo: Echo Value
        Returns:
            None
        """
        pass
<commit_msg>add all call functions<commit_after>from .. import utils

class TCPCall(object):
    """Command
    Provides ways to send commands
    """
    def __init__(self, loop, service):
        self.loop = loop
        self.service = service

    async def ping(self, remote):
        """Ping

        Args:
            remote: Remote Destination
        Returns:
            Remote Node
        """
        reader, writer = await remote.connect_tcp(self.loop)
        echo = utils.get_echo_bytes()
        await self.service.protocol._do_ping(writer, echo)
        writer.close()

        await self.service.event.do_ping(remote, echo)

    async def store(self, remote, key, value):
        """Store

        Args:
            remote: Remote Destination
            key: Key
            value: Value
        Returns:
            None
        """
        reader, writer = await remote.connect_tcp(self.loop)
        echo = utils.get_echo_bytes()
        await self.service.protocol._do_store(writer, echo, key, value)
        writer.close()

        await self.service.event.do_store(remote, echo, key, value)

    async def findNode(self, remote, remoteId):
        """findNode

        Args:
            remote: Remote Destination
            remoteId: Remote Hash
        Returns:
            (Remote ID, Remote Node)
        """
        pass

    async def findValue(self, remote, key):
        """findValue

        Args:
            remote: Remote Destination
            key: Key
        Returns:
            (key, value)
        """
        pass

    async def findReduce(self, remote, keyStart, keyEnd):
        """findReduce

        Args:
            remote: Remote Destination
            keyStart: Start Key
            keyEnd: End Key
        Returns:
            (keyStart, keyEnd, value)
        """
        pass

    async def pong_ping(self, remote, echo):
        """pong_ping

        Args:
            remote: Remote Destination
            echo: Echo Value
        Returns:
            None
        """
        reader, writer = await remote.connect_tcp(self.loop)
        await self.service.protocol._do_pong_ping(writer, echo)
        writer.close()
        await self.service.event.do_pong_ping(remote, echo)

    async def pong_store(self, remote, echo, key):
        """pong_store

        Args:
            remote: Remote Destination
            echo: Echo Value
            key: Key Saved
        Returns:
            None
        """
        reader, writer = await remote.connect_tcp(self.loop)
        await self.service.protocol._do_pong_store(writer, echo, key)
        writer.close()

        await self.service.event.do_pong_store(remote, echo, key)

    async def pong_findNode(self, remote, echo, remoteId, remoteNode):
        """pong_findNode

        Args:
            remote: Remote Destination
            echo: Echo Value
            remoteId: Remote ID found
            remoteNode: Remote Address
        Returns:
            None
        """
        pass

    async def pong_findValue(self, remote, echo, key, value):
        """pong_findeValue

        Args:
            remote: Remote Destination
            echo: Echo Value
            key: Key found
            value: value found
        Returns:
            None
        """
        pass

    async def pong_findReduce(self, remote, echo, keyStart, keyEnd, value):
        """pong_findReduce

        Args:
            remote: Remote Destination
            keyStart: Start Key
            keyEnd: End Key
            value: Reduced value
        Returns:
            None
        """
        pass
<|endoftext|>


<commit_before>from __future__ import division, print_function

import os, json
from glob import glob
import numpy as np
from scipy import misc, ndimage
from scipy.ndimage.interpolation import zoom

from keras import backend as K
from keras.layers.normalization import BatchNormalization
from keras.utils.data_utils import get_file
from keras.models import Sequential
from keras.layers.core import Flatten, Dense, Dropout, Lambda
from keras.layers.convolutional import Conv2D, MaxPooling2D, ZeroPadding2D  # Conv2D: Keras2
from keras.layers.pooling import GlobalAveragePooling2D
from keras.optimizers import SGD, RMSprop, Adam
from keras.preprocessing import image

vgg_mean = np.array([123.68, 116.779, 103.939], dtype=np.float32).reshape((3,1,1))
def vgg_preprocess(x):
    x = x - vgg_mean
    return x[:, ::-1] # reverse axis rgb->bgr

class Vgg16():
    """The VGG 16 Imagenet model"""

    def __init__(self):
        self.FILE_PATH = 'http://files.fast.ai/models/'
        self.create()
        self.get_classes()

    def get_classes(self):
        fname = 'imagenet_class_index.json'
        fpath = get_file(fname, self.FILE_PATH+fname, cache_subdir='models')
        with open(fpath) as f:
            class_dict = json.load(f)
        self.classes = [class_dict[str(i)][1] for i in range(len(class_dict))]

    def predict(self, imgs, details=False):
        all_preds = self.model.predict(imgs)
        idxs = np.argmax(all_preds, axis=1)
        preds = [all_preds[i, idxs[i]] for i in range(len(idxs))]
        classes = [self.classes[idx] for idx in idxs]
        return np.array(preds), idxs, classes

    def ConvBlock(self, layers, filters):
        model = self.model
        for i in range(layers):
            model.add(ZeroPadding2D((1, 1)))
            model.add(Conv2D(filters, kernel_size=(3, 3), activation='relu'))  # Keras2
        model.add(MaxPooling2D((2, 2), strides=(2, 2)))

    def FCBlock(self):
        model = self.model
        model.add(Dense(4096, activation='relu'))
        model.add(Dropout(0.5))

    def create(self):
        model = self.model = Sequential()
        model.add(Lambda(vgg_preprocess, input_shape=(3,224,224), output_shape=(3,224,224)))

        self.ConvBlock(2, 64)
        self.ConvBlock(2, 128)
        self.ConvBlock(3, 256)
        self.ConvBlock(3, 512)
        self.ConvBlock(3, 512)

        model.add(Flatten())
        self.FCBlock()
        self.FCBlock()
        model.add(Dense(1000, activation='softmax'))

        fname = 'vgg16.h5'
        model.load_weights(get_file(fname, self.FILE_PATH+fname, cache_subdir='models'))

    def get_batches(self, path, gen=image.ImageDataGenerator(), shuffle=True, batch_size=8, class_mode='categorical'):
        return gen.flow_from_directory(path, target_size=(224,224),
                class_mode=class_mode, shuffle=shuffle, batch_size=batch_size)

    def ft(self, num):
        model = self.model
        model.pop()
        for layer in model.layers: layer.trainable=False
        model.add(Dense(num, activation='softmax'))
        self.compile()

    def finetune(self, batches):
        self.ft(batches.num_class)  # Keras2
        classes = list(iter(batches.class_indices))
        for c in batches.class_indices:
            classes[batches.class_indices[c]] = c
        self.classes = classes

    def compile(self, lr=0.001):
        self.model.compile(optimizer=Adam(lr=lr),
                loss='categorical_crossentropy', metrics=['accuracy'])

    # Keras2
    def fit_data(self, trn, labels,  val, val_labels,  nb_epoch=1, batch_size=64):
        self.model.fit(trn, labels, epochs=nb_epoch,
                validation_data=(val, val_labels), batch_size=batch_size)

    # Keras2
    def fit(self, batches, val_batches, batch_size, nb_epoch=1):
        self.model.fit_generator(batches, steps_per_epoch=int(np.ceil(batches.samples/batch_size)), epochs=nb_epoch,
                validation_data=val_batches, validation_steps=int(np.ceil(val_batches.samples/batch_size)))

    # Keras2
    def test(self, path, batch_size=8):
        test_batches = self.get_batches(path, shuffle=False, batch_size=batch_size, class_mode=None)
        return test_batches, self.model.predict_generator(test_batches, int(np.ceil(test_batches.samples/batch_size)))
<commit_msg>read and commented vgg16.py<commit_after>from __future__ import division, print_function

import os, json
from glob import glob
import numpy as np
from scipy import misc, ndimage
from scipy.ndimage.interpolation import zoom

from keras import backend as K
from keras.layers.normalization import BatchNormalization
from keras.utils.data_utils import get_file
from keras.models import Sequential
from keras.layers.core import Flatten, Dense, Dropout, Lambda
from keras.layers.convolutional import Conv2D, MaxPooling2D, ZeroPadding2D  # Conv2D: Keras2
from keras.layers.pooling import GlobalAveragePooling2D
from keras.optimizers import SGD, RMSprop, Adam
from keras.preprocessing import image

vgg_mean = np.array([123.68, 116.779, 103.939], dtype=np.float32).reshape((3,1,1))

def vgg_preprocess(x):
    x = x - vgg_mean
    # x[:, ::-1] is equivalent to x[:, ::-1, :, :]
    # it reverses order of second dimension (channels in this case)
    return x[:, ::-1] # reverse axis rgb->bgr

class Vgg16():
    """The VGG 16 Imagenet model"""

    def __init__(self):
        self.FILE_PATH = 'http://files.fast.ai/models/'
        self.create()
        self.get_classes()

    def get_classes(self):
        fname = 'imagenet_class_index.json'
        fpath = get_file(fname, self.FILE_PATH+fname, cache_subdir='models')
        with open(fpath) as f:
            class_dict = json.load(f)
        self.classes = [class_dict[str(i)][1] for i in range(len(class_dict))]

    def predict(self, imgs, details=False):
        all_preds = self.model.predict(imgs)
        idxs = np.argmax(all_preds, axis=1)
        preds = [all_preds[i, idxs[i]] for i in range(len(idxs))]
        classes = [self.classes[idx] for idx in idxs]
        return np.array(preds), idxs, classes

    def ConvBlock(self, layers, filters):
        model = self.model
        for i in range(layers):
            model.add(ZeroPadding2D((1, 1))) # adds padding to the input volume, to keep spatial dimension same across layers
            model.add(Conv2D(filters, kernel_size=(3, 3), activation='relu'))  # Keras2
        model.add(MaxPooling2D((2, 2), strides=(2, 2))) # reduces spacial dimensions by 2

    def FCBlock(self):
        model = self.model
        model.add(Dense(4096, activation='relu'))
        model.add(Dropout(0.5))

    def create(self):
        model = self.model = Sequential() # why double assignment? just `model` only for scope of this method (to make shorter?)
        # 1st layer must have input_shape parameter
        model.add(Lambda(vgg_preprocess, input_shape=(3,224,224), output_shape=(3,224,224)))

        self.ConvBlock(2, 64)
        self.ConvBlock(2, 128)
        self.ConvBlock(3, 256)
        self.ConvBlock(3, 512)
        self.ConvBlock(3, 512) # total of 13 conv layers

        model.add(Flatten())
        self.FCBlock() # with dropout 0.5
        self.FCBlock()
        model.add(Dense(1000, activation='softmax'))

        fname = 'vgg16.h5'
        # get_file() returns path to downloaded file
        # as per: https://github.com/fchollet/keras/blob/master/keras/utils/data_utils.py
        model.load_weights(get_file(fname, self.FILE_PATH+fname, cache_subdir='models')) # stores at ~/.keras/models

    # wrapper for flow_from_dir, returns a generator
    def get_batches(self, path, gen=image.ImageDataGenerator(), shuffle=True, batch_size=8, class_mode='categorical'):
        return gen.flow_from_directory(path, target_size=(224,224),
                class_mode=class_mode, shuffle=shuffle, batch_size=batch_size)

    # replaces last layer with new, having num softmax activations
    # makes layers not trainable (hence finetuning)
    def ft(self, num):
        model = self.model # this assignment to make shorter?
        model.pop()
        for layer in model.layers: layer.trainable=False
        model.add(Dense(num, activation='softmax'))
        self.compile()

    def finetune(self, batches):
        # prepares model and makes classes, but no training!
        self.ft(batches.num_class)  # Keras2
        '''
        batches.class_indices returns a dict {class_name: class_index}
        list(iter(batches.class_indices)) should be list of keys?
        same as list(batches.class_indices)?
        '''
        classes = list(iter(batches.class_indices))
        for c in batches.class_indices: # loops through keys of dict
            # appends classname at position of that class integer representation in the model
            classes[batches.class_indices[c]] = c
        self.classes = classes

    def compile(self, lr=0.001):
        self.model.compile(optimizer=Adam(lr=lr),
                loss='categorical_crossentropy', metrics=['accuracy'])

    # Keras2, fit from data loaded in memory
    def fit_data(self, trn, labels,  val, val_labels,  nb_epoch=1, batch_size=64):
        self.model.fit(trn, labels, epochs=nb_epoch,
                validation_data=(val, val_labels), batch_size=batch_size)

    # Keras2, fit from data generator
    # need to manually calculate steps_per_epoch and validation_steps
    def fit(self, batches, val_batches, batch_size, nb_epoch=1):
        self.model.fit_generator(batches, steps_per_epoch=int(np.ceil(batches.samples/batch_size)), epochs=nb_epoch,
                validation_data=val_batches, validation_steps=int(np.ceil(val_batches.samples/batch_size)))

    # Keras2
    def test(self, path, batch_size=8):
        test_batches = self.get_batches(path, shuffle=False, batch_size=batch_size, class_mode=None)
        return test_batches, self.model.predict_generator(test_batches, int(np.ceil(test_batches.samples/batch_size)))
<|endoftext|>


<commit_before>#!/usr/bin/env python3

from concurrent.futures import ProcessPoolExecutor
from multiprocessing import cpu_count
from argparse import ArgumentParser
import functools

from .lib.json_folder_map import json_folder_map
from .lib.maintain_lists_of_entries import maintain_lists_of_entries
from .lib.fetch_course_details import fetch_course_details
from .lib.save_term import save_term
from .lib.calculate_terms import calculate_terms
from .lib.process_courses import process_courses
from .lib.fetch_term_data import load_term
from .lib.log import log

def get_years_and_terms(terms_or_years):
    years, terms = [], []
    for item in terms_or_years:
        str_item = str(item)
        if len(str_item) is 4:
            years.append(item)
        elif len(str_item) is 5:
            terms.append(item)

    return years, terms

def one_term(term, **kwargs):
    str_term = str(term)
    pretty_term = str_term[0:4] + ':' + str_term[4]

    log(pretty_term, 'Loading term')
    raw_term_data = load_term(term,
                              force_download=kwargs['force_download_terms'])

    if not raw_term_data:
        return []

    log(pretty_term, 'Extracting courses')
    courses = raw_term_data['searchresults']['course']

    log(pretty_term, 'Loading details')
    details = fetch_course_details([c['clbid'] for c in courses],
                                   dry_run=kwargs['dry_run'],
                                   force_download=kwargs['force_download_details'])

    log(pretty_term, 'Processing courses')
    final_courses = process_courses(courses, details,
                                    dry_run=kwargs['dry_run'],
                                    find_revisions=kwargs['find_revisions'],
                                    ignore_revisions=kwargs['ignore_revisions'])

    return final_courses

def run(args):
    years, terms = get_years_and_terms(args.term_or_year)

    terms = calculate_terms(years=years, terms=terms)

    edit_one_term = functools.partial(
        one_term, find_revisions=args.no_revisions, **vars(args))

    if args.workers > 1:
        with ProcessPoolExecutor(max_workers=args.workers) as pool:
            processed_terms = list(pool.map(edit_one_term, terms))
    else:
        processed_terms = list(map(edit_one_term, terms))

    [save_term(term,
               path=args.output_dir,
               kind=args.output_type)
        for term in processed_terms]

    json_folder_map(folder=args.output_dir + 'terms/',
                    kind='courses',
                    dry_run=args.dry_run,
                    path=args.output_dir)

    maintain_lists_of_entries(
        [course for term in processed_terms for course in term],
        dry_run=args.dry_run)

def main():
    argparser = ArgumentParser(description='Fetch term data from the SIS.')

    argparser.add_argument('term_or_year',
                           type=int,
                           nargs='*',
                           help='Terms (or entire years) for which to request data from the SIS.')
    argparser.add_argument('--workers', '-w',
                           type=int,
                           default=cpu_count(),
                           help='Control the number of operations to perform in parallel')
    argparser.add_argument('--force-download-terms',
                           action='store_true',
                           help='Force reloading of all specified terms.')
    argparser.add_argument('--force-download-details',
                           action='store_true',
                           help='Force reloading of any course details from all specified terms.')
    argparser.add_argument('--dry-run', '-d',
                           action='store_true',
                           help='Only print output; don\'t write files.')
    argparser.add_argument('--no-revisions', '-n',
                           action='store_false',
                           help='Prevent searching for revisions of courses.')
    argparser.add_argument('--ignore-revisions',
                           metavar='PROP',
                           nargs='+',
                           default=[],
                           help='Prevent storing revisions within property $PROP.')
    argparser.add_argument('--quiet', '-q',
                           action='store_true',
                           help='Silence logging; mostly used when looking for data.')
    argparser.add_argument('--output-type',
                           action='store',
                           default='json',
                           choices=['json', 'csv'],
                           help='Change the output filetype.')
    argparser.add_argument('--output-dir',
                           action='store',
                           default='build/',
                           help='Choose an output directory.')

    argparser.allow_abbrev = False

    args = argparser.parse_args()
    if args.output_dir[-1] != '/':
        args.output_dir += '/'

    run(args)

if __name__ == '__main__':
    main()
<commit_msg>update update.py's import paths<commit_after>#!/usr/bin/env python3

from concurrent.futures import ProcessPoolExecutor
from multiprocessing import cpu_count
from argparse import ArgumentParser
import functools

from ..lib.json_folder_map import json_folder_map
from ..lib.maintain_lists_of_entries import maintain_lists_of_entries
from ..lib.fetch_course_details import fetch_course_details
from ..lib.save_term import save_term
from ..lib.calculate_terms import calculate_terms
from ..lib.process_courses import process_courses
from ..lib.fetch_term_data import load_term
from ..lib.log import log

def get_years_and_terms(terms_or_years):
    years, terms = [], []
    for item in terms_or_years:
        str_item = str(item)
        if len(str_item) is 4:
            years.append(item)
        elif len(str_item) is 5:
            terms.append(item)

    return years, terms

def one_term(term, **kwargs):
    str_term = str(term)
    pretty_term = str_term[0:4] + ':' + str_term[4]

    log(pretty_term, 'Loading term')
    raw_term_data = load_term(term,
                              force_download=kwargs['force_download_terms'])

    if not raw_term_data:
        return []

    log(pretty_term, 'Extracting courses')
    courses = raw_term_data['searchresults']['course']

    log(pretty_term, 'Loading details')
    details = fetch_course_details([c['clbid'] for c in courses],
                                   dry_run=kwargs['dry_run'],
                                   force_download=kwargs['force_download_details'])

    log(pretty_term, 'Processing courses')
    final_courses = process_courses(courses, details,
                                    dry_run=kwargs['dry_run'],
                                    find_revisions=kwargs['find_revisions'],
                                    ignore_revisions=kwargs['ignore_revisions'])

    return final_courses

def run(args):
    years, terms = get_years_and_terms(args.term_or_year)

    terms = calculate_terms(years=years, terms=terms)

    edit_one_term = functools.partial(
        one_term, find_revisions=args.no_revisions, **vars(args))

    if args.workers > 1:
        with ProcessPoolExecutor(max_workers=args.workers) as pool:
            processed_terms = list(pool.map(edit_one_term, terms))
    else:
        processed_terms = list(map(edit_one_term, terms))

    [save_term(term,
               path=args.output_dir,
               kind=args.output_type)
        for term in processed_terms]

    json_folder_map(folder=args.output_dir + 'terms/',
                    kind='courses',
                    dry_run=args.dry_run,
                    path=args.output_dir)

    maintain_lists_of_entries(
        [course for term in processed_terms for course in term],
        dry_run=args.dry_run)

def main():
    argparser = ArgumentParser(description='Fetch term data from the SIS.')

    argparser.add_argument('term_or_year',
                           type=int,
                           nargs='*',
                           help='Terms (or entire years) for which to request data from the SIS.')
    argparser.add_argument('--workers', '-w',
                           type=int,
                           default=cpu_count(),
                           help='Control the number of operations to perform in parallel')
    argparser.add_argument('--force-download-terms',
                           action='store_true',
                           help='Force reloading of all specified terms.')
    argparser.add_argument('--force-download-details',
                           action='store_true',
                           help='Force reloading of any course details from all specified terms.')
    argparser.add_argument('--dry-run', '-d',
                           action='store_true',
                           help='Only print output; don\'t write files.')
    argparser.add_argument('--no-revisions', '-n',
                           action='store_false',
                           help='Prevent searching for revisions of courses.')
    argparser.add_argument('--ignore-revisions',
                           metavar='PROP',
                           nargs='+',
                           default=[],
                           help='Prevent storing revisions within property $PROP.')
    argparser.add_argument('--quiet', '-q',
                           action='store_true',
                           help='Silence logging; mostly used when looking for data.')
    argparser.add_argument('--output-type',
                           action='store',
                           default='json',
                           choices=['json', 'csv'],
                           help='Change the output filetype.')
    argparser.add_argument('--output-dir',
                           action='store',
                           default='build/',
                           help='Choose an output directory.')

    argparser.allow_abbrev = False

    args = argparser.parse_args()
    if args.output_dir[-1] != '/':
        args.output_dir += '/'

    run(args)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from dlcs.queue_response import Batch
from requests import post, auth
import settings

def register_collection(image_collection):

    authorisation = auth.HTTPBasicAuth(settings.DLCS_API_KEY, settings.DLCS_API_SECRET)
    url = settings.DLCS_ENTRY + 'customers/' + str(settings.DLCS_CUSTOMER_ID) + '/queue'
    json = image_collection.as_json()
    response = post(url, json=json, auth=authorisation)
    batch = Batch(response.json())

    return batch
<commit_msg>fix for correct json data to use.<commit_after>from dlcs.queue_response import Batch
from requests import post, auth
import settings

def register_collection(image_collection):

    authorisation = auth.HTTPBasicAuth(settings.DLCS_API_KEY, settings.DLCS_API_SECRET)
    url = settings.DLCS_ENTRY + 'customers/' + str(settings.DLCS_CUSTOMER_ID) + '/queue'
    json = image_collection.to_json_dict()
    response = post(url, json=json, auth=authorisation)
    batch = Batch(response.json())

    return batch
<|endoftext|>


<commit_before>import copy
import os
import subprocess
import zipfile
from functools import partial

import yaml
from yaml.parser import ParserError
from yaml.scanner import ScannerError

from dmoj import checkers
from dmoj.generator import GeneratorManager
from dmoj.judgeenv import get_problem_root
from dmoj.utils.module import load_module_from_file

class InvalidInitException(Exception):
    """
    Your init.yml is bad and you should feel bad.
    """

    def __init__(self, message):
        super(InvalidInitException, self).__init__(message)

class ConfigNode(object):
    """
    A wrapper around a YAML configuration object for easier use.

    Consider the following YAML object, stored in a ConfigNode "node":

    output_prefix_length: 5
    test_cases: [
        {
            batched: [
                {
                    in: ruby.1.in
                },
                {
                    in: ruby.2.in,
                    output_prefix_length: 0,
                }
            ],
            out: ruby.out,
            points: 10
        },
        {
            in: ruby.4.in,
            out: ruby.4.out,
            points: 15
        }
    ]

    node.test_cases[0].batched[0]['in'] == 'ruby.1.in'
    node.test_cases[0].batched[0].out == 'ruby.out'
    node.test_cases[0].batched[0].points == 10
    node.test_cases[1].points == 15
    node.test_cases[1].output_prefix_length == 5
    node.test_cases[0].batched[0].output_prefix_length == 5
    node.test_cases[0].batched[1].output_prefix_length == 0
    """

    def __init__(self, raw_config, parent=None, defaults=None):
        self.raw_config = copy.deepcopy(raw_config)
        if defaults:
            self.raw_config.update(defaults)
        self.parent = parent

    def __getattr__(self, item):
        return self[item]

    def __getitem__(self, item):
        try:
            def run_dynamic_key(dynamic_key, run_func):
                # Wrap in a ConfigNode so dynamic keys can benefit from the nice features of ConfigNode
                local = {'node': ConfigNode(self.raw_config.get(item, {}), self)}
                try:
                    cfg = run_func(self.raw_config[dynamic_key], local)
                except Exception as e:
                    import traceback

                    traceback.print_exc()
                    raise InvalidInitException('exception executing dynamic key ' + str(dynamic_key) + ': ' + e.message)
                del self.raw_config[dynamic_key]
                self.raw_config[item] = cfg

            if item + '++' in self.raw_config:
                def full(code, local):
                    exec code in local
                    return local['node']

                run_dynamic_key(item + '++', full)
            elif item + '+' in self.raw_config:
                run_dynamic_key(item + '+', lambda code, local: eval(code, local))
            cfg = self.raw_config[item]

            if isinstance(cfg, list) or isinstance(cfg, dict):
                cfg = ConfigNode(cfg, self)
        except (KeyError, IndexError, TypeError):
            cfg = self.parent[item] if self.parent else None
        return cfg

    def keys(self):
        return self.raw_config.keys()

    def __iter__(self):
        for cfg in self.raw_config:
            if isinstance(cfg, list) or isinstance(cfg, dict):
                cfg = ConfigNode(cfg, self)
            yield cfg

    def __str__(self):
        return '<ConfigNode(%s)>' % str(self.raw_config)

class TestCase(object):
    def __init__(self, count, config, problem):
        self.position = count
        self.config = config
        self.problem = problem
        self.points = config.points
        self.output_prefix_length = config.output_prefix_length
        self._generated = None

    def _normalize(self, data):
        return data.replace('\r\n', '\n')

    def _run_generator(self, gen, args=None):
        flags = []
        args = args or []
        if isinstance(gen, str):
            filename = os.path.join(get_problem_root(self.problem.id), gen)
        else:
            filename = gen.source
            if gen.flags:
                flags += gen.flags
            if not args and gen.args:
                args += gen.args

        executor = self.problem.generator_manager.get_generator(filename, flags)
        proc = executor.launch_unsafe(*args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        try:
            input = self.problem.problem_data[self.config['in']]
        except KeyError:
            input = None
        self._generated = map(self._normalize, proc.communicate(input))

    def input_data(self):
        gen = self.config.generator
        if gen:
            if self._generated is None:
                self._run_generator(gen, args=self.config.generator_args)
            return self._generated[0]
        # in file is optional
        return self._normalize(self.problem.problem_data[self.config['in']]) if self.config['in'] else ''

    def output_data(self):
        if self.config.out:
            return self._normalize(self.problem.problem_data[self.config.out])
        gen = self.config.generator
        if gen:
            if self._generated is None:
                self._run_generator(gen, args=self.config.generator_args)
            return self._generated[1]

    def checker(self):
        try:
            name = self.config['checker'] or 'standard'
            if isinstance(name, ConfigNode):
                params = name['args'] or {}
                name = name['name']
            else:
                params = {}
            if '.' in name:
                try:
                    checker = self.problem.load_checker(name)
                except IOError:
                    raise InvalidInitException('checker module path does not exist: %s' % name)
            else:
                checker = getattr(checkers, name)
        except AttributeError as e:
            raise InvalidInitException('error loading checker: ' + e.message)
        if not hasattr(checker, 'check') or not callable(checker.check):
            raise InvalidInitException('malformed checker: no check method found')

        return partial(checker.check, **params)

    def __str__(self):
        return 'TestCase{in=%s,out=%s,points=%s}' % (self.config['in'], self.config['out'], self.config['points'])

class BatchedTestCase(object):
    def __init__(self, config, problem):
        self.config = config
        self.batched_cases = problem._resolve_testcases(config['batched'])
        if any(isinstance(case, BatchedTestCase) for case in self.batched_cases):
            raise InvalidInitException("nested batches")
        self.problem = problem

    def __str__(self):
        return 'BatchedTestCase{cases=%s}' % str(self.batched_cases)

class ProblemDataManager(dict):
    def __init__(self, problem_id, **kwargs):
        super(ProblemDataManager, self).__init__(**kwargs)
        self.problem_id = problem_id
        self.archive = None

    def __missing__(self, key):
        try:
            return open(os.path.join(get_problem_root(self.problem_id), key), 'r').read()
        except IOError:
            if self.archive:
                zipinfo = self.archive.getinfo(key)
                return self.archive.open(zipinfo).read()
            raise KeyError()

    def __del__(self):
        if self.archive:
            self.archive.close()

class Problem(object):
    def __init__(self, problem_id, time_limit, memory_limit):
        self.id = problem_id
        self.time_limit = time_limit
        self.memory_limit = memory_limit
        self.generator_manager = GeneratorManager()

        self.problem_data = ProblemDataManager(problem_id)

        # Checkers modules must be stored in a dict, for the duration of execution,
        # lest globals be deleted with the module.
        self._checkers = {}
        self._testcase_counter = 0

        try:
            self.config = ConfigNode(yaml.safe_load(self.problem_data['init.yml']), defaults={
                'output_prefix_length': 64
            })
        except (IOError, ParserError, ScannerError) as e:
            raise InvalidInitException(str(e))

        self.problem_data.archive = self._resolve_archive_files()
        self.cases = self._resolve_testcases(self.config['test_cases'])

    def load_checker(self, name):
        if name in self._checkers:
            return self._checkers[name]
        self._checkers[name] = checker = load_module_from_file(os.path.join(get_problem_root(self.id), name))
        return checker

    def _resolve_archive_files(self):
        if self.config.archive:
            archive_path = os.path.join(get_problem_root(self.id), self.config.archive)
            if not os.path.exists(archive_path):
                raise InvalidInitException('archive file "%s" does not exist' % archive_path)
            try:
                archive = zipfile.ZipFile(archive_path, 'r')
            except zipfile.BadZipfile:
                raise InvalidInitException('bad archive: "%s"' % archive_path)
            return archive
        return None

    def _resolve_testcases(self, cfg):
        cases = []
        for case_config in cfg:
            if 'batched' in case_config.raw_config:
                cases.append(BatchedTestCase(case_config, self))
            else:
                cases.append(TestCase(self._testcase_counter, case_config, self))
        self._testcase_counter += 1
        return cases
<commit_msg>Use file for input if generator produces none<commit_after>import copy
import os
import subprocess
import zipfile
from functools import partial

import yaml
from yaml.parser import ParserError
from yaml.scanner import ScannerError

from dmoj import checkers
from dmoj.generator import GeneratorManager
from dmoj.judgeenv import get_problem_root
from dmoj.utils.module import load_module_from_file

class InvalidInitException(Exception):
    """
    Your init.yml is bad and you should feel bad.
    """

    def __init__(self, message):
        super(InvalidInitException, self).__init__(message)

class ConfigNode(object):
    """
    A wrapper around a YAML configuration object for easier use.

    Consider the following YAML object, stored in a ConfigNode "node":

    output_prefix_length: 5
    test_cases: [
        {
            batched: [
                {
                    in: ruby.1.in
                },
                {
                    in: ruby.2.in,
                    output_prefix_length: 0,
                }
            ],
            out: ruby.out,
            points: 10
        },
        {
            in: ruby.4.in,
            out: ruby.4.out,
            points: 15
        }
    ]

    node.test_cases[0].batched[0]['in'] == 'ruby.1.in'
    node.test_cases[0].batched[0].out == 'ruby.out'
    node.test_cases[0].batched[0].points == 10
    node.test_cases[1].points == 15
    node.test_cases[1].output_prefix_length == 5
    node.test_cases[0].batched[0].output_prefix_length == 5
    node.test_cases[0].batched[1].output_prefix_length == 0
    """

    def __init__(self, raw_config, parent=None, defaults=None):
        self.raw_config = copy.deepcopy(raw_config)
        if defaults:
            self.raw_config.update(defaults)
        self.parent = parent

    def __getattr__(self, item):
        return self[item]

    def __getitem__(self, item):
        try:
            def run_dynamic_key(dynamic_key, run_func):
                # Wrap in a ConfigNode so dynamic keys can benefit from the nice features of ConfigNode
                local = {'node': ConfigNode(self.raw_config.get(item, {}), self)}
                try:
                    cfg = run_func(self.raw_config[dynamic_key], local)
                except Exception as e:
                    import traceback

                    traceback.print_exc()
                    raise InvalidInitException('exception executing dynamic key ' + str(dynamic_key) + ': ' + e.message)
                del self.raw_config[dynamic_key]
                self.raw_config[item] = cfg

            if item + '++' in self.raw_config:
                def full(code, local):
                    exec code in local
                    return local['node']

                run_dynamic_key(item + '++', full)
            elif item + '+' in self.raw_config:
                run_dynamic_key(item + '+', lambda code, local: eval(code, local))
            cfg = self.raw_config[item]

            if isinstance(cfg, list) or isinstance(cfg, dict):
                cfg = ConfigNode(cfg, self)
        except (KeyError, IndexError, TypeError):
            cfg = self.parent[item] if self.parent else None
        return cfg

    def keys(self):
        return self.raw_config.keys()

    def __iter__(self):
        for cfg in self.raw_config:
            if isinstance(cfg, list) or isinstance(cfg, dict):
                cfg = ConfigNode(cfg, self)
            yield cfg

    def __str__(self):
        return '<ConfigNode(%s)>' % str(self.raw_config)

class TestCase(object):
    def __init__(self, count, config, problem):
        self.position = count
        self.config = config
        self.problem = problem
        self.points = config.points
        self.output_prefix_length = config.output_prefix_length
        self._generated = None

    def _normalize(self, data):
        return data.replace('\r\n', '\n')

    def _run_generator(self, gen, args=None):
        flags = []
        args = args or []
        if isinstance(gen, str):
            filename = os.path.join(get_problem_root(self.problem.id), gen)
        else:
            filename = gen.source
            if gen.flags:
                flags += gen.flags
            if not args and gen.args:
                args += gen.args

        executor = self.problem.generator_manager.get_generator(filename, flags)
        proc = executor.launch_unsafe(*args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        try:
            input = self.problem.problem_data[self.config['in']]
        except KeyError:
            input = None
        self._generated = map(self._normalize, proc.communicate(input))

    def input_data(self):
        gen = self.config.generator
        if gen:
            if self._generated is None:
                self._run_generator(gen, args=self.config.generator_args)
            if self._generated[0]:
                return self._generated[0]
        # in file is optional
        return self._normalize(self.problem.problem_data[self.config['in']]) if self.config['in'] else ''

    def output_data(self):
        if self.config.out:
            return self._normalize(self.problem.problem_data[self.config.out])
        gen = self.config.generator
        if gen:
            if self._generated is None:
                self._run_generator(gen, args=self.config.generator_args)
            return self._generated[1]

    def checker(self):
        try:
            name = self.config['checker'] or 'standard'
            if isinstance(name, ConfigNode):
                params = name['args'] or {}
                name = name['name']
            else:
                params = {}
            if '.' in name:
                try:
                    checker = self.problem.load_checker(name)
                except IOError:
                    raise InvalidInitException('checker module path does not exist: %s' % name)
            else:
                checker = getattr(checkers, name)
        except AttributeError as e:
            raise InvalidInitException('error loading checker: ' + e.message)
        if not hasattr(checker, 'check') or not callable(checker.check):
            raise InvalidInitException('malformed checker: no check method found')

        return partial(checker.check, **params)

    def __str__(self):
        return 'TestCase{in=%s,out=%s,points=%s}' % (self.config['in'], self.config['out'], self.config['points'])

class BatchedTestCase(object):
    def __init__(self, config, problem):
        self.config = config
        self.batched_cases = problem._resolve_testcases(config['batched'])
        if any(isinstance(case, BatchedTestCase) for case in self.batched_cases):
            raise InvalidInitException("nested batches")
        self.problem = problem

    def __str__(self):
        return 'BatchedTestCase{cases=%s}' % str(self.batched_cases)

class ProblemDataManager(dict):
    def __init__(self, problem_id, **kwargs):
        super(ProblemDataManager, self).__init__(**kwargs)
        self.problem_id = problem_id
        self.archive = None

    def __missing__(self, key):
        try:
            return open(os.path.join(get_problem_root(self.problem_id), key), 'r').read()
        except IOError:
            if self.archive:
                zipinfo = self.archive.getinfo(key)
                return self.archive.open(zipinfo).read()
            raise KeyError()

    def __del__(self):
        if self.archive:
            self.archive.close()

class Problem(object):
    def __init__(self, problem_id, time_limit, memory_limit):
        self.id = problem_id
        self.time_limit = time_limit
        self.memory_limit = memory_limit
        self.generator_manager = GeneratorManager()

        self.problem_data = ProblemDataManager(problem_id)

        # Checkers modules must be stored in a dict, for the duration of execution,
        # lest globals be deleted with the module.
        self._checkers = {}
        self._testcase_counter = 0

        try:
            self.config = ConfigNode(yaml.safe_load(self.problem_data['init.yml']), defaults={
                'output_prefix_length': 64
            })
        except (IOError, ParserError, ScannerError) as e:
            raise InvalidInitException(str(e))

        self.problem_data.archive = self._resolve_archive_files()
        self.cases = self._resolve_testcases(self.config['test_cases'])

    def load_checker(self, name):
        if name in self._checkers:
            return self._checkers[name]
        self._checkers[name] = checker = load_module_from_file(os.path.join(get_problem_root(self.id), name))
        return checker

    def _resolve_archive_files(self):
        if self.config.archive:
            archive_path = os.path.join(get_problem_root(self.id), self.config.archive)
            if not os.path.exists(archive_path):
                raise InvalidInitException('archive file "%s" does not exist' % archive_path)
            try:
                archive = zipfile.ZipFile(archive_path, 'r')
            except zipfile.BadZipfile:
                raise InvalidInitException('bad archive: "%s"' % archive_path)
            return archive
        return None

    def _resolve_testcases(self, cfg):
        cases = []
        for case_config in cfg:
            if 'batched' in case_config.raw_config:
                cases.append(BatchedTestCase(case_config, self))
            else:
                cases.append(TestCase(self._testcase_counter, case_config, self))
        self._testcase_counter += 1
        return cases
<|endoftext|>