<commit_before>import pandas as pd
import numpy as np
import logging

from edisgo.grid.tools import get_gen_info, \
    get_capacities_by_type, \
    get_capacities_by_type_and_weather_cell

def curtail_voltage(feedin, total_curtailment_ts, edisgo_object, **kwargs):
    """
    Implements curtailment methodology 'curtail_voltage'.

    The curtailment that has to be met in each step is allocated
    depending on the voltage at the nodes of the generators. The voltage
    at the node is used as an input to calculate a feedin_factor that changes
    the curtailment by modifying the feed-in at the points where
    there are very high voltages. This is only used to manipulate the resulting
    curtailment and does not change the feed-in timeseries itself.

    The lower voltage threshold is the node voltage below which no
    curtailment is assigned to the respective generator connected
    to the node. This assignment can be done by using the keyword
    argument 'voltage_threshold_lower'. By default, this voltage
    is set to 1.0 per unit.

    Above the lower voltage threshold, the curtailment is proportional
    to the difference between the node voltage and the lower
    voltage threshold. Thus the higher the voltage, the greater the
    difference from the lower voltage threshold and thereby the higher the
    curtailment.

    Lowering this voltage will increase the amount of curtailment to
    generators with higher node voltages as well as possibly increase the
    number of generators the curtailment is spread over.

    This method runs an edisgo_object.analyze internally to find out
    the voltage at the nodes if an :meth:`edisgo.grid.network.EDisGo.analyze`
    has not already been performed and the results saved in
    :meth:`edisgo.grid.network.Results.v_res`.

    Parameters
    ----------
    feedin : : pandas:`pandas.DataFrame<dataframe>`
        Obtains the feedin of each and every generator in the grid from the
        :class:`edisgo.grid.network.CurtailmentControl` class. The feedin
        dataframe has a Datetimeindex which is the same as the timeindex
        in the edisgo_object for simulation. The columns of the feedin dataframe
        are a MultiIndex column which consists of the following levels:

        * generator : :class:`edisgo.grid.components.GeneratorFluctuating`,
          essentially all the generator objects in the MV grid and the LV grid
        * gen_repr : :obj:`str`
          the repr strings of the generator objects from above
        * type : :obj:`str`
          the type of the generator object e.g. 'solar' or 'wind'
        * weather_cell_id : :obj:`int`
          the weather_cell_id that the generator object belongs to.

        All the information is gathered in the
        :class:`edisgo.grid.network.CurtailmentControl` class and available
        through the :class:`edisgo.grid.network.CurtailmentControl.feedin`
        attribute. See :class:`edisgo.grid.network.CurtailmentControl` for more
        details.
    total_curtailment_ts : :pandas:`pandas.Series<series>` or :pandas:`pandas.DataFrame<dataframe>`
        The curtailment to be distributed amongst the generators in the
        edisgo_objects' network. This is input through the edisgo_object.
        See class definition for further information.
    edisgo_object : :class:`edisgo.EDisGo`
        The edisgo object in which this function was called through the
        respective :class:`edisgo.network.CurtailmentControl` instance.
    voltage_threshold_lower: :float
        The node voltage below which no curtailment would be assigned to the
        respective generator.
    """
    voltage_threshold_lower = kwargs.get('voltage_threshold_lower', 1.0)
    difference_scaling = kwargs.get('difference_scaling', 1.0)

    # get the results of a load flow
    # get the voltages at the nodes
    feedin_gen_reprs = feedin.columns.levels[1].values.copy()

    try:
        v_pu = edisgo_object.network.results.v_res()

    except AttributeError:
        # if the load flow hasn't been done,
        # do it!
        edisgo_object.analyze()
        v_pu = edisgo_object.network.results.v_res()

    if not(v_pu.empty):
        # get only the specific feedin objects
        v_pu = v_pu.loc[:, (slice(None), feedin_gen_reprs)]

        # curtailment calculation by inducing a reduced or increased feedin
        # find out the difference from lower threshold
        feedin_factor = v_pu - voltage_threshold_lower
        # make sure the difference is positive
        # zero the curtailment of those generators below the voltage_threshold_lower
        feedin_factor = feedin_factor[feedin_factor >= 0].fillna(0)
        # after being normalized to maximum difference being 1 and minimum being 0
        feedin_factor = feedin_factor.divide(feedin_factor.max(axis=1), axis=0)
        feedin_factor = difference_scaling*2.0*feedin_factor
        # the curtailment here would be directly multplying the difference
        # with a difference_scaling factor gi

        feedin_factor.columns = feedin_factor.columns.droplevel(0)  # drop the 'mv' 'lv' labels

        # multiply feedin_factor to feedin to modify the amount of curtailment
        modified_feedin = feedin_factor.multiply(feedin, level=1)

        # total_curtailment
        curtailment = modified_feedin.divide(modified_feedin.sum(axis=1), axis=0). \
            multiply(total_curtailment_ts, axis=0)

        # assign curtailment to individual generators
        assign_curtailment(curtailment, edisgo_object)
    else:
        message = "There is no resulting node voltages after the PFA calculation" +\
            " which correspond to the generators in the columns of the given feedin data"
        logging.warning(message)

def curtail_all(feedin, total_curtailment_ts, edisgo_object, **kwargs):
    """
    Implements curtailment methodology 'curtail_all'.

    The curtailment that has to be met in each time step is allocated
    equally to all generators depending on their share of total
    feed-in in that time step. This is a simple curtailment method where
    the feedin is summed up and normalized, multiplied with `total_curtailment_ts`
    and assigned to each generator directly based on the columns in
    `total_curtailment_ts`.

    Parameters
    ----------
    feedin : : pandas:`pandas.DataFrame<dataframe>`
        Obtains the feedin of each and every generator in the grid from the
        : :class:`edisgo.grid.network.CurtailmentControl` class. The feedin dataframe
        has a Datetimeindex which is the same as the timeindex
        in the edisgo_object for simulation. The columns of the feedin dataframe
        is a MultiIndex column which consists of the following levels:

        * generator : :class:`edisgo.grid.components.GeneratorFluctuating`,
          essentially all the generator objects in the MV grid and the LV grid
        * gen_repr : :obj:`str`
          the repr strings of the generator objects from above
        * type : :obj:`str`
          the type of the generator object e.g. 'solar' or 'wind'
        * weather_cell_id : :obj:`int`
          the weather_cell_id that the generator object belongs to.

        All the information is gathered in the :class:`edisgo.grid.network.CurtailmentControl`
        class and available through the :class:`edisgo.grid.network.CurtailmentControl.feedin`
        attribute. See :class:`edisgo.grid.network.CurtailmentControl` for more details.
    total_curtailment_ts : :pandas:`pandas.Series<series>` or :pandas:`pandas.DataFrame<dataframe>`
        The curtailment to be distributed amongst the generators in the edisgo_objects'
        network. This is input through the edisgo_object. See class definition for further information.
    edisgo_object : :class:`edisgo.EDisGo`
        The edisgo object in which this function was called through the respective
        :class:`edisgo.network.CurtailmentControl` instance.
    """
    # create a feedin factor of 1
    # make sure the nans are filled in
    # this is a work around to ensure the
    # type of total_curtailment_ts (either series or dataframe)
    # doesn't affect the calculation depending on the input
    # and the timestamps are maintained

    feedin_factor = total_curtailment_ts.copy()
    feedin_factor = feedin_factor / feedin_factor
    feedin_factor.fillna(1.0, inplace=True)

    feedin.mul(feedin_factor, axis=0, level=1)

    # total_curtailment
    curtailment = feedin.divide(feedin.sum(axis=1), axis=0). \
        multiply(total_curtailment_ts, axis=0)

    # assign curtailment to individual generators
    assign_curtailment(curtailment, edisgo_object)

def assign_curtailment(curtailment, edisgo_object):
    """
    Implements curtailment helper function to assign the curtailment time series
    to each and every individual generator and ensure that they get processed
    and included in the edisgo_object.timeseries.curtailment correctly

    Parameters
    ----------
    curtailment : : pandas:`pandas.DataFrame<dataframe>`
        final curtailment dataframe with generator objects as column
        labels and a DatetimeIndex as the index
    edisgo_object : :class:`edisgo.EDisGo`
        The edisgo object created
    """
    # pre-process curtailment before assigning it to generators
    curtailment.fillna(0, inplace=True)

    # drop extra column levels that were present in feedin
    for r in range(len(curtailment.columns.levels) - 1):
        curtailment.columns = curtailment.columns.droplevel(1)

    # assign curtailment to individual generators
    for gen in curtailment.columns:
        gen.curtailment = curtailment.loc[:, gen]

    if not edisgo_object.network.timeseries._curtailment:
        edisgo_object.network.timeseries._curtailment = list(curtailment.columns)
    else:
        edisgo_object.network.timeseries._curtailment.extend(list(curtailment.columns))
<commit_msg>minor change to difference scaling functionality, don't double<commit_after>import pandas as pd
import numpy as np
import logging

from edisgo.grid.tools import get_gen_info, \
    get_capacities_by_type, \
    get_capacities_by_type_and_weather_cell

def curtail_voltage(feedin, total_curtailment_ts, edisgo_object, **kwargs):
    """
    Implements curtailment methodology 'curtail_voltage'.

    The curtailment that has to be met in each step is allocated
    depending on the voltage at the nodes of the generators. The voltage
    at the node is used as an input to calculate a feedin_factor that changes
    the curtailment by modifying the feed-in at the points where
    there are very high voltages. This is only used to manipulate the resulting
    curtailment and does not change the feed-in timeseries itself.

    The lower voltage threshold is the node voltage below which no
    curtailment is assigned to the respective generator connected
    to the node. This assignment can be done by using the keyword
    argument 'voltage_threshold_lower'. By default, this voltage
    is set to 1.0 per unit.

    Above the lower voltage threshold, the curtailment is proportional
    to the difference between the node voltage and the lower
    voltage threshold. Thus the higher the voltage, the greater the
    difference from the lower voltage threshold and thereby the higher the
    curtailment.

    Lowering this voltage will increase the amount of curtailment to
    generators with higher node voltages as well as possibly increase the
    number of generators the curtailment is spread over.

    This method runs an edisgo_object.analyze internally to find out
    the voltage at the nodes if an :meth:`edisgo.grid.network.EDisGo.analyze`
    has not already been performed and the results saved in
    :meth:`edisgo.grid.network.Results.v_res`.

    Parameters
    ----------
    feedin : : pandas:`pandas.DataFrame<dataframe>`
        Obtains the feedin of each and every generator in the grid from the
        :class:`edisgo.grid.network.CurtailmentControl` class. The feedin
        dataframe has a Datetimeindex which is the same as the timeindex
        in the edisgo_object for simulation. The columns of the feedin dataframe
        are a MultiIndex column which consists of the following levels:

        * generator : :class:`edisgo.grid.components.GeneratorFluctuating`,
          essentially all the generator objects in the MV grid and the LV grid
        * gen_repr : :obj:`str`
          the repr strings of the generator objects from above
        * type : :obj:`str`
          the type of the generator object e.g. 'solar' or 'wind'
        * weather_cell_id : :obj:`int`
          the weather_cell_id that the generator object belongs to.

        All the information is gathered in the
        :class:`edisgo.grid.network.CurtailmentControl` class and available
        through the :class:`edisgo.grid.network.CurtailmentControl.feedin`
        attribute. See :class:`edisgo.grid.network.CurtailmentControl` for more
        details.
    total_curtailment_ts : :pandas:`pandas.Series<series>` or :pandas:`pandas.DataFrame<dataframe>`
        The curtailment to be distributed amongst the generators in the
        edisgo_objects' network. This is input through the edisgo_object.
        See class definition for further information.
    edisgo_object : :class:`edisgo.EDisGo`
        The edisgo object in which this function was called through the
        respective :class:`edisgo.network.CurtailmentControl` instance.
    voltage_threshold_lower: :float
        The node voltage below which no curtailment would be assigned to the
        respective generator.
    difference_scaling: :float
        The factor which changes the slope of the per unit curtailment vs the
        node voltage characteristic.
    """
    voltage_threshold_lower = kwargs.get('voltage_threshold_lower', 1.0)
    difference_scaling = kwargs.get('difference_scaling', 1.0)

    # get the results of a load flow
    # get the voltages at the nodes
    feedin_gen_reprs = feedin.columns.levels[1].values.copy()

    try:
        v_pu = edisgo_object.network.results.v_res()

    except AttributeError:
        # if the load flow hasn't been done,
        # do it!
        edisgo_object.analyze()
        v_pu = edisgo_object.network.results.v_res()

    if not(v_pu.empty):
        # get only the specific feedin objects
        v_pu = v_pu.loc[:, (slice(None), feedin_gen_reprs)]

        # curtailment calculation by inducing a reduced or increased feedin
        # find out the difference from lower threshold
        feedin_factor = v_pu - voltage_threshold_lower
        # make sure the difference is positive
        # zero the curtailment of those generators below the voltage_threshold_lower
        feedin_factor = feedin_factor[feedin_factor >= 0].fillna(0)
        # after being normalized to maximum difference being 1 and minimum being 0
        feedin_factor = feedin_factor.divide(feedin_factor.max(axis=1), axis=0)
        feedin_factor = difference_scaling*feedin_factor
        # the curtailment here would be directly multplying the difference
        # with a difference_scaling factor gi

        feedin_factor.columns = feedin_factor.columns.droplevel(0)  # drop the 'mv' 'lv' labels

        # multiply feedin_factor to feedin to modify the amount of curtailment
        modified_feedin = feedin_factor.multiply(feedin, level=1)

        # total_curtailment
        curtailment = modified_feedin.divide(modified_feedin.sum(axis=1), axis=0). \
            multiply(total_curtailment_ts, axis=0)

        # assign curtailment to individual generators
        assign_curtailment(curtailment, edisgo_object)
    else:
        message = "There is no resulting node voltages after the PFA calculation" +\
            " which correspond to the generators in the columns of the given feedin data"
        logging.warning(message)

def curtail_all(feedin, total_curtailment_ts, edisgo_object, **kwargs):
    """
    Implements curtailment methodology 'curtail_all'.

    The curtailment that has to be met in each time step is allocated
    equally to all generators depending on their share of total
    feed-in in that time step. This is a simple curtailment method where
    the feedin is summed up and normalized, multiplied with `total_curtailment_ts`
    and assigned to each generator directly based on the columns in
    `total_curtailment_ts`.

    Parameters
    ----------
    feedin : : pandas:`pandas.DataFrame<dataframe>`
        Obtains the feedin of each and every generator in the grid from the
        : :class:`edisgo.grid.network.CurtailmentControl` class. The feedin dataframe
        has a Datetimeindex which is the same as the timeindex
        in the edisgo_object for simulation. The columns of the feedin dataframe
        is a MultiIndex column which consists of the following levels:

        * generator : :class:`edisgo.grid.components.GeneratorFluctuating`,
          essentially all the generator objects in the MV grid and the LV grid
        * gen_repr : :obj:`str`
          the repr strings of the generator objects from above
        * type : :obj:`str`
          the type of the generator object e.g. 'solar' or 'wind'
        * weather_cell_id : :obj:`int`
          the weather_cell_id that the generator object belongs to.

        All the information is gathered in the :class:`edisgo.grid.network.CurtailmentControl`
        class and available through the :class:`edisgo.grid.network.CurtailmentControl.feedin`
        attribute. See :class:`edisgo.grid.network.CurtailmentControl` for more details.
    total_curtailment_ts : :pandas:`pandas.Series<series>` or :pandas:`pandas.DataFrame<dataframe>`
        The curtailment to be distributed amongst the generators in the edisgo_objects'
        network. This is input through the edisgo_object. See class definition for further information.
    edisgo_object : :class:`edisgo.EDisGo`
        The edisgo object in which this function was called through the respective
        :class:`edisgo.network.CurtailmentControl` instance.
    """
    # create a feedin factor of 1
    # make sure the nans are filled in
    # this is a work around to ensure the
    # type of total_curtailment_ts (either series or dataframe)
    # doesn't affect the calculation depending on the input
    # and the timestamps are maintained

    feedin_factor = total_curtailment_ts.copy()
    feedin_factor = feedin_factor / feedin_factor
    feedin_factor.fillna(1.0, inplace=True)

    feedin.mul(feedin_factor, axis=0, level=1)

    # total_curtailment
    curtailment = feedin.divide(feedin.sum(axis=1), axis=0). \
        multiply(total_curtailment_ts, axis=0)

    # assign curtailment to individual generators
    assign_curtailment(curtailment, edisgo_object)

def assign_curtailment(curtailment, edisgo_object):
    """
    Implements curtailment helper function to assign the curtailment time series
    to each and every individual generator and ensure that they get processed
    and included in the edisgo_object.timeseries.curtailment correctly

    Parameters
    ----------
    curtailment : : pandas:`pandas.DataFrame<dataframe>`
        final curtailment dataframe with generator objects as column
        labels and a DatetimeIndex as the index
    edisgo_object : :class:`edisgo.EDisGo`
        The edisgo object created
    """
    # pre-process curtailment before assigning it to generators
    curtailment.fillna(0, inplace=True)

    # drop extra column levels that were present in feedin
    for r in range(len(curtailment.columns.levels) - 1):
        curtailment.columns = curtailment.columns.droplevel(1)

    # assign curtailment to individual generators
    for gen in curtailment.columns:
        gen.curtailment = curtailment.loc[:, gen]

    if not edisgo_object.network.timeseries._curtailment:
        edisgo_object.network.timeseries._curtailment = list(curtailment.columns)
    else:
        edisgo_object.network.timeseries._curtailment.extend(list(curtailment.columns))
<|endoftext|>


<commit_before>from django.contrib import admin

# Register your models here.
<commit_msg>UserProfile added to admin.<commit_after>from django.contrib import admin
from ed_news.models import UserProfile

admin.site.register(UserProfile)
<|endoftext|>


<commit_before>from functools import wraps

class AttributeDescription(object):
    def __init__(self, text, value=None, *args, **kwargs):
        self.name = None
        self.text = text
        self.value = value

    def __call__(self, attr, model):
        self.name = attr

    def __get__(self, obj, type=None):
        return self.value

    def __set__(self, obj, val):
        self.value = val

class Dimension(AttributeDescription):
    def __get__(self, obj, type=None):
        return obj._dims.get(self.name, None)

    def __set__(self, obj, value):
        obj._dims[self.name] = value

class Weights(AttributeDescription):
    def __init__(self, text, get_shape, init=None):
        self.name = None
        self.text = text
        self.get_shape = get_shape
        self.init = init

    def __get__(self, obj, type=None):
        key = (id(obj), self.name)
        if key in obj._mem:
            return obj._mem[key]
        else:
            shape = self.get_shape(obj)
            data = obj._mem.add(key, shape)
            if self.init is not None:
                self.init(data, obj.ops)
            return data

    def __set__(self, obj, val):
        data = obj._mem.get((id(obj), self.name))
        data[:] = val

class Gradient(AttributeDescription):
    def __init__(self, param_name):
        self.name = None
        self.text = "Gradient of %s" % param_name
        self.param_name = param_name

    def __get__(self, obj, type=None):
        key = (id(obj), self.name)
        if key in obj._mem:
            return obj._mem.get(key)
        else:
            param_key = (id(obj), self.param_name)
            grad = obj._mem.add_gradient(key, param_key)
            return grad

    def __set__(self, obj, val):
        data = obj._mem.get((id(obj), self.name))
        data[:] = val

class Synapses(Weights):
    pass

class Biases(Weights):
    pass

def attributes(**specs):
    if not specs: # pragma: no cover
        raise ValueError("Must describe at least one attribute")
    def wrapped(cls):
        cls.descriptions = dict(cls.descriptions)
        cls.descriptions.update(specs)
        for attr, desc in cls.descriptions.items():
            setattr(cls, attr, desc)
            desc.name = attr
        return cls
    return wrapped

def on_init(*callbacks):
    def wrapped(cls):
        cls.on_init_hooks = list(cls.on_init_hooks)
        cls.on_init_hooks.extend(callbacks)
        return cls
    return wrapped

def on_data(*callbacks):
    def wrapped(cls):
        cls.on_data_hooks = list(cls.on_data_hooks)
        cls.on_data_hooks.extend(callbacks)
        return cls
    return wrapped

def input(getter):
    def wrapped(cls):
        cls.describe_input = getter
        return cls
    return wrapped

def output(getter):
    def wrapped(cls):
        cls.describe_output = getter
        return cls
    return wrapped
<commit_msg>Exclude __get__ from coverage<commit_after>from functools import wraps

class AttributeDescription(object):
    def __init__(self, text, value=None, *args, **kwargs):
        self.name = None
        self.text = text
        self.value = value

    def __call__(self, attr, model):
        self.name = attr

    def __get__(self, obj, type=None): # pragma: no cover
        return self.value

    def __set__(self, obj, val):
        self.value = val

class Dimension(AttributeDescription):
    def __get__(self, obj, type=None):
        return obj._dims.get(self.name, None)

    def __set__(self, obj, value):
        obj._dims[self.name] = value

class Weights(AttributeDescription):
    def __init__(self, text, get_shape, init=None):
        self.name = None
        self.text = text
        self.get_shape = get_shape
        self.init = init

    def __get__(self, obj, type=None):
        key = (id(obj), self.name)
        if key in obj._mem:
            return obj._mem[key]
        else:
            shape = self.get_shape(obj)
            data = obj._mem.add(key, shape)
            if self.init is not None:
                self.init(data, obj.ops)
            return data

    def __set__(self, obj, val):
        data = obj._mem.get((id(obj), self.name))
        data[:] = val

class Gradient(AttributeDescription):
    def __init__(self, param_name):
        self.name = None
        self.text = "Gradient of %s" % param_name
        self.param_name = param_name

    def __get__(self, obj, type=None):
        key = (id(obj), self.name)
        if key in obj._mem:
            return obj._mem.get(key)
        else:
            param_key = (id(obj), self.param_name)
            grad = obj._mem.add_gradient(key, param_key)
            return grad

    def __set__(self, obj, val):
        data = obj._mem.get((id(obj), self.name))
        data[:] = val

class Synapses(Weights):
    pass

class Biases(Weights):
    pass

def attributes(**specs):
    if not specs: # pragma: no cover
        raise ValueError("Must describe at least one attribute")
    def wrapped(cls):
        cls.descriptions = dict(cls.descriptions)
        cls.descriptions.update(specs)
        for attr, desc in cls.descriptions.items():
            setattr(cls, attr, desc)
            desc.name = attr
        return cls
    return wrapped

def on_init(*callbacks):
    def wrapped(cls):
        cls.on_init_hooks = list(cls.on_init_hooks)
        cls.on_init_hooks.extend(callbacks)
        return cls
    return wrapped

def on_data(*callbacks):
    def wrapped(cls):
        cls.on_data_hooks = list(cls.on_data_hooks)
        cls.on_data_hooks.extend(callbacks)
        return cls
    return wrapped

def input(getter):
    def wrapped(cls):
        cls.describe_input = getter
        return cls
    return wrapped

def output(getter):
    def wrapped(cls):
        cls.describe_output = getter
        return cls
    return wrapped
<|endoftext|>


<commit_before>"""
Main migration logic.
"""

import sys

from django.core.exceptions import ImproperlyConfigured

import south.db
from south import exceptions
from south.models import MigrationHistory
from south.db import db, DEFAULT_DB_ALIAS
from south.migration.migrators import (Backwards, Forwards,
                                       DryRunMigrator, FakeMigrator,
                                       LoadInitialDataMigrator)
from south.migration.base import Migration, Migrations
from south.migration.utils import SortedSet
from south.migration.base import all_migrations
from south.signals import pre_migrate, post_migrate

def to_apply(forwards, done):
    return [m for m in forwards if m not in done]

def to_unapply(backwards, done):
    return [m for m in backwards if m in done]

def problems(pending, done):
    last = None
    if not pending:
        raise StopIteration()
    for migration in pending:
        if migration in done:
            last = migration
            continue
        if last and migration not in done:
            yield last, migration

def forwards_problems(pending, done, verbosity):
    """
    Takes the list of linearised pending migrations, and the set of done ones,
    and returns the list of problems, if any.
    """
    return inner_problem_check(problems(reversed(pending), done), done, verbosity)

def backwards_problems(pending, done, verbosity):
    return inner_problem_check(problems(pending, done), done, verbosity)

def inner_problem_check(problems, done, verbosity):
    "Takes a set of possible problems and gets the actual issues out of it."
    result = []
    for last, migration in problems:
        # 'Last' is the last applied migration. Step back from it until we
        # either find nothing wrong, or we find something.
        to_check = list(last.dependencies)
        while to_check:
            checking = to_check.pop()
            if checking not in done:
                # That's bad. Error.
                if verbosity:
                    print (" ! Migration %s should not have been applied "
                           "before %s but was." % (last, checking))
                result.append((last, checking))
            else:
                to_check.extend(checking.dependencies)
    return result

def check_migration_histories(histories, delete_ghosts=False, ignore_ghosts=False):
    "Checks that there's no 'ghost' migrations in the database."
    exists = SortedSet()
    ghosts = []
    for h in histories:
        try:
            m = h.get_migration()
            m.migration()
        except exceptions.UnknownMigration:
            ghosts.append(h)
        except ImproperlyConfigured:
            pass                        # Ignore missing applications
        else:
            exists.add(m)
    if ghosts:
        # They may want us to delete ghosts.
        if delete_ghosts:
            for h in ghosts:
                h.delete()
        elif not ignore_ghosts:
            raise exceptions.GhostMigrations(ghosts)
    return exists

def get_dependencies(target, migrations):
    forwards = list
    backwards = list
    if target is None:
        backwards = migrations[0].backwards_plan
    else:
        forwards = target.forwards_plan
        # When migrating backwards we want to remove up to and
        # including the next migration up in this app (not the next
        # one, that includes other apps)
        migration_before_here = target.next()
        if migration_before_here:
            backwards = migration_before_here.backwards_plan
    return forwards, backwards

def get_direction(target, applied, migrations, verbosity, interactive):
    # Get the forwards and reverse dependencies for this target
    forwards, backwards = get_dependencies(target, migrations)
    # Is the whole forward branch applied?
    problems = None
    forwards = forwards()
    workplan = to_apply(forwards, applied)
    if not workplan:
        # If they're all applied, we only know it's not backwards
        direction = None
    else:
        # If the remaining migrations are strictly a right segment of
        # the forwards trace, we just need to go forwards to our
        # target (and check for badness)
        problems = forwards_problems(forwards, applied, verbosity)
        direction = Forwards(verbosity=verbosity, interactive=interactive)
    if not problems:
        # What about the whole backward trace then?
        backwards = backwards()
        missing_backwards = to_apply(backwards, applied)
        if missing_backwards != backwards:
            # If what's missing is a strict left segment of backwards (i.e.
            # all the higher migrations) then we need to go backwards
            workplan = to_unapply(backwards, applied)
            problems = backwards_problems(backwards, applied, verbosity)
            direction = Backwards(verbosity=verbosity, interactive=interactive)
    return direction, problems, workplan

def get_migrator(direction, db_dry_run, fake, load_initial_data):
    if not direction:
        return direction
    if db_dry_run:
        direction = DryRunMigrator(migrator=direction, ignore_fail=False)
    elif fake:
        direction = FakeMigrator(migrator=direction)
    elif load_initial_data:
        direction = LoadInitialDataMigrator(migrator=direction)
    return direction

def get_unapplied_migrations(migrations, applied_migrations):
    applied_migration_names = ['%s.%s' % (mi.app_name,mi.migration) for mi in applied_migrations]

    for migration in migrations:
        is_applied = '%s.%s' % (migration.app_label(), migration.name()) in applied_migration_names
        if not is_applied:
            yield migration

def migrate_app(migrations, target_name=None, merge=False, fake=False, db_dry_run=False, yes=False, verbosity=0, load_initial_data=False, skip=False, database=DEFAULT_DB_ALIAS, delete_ghosts=False, ignore_ghosts=False, interactive=False):
    app_label = migrations.app_label()

    verbosity = int(verbosity)
    # Fire off the pre-migrate signal
    pre_migrate.send(None, app=app_label)

    # If there aren't any, quit quizically
    if not migrations:
        print "? You have no migrations for the '%s' app. You might want some." % app_label
        return

    # Load the entire dependency graph
    Migrations.calculate_dependencies()

    # Check there's no strange ones in the database
    applied_all = MigrationHistory.objects.filter(applied__isnull=False).order_by('applied')
    applied = applied_all.filter(app_name=app_label)
    # If we're using a different database, use that
    if database != DEFAULT_DB_ALIAS:
        applied_all = applied_all.using(database)
        applied = applied.using(database)
        south.db.db = south.db.dbs[database]
        # We now have to make sure the migrations are all reloaded, as they'll
        # have imported the old value of south.db.db.
        Migrations.invalidate_all_modules()

    south.db.db.debug = (verbosity > 1)

    if target_name == 'current-1':
        if applied.count() > 1:
            previous_migration = applied[applied.count() - 2]
            if verbosity:
                print 'previous_migration: %s (applied: %s)' % (previous_migration.migration, previous_migration.applied)
            target_name = previous_migration.migration
        else:
            if verbosity:
                print 'previous_migration: zero'
            target_name = 'zero'
    elif target_name == 'current+1':
        try:
            first_unapplied_migration = get_unapplied_migrations(migrations, applied).next()
            target_name = first_unapplied_migration.name()
        except StopIteration:
            target_name = None

    applied_all = check_migration_histories(applied_all, delete_ghosts, ignore_ghosts)

    # Guess the target_name
    target = migrations.guess_migration(target_name)
    if verbosity:
        if target_name not in ('zero', None) and target.name() != target_name:
            print " - Soft matched migration %s to %s." % (target_name,
                                                           target.name())
        print "Running migrations for %s:" % app_label

    # Get the forwards and reverse dependencies for this target
    direction, problems, workplan = get_direction(target, applied_all, migrations,
                                                  verbosity, interactive)
    if problems and not (merge or skip):
        raise exceptions.InconsistentMigrationHistory(problems)

    # Perform the migration
    migrator = get_migrator(direction, db_dry_run, fake, load_initial_data)
    if migrator:
        migrator.print_title(target)
        success = migrator.migrate_many(target, workplan, database)
        # Finally, fire off the post-migrate signal
        if success:
            post_migrate.send(None, app=app_label)
    else:
        if verbosity:
            # Say there's nothing.
            print '- Nothing to migrate.'
        # If we have initial data enabled, and we're at the most recent
        # migration, do initial data.
        # Note: We use a fake Forwards() migrator here. It's never used really.
        if load_initial_data:
            migrator = LoadInitialDataMigrator(migrator=Forwards(verbosity=verbosity))
            migrator.load_initial_data(target, db=database)
        # Send signal.
        post_migrate.send(None, app=app_label)
<commit_msg>Fix for switching between databases<commit_after>"""
Main migration logic.
"""

import sys

from django.core.exceptions import ImproperlyConfigured

import south.db
from south import exceptions
from south.models import MigrationHistory
from south.db import db, DEFAULT_DB_ALIAS
from south.migration.migrators import (Backwards, Forwards,
                                       DryRunMigrator, FakeMigrator,
                                       LoadInitialDataMigrator)
from south.migration.base import Migration, Migrations
from south.migration.utils import SortedSet
from south.migration.base import all_migrations
from south.signals import pre_migrate, post_migrate

def to_apply(forwards, done):
    return [m for m in forwards if m not in done]

def to_unapply(backwards, done):
    return [m for m in backwards if m in done]

def problems(pending, done):
    last = None
    if not pending:
        raise StopIteration()
    for migration in pending:
        if migration in done:
            last = migration
            continue
        if last and migration not in done:
            yield last, migration

def forwards_problems(pending, done, verbosity):
    """
    Takes the list of linearised pending migrations, and the set of done ones,
    and returns the list of problems, if any.
    """
    return inner_problem_check(problems(reversed(pending), done), done, verbosity)

def backwards_problems(pending, done, verbosity):
    return inner_problem_check(problems(pending, done), done, verbosity)

def inner_problem_check(problems, done, verbosity):
    "Takes a set of possible problems and gets the actual issues out of it."
    result = []
    for last, migration in problems:
        # 'Last' is the last applied migration. Step back from it until we
        # either find nothing wrong, or we find something.
        to_check = list(last.dependencies)
        while to_check:
            checking = to_check.pop()
            if checking not in done:
                # That's bad. Error.
                if verbosity:
                    print (" ! Migration %s should not have been applied "
                           "before %s but was." % (last, checking))
                result.append((last, checking))
            else:
                to_check.extend(checking.dependencies)
    return result

def check_migration_histories(histories, delete_ghosts=False, ignore_ghosts=False):
    "Checks that there's no 'ghost' migrations in the database."
    exists = SortedSet()
    ghosts = []
    for h in histories:
        try:
            m = h.get_migration()
            m.migration()
        except exceptions.UnknownMigration:
            ghosts.append(h)
        except ImproperlyConfigured:
            pass                        # Ignore missing applications
        else:
            exists.add(m)
    if ghosts:
        # They may want us to delete ghosts.
        if delete_ghosts:
            for h in ghosts:
                h.delete()
        elif not ignore_ghosts:
            raise exceptions.GhostMigrations(ghosts)
    return exists

def get_dependencies(target, migrations):
    forwards = list
    backwards = list
    if target is None:
        backwards = migrations[0].backwards_plan
    else:
        forwards = target.forwards_plan
        # When migrating backwards we want to remove up to and
        # including the next migration up in this app (not the next
        # one, that includes other apps)
        migration_before_here = target.next()
        if migration_before_here:
            backwards = migration_before_here.backwards_plan
    return forwards, backwards

def get_direction(target, applied, migrations, verbosity, interactive):
    # Get the forwards and reverse dependencies for this target
    forwards, backwards = get_dependencies(target, migrations)
    # Is the whole forward branch applied?
    problems = None
    forwards = forwards()
    workplan = to_apply(forwards, applied)
    if not workplan:
        # If they're all applied, we only know it's not backwards
        direction = None
    else:
        # If the remaining migrations are strictly a right segment of
        # the forwards trace, we just need to go forwards to our
        # target (and check for badness)
        problems = forwards_problems(forwards, applied, verbosity)
        direction = Forwards(verbosity=verbosity, interactive=interactive)
    if not problems:
        # What about the whole backward trace then?
        backwards = backwards()
        missing_backwards = to_apply(backwards, applied)
        if missing_backwards != backwards:
            # If what's missing is a strict left segment of backwards (i.e.
            # all the higher migrations) then we need to go backwards
            workplan = to_unapply(backwards, applied)
            problems = backwards_problems(backwards, applied, verbosity)
            direction = Backwards(verbosity=verbosity, interactive=interactive)
    return direction, problems, workplan

def get_migrator(direction, db_dry_run, fake, load_initial_data):
    if not direction:
        return direction
    if db_dry_run:
        direction = DryRunMigrator(migrator=direction, ignore_fail=False)
    elif fake:
        direction = FakeMigrator(migrator=direction)
    elif load_initial_data:
        direction = LoadInitialDataMigrator(migrator=direction)
    return direction

def get_unapplied_migrations(migrations, applied_migrations):
    applied_migration_names = ['%s.%s' % (mi.app_name,mi.migration) for mi in applied_migrations]

    for migration in migrations:
        is_applied = '%s.%s' % (migration.app_label(), migration.name()) in applied_migration_names
        if not is_applied:
            yield migration

def migrate_app(migrations, target_name=None, merge=False, fake=False, db_dry_run=False, yes=False, verbosity=0, load_initial_data=False, skip=False, database=DEFAULT_DB_ALIAS, delete_ghosts=False, ignore_ghosts=False, interactive=False):
    app_label = migrations.app_label()

    verbosity = int(verbosity)
    # Fire off the pre-migrate signal
    pre_migrate.send(None, app=app_label)

    # If there aren't any, quit quizically
    if not migrations:
        print "? You have no migrations for the '%s' app. You might want some." % app_label
        return

    # Load the entire dependency graph
    Migrations.calculate_dependencies()

    # Check there's no strange ones in the database
    applied_all = MigrationHistory.objects.filter(applied__isnull=False).order_by('applied').using(database)
    applied = applied_all.filter(app_name=app_label).using(database)
    south.db.db = south.db.dbs[database]
    Migrations.invalidate_all_modules()

    south.db.db.debug = (verbosity > 1)

    if target_name == 'current-1':
        if applied.count() > 1:
            previous_migration = applied[applied.count() - 2]
            if verbosity:
                print 'previous_migration: %s (applied: %s)' % (previous_migration.migration, previous_migration.applied)
            target_name = previous_migration.migration
        else:
            if verbosity:
                print 'previous_migration: zero'
            target_name = 'zero'
    elif target_name == 'current+1':
        try:
            first_unapplied_migration = get_unapplied_migrations(migrations, applied).next()
            target_name = first_unapplied_migration.name()
        except StopIteration:
            target_name = None

    applied_all = check_migration_histories(applied_all, delete_ghosts, ignore_ghosts)

    # Guess the target_name
    target = migrations.guess_migration(target_name)
    if verbosity:
        if target_name not in ('zero', None) and target.name() != target_name:
            print " - Soft matched migration %s to %s." % (target_name,
                                                           target.name())
        print "Running migrations for %s:" % app_label

    # Get the forwards and reverse dependencies for this target
    direction, problems, workplan = get_direction(target, applied_all, migrations,
                                                  verbosity, interactive)
    if problems and not (merge or skip):
        raise exceptions.InconsistentMigrationHistory(problems)

    # Perform the migration
    migrator = get_migrator(direction, db_dry_run, fake, load_initial_data)
    if migrator:
        migrator.print_title(target)
        success = migrator.migrate_many(target, workplan, database)
        # Finally, fire off the post-migrate signal
        if success:
            post_migrate.send(None, app=app_label)
    else:
        if verbosity:
            # Say there's nothing.
            print '- Nothing to migrate.'
        # If we have initial data enabled, and we're at the most recent
        # migration, do initial data.
        # Note: We use a fake Forwards() migrator here. It's never used really.
        if load_initial_data:
            migrator = LoadInitialDataMigrator(migrator=Forwards(verbosity=verbosity))
            migrator.load_initial_data(target, db=database)
        # Send signal.
        post_migrate.send(None, app=app_label)
<|endoftext|>


<commit_before># coding=utf-8
"""
"""
from __future__ import absolute_import

from redis import from_url as redis_from_url

class Extension(object):
  """
  Redis extension for flask
  """
  client = None

  def __init__(self, app=None):
    if app is not None:
      self.init_app(app)

  def init_app(self, app):
    app.extensions['redis'] = self
    self.setup_client()

  def setup_client(self, uri=None):
    if uri is None:
      uri = app.config.get('REDIS_URI')
    if uri:
      self.client = redis_from_url(uri)
    elif not (app.configured or app.testing):
      raise ValueError('Redis extension: REDIS_URI is not defined in '
                       'application configuration')
<commit_msg>redis: fix setup_client<commit_after># coding=utf-8
"""
"""
from __future__ import absolute_import

from redis import from_url as redis_from_url
from flask import current_app

class Extension(object):
  """
  Redis extension for flask
  """
  client = None

  def __init__(self, app=None):
    if app is not None:
      self.init_app(app)

  def init_app(self, app):
    app.extensions['redis'] = self
    self.setup_client(app)

  def setup_client(self, app=None, uri=None):
    if app is None:
      app = current_app
    if uri is None:
      uri = app.config.get('REDIS_URI')
    if uri:
      self.client = redis_from_url(uri)
    elif not (app.configured or app.testing):
      raise ValueError('Redis extension: REDIS_URI is not defined in '
                       'application configuration')
<|endoftext|>


<commit_before>#!/usr/bin/python
#

import os
import cgi
import cgitb
import sys
import meowaux as mew
import urllib
import Cookie
import json
import uuid
import subprocess as sp
import zipfile as zf

cgitb.enable();

jsonsch_exec = "/home/meow/pykicad/jsonsch.py"
jsonbrd_exec = "/home/meow/pykicad/jsonbrd.py"

brdgrb_exec = "/home/meow/pykicad/brdgerber.py"
grbngc_exec = "/home/meow/bin/gbl2ngc"
font_file = "/home/meow/pykicad/aux/hershey_ascii.json"

staging_base = "/home/meow/stage"

#print "Content-Type: text/html"
#print

cookie = Cookie.SimpleCookie()
cookie_hash = mew.getCookieHash( os.environ )

g_debug = False
def log_line( l ):
  logf = open("/tmp/downloadProject.log", "a")
  logf.write( l  + "\n")
  logf.close()

def error_and_quit():
  if g_debug:
    log_line("error, quitting")
  print "Status: 404 Not Found"
  print
  print "File not found"
  sys.exit(0)

def dumpToFile( data ):
  u_id = uuid.uuid4()
  fn = staging_base + "/" + str(u_id)
  f = open( fn, "w" )
  #f.write( json.dumps(json_message["data"], indent=2) )
  f.write( data )
  f.close()

  return u_id, fn

def readyProjectZipfile( json_message ):

  try:
    zip_uid = uuid.uuid4()

    # Put json data structures into files
    #
    sch_json_uid, brd_json_fn = dumpToFile( json.dumps(json_message["board"], indent=2) )
    brd_json_uid, sch_json_fn = dumpToFile( json.dumps(json_message["schematic"], indent=2) )

    log_line( "brd: " + brd_json_fn )
    log_line( "sch: " + sch_json_fn )

    # Create KiCAD schematic and board files
    #
    kicad_sch = sp.check_output( [jsonsch_exec, sch_json_fn ] );
    sch_uid, sch_fn  = dumpToFile( kicad_sch )

    kicad_brd = sp.check_output( [jsonbrd_exec, brd_json_fn ] );
    brd_uid, brd_fn = dumpToFile( kicad_brd )

    # Go through relevant layers and create gerber and gcode files for each
    #
    layers = { -1: ".drl", 0 : "-B_Cu.gbl", 15 : "-F_Cu.gtl", 20 : "-B_SilkS.gbo", 21 : "-F_SilkS.gto", 28 : "-Edge_Cuts.gbr" }
    gerber_files = []
    gcode_files = []
    for layer in layers:

      grbr = sp.check_output( [brdgrb_exec, str(brd_fn), str(layer), str(font_file) ] )
      gerber_uid, gerber_fn = dumpToFile( grbr )
      gerber_files.append( { "id": gerber_uid , "filename" : gerber_fn, "layer" : layer } )

      radius_decithou = "0.002"
      gcode = sp.check_output( [ grbngc_exec, "--radius", radius_decithou, "--input", gerber_fn  ] )
      gcode_uid, gcode_fn = dumpToFile( gcode )
      gcode_files.append( { "id" : gcode_uid, "filename" : gcode_fn, "layer" : layer } )

    projname = "project"

    zip_fn = str(staging_base) + "/" + str(zip_uid)

    # http://stackoverflow.com/questions/9289734/linux-how-to-add-a-file-to-a-specific-folder-within-a-zip-file
    z = zf.ZipFile( str(zip_fn), "a" )
    z.write( sch_json_fn, projname + "/json/schematic.json" )
    z.write( brd_json_fn, projname + "/json/board.json" )
    z.write( sch_fn, projname + "/KiCAD/schematic.sch" )
    z.write( brd_fn, projname + "/KiCAD/board.brd" )

    for f in gerber_files:
      z.write( f["filename"], projname + "/gerber/board" + layers[ int(f["layer"]) ] )

    for f in gcode_files:
      z.write( f["filename"], projname + "/gcode/board" + layers[ int(f["layer"]) ] )

    z.close()

  except Exception as e:
    error_and_quit(e, "...")

  obj = { "type": "id", "id" : str(zip_uid), "notes" : "zip project file",
          "name" : json_message["name"] }
  return obj

userId = None
sessionId = None
projectId = None

fields = cgi.FieldStorage()
if ("userId" in fields) and ("sessionId" in fields):
  if mew.authenticateSession( fields["userId"].value, fields["sessionId"].value ):
    userId = fields["userId"].value
    sessionId = fields["sessionId"].value
    if "projectId" in fields:
      projectId = fields["projectId"].value

if ( ("userId" in cookie_hash) and ("sessionId" in cookie_hash) and
     ( mew.authenticateSession( cookie_hash["userId"], cookie_hash["sessionId"] ) == 1) ):
    userId = cookie_hash["userId"]
    sessionId = cookie_hash["sessionId"]
    if "projectId" in fields:
      projectId = fields["projectId"].value

#log_line( "u: " + str(userId) )
#log_line( "s: " + str(sessionId) )
#log_line( "p: " + str(projectId) )

if projectId is None:

  log_line( "--> None projectId" )

  error_and_quit()

proj = mew.getProject( projectId )
if proj["permission"] != "world-read":
  if (userId is None) or (userId != proj["userId"]):

    log_line("---> permission error")

    error_and_quit()

snapshot = {}
json_container = {}
try:
  snapshot = mew.getProjectSnapshot( projectId )
  json_container = { "board" : json.loads(snapshot["json_brd"]),
                     "schematic" : json.loads(snapshot["json_sch"]),
                     "name" : mew.safe_str( proj["name"] ) + ".zip"  }
except Exception as e:

  log_line( "----> loads error: " + str(e) )
  error_and_quit()

obj = readyProjectZipfile( json_container )

print "Content-Type: application/json; charset=utf-8"
print
print json.dumps(obj)

<commit_msg>using drl2ngc for drill layer.<commit_after>#!/usr/bin/python
#

import os
import cgi
import cgitb
import sys
import meowaux as mew
import urllib
import Cookie
import json
import uuid
import subprocess as sp
import zipfile as zf

cgitb.enable();

jsonsch_exec = "/home/meow/pykicad/jsonsch.py"
jsonbrd_exec = "/home/meow/pykicad/jsonbrd.py"

brdgrb_exec = "/home/meow/pykicad/brdgerber.py"
grbngc_exec = "/home/meow/bin/gbl2ngc"
drlngc_exec = "/home/meow/bin/drl2ngc"
font_file = "/home/meow/pykicad/aux/hershey_ascii.json"

staging_base = "/home/meow/stage"

#print "Content-Type: text/html"
#print

cookie = Cookie.SimpleCookie()
cookie_hash = mew.getCookieHash( os.environ )

g_debug = False
def log_line( l ):
  logf = open("/tmp/downloadProject.log", "a")
  logf.write( l  + "\n")
  logf.close()

def error_and_quit():
  if g_debug:
    log_line("error, quitting")
  print "Status: 404 Not Found"
  print
  print "File not found"
  sys.exit(0)

def dumpToFile( data ):
  u_id = uuid.uuid4()
  fn = staging_base + "/" + str(u_id)
  f = open( fn, "w" )
  #f.write( json.dumps(json_message["data"], indent=2) )
  f.write( data )
  f.close()

  return u_id, fn

def readyProjectZipfile( json_message ):

  try:
    zip_uid = uuid.uuid4()

    # Put json data structures into files
    #
    sch_json_uid, brd_json_fn = dumpToFile( json.dumps(json_message["board"], indent=2) )
    brd_json_uid, sch_json_fn = dumpToFile( json.dumps(json_message["schematic"], indent=2) )

    log_line( "brd: " + brd_json_fn )
    log_line( "sch: " + sch_json_fn )

    # Create KiCAD schematic and board files
    #
    kicad_sch = sp.check_output( [jsonsch_exec, sch_json_fn ] );
    sch_uid, sch_fn  = dumpToFile( kicad_sch )

    kicad_brd = sp.check_output( [jsonbrd_exec, brd_json_fn ] );
    brd_uid, brd_fn = dumpToFile( kicad_brd )

    # Go through relevant layers and create gerber and gcode files for each
    #
    layers = { -1: ".drl", 0 : "-B_Cu.gbl", 15 : "-F_Cu.gtl", 20 : "-B_SilkS.gbo", 21 : "-F_SilkS.gto", 28 : "-Edge_Cuts.gbr" }
    gerber_files = []
    gcode_files = []
    for layer in layers:

      grbr = sp.check_output( [brdgrb_exec, str(brd_fn), str(layer), str(font_file) ] )
      gerber_uid, gerber_fn = dumpToFile( grbr )
      gerber_files.append( { "id": gerber_uid , "filename" : gerber_fn, "layer" : layer } )

      if layer >= 0:
        radius_decithou = "0.002"
        gcode = sp.check_output( [ grbngc_exec, "--radius", radius_decithou, "--input", gerber_fn  ] )
        gcode_uid, gcode_fn = dumpToFile( gcode )
        gcode_files.append( { "id" : gcode_uid, "filename" : gcode_fn, "layer" : layer } )
      else:
        gcode = sp.check_output( [ drlngc_exec, "--input", gerber_fn  ] )
        gcode_uid, gcode_fn = dumpToFile( gcode )
        gcode_files.append( { "id" : gcode_uid, "filename" : gcode_fn, "layer" : layer } )

    projname = "project"

    zip_fn = str(staging_base) + "/" + str(zip_uid)

    # http://stackoverflow.com/questions/9289734/linux-how-to-add-a-file-to-a-specific-folder-within-a-zip-file
    z = zf.ZipFile( str(zip_fn), "a" )
    z.write( sch_json_fn, projname + "/json/schematic.json" )
    z.write( brd_json_fn, projname + "/json/board.json" )
    z.write( sch_fn, projname + "/KiCAD/schematic.sch" )
    z.write( brd_fn, projname + "/KiCAD/board.brd" )

    for f in gerber_files:
      z.write( f["filename"], projname + "/gerber/board" + layers[ int(f["layer"]) ] )

    for f in gcode_files:
      z.write( f["filename"], projname + "/gcode/board" + layers[ int(f["layer"]) ] )

    z.close()

  except Exception as e:
    error_and_quit(e, "...")

  obj = { "type": "id", "id" : str(zip_uid), "notes" : "zip project file",
          "name" : json_message["name"] }
  return obj

userId = None
sessionId = None
projectId = None

fields = cgi.FieldStorage()
if ("userId" in fields) and ("sessionId" in fields):
  if mew.authenticateSession( fields["userId"].value, fields["sessionId"].value ):
    userId = fields["userId"].value
    sessionId = fields["sessionId"].value
    if "projectId" in fields:
      projectId = fields["projectId"].value

if ( ("userId" in cookie_hash) and ("sessionId" in cookie_hash) and
     ( mew.authenticateSession( cookie_hash["userId"], cookie_hash["sessionId"] ) == 1) ):
    userId = cookie_hash["userId"]
    sessionId = cookie_hash["sessionId"]
    if "projectId" in fields:
      projectId = fields["projectId"].value

#log_line( "u: " + str(userId) )
#log_line( "s: " + str(sessionId) )
#log_line( "p: " + str(projectId) )

if projectId is None:

  log_line( "--> None projectId" )

  error_and_quit()

proj = mew.getProject( projectId )
if proj["permission"] != "world-read":
  if (userId is None) or (userId != proj["userId"]):

    log_line("---> permission error")

    error_and_quit()

snapshot = {}
json_container = {}
try:
  snapshot = mew.getProjectSnapshot( projectId )
  json_container = { "board" : json.loads(snapshot["json_brd"]),
                     "schematic" : json.loads(snapshot["json_sch"]),
                     "name" : mew.safe_str( proj["name"] ) + ".zip"  }
except Exception as e:

  log_line( "----> loads error: " + str(e) )
  error_and_quit()

obj = readyProjectZipfile( json_container )

print "Content-Type: application/json; charset=utf-8"
print
print json.dumps(obj)

<|endoftext|>


<commit_before>STOP_WORDS = set(
    """
alle allerede alt and andre annen annet at av

bak bare bedre beste blant ble bli blir blitt bris by både

da dag de del dem den denne der dermed det dette disse drept du

eller en enn er et ett etter

fem fikk fire fjor flere folk for fortsatt fotball fra fram frankrike fredag
funnet få får fått før først første

gang gi gikk gjennom gjorde gjort gjør gjøre god godt grunn gå går

ha hadde ham han hans har hele helt henne hennes her hun hva hvor hvordan
hvorfor

i ifølge igjen ikke ingen inn

ja jeg

kamp kampen kan kl klart kom komme kommer kontakt kort kroner kunne kveld
kvinner

la laget land landet langt leder ligger like litt løpet lørdag

man mandag mange mannen mars med meg mellom men mener menn mennesker mens mer
millioner minutter mot msci mye må mål måtte

ned neste noe noen nok norge norsk norske ntb ny nye nå når

og også om onsdag opp opplyser oslo oss over

personer plass poeng politidistrikt politiet president prosent på

regjeringen runde rundt russland

sa saken samme sammen samtidig satt se seg seks selv senere september ser sett
siden sier sin sine siste sitt skal skriver skulle slik som sted stedet stor
store står sverige svært så søndag

ta tatt tid tidligere til tilbake tillegg tirsdag to tok torsdag tre tror
tyskland

under usa ut uten utenfor

vant var ved veldig vi videre viktig vil ville viser vår være vært

å år

ønsker
""".split()
)
<commit_msg>Remove NER words from stop words in Norwegian (#9820)<commit_after>STOP_WORDS = set(
    """
alle allerede alt and andre annen annet at av

bak bare bedre beste blant ble bli blir blitt bris by både

da dag de del dem den denne der dermed det dette disse du

eller en enn er et ett etter

fem fikk fire fjor flere folk for fortsatt fra fram
funnet få får fått før først første

gang gi gikk gjennom gjorde gjort gjør gjøre god godt grunn gå går

ha hadde ham han hans har hele helt henne hennes her hun

i ifølge igjen ikke ingen inn

ja jeg

kamp kampen kan kl klart kom komme kommer kontakt kort kroner kunne kveld

la laget land landet langt leder ligger like litt løpet

man mange med meg mellom men mener mennesker mens mer mot mye må mål måtte

ned neste noe noen nok ny nye nå når

og også om opp opplyser oss over

personer plass poeng på

runde rundt

sa saken samme sammen samtidig satt se seg seks selv senere ser sett
siden sier sin sine siste sitt skal skriver skulle slik som sted stedet stor
store står svært så

ta tatt tid tidligere til tilbake tillegg tok tror

under ut uten utenfor

vant var ved veldig vi videre viktig vil ville viser vår være vært

å år

ønsker
""".split()
)
<|endoftext|>


<commit_before>import itertools
import collections

def postfix_encoded_data(encoded_data):
    field_names_to_postfix = ['extra']
    sentry_fields = keys_starting_with(encoded_data, 'sentry.')
    field_names_to_postfix.extend(sentry_fields)

    fields_to_postfix = filter(lambda x: x in field_names_to_postfix,
                               encoded_data)

    for field in fields_to_postfix:
        _, encoded_data[field] = next(postfix_types(
            ('', encoded_data[field])))

def keys_starting_with(dictionary, word):
    return (key for key in dictionary.keys() if key.startswith(word))

def postfix_types(row):
    name, data = row
    if data is None:
        return postfix_none(name, data)
    type_postfix = postfixes.get(type(data), postfix_other)
    return type_postfix(name, data)

def postfix_dict(name, data):
    if name.endswith(">"):
        name = name + "<dict>"
    postfix_items = list(map(postfix_types, data.items()))
    yield name, dict(itertools.chain(*postfix_items))

def postfix_str(name, data):
    yield name + '<string>', data

def postfix_list(name, data):
    for k, v in _split_list_by_type(data).items():
        yield name + k, v

def postfix_none(name, data):
    yield name, None

def postfix_other(name, data):
    yield ('%s<%s>' % (name, type(data).__name__)), data

postfixes = {
    dict: postfix_dict,
    str: postfix_str,
    list: postfix_list,
    }

def _split_list_by_type(data):
    result = collections.defaultdict(list)
    for element in data:
        for type, value in postfix_types(('', element)):
            result[type].append(value)
    if result:
        return result
    else:
        return {'': []}
<commit_msg>Refactor postfix.<commit_after>import itertools
import collections

def postfix_encoded_data(encoded_data):
    field_names_to_postfix = ['extra']
    sentry_fields = keys_starting_with(encoded_data, 'sentry.')
    field_names_to_postfix.extend(sentry_fields)

    fields_to_postfix = filter(lambda x: x in field_names_to_postfix,
                               encoded_data)

    for field in fields_to_postfix:
        _, encoded_data[field] = next(postfix_types(
            ('', encoded_data[field])))

def keys_starting_with(dictionary, word):
    return (key for key in dictionary.keys() if key.startswith(word))

def postfix_types(row):
    name, data = row
    if data is None:
        return postfix_none(name, data)
    type_postfix = postfixes.get(type(data), postfix_other)
    return type_postfix(name, data)

def postfix_none(name, _):
    yield name, None

def postfix_other(name, data):
    yield ('%s<%s>' % (name, type(data).__name__)), data

def postfix_dict(name, data):
    if name.endswith(">"):
        name += "<dict>"
    postfix_items = list(map(postfix_types, data.items()))
    yield name, dict(itertools.chain(*postfix_items))

def postfix_str(name, data):
    yield name + '<string>', data

def postfix_list(name, data):
    for k, v in _split_list_by_type(data).items():
        yield name + k, v

def _split_list_by_type(data):
    result = collections.defaultdict(list)
    for element in data:
        for value_type, value in postfix_types(('', element)):
            result[value_type].append(value)
    if result:
        return result
    else:
        return {'': []}

postfixes = {
    dict: postfix_dict,
    str: postfix_str,
    list: postfix_list,
}
<|endoftext|>


<commit_before>import pickle, geojson, json
import sys, getopt, math
import numpy as np
import shapely.geometry as sg
from scipy import spatial
import shapefile

from spat.spatial_graph import SpatialGraph
from spat.utility import *

def extract_length(trajectory, graph, predicate):
  length = 0
  for segment in trajectory:
    if (predicate(segment['link'])):
      if segment['link'] == None:
        length += sg.LineString(segment['geometry']).length
      else:
        length += segment['bounds'][1][1] - segment['bounds'][0][1]
  return length

def load_discontinuity(file):
  sf = shapefile.Reader(file)
  points = {}

  id_field = next(i for i,v in enumerate(sf.fields) if v[0] == "id_link")-1
  for s, r in zip(sf.shapes(), sf.records()):
    points[r[id_field]] = {
      'geometry': sg.Point(s.points[0]),
      'properties': {'id': r[id_field]}}
  return points

def load_discontinuity_inter(file):
  sf = shapefile.Reader(file)
  points = {}

  id_field = next(i for i,v in enumerate(sf.fields) if v[0] == "nb_classes")-1
  for s, r in zip(sf.shapes(), sf.records()):
    points[r[id_field]] = {
      'geometry': sg.Point(s.points[0])}
  return points

def match_discontinuity(points, graph):
  discontinuity_index = {}
  for i, p in points.iteritems():
    geom = p['geometry']
    nearby = graph.node_intersection(bb_bounds(geom.x, geom.y, 2, 2));

    min_distance = np.inf
    best_node = None
    for node in nearby:
      distance = graph.intersection(node).distance(geom)
      if distance < min_distance:
        min_distance = distance
        best_node = node
    if best_node not in discontinuity_index:
      discontinuity_index[best_node] = []
    discontinuity_index[best_node].append(i)
  return discontinuity_index

def extract_discontinuity(trajectory, discontinuity_index):
  count = 0

  for segment in trajectory:
    if segment['link'] != None and segment['bounds'][1][0] == True:
      node = segment['link'][1]
      if node in discontinuity_index:
        count += len(discontinuity_index[node])
  return count

def extract_features(trajectories, graph):
  end_of_facility = match_discontinuity(
    load_discontinuity("data/discontinuity/end_of_facility"), graph)
  change_of_facility_type = match_discontinuity(
    load_discontinuity("data/discontinuity/change_of_facility_type"), graph)
  intersections_disc = match_discontinuity(
    load_discontinuity_inter("data/intersections/intersections_on_bike_network_with_change_in_road_type"), graph)

  def any_link(link):
    return True
  def any_cycling_link(link):
    if link == None:
      return False
    return graph.edge(link)['type'] > 10
  def designated_roadway(link):
    if link == None:
      return False
    return graph.edge(link)['type'] == 11
  def bike_lane(link):
    if link == None:
      return False
    return graph.edge(link)['type'] == 13
  def seperate_cycling_link(link):
    if link == None:
      return False
    return graph.edge(link)['type'] in {14, 15, 16}
  def offroad(link):
    if link == None:
      return False
    return graph.edge(link)['type'] == 17

  features = {}
  for i, trajectory in enumerate(trajectories):
    features[i] = {}
    features[i]['length'] = extract_length(trajectory, graph,
      any_link)
    features[i]['length_cycling'] = extract_length(trajectory, graph,
      any_cycling_link)
    features[i]['length_designated_roadway'] = extract_length(trajectory, graph,
      designated_roadway)
    features[i]['length_bike_lane'] = extract_length(trajectory, graph,
      bike_lane)
    features[i]['length_seperate_cycling_link'] = extract_length(trajectory, graph,
      seperate_cycling_link)
    features[i]['length_offroad'] = extract_length(trajectory, graph,
      offroad)

    features[i]['end_of_facility'] = extract_discontinuity(
      trajectory, end_of_facility)
    features[i]['change_of_facility_type'] = extract_discontinuity(
      trajectory, change_of_facility_type)
    features[i]['intersections_disc'] = extract_discontinuity(
      trajectory, intersections_disc)

    print features[i].values()
  return features

def main(argv):
  inputfile = 'data/bike_path/mm.pickle'
  facilityfile = 'data/mtl_geobase/mtl.pickle'
  outputfile = 'data/bike_path/features.json'
  try:
    opts, args = getopt.getopt(argv,"hi:o:",["ifile=","ofile=","facility="])
  except getopt.GetoptError:
    print 'features [-i <inputfile>] [-o <outputfile>]'
  for opt, arg in opts:
    if opt == '-h':
      print 'features [-i <inputfile>] [-o <outputfile>]'
      sys.exit()
    if opt in ("-i", "--ifile"):
      inputfile = arg
    if opt in ("--facility"):
      facilityfile = arg
    elif opt in ("-o", "--ofile"):
      outputfile = arg

  print 'input file:', inputfile
  print 'facility:', facilityfile
  print 'output file:', outputfile

  with open(facilityfile, 'r') as f:
    graph = pickle.load(f)
  graph.build_spatial_node_index()

  with open(inputfile, 'r') as f:
    data = pickle.load(f)

  features = extract_features(data, graph)
  print features

  with open(outputfile, 'w+') as f:
    json.dump(features, f, indent=2)

if __name__ == "__main__":
  main(sys.argv[1:])
<commit_msg>convert trajectory.features to argparse<commit_after>import sys, getopt, argparse, fnmatch
import pickle, csv, geojson, json, math
import numpy as np
import shapely.geometry as sg
from scipy import spatial
import shapefile

from spat.spatial_graph import SpatialGraph
from spat.utility import *

def extract_length(trajectory, graph, predicate):
  length = 0
  for segment in trajectory:
    if (predicate(segment['link'])):
      if segment['link'] == None:
        length += sg.LineString(segment['geometry']).length
      else:
        length += segment['bounds'][1][1] - segment['bounds'][0][1]
  return length

def load_discontinuity(file):
  sf = shapefile.Reader(file)
  points = []

  for s, r in zip(sf.shapes(), sf.records()):
    points.append({'geometry': sg.Point(s.points[0])})
  return points

def match_discontinuity(points, graph):
  discontinuity_index = {}
  for i, p in enumerate(points):
    geom = p['geometry']
    nearby = graph.node_intersection(bb_bounds(geom.x, geom.y, 2, 2));

    min_distance = np.inf
    best_node = None
    for node in nearby:
      distance = graph.intersection(node).distance(geom)
      if distance < min_distance:
        min_distance = distance
        best_node = node
    if best_node not in discontinuity_index:
      discontinuity_index[best_node] = []
    discontinuity_index[best_node].append(i)
  return discontinuity_index

def extract_discontinuity(trajectory, discontinuity_index):
  count = 0

  for segment in trajectory:
    if segment['link'] != None and segment['bounds'][1][0] == True:
      node = segment['link'][1]
      if node in discontinuity_index:
        count += len(discontinuity_index[node])
  return count

def extract_features(trajectories, graph):
  end_of_facility = match_discontinuity(
    load_discontinuity("data/discontinuity/end_of_facility"), graph)
  change_of_facility_type = match_discontinuity(
    load_discontinuity("data/discontinuity/change_of_facility_type"), graph)
  intersections_disc = match_discontinuity(
    load_discontinuity("data/intersections/intersections_on_bike_network_with_change_in_road_type"), graph)

  def any_link(link):
    return True
  def any_cycling_link(link):
    if link == None:
      return False
    return graph.edge(link)['type'] > 10
  def designated_roadway(link):
    if link == None:
      return False
    return graph.edge(link)['type'] == 11
  def bike_lane(link):
    if link == None:
      return False
    return graph.edge(link)['type'] == 13
  def seperate_cycling_link(link):
    if link == None:
      return False
    return graph.edge(link)['type'] in {14, 15, 16}
  def offroad(link):
    if link == None:
      return False
    return graph.edge(link)['type'] == 17

  features = []
  for i, trajectory in enumerate(trajectories):
    features.append({})
    features[i]['length'] = extract_length(trajectory, graph,
      any_link)
    features[i]['length_cycling'] = extract_length(trajectory, graph,
      any_cycling_link)
    features[i]['length_designated_roadway'] = extract_length(trajectory, graph,
      designated_roadway)
    features[i]['length_bike_lane'] = extract_length(trajectory, graph,
      bike_lane)
    features[i]['length_seperate_cycling_link'] = extract_length(trajectory, graph,
      seperate_cycling_link)
    features[i]['length_offroad'] = extract_length(trajectory, graph,
      offroad)

    features[i]['end_of_facility'] = extract_discontinuity(
      trajectory, end_of_facility)
    features[i]['change_of_facility_type'] = extract_discontinuity(
      trajectory, change_of_facility_type)
    features[i]['intersections_disc'] = extract_discontinuity(
      trajectory, intersections_disc)

    features[i]['left_turn'] = extract_turn(
      trajectory, left_turn)

    print features[i].values()
  return features

def main(argv):
  parser = argparse.ArgumentParser(description="""
    Extract several features from mapmatched bike trajectories.
    Features are:
    length
    length_cycling
    length_designated_roadway
    length_bike_lane
    length_seperate_cycling_link
    length_offroad

    end_of_facility
    change_of_facility_type
    intersections_disc
    """, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
  parser.add_argument('-i', '--ifile', default = 'data/bike_path/mm.pickle',
    help='input pickle file of mapmatched (with spat.trajectory.mapmatch) data.');
  parser.add_argument('--facility', default = 'data/mtl_geobase/mtl.pickle',
    help="""input pickle file containing the facility graph
    (with spat.geobase.preprocess) representing the road network""");
  parser.add_argument('-o', '--ofile',
    default = ['data/bike_path/features.json'], nargs='+',
    help="""output file containing an array of segments.
    Supported formats include *.json, *.csv""");

  args = parser.parse_args()
  print 'input file:', args.ifile
  print 'facility:', args.facility
  print 'output file:', args.ofile

  with open(args.facility, 'r') as f:
    graph = pickle.load(f)
  graph.build_spatial_node_index()

  with open(args.ifile, 'r') as f:
    data = pickle.load(f)

  features = extract_features(data, graph)

  for output in args.ofile:
    with open(output, 'w+') as f:
      if fnmatch.fnmatch(output, '*.csv'):
        writer = csv.writer(f)
        for row in features:
          print row.values()
          writer.writerow(row.values())
      elif fnmatch.fnmatch(output, '*.json'):
        json.dump(features, f, indent=2)

if __name__ == "__main__":
  main(sys.argv[1:])
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# pylint:disable=unused-wildcard-import,wildcard-import,invalid-name,attribute-defined-outside-init
"""Advanced tab for the TKinter GUI."""
from __future__ import print_function, unicode_literals, absolute_import

from . import controls, binding
from .layout import GridLayouter
from .tab import Tab
import sys

from core import launcher, legends_processor
from core.lnp import lnp

if sys.version_info[0] == 3:  # Alternate import names
    # pylint:disable=import-error
    from tkinter import *
    from tkinter.ttk import *
    import tkinter.messagebox as messagebox
else:
    # pylint:disable=import-error
    from Tkinter import *
    from ttk import *
    import tkMessageBox as messagebox

#pylint: disable=too-many-public-methods
class AdvancedTab(Tab):
    """Advanced tab for the TKinter GUI."""
    def create_variables(self):
        self.volume_var = StringVar()
        self.fps_var = StringVar()
        self.gps_var = StringVar()
        self.winX_var = StringVar()
        self.winY_var = StringVar()
        self.fullX_var = StringVar()
        self.fullY_var = StringVar()

    def create_controls(self):
        Grid.columnconfigure(self, 0, weight=1)
        Grid.columnconfigure(self, 1, weight=1)

        main_grid = GridLayouter(2)

        if lnp.settings.version_has_option('sound'):
            sound = controls.create_control_group(self, 'Sound')
            main_grid.add(sound)

            controls.create_option_button(
                sound, 'Sound', 'Turn game music on/off', 'sound').pack(
                    side=LEFT)
            if lnp.settings.version_has_option('volume'):
                controls.create_numeric_entry(
                    sound, self.volume_var, 'volume',
                    'Music volume (0 to 255)').pack(side=LEFT)
                Label(sound, text='/255').pack(side=LEFT)
        if lnp.settings.version_has_option('fpsCounter'):
            fps = controls.create_control_group(self, 'FPS')
            main_grid.add(fps, rowspan=2)

            controls.create_option_button(
                fps, 'FPS Counter', 'Whether or not to display your FPS',
                'fpsCounter').pack(fill=BOTH)
            if lnp.settings.version_has_option('fpsCap'):
                Label(fps, text='Calculation FPS Cap:').pack(anchor="w")
                controls.create_numeric_entry(
                    fps, self.fps_var, 'fpsCap', 'How fast the game runs').pack(
                        anchor="w")
            if lnp.settings.version_has_option('gpsCap'):
                Label(fps, text='Graphical FPS Cap:').pack(anchor="w")
                controls.create_numeric_entry(
                    fps, self.gps_var, 'gpsCap', 'How fast the game visually '
                    'updates.\nLower value may give small boost to FPS but '
                    'will be less reponsive.').pack(anchor="w")

        if lnp.settings.version_has_option('introMovie'):
            startup = controls.create_control_group(self, 'Startup')
            main_grid.add(startup)
            Grid.columnconfigure(startup, 0, weight=1)

            controls.create_option_button(
                startup, 'Intro Movie',
                'Do you want to see the beautiful ASCII intro movie?',
                'introMovie').grid(column=0, row=0, sticky="nsew")
            controls.create_option_button(
                startup, 'Windowed', 'Start windowed or fullscreen',
                'startWindowed').grid(column=0, row=1, sticky="nsew")

        resolution = controls.create_control_group(self, 'Resolution', True)
        main_grid.add(resolution, 2)

        grid = GridLayouter(2)
        grid.add(Label(resolution, text='Windowed X'))
        grid.add(Label(resolution, text='Windowed Y'))
        grid.add(controls.create_numeric_entry(
            resolution, self.winX_var, ('WINDOWEDX', 'GRAPHICS_WINDOWEDX'),
            'Horizontal resolution in '
            'windowed mode. Values <= 255 represent number of tiles, values > '
            '255 represent number of pixels.'))
        grid.add(controls.create_numeric_entry(
            resolution, self.winY_var, ('WINDOWEDY', 'GRAPHICS_WINDOWEDY'),
            'Vertical resolution in '
            'windowed mode. Values <= 255 represent number of tiles, values > '
            '255 represent number of pixels.'))
        grid.add(Label(resolution, text='Fullscreen X'))
        grid.add(Label(resolution, text='Fullscreen Y'))
        grid.add(controls.create_numeric_entry(
            resolution, self.fullX_var, ('FULLSCREENX', 'GRAPHICS_FULLSCREENX'),
            'Horizontal resolution '
            'in fullscreen. Values <= 255 represent number of tiles, values > '
            '255 represent number of pixels.'))
        grid.add(controls.create_numeric_entry(
            resolution, self.fullY_var, ('FULLSCREENY', 'GRAPHICS_FULLSCREENY'),
            'Vertical resolution in '
            'fullscreen. Values <= 255 represent number of tiles, values > 255 '
            'represent number of pixels.'))
        saverelated = controls.create_control_group(
            self, 'Save-related', True)
        main_grid.add(saverelated, 2)

        grid = GridLayouter(2)
        grid.add(controls.create_option_button(
            saverelated, 'Autosave',
            'How often the game will automatically save', 'autoSave'))
        grid.add(controls.create_option_button(
            saverelated, 'Initial Save', 'Saves as soon as you embark',
            'initialSave'))
        grid.add(controls.create_option_button(
            saverelated, 'Pause on Save', 'Pauses the game after auto-saving',
            'autoSavePause'))
        grid.add(controls.create_option_button(
            saverelated, 'Pause on Load', 'Pauses the game as soon as it loads',
            'pauseOnLoad'))
        grid.add(controls.create_option_button(
            saverelated, 'Backup Saves', 'Makes a backup of every autosave',
            'autoBackup'))
        if lnp.df_info.version >= '0.31.01':
            grid.add(controls.create_option_button(
                saverelated, 'Compress Saves', 'Whether to compress the '
                'savegames (keep this on unless you experience problems with '
                'your saves', 'compressSaves'))
        grid.add(controls.create_trigger_button(
            saverelated, 'Open Savegame Folder', 'Open the savegame folder',
            launcher.open_savegames))

        main_grid.add(Frame(self, height=30), 2)
        main_grid.add(controls.create_option_button(
            self, 'Processor Priority',
            'Adjusts the priority given to Dwarf Fortress by your OS',
            'procPriority'), 2)

        main_grid.add(controls.create_trigger_option_button(
            self, 'Close GUI on launch',
            'Whether this GUI should close when Dwarf Fortress is launched',
            self.toggle_autoclose, 'autoClose', lambda v: ('NO', 'YES')[
                lnp.userconfig.get_bool('autoClose')]), 2)

        if lnp.df_info.version >= '0.40.09':
            main_grid.add(controls.create_trigger_button(
                self, 'Process Legends Exports',
                'Compress and sort files exported from legends mode',
                self.process_legends), 2)

    @staticmethod
    def process_legends():
        """Process legends exports."""
        if not legends_processor.get_region_info():
            messagebox.showinfo('No legends exports',
                                'There were no legends exports to process.')
        else:
            messagebox.showinfo('Exports will be compressed',
                                'Maps exported from legends mode will be '
                                'converted to .png format, a compressed archive'
                                ' will be made, and files will be sorted and '
                                'moved to a subfolder.  Please wait...')
            i = legends_processor.process_legends()
            string = str(i) + ' region'
            if i > 1:
                string += 's'
            messagebox.showinfo(string + ' processed',
                                'Legends exported from ' + string +
                                ' were found and processed')

    @staticmethod
    def toggle_autoclose():
        """Toggle automatic closing of the UI when launching DF."""
        launcher.toggle_autoclose()
        binding.update()

<commit_msg>Minor formatting change<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# pylint:disable=unused-wildcard-import,wildcard-import,invalid-name,attribute-defined-outside-init
"""Advanced tab for the TKinter GUI."""
from __future__ import print_function, unicode_literals, absolute_import

from . import controls, binding
from .layout import GridLayouter
from .tab import Tab
import sys

from core import launcher, legends_processor
from core.lnp import lnp

if sys.version_info[0] == 3:  # Alternate import names
    # pylint:disable=import-error
    from tkinter import *
    from tkinter.ttk import *
    import tkinter.messagebox as messagebox
else:
    # pylint:disable=import-error
    from Tkinter import *
    from ttk import *
    import tkMessageBox as messagebox

#pylint: disable=too-many-public-methods
class AdvancedTab(Tab):
    """Advanced tab for the TKinter GUI."""
    def create_variables(self):
        self.volume_var = StringVar()
        self.fps_var = StringVar()
        self.gps_var = StringVar()
        self.winX_var = StringVar()
        self.winY_var = StringVar()
        self.fullX_var = StringVar()
        self.fullY_var = StringVar()

    def create_controls(self):
        Grid.columnconfigure(self, 0, weight=1)
        Grid.columnconfigure(self, 1, weight=1)

        main_grid = GridLayouter(2)

        if lnp.settings.version_has_option('sound'):
            sound = controls.create_control_group(self, 'Sound')
            main_grid.add(sound)

            controls.create_option_button(
                sound, 'Sound', 'Turn game music on/off', 'sound').pack(
                    side=LEFT)
            if lnp.settings.version_has_option('volume'):
                controls.create_numeric_entry(
                    sound, self.volume_var, 'volume',
                    'Music volume (0 to 255)').pack(side=LEFT)
                Label(sound, text='/255').pack(side=LEFT)
        if lnp.settings.version_has_option('fpsCounter'):
            fps = controls.create_control_group(self, 'FPS')
            main_grid.add(fps, rowspan=2)

            controls.create_option_button(
                fps, 'FPS Counter', 'Whether or not to display your FPS',
                'fpsCounter').pack(fill=BOTH)
            if lnp.settings.version_has_option('fpsCap'):
                Label(fps, text='Calculation FPS Cap:').pack(anchor="w")
                controls.create_numeric_entry(
                    fps, self.fps_var, 'fpsCap', 'How fast the game runs').pack(
                        anchor="w")
            if lnp.settings.version_has_option('gpsCap'):
                Label(fps, text='Graphical FPS Cap:').pack(anchor="w")
                controls.create_numeric_entry(
                    fps, self.gps_var, 'gpsCap', 'How fast the game visually '
                    'updates.\nLower value may give small boost to FPS but '
                    'will be less reponsive.').pack(anchor="w")

        if lnp.settings.version_has_option('introMovie'):
            startup = controls.create_control_group(self, 'Startup')
            main_grid.add(startup)
            Grid.columnconfigure(startup, 0, weight=1)

            controls.create_option_button(
                startup, 'Intro Movie',
                'Do you want to see the beautiful ASCII intro movie?',
                'introMovie').grid(column=0, row=0, sticky="nsew")
            controls.create_option_button(
                startup, 'Windowed', 'Start windowed or fullscreen',
                'startWindowed').grid(column=0, row=1, sticky="nsew")

        resolution = controls.create_control_group(self, 'Resolution', True)
        main_grid.add(resolution, 2)

        grid = GridLayouter(2)
        grid.add(Label(resolution, text='Windowed X'))
        grid.add(Label(resolution, text='Windowed Y'))
        grid.add(controls.create_numeric_entry(
            resolution, self.winX_var, ('WINDOWEDX', 'GRAPHICS_WINDOWEDX'),
            'Horizontal resolution in windowed mode. Values <= 255 represent '
            'number of tiles, values > 255 represent number of pixels.'))
        grid.add(controls.create_numeric_entry(
            resolution, self.winY_var, ('WINDOWEDY', 'GRAPHICS_WINDOWEDY'),
            'Vertical resolution in windowed mode. Values <= 255 represent '
            'number of tiles, values > 255 represent number of pixels.'))
        grid.add(Label(resolution, text='Fullscreen X'))
        grid.add(Label(resolution, text='Fullscreen Y'))
        grid.add(controls.create_numeric_entry(
            resolution, self.fullX_var, ('FULLSCREENX', 'GRAPHICS_FULLSCREENX'),
            'Horizontal resolution in fullscreen. Values <= 255 represent '
            'number of tiles, values > 255 represent number of pixels.'))
        grid.add(controls.create_numeric_entry(
            resolution, self.fullY_var, ('FULLSCREENY', 'GRAPHICS_FULLSCREENY'),
            'Vertical resolution in fullscreen. Values <= 255 represent '
            'number of tiles, values > 255 represent number of pixels.'))
        saverelated = controls.create_control_group(
            self, 'Save-related', True)
        main_grid.add(saverelated, 2)

        grid = GridLayouter(2)
        grid.add(controls.create_option_button(
            saverelated, 'Autosave',
            'How often the game will automatically save', 'autoSave'))
        grid.add(controls.create_option_button(
            saverelated, 'Initial Save', 'Saves as soon as you embark',
            'initialSave'))
        grid.add(controls.create_option_button(
            saverelated, 'Pause on Save', 'Pauses the game after auto-saving',
            'autoSavePause'))
        grid.add(controls.create_option_button(
            saverelated, 'Pause on Load', 'Pauses the game as soon as it loads',
            'pauseOnLoad'))
        grid.add(controls.create_option_button(
            saverelated, 'Backup Saves', 'Makes a backup of every autosave',
            'autoBackup'))
        if lnp.df_info.version >= '0.31.01':
            grid.add(controls.create_option_button(
                saverelated, 'Compress Saves', 'Whether to compress the '
                'savegames (keep this on unless you experience problems with '
                'your saves', 'compressSaves'))
        grid.add(controls.create_trigger_button(
            saverelated, 'Open Savegame Folder', 'Open the savegame folder',
            launcher.open_savegames))

        main_grid.add(Frame(self, height=30), 2)
        main_grid.add(controls.create_option_button(
            self, 'Processor Priority',
            'Adjusts the priority given to Dwarf Fortress by your OS',
            'procPriority'), 2)

        main_grid.add(controls.create_trigger_option_button(
            self, 'Close GUI on launch',
            'Whether this GUI should close when Dwarf Fortress is launched',
            self.toggle_autoclose, 'autoClose', lambda v: ('NO', 'YES')[
                lnp.userconfig.get_bool('autoClose')]), 2)

        if lnp.df_info.version >= '0.40.09':
            main_grid.add(controls.create_trigger_button(
                self, 'Process Legends Exports',
                'Compress and sort files exported from legends mode',
                self.process_legends), 2)

    @staticmethod
    def process_legends():
        """Process legends exports."""
        if not legends_processor.get_region_info():
            messagebox.showinfo('No legends exports',
                                'There were no legends exports to process.')
        else:
            messagebox.showinfo('Exports will be compressed',
                                'Maps exported from legends mode will be '
                                'converted to .png format, a compressed archive'
                                ' will be made, and files will be sorted and '
                                'moved to a subfolder.  Please wait...')
            i = legends_processor.process_legends()
            string = str(i) + ' region'
            if i > 1:
                string += 's'
            messagebox.showinfo(string + ' processed',
                                'Legends exported from ' + string +
                                ' were found and processed')

    @staticmethod
    def toggle_autoclose():
        """Toggle automatic closing of the UI when launching DF."""
        launcher.toggle_autoclose()
        binding.update()

<|endoftext|>


<commit_before># This code is supporting material for the book
# Building Machine Learning Systems with Python
# by Willi Richert and Luis Pedro Coelho
# published by PACKT Publishing
#
# It is made available under the MIT License

from __future__ import print_function
from all_correlations import all_correlations
import numpy as np
from scipy import sparse
from load_ml100k import load
reviews = load()

def estimate_user(user, rest):
    bu = user > 0
    br = rest > 0
    ws = all_correlations(bu, br)
    selected = ws.argsort()[-100:]
    estimates = rest[selected].mean(0)
    estimates /= (.1 + br[selected].mean(0))
    return estimates

def train_test(user, rest):
    estimates = estimate_user(user, rest)
    bu = user > 0
    br = rest > 0
    err = estimates[bu] - user[bu]
    null = rest.mean(0)
    null /= (.1 + br.mean(0))
    nerr = null[bu] - user[bu]
    return np.dot(err, err), np.dot(nerr, nerr)

def cross_validate_all():
    err = []
    for i in xrange(reviews.shape[0]):
        err.append(
            train_test(reviews[i], np.delete(reviews, i, 0))
        )
    revs = (reviews > 0).sum(1)
    err = np.array(err)
    rmse = np.sqrt(err / revs[:, None])
    print(np.mean(rmse, 0))
    print(np.mean(rmse[revs > 60], 0))

def all_estimates(reviews):
    reviews = reviews.toarray()
    estimates = np.zeros_like(reviews)
    for i in xrange(reviews.shape[0]):
        estimates[i] = estimate_user(reviews[i], np.delete(reviews, i, 0))
    return estimates
<commit_msg>RFCT Only load data when necessary<commit_after># This code is supporting material for the book
# Building Machine Learning Systems with Python
# by Willi Richert and Luis Pedro Coelho
# published by PACKT Publishing
#
# It is made available under the MIT License

from __future__ import print_function
from all_correlations import all_correlations
import numpy as np
from load_ml100k import load
def estimate_user(user, rest):
    bu = user > 0
    br = rest > 0
    ws = all_correlations(bu, br)
    selected = ws.argsort()[-100:]
    estimates = rest[selected].mean(0)
    estimates /= (.1 + br[selected].mean(0))
    return estimates

def train_test(user, rest):
    estimates = estimate_user(user, rest)
    bu = user > 0
    br = rest > 0
    err = estimates[bu] - user[bu]
    null = rest.mean(0)
    null /= (.1 + br.mean(0))
    nerr = null[bu] - user[bu]
    return np.dot(err, err), np.dot(nerr, nerr)

def all_estimates(reviews):
    reviews = reviews.toarray()
    estimates = np.zeros_like(reviews)
    for i in xrange(reviews.shape[0]):
        estimates[i] = estimate_user(reviews[i], np.delete(reviews, i, 0))
    return estimates

def main():
    reviews = load()
    reviews = reviews.toarray()

    err = []
    for i in xrange(reviews.shape[0]):
        err.append(
            train_test(reviews[i], np.delete(reviews, i, 0))
        )
    revs = (reviews > 0).sum(1)
    err = np.array(err)
    rmse = np.sqrt(err / revs[:, None])
    print(np.mean(rmse, 0))
    print(np.mean(rmse[revs > 60], 0))

if __name__ == '__main__':
    main()
<|endoftext|>