<commit_before>from dataclasses import dataclass

from boltons.cacheutils import cachedproperty

from cltkv1.core.data_types import Doc, Process, Word
from cltkv1.dependency.stanford import StanfordNLPWrapper

@dataclass
class StanfordNLPProcess(Process):
    """A ``Process`` type to capture everything
    that the ``stanfordnlp`` project can do for a
    given language.

    .. note::
        ``stanfordnlp`` has only partial functionality available for some languages.

    >>> from cltkv1.dependency.stanford import StanfordNLPProcess
    >>> from cltkv1.utils.example_texts import get_example_text
    >>> process_stanford = StanfordNLPProcess(input_doc=Doc(raw=get_example_text("lat")), language="lat")
    >>> isinstance(process_stanford, StanfordNLPProcess)
    True
    >>> from stanfordnlp.pipeline.doc import Document
    >>> process_stanford.run()
    >>> isinstance(process_stanford.output_doc.stanfordnlp_doc, Document)
    True
    """

    language: str = None

    @cachedproperty
    def algorithm(self):
        return StanfordNLPWrapper.get_nlp(language=self.language)

    def run(self):
        tmp_doc = self.input_doc
        stanfordnlp_wrapper = self.algorithm
        stanfordnlp_doc = stanfordnlp_wrapper.parse(tmp_doc.raw)
        cltk_words = self.stanfordnlp_to_cltk_word_type(stanfordnlp_doc)
        tmp_doc.words = cltk_words
        tmp_doc.stanfordnlp_doc = stanfordnlp_doc
        self.output_doc = tmp_doc

    """
    @cachedproperty
    def algorithm(self):
        return FastTextEmbeddings(iso_code=self.language)

    def run(self):
        tmp_doc = self.input_doc
        embedding_length = None
        embeddings_obj = self.algorithm
        for index, word_obj in enumerate(tmp_doc.words):
            if not embedding_length:
                embedding_length = embeddings_obj.get_embedding_length()
            word_embedding = embeddings_obj.get_word_vector(word=word_obj.string)
            if not isinstance(word_embedding, np.ndarray):
                word_embedding = np.zeros([embedding_length])
            word_obj.embedding = word_embedding
            tmp_doc.words[index] = word_obj
        self.output_doc = tmp_doc
    """

    '''
    def __init__(self, input_doc, language):
        """Constructor."""
        super().__init__(input_doc=input_doc, language=language)
        self.stanfordnlp_wrapper = StanfordNLPWrapper.get_nlp(language=self.language)

    def algorithm(self, doc):
        stanfordnlp_doc = self.stanfordnlp_wrapper.parse(doc.raw)
        cltk_words = StanfordNLPProcess.stanfordnlp_to_cltk_word_type(stanfordnlp_doc)
        doc.words = cltk_words
        doc.stanfordnlp_doc = stanfordnlp_doc

        return doc
    '''

    @staticmethod
    def stanfordnlp_to_cltk_word_type(stanfordnlp_doc):

        """Take an entire ``stanfordnlp`` document, extract
        each word, and encode it in the way expected by
        the CLTK's ``Word`` type.

        >>> from cltkv1.dependency.stanford import StanfordNLPProcess
        >>> from cltkv1.utils.example_texts import get_example_text
        >>> process_stanford = StanfordNLPProcess(input_doc=Doc(raw=get_example_text("lat")), language="lat")
        >>> process_stanford.run()
        >>> cltk_words = process_stanford.output_doc.words
        >>> isinstance(cltk_words, list)
        True
        >>> isinstance(cltk_words[0], Word)
        True
        >>> cltk_words[0]
        Word(index_char_start=None, index_char_stop=None, index_token=1, index_sentence=0, string='Gallia', pos='A1|grn1|casA|gen2|stAM', lemma='aallius', scansion=None, xpos='A1|grn1|casA|gen2|stAM', upos='NOUN', dependency_relation='nsubj', governor=Word(index_char_start=None, index_char_stop=None, index_token=4, index_sentence=0, string='divisa', pos='L2', lemma='divido', scansion=None, xpos='L2', upos='VERB', dependency_relation='root', governor=None, parent=None, features={'Aspect': 'Perf', 'Case': 'Nom', 'Degree': 'Pos', 'Gender': 'Fem', 'Number': 'Sing', 'Tense': 'Past', 'VerbForm': 'Part', 'Voice': 'Pass'}, embedding=None), parent=None, features={'Case': 'Nom', 'Degree': 'Pos', 'Gender': 'Fem', 'Number': 'Sing'}, embedding=None)
        """
        words_list = list()

        for sentence_index, sentence in enumerate(stanfordnlp_doc.sentences):
            sent_words = dict()
            indices = list()

            for token_index, token in enumerate(sentence.tokens):
                stanfordnlp_word = token.words[0]
                cltk_word = Word(
                    index_token=int(stanfordnlp_word.index),  # same as ``token.index``
                    index_sentence=sentence_index,
                    string=stanfordnlp_word.text,  # same as ``token.text``
                    pos=stanfordnlp_word.pos,
                    xpos=stanfordnlp_word.xpos,
                    upos=stanfordnlp_word.upos,
                    lemma=stanfordnlp_word.lemma,
                    dependency_relation=stanfordnlp_word.dependency_relation,
                    features={}
                    if stanfordnlp_word.feats == "_"
                    else dict(
                        [f.split("=") for f in stanfordnlp_word.feats.split("|")]
                    ),
                )
                sent_words[cltk_word.index_token] = cltk_word
                indices.append(
                    (
                        int(stanfordnlp_word.governor),
                        int(stanfordnlp_word.parent_token.index),
                    )
                )
                words_list.append(cltk_word)

            for i, cltk_word in enumerate(sent_words.values()):
                (governor_index, parent_index) = indices[i]
                cltk_word.governor = (
                    sent_words[governor_index] if governor_index > 0 else None
                )
                if cltk_word.index_token != sent_words[parent_index].index_token:
                    cltk_word.parent = sent_words[parent_index]

        return words_list

@dataclass
class GreekStanfordNLPProcess(StanfordNLPProcess):
    language: str = "grc"
    description: str = "Default process for StanfordNLP for the Ancient Greek language."

@dataclass
class LatinStanfordNLPProcess(StanfordNLPProcess):
    language: str = "lat"
    description: str = "Default process for StanfordNLP for the Latin language."

@dataclass
class OCSStanfordNLPProcess(StanfordNLPProcess):
    language: str = "chu"
    description: str = "Default process for StanfordNLP for the Old Church Slavonic language."

@dataclass
class OldFrenchStanfordNLPProcess(StanfordNLPProcess):
    language: str = "fro"
    description: str = "Default process for StanfordNLP for the Old French language."

@dataclass
class GothicStanfordNLPProcess(StanfordNLPProcess):
    language: str = "got"
    description: str = "Default process for StanfordNLP for the Gothic language."
<commit_msg>make pass all doctests in dependency.processes<commit_after>from dataclasses import dataclass

from boltons.cacheutils import cachedproperty

from cltkv1.core.data_types import Doc, Process, Word
from cltkv1.dependency.stanford import StanfordNLPWrapper

@dataclass
class StanfordNLPProcess(Process):
    """A ``Process`` type to capture everything
    that the ``stanfordnlp`` project can do for a
    given language.

    .. note::
        ``stanfordnlp`` has only partial functionality available for some languages.

    >>> from cltkv1.dependency.processes import StanfordNLPProcess
    >>> from cltkv1.utils.example_texts import get_example_text
    >>> process_stanford = StanfordNLPProcess(input_doc=Doc(raw=get_example_text("lat")), language="lat")
    >>> isinstance(process_stanford, StanfordNLPProcess)
    True
    >>> from stanfordnlp.pipeline.doc import Document
    >>> process_stanford.run()
    >>> isinstance(process_stanford.output_doc.stanfordnlp_doc, Document)
    True
    """

    language: str = None

    @cachedproperty
    def algorithm(self):
        return StanfordNLPWrapper.get_nlp(language=self.language)

    def run(self):
        tmp_doc = self.input_doc
        stanfordnlp_wrapper = self.algorithm
        stanfordnlp_doc = stanfordnlp_wrapper.parse(tmp_doc.raw)
        cltk_words = self.stanfordnlp_to_cltk_word_type(stanfordnlp_doc)
        tmp_doc.words = cltk_words
        tmp_doc.stanfordnlp_doc = stanfordnlp_doc
        self.output_doc = tmp_doc

    @staticmethod
    def stanfordnlp_to_cltk_word_type(stanfordnlp_doc):

        """Take an entire ``stanfordnlp`` document, extract
        each word, and encode it in the way expected by
        the CLTK's ``Word`` type.

        >>> from cltkv1.dependency.processes import StanfordNLPProcess
        >>> from cltkv1.utils.example_texts import get_example_text
        >>> process_stanford = StanfordNLPProcess(input_doc=Doc(raw=get_example_text("lat")), language="lat")
        >>> process_stanford.run()
        >>> cltk_words = process_stanford.output_doc.words
        >>> isinstance(cltk_words, list)
        True
        >>> isinstance(cltk_words[0], Word)
        True
        >>> cltk_words[0]
        Word(index_char_start=None, index_char_stop=None, index_token=1, index_sentence=0, string='Gallia', pos='A1|grn1|casA|gen2|stAM', lemma='aallius', scansion=None, xpos='A1|grn1|casA|gen2|stAM', upos='NOUN', dependency_relation='nsubj', governor=Word(index_char_start=None, index_char_stop=None, index_token=4, index_sentence=0, string='divisa', pos='L2', lemma='divido', scansion=None, xpos='L2', upos='VERB', dependency_relation='root', governor=None, parent=None, features={'Aspect': 'Perf', 'Case': 'Nom', 'Degree': 'Pos', 'Gender': 'Fem', 'Number': 'Sing', 'Tense': 'Past', 'VerbForm': 'Part', 'Voice': 'Pass'}, embedding=None), parent=None, features={'Case': 'Nom', 'Degree': 'Pos', 'Gender': 'Fem', 'Number': 'Sing'}, embedding=None)
        """
        words_list = list()

        for sentence_index, sentence in enumerate(stanfordnlp_doc.sentences):
            sent_words = dict()
            indices = list()

            for token_index, token in enumerate(sentence.tokens):
                stanfordnlp_word = token.words[0]
                cltk_word = Word(
                    index_token=int(stanfordnlp_word.index),  # same as ``token.index``
                    index_sentence=sentence_index,
                    string=stanfordnlp_word.text,  # same as ``token.text``
                    pos=stanfordnlp_word.pos,
                    xpos=stanfordnlp_word.xpos,
                    upos=stanfordnlp_word.upos,
                    lemma=stanfordnlp_word.lemma,
                    dependency_relation=stanfordnlp_word.dependency_relation,
                    features={}
                    if stanfordnlp_word.feats == "_"
                    else dict(
                        [f.split("=") for f in stanfordnlp_word.feats.split("|")]
                    ),
                )
                sent_words[cltk_word.index_token] = cltk_word
                indices.append(
                    (
                        int(stanfordnlp_word.governor),
                        int(stanfordnlp_word.parent_token.index),
                    )
                )
                words_list.append(cltk_word)

            for i, cltk_word in enumerate(sent_words.values()):
                (governor_index, parent_index) = indices[i]
                cltk_word.governor = (
                    sent_words[governor_index] if governor_index > 0 else None
                )
                if cltk_word.index_token != sent_words[parent_index].index_token:
                    cltk_word.parent = sent_words[parent_index]

        return words_list

@dataclass
class GreekStanfordNLPProcess(StanfordNLPProcess):
    language: str = "grc"
    description: str = "Default process for StanfordNLP for the Ancient Greek language."

@dataclass
class LatinStanfordNLPProcess(StanfordNLPProcess):
    language: str = "lat"
    description: str = "Default process for StanfordNLP for the Latin language."

@dataclass
class OCSStanfordNLPProcess(StanfordNLPProcess):
    language: str = "chu"
    description: str = "Default process for StanfordNLP for the Old Church Slavonic language."

@dataclass
class OldFrenchStanfordNLPProcess(StanfordNLPProcess):
    language: str = "fro"
    description: str = "Default process for StanfordNLP for the Old French language."

@dataclass
class GothicStanfordNLPProcess(StanfordNLPProcess):
    language: str = "got"
    description: str = "Default process for StanfordNLP for the Gothic language."
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
"""
This script is to extract sift features from object data
distributed by Robot Learning Lab, UC Berkeley.
Object data is available from here::

    * http://rll.berkeley.edu/amazon_picking_challenge/

Usage
-----
1. Download dataset to data dir, and extract it.
2. Execute following::

    $ roscore
    $ rosrun imagesift imagesift
    $ rosrun extract_sift_from_objdata.py _object:=oreo_mega_stuf

Attention
---------
You should change dirname for following items manually::

    * kygen_squeakin_eggs_plush_puppies  -> kyjen_squeakin_eggs_plush_puppies
    * rollodex_mesh_collection_jumbo_pencil_cup -> rolodex_jumbo_pencil_cup

"""
import os
import sys
import collections
import gzip
import cPickle as pickle

import cv2
import numpy as np
import yaml
import progressbar

import rospy
from sensor_msgs.msg import CameraInfo
from posedetection_msgs.srv import Feature0DDetect
from posedetection_msgs.msg import ImageFeature0D

from sift_matcher_oneimg import SiftMatcherOneImage

def get_train_imgpaths(obj_name):
    """Find train image paths from data/obj_name"""
    dirname = os.path.dirname(os.path.abspath(__file__))
    obj_dir = os.path.join(dirname, '../data/', obj_name)
    if not os.path.exists(obj_dir):
        rospy.logwarn('Object data does not exists: {}'.format(obj_name))
        return
    os.chdir(obj_dir)
    imgpaths = []
    for imgfile in os.listdir('.'):
        if not imgfile.endswith('.jpg'):
            continue
        raw_path = os.path.join(obj_dir, imgfile)
        maskfile = os.path.splitext(imgfile)[0] + '_mask.pbm'
        mask_path = os.path.join(obj_dir, 'masks', maskfile)
        imgpaths.append((raw_path, mask_path))
    os.chdir(dirname)
    return imgpaths

def save_siftdata(siftdata, obj_name):
    """Save sift data to data/siftdata/{obj_name}.pkl.gz"""
    dirname = os.path.dirname(os.path.abspath(__file__))
    siftdata_dir = os.path.join(dirname, '../data/siftdata')
    if not os.path.exists(siftdata_dir):
        os.mkdir(siftdata_dir)
    filename = os.path.join(siftdata_dir, obj_name+'.pkl.gz')
    with gzip.open(filename, 'wb') as f:
        pickle.dump(siftdata, f)

def extract_sift_from_objdata(obj_name):
    """Extract sift data from object images"""
    positions = []
    descriptors = []
    imgpaths = get_train_imgpaths(obj_name)
    if imgpaths is None:
        return   # skip if img does not exists
    progress = progressbar.ProgressBar(widgets=['{o}: '.format(o=obj_name),
        progressbar.Bar(), progressbar.Percentage(), ' ', progressbar.ETA()])
    for raw_path, mask_path in progress(imgpaths):
        raw_img = cv2.imread(raw_path)
        mask_img = cv2.imread(mask_path)
        train_img = cv2.add(mask_img, raw_img)
        train_features = SiftMatcherOneImage.imgsift_client(train_img)
        train_pos = np.array(train_features.positions)
        train_des = np.array(train_features.descriptors)
        positions.append(train_pos)
        descriptors.append(train_des)
    positions, descriptors = map(np.array, [positions, descriptors])
    siftdata = dict(positions=positions, descriptors=descriptors)
    # save sift data
    save_siftdata(siftdata, obj_name)

def main():
    rospy.init_node('extract_sift_from_objdata')
    pub = rospy.Publisher('/camera_info', CameraInfo, queue_size=1)
    pub.publish()  # to enable imagesift service

    dirname = os.path.dirname(os.path.abspath(__file__))
    ymlfile = os.path.join(dirname, '../data/object_list.yml')
    all_objects = yaml.load(open(ymlfile))

    obj_names = rospy.get_param('~object',
                                'oreo_mega_stuf,safety_works_safety_glasses')
    obj_names = obj_names.split(',')
    if len(obj_names) == 1 and obj_names[0] == 'all':
        obj_names = all_objects
    rospy.loginfo('objects: {obj}'.format(obj=obj_names))

    for obj_name in obj_names:
        if obj_name not in all_objects:
            rospy.logwarn('Unknown object, skipping: {}'.format(obj_name))
            continue
        extract_sift_from_objdata(obj_name)

if __name__ == '__main__':
    print(__doc__)
    main()

<commit_msg>match changed class name<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
"""
This script is to extract sift features from object data
distributed by Robot Learning Lab, UC Berkeley.
Object data is available from here::

    * http://rll.berkeley.edu/amazon_picking_challenge/

Usage
-----
1. Download dataset to data dir, and extract it.
2. Execute following::

    $ roscore
    $ rosrun imagesift imagesift
    $ rosrun extract_sift_from_objdata.py _object:=oreo_mega_stuf

Attention
---------
You should change dirname for following items manually::

    * kygen_squeakin_eggs_plush_puppies  -> kyjen_squeakin_eggs_plush_puppies
    * rollodex_mesh_collection_jumbo_pencil_cup -> rolodex_jumbo_pencil_cup

"""
import os
import sys
import collections
import gzip
import cPickle as pickle

import cv2
import numpy as np
import yaml
import progressbar

import rospy
from sensor_msgs.msg import CameraInfo
from posedetection_msgs.srv import Feature0DDetect
from posedetection_msgs.msg import ImageFeature0D

from sift_matcher_oneimg import SiftMatcherOneImg

def get_train_imgpaths(obj_name):
    """Find train image paths from data/obj_name"""
    dirname = os.path.dirname(os.path.abspath(__file__))
    obj_dir = os.path.join(dirname, '../data/', obj_name)
    if not os.path.exists(obj_dir):
        rospy.logwarn('Object data does not exists: {}'.format(obj_name))
        return
    os.chdir(obj_dir)
    imgpaths = []
    for imgfile in os.listdir('.'):
        if not imgfile.endswith('.jpg'):
            continue
        raw_path = os.path.join(obj_dir, imgfile)
        maskfile = os.path.splitext(imgfile)[0] + '_mask.pbm'
        mask_path = os.path.join(obj_dir, 'masks', maskfile)
        imgpaths.append((raw_path, mask_path))
    os.chdir(dirname)
    return imgpaths

def save_siftdata(siftdata, obj_name):
    """Save sift data to data/siftdata/{obj_name}.pkl.gz"""
    dirname = os.path.dirname(os.path.abspath(__file__))
    siftdata_dir = os.path.join(dirname, '../data/siftdata')
    if not os.path.exists(siftdata_dir):
        os.mkdir(siftdata_dir)
    filename = os.path.join(siftdata_dir, obj_name+'.pkl.gz')
    with gzip.open(filename, 'wb') as f:
        pickle.dump(siftdata, f)

def extract_sift_from_objdata(obj_name):
    """Extract sift data from object images"""
    positions = []
    descriptors = []
    imgpaths = get_train_imgpaths(obj_name)
    if imgpaths is None:
        return   # skip if img does not exists
    progress = progressbar.ProgressBar(widgets=['{o}: '.format(o=obj_name),
        progressbar.Bar(), progressbar.Percentage(), ' ', progressbar.ETA()])
    for raw_path, mask_path in progress(imgpaths):
        raw_img = cv2.imread(raw_path)
        mask_img = cv2.imread(mask_path)
        train_img = cv2.add(mask_img, raw_img)
        train_features = SiftMatcherOneImg.imgsift_client(train_img)
        train_pos = np.array(train_features.positions)
        train_des = np.array(train_features.descriptors)
        positions.append(train_pos)
        descriptors.append(train_des)
    positions, descriptors = map(np.array, [positions, descriptors])
    siftdata = dict(positions=positions, descriptors=descriptors)
    # save sift data
    save_siftdata(siftdata, obj_name)

def main():
    rospy.init_node('extract_sift_from_objdata')
    pub = rospy.Publisher('/camera_info', CameraInfo, queue_size=1)
    pub.publish()  # to enable imagesift service

    dirname = os.path.dirname(os.path.abspath(__file__))
    ymlfile = os.path.join(dirname, '../data/object_list.yml')
    all_objects = yaml.load(open(ymlfile))

    obj_names = rospy.get_param('~object',
                                'oreo_mega_stuf,safety_works_safety_glasses')
    obj_names = obj_names.split(',')
    if len(obj_names) == 1 and obj_names[0] == 'all':
        obj_names = all_objects
    rospy.loginfo('objects: {obj}'.format(obj=obj_names))

    for obj_name in obj_names:
        if obj_name not in all_objects:
            rospy.logwarn('Unknown object, skipping: {}'.format(obj_name))
            continue
        extract_sift_from_objdata(obj_name)

if __name__ == '__main__':
    print(__doc__)
    main()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
stream_power_smooth_threshold.py: Defines the StreamPowerSmoothThresholdEroder,
which is a version of the FastscapeEroder (derived from it).

StreamPowerSmoothThresholdEroder uses a mathematically smooth threshold
formulation, rather than one with a singularity. The erosion rate is defined as
follows:

$\omega = K A^m S$

$E = \omega - \omega_c \left[ 1 - \exp ( -\omega / \omega_c ) \right]$

Created on Sat Nov 26 08:36:49 2016

@author: gtucker
"""

if __name__ == '__main__':
    from landlab.components import FastscapeEroder
else:
    from .fastscape_stream_power import FastscapeEroder
import numpy as np
from scipy.optimize import newton

UNDEFINED_INDEX = -1

def new_elev(x, a, b, c, d):
    """Equation for elevation of a node at timestep t+1.

    Parameters
    ----------
    x : float
        Value of new elevation
    a : float
        Parameter = K A^m dt / L
    b : float
        Parameter = K A^m / wc L
    c : float
        Elevation of downstream node, z_j
    d : float
        z(t) + a z_j + wc
    """
    return x * (1.0 + a) + np.exp(-b * (x - c)) - d

def new_elev_prime(x, a, b, c, d=0.0):
    """Equation for elevation of a node at timestep t+1.

    Parameters
    ----------
    x : float
        Value of new elevation
    a : float
        Parameter = K A^m dt / L
    b : float
        Parameter = K A^m / wc L
    c : float
        Elevation of downstream node, z_j
    d : n/a
        Placeholder; not used
    """
    return (1.0 + a) - b * np.exp(-b * (x - c))

def new_elev_prime2(x, a, b, c, d=0.0):
    """Equation for elevation of a node at timestep t+1.

    Parameters
    ----------
    x : float
        Value of new elevation
    a : n/a
        Placeholder; not used
    b : float
        Parameter = K A^m / wc L
    c : float
        Elevation of downstream node, z_j
    d : n/a
        Placeholder; not used
    """
    return b * b * np.exp(-b * (x - c))

class StreamPowerSmoothThresholdEroder(FastscapeEroder):
    """Stream erosion component with smooth threshold function.

    Parameters
    ----------
    grid : ModelGrid
        A grid.
    K_sp : float, array, or field name
        K in the stream power equation (units vary with other parameters).
    m_sp : float, optional
        m in the stream power equation (power on drainage area).
    n_sp : float, optional, ~ 0.5<n_sp<4.
        n in the stream power equation (power on slope). NOTE: NOT PRESENTLY
        HONORED BY StreamPowerSmoothThresholdEroder (TODO)
    threshold_sp : float (TODO: array, or field name)
        The threshold stream power.
    rainfall_intensity : float; optional
        NOT PRESENTLY HONORED (TODO)

    Examples
    --------
    >>> from landlab import RasterModelGrid
    >>> rg = RasterModelGrid((3, 4), 1.0)
    >>> rg.set_closed_boundaries_at_grid_edges(False, True, True, True)
    >>> z = rg.add_zeros('node', 'topographic__elevation')
    >>> z[5] = 2.0
    >>> z[6] = 1.0
    >>> from landlab.components import FlowRouter
    >>> fr = FlowRouter(rg, method='D4')
    >>> fr.run_one_step()
    >>> from landlab.components import StreamPowerSmoothThresholdEroder
    >>> sp = StreamPowerSmoothThresholdEroder(rg, K_sp=1.0)
    >>> sp.thresholds
    1.0
    >>> sp.run_one_step(dt=1.0)
    >>> import numpy as np
    >>> np.round(z, 3)
    array([ 0.   ,  0.   ,  0.   ,  0.   ,  0.   ,  1.846,  0.667,  0.   ,
            0.   ,  0.   ,  0.   ,  0.   ])
    """

    def __init__(self, grid, K_sp=None, m_sp=0.5, n_sp=1., threshold_sp=1.,
                 rainfall_intensity=1., **kwargs):
        """Initialize StreamPowerSmoothThresholdEroder."""

        # Call base-class init
        super(StreamPowerSmoothThresholdEroder,
              self).__init__(grid, K_sp=K_sp, m_sp=m_sp, n_sp=n_sp,
                             threshold_sp=threshold_sp,
                             rainfall_intensity=rainfall_intensity, **kwargs)

        # Arrays with parameters for use in implicit solver
        self.beta = grid.empty(at='node')
        self.delta = grid.empty(at='node')

    def run_one_step(self, dt, flooded_nodes=None, runoff_rate=None, **kwds):
        """Run one forward iteration of duration dt.

        Parameters
        ----------
        dt : float
            Time step size
        flooded_nodes : ndarray of int (optional)
            Indices of nodes in lakes/depressions
        runoff_rate : (not used yet)
            (to be added later)

        Examples
        --------
        >>> from landlab import RasterModelGrid
        >>> rg = RasterModelGrid((3, 3), 1.0)
        >>> rg.set_closed_boundaries_at_grid_edges(False, True, True, True)
        >>> z = rg.add_zeros('node', 'topographic__elevation')
        >>> z[4] = 1.0
        >>> from landlab.components import FlowRouter
        >>> fr = FlowRouter(rg, method='D4')
        >>> fr.run_one_step()
        >>> from landlab.components import StreamPowerSmoothThresholdEroder
        >>> sp = StreamPowerSmoothThresholdEroder(rg, K_sp=1.0)
        >>> sp.run_one_step(dt=1.0)
        >>> sp.alpha
        array([ 0.,  0.,  0.,  0.,  1.,  0.,  0.,  0.,  0.])
        >>> sp.beta
        array([ 0.,  0.,  0.,  0.,  1.,  0.,  0.,  0.,  0.])
        >>> sp.delta
        array([ 0.,  0.,  0.,  0.,  2.,  0.,  0.,  0.,  0.])
        """

        # Set up needed arrays
        #
        # Get shorthand for elevation field ("z"), and for up-to-downstream
        # ordered array of node IDs ("upstream_order_IDs")
        upstream_order_IDs = self._grid['node']['flow__upstream_node_order']
        z = self._grid['node']['topographic__elevation']
        flow_receivers = self._grid['node']['flow__receiver_node']

        # Get an array of flow-link length for each node that has a defined
        # receiver (i.e., that drains to another node).
        defined_flow_receivers = np.not_equal(self._grid['node'][
            'flow__link_to_receiver_node'], UNDEFINED_INDEX)
        flow_link_lengths = self._grid._length_of_link_with_diagonals[
            self._grid['node']['flow__link_to_receiver_node'][
                defined_flow_receivers]]

        # (Later on, add functionality for a runoff rate, or discharge, or
        # variable K)

        # Set up alpha, beta, delta arrays
        #
        #   First, compute drainage area raised to the power m.
        np.power(self._grid['node']['drainage_area'], self.m,
                 out=self.A_to_the_m)

        #   Alpha
        self.alpha[defined_flow_receivers] = (self.K * dt
            * self.A_to_the_m[defined_flow_receivers] / flow_link_lengths)

        #   Beta
        self.beta[defined_flow_receivers] = ((self.K
            * self.A_to_the_m[defined_flow_receivers] )
            / (self.thresholds * flow_link_lengths))

        #   Gamma
        self.delta[defined_flow_receivers] = (z[defined_flow_receivers]
            + (self.alpha[defined_flow_receivers]
               * z[flow_receivers[defined_flow_receivers]])
            + self.thresholds)

        # Iterate over nodes from downstream to upstream, using scipy's
        # 'newton' function to find new elevation at each node in turn.
        for node in upstream_order_IDs:
            if defined_flow_receivers[node]:
                z[node] = newton(new_elev, z[node],
                                 fprime=new_elev_prime,
                                 args=(self.alpha[node], self.beta[node],
                                       z[flow_receivers[node]],
                                       self.delta[node]),
                                 fprime2=new_elev_prime2)

        # TODO: handle case self.thresholds = 0

#if __name__ == '__main__':
#    from landlab import RasterModelGrid
#    rg = RasterModelGrid((3, 4), 1.0)
#    z = rg.add_zeros('node', 'topographic__elevation')
#    rg.set_closed_boundaries_at_grid_edges(False, True, True, True)
#    z[5] = 2.0
#    z[6] = 1.0
#    from landlab.components import FlowRouter
#    fr = FlowRouter(rg, method='D4')
#    fr.run_one_step()
#    sp = StreamPowerSmoothThresholdEroder(rg, K_sp=1.0)
#    sp.run_one_step(dt=1.0)
#    print(z)<commit_msg>correct error in solver<commit_after># -*- coding: utf-8 -*-
"""
stream_power_smooth_threshold.py: Defines the StreamPowerSmoothThresholdEroder,
which is a version of the FastscapeEroder (derived from it).

StreamPowerSmoothThresholdEroder uses a mathematically smooth threshold
formulation, rather than one with a singularity. The erosion rate is defined as
follows:

$\omega = K A^m S$

$E = \omega - \omega_c \left[ 1 - \exp ( -\omega / \omega_c ) \right]$

Created on Sat Nov 26 08:36:49 2016

@author: gtucker
"""

if __name__ == '__main__':
    from landlab.components import FastscapeEroder
else:
    from .fastscape_stream_power import FastscapeEroder
import numpy as np
from scipy.optimize import newton

UNDEFINED_INDEX = -1

def new_elev(x, a, b, c, d, e):
    """Equation for elevation of a node at timestep t+1.

    Parameters
    ----------
    x : float
        Value of new elevation
    a : float
        Parameter = K A^m dt / L (nondimensional)
    b : float
        Elevation of downstream node, z_j
    c : float
        Parameter = omega_c * dt (dimension of L, because omega_c [=] L/T)
    d : float
        Parameter = K A^m / (L * wc) [=] L^{-1} (so d * z [=] [-])
    e : float
        z(t) + a z_j + (wc * dt)
    """
    return x * (1.0 + a) + c * np.exp(-d * (x - b)) - e

def new_elev_prime(x, a, b, c, d=0.0):
    """Equation for elevation of a node at timestep t+1.

    Parameters
    ----------
    x : float
        Value of new elevation
    a : float
        Parameter = K A^m dt / L
    b : float
        Parameter = K A^m / wc L
    c : float
        Elevation of downstream node, z_j
    d : n/a
        Placeholder; not used
    """
    return (1.0 + a) - b * np.exp(-b * (x - c))

def new_elev_prime2(x, a, b, c, d=0.0):
    """Equation for elevation of a node at timestep t+1.

    Parameters
    ----------
    x : float
        Value of new elevation
    a : n/a
        Placeholder; not used
    b : float
        Parameter = K A^m / wc L
    c : float
        Elevation of downstream node, z_j
    d : n/a
        Placeholder; not used
    """
    return b * b * np.exp(-b * (x - c))

class StreamPowerSmoothThresholdEroder(FastscapeEroder):
    """Stream erosion component with smooth threshold function.

    Parameters
    ----------
    grid : ModelGrid
        A grid.
    K_sp : float, array, or field name
        K in the stream power equation (units vary with other parameters).
    m_sp : float, optional
        m in the stream power equation (power on drainage area).
    n_sp : float, optional, ~ 0.5<n_sp<4.
        n in the stream power equation (power on slope). NOTE: NOT PRESENTLY
        HONORED BY StreamPowerSmoothThresholdEroder (TODO)
    threshold_sp : float (TODO: array, or field name)
        The threshold stream power.
    rainfall_intensity : float; optional
        NOT PRESENTLY HONORED (TODO)

    Examples
    --------
    >>> from landlab import RasterModelGrid
    >>> rg = RasterModelGrid((3, 4), 1.0)
    >>> rg.set_closed_boundaries_at_grid_edges(False, True, True, True)
    >>> z = rg.add_zeros('node', 'topographic__elevation')
    >>> z[5] = 2.0
    >>> z[6] = 1.0
    >>> from landlab.components import FlowRouter
    >>> fr = FlowRouter(rg, method='D4')
    >>> fr.run_one_step()
    >>> from landlab.components import StreamPowerSmoothThresholdEroder
    >>> sp = StreamPowerSmoothThresholdEroder(rg, K_sp=1.0)
    >>> sp.thresholds
    1.0
    >>> sp.run_one_step(dt=1.0)
    >>> import numpy as np
    >>> np.round(z, 3)
    array([ 0.   ,  0.   ,  0.   ,  0.   ,  0.   ,  1.846,  0.667,  0.   ,
            0.   ,  0.   ,  0.   ,  0.   ])
    """

    def __init__(self, grid, K_sp=None, m_sp=0.5, n_sp=1., threshold_sp=1.,
                 rainfall_intensity=1., **kwargs):
        """Initialize StreamPowerSmoothThresholdEroder."""

        # Call base-class init
        super(StreamPowerSmoothThresholdEroder,
              self).__init__(grid, K_sp=K_sp, m_sp=m_sp, n_sp=n_sp,
                             threshold_sp=threshold_sp,
                             rainfall_intensity=rainfall_intensity, **kwargs)

        # Arrays with parameters for use in implicit solver
        self.gamma = grid.empty(at='node')
        self.delta = grid.empty(at='node')
        self.epsilon = grid.empty(at='node')

    def run_one_step(self, dt, flooded_nodes=None, runoff_rate=None, **kwds):
        """Run one forward iteration of duration dt.

        Parameters
        ----------
        dt : float
            Time step size
        flooded_nodes : ndarray of int (optional)
            Indices of nodes in lakes/depressions
        runoff_rate : (not used yet)
            (to be added later)

        Examples
        --------
        >>> from landlab import RasterModelGrid
        >>> rg = RasterModelGrid((3, 3), 1.0)
        >>> rg.set_closed_boundaries_at_grid_edges(False, True, True, True)
        >>> z = rg.add_zeros('node', 'topographic__elevation')
        >>> z[4] = 1.0
        >>> from landlab.components import FlowRouter
        >>> fr = FlowRouter(rg, method='D4')
        >>> fr.run_one_step()
        >>> from landlab.components import StreamPowerSmoothThresholdEroder
        >>> sp = StreamPowerSmoothThresholdEroder(rg, K_sp=1.0)
        >>> sp.run_one_step(dt=1.0)
        >>> sp.alpha
        array([ 0.,  0.,  0.,  0.,  1.,  0.,  0.,  0.,  0.])
        >>> sp.gamma
        array([ 0.,  0.,  0.,  0.,  1.,  0.,  0.,  0.,  0.])
        >>> sp.delta
        array([ 0.,  0.,  0.,  0.,  1.,  0.,  0.,  0.,  0.])
        >>> sp.epsilon
        array([ 0.,  0.,  0.,  0.,  2.,  0.,  0.,  0.,  0.])
        """

        # Set up needed arrays
        #
        # Get shorthand for elevation field ("z"), and for up-to-downstream
        # ordered array of node IDs ("upstream_order_IDs")
        upstream_order_IDs = self._grid['node']['flow__upstream_node_order']
        z = self._grid['node']['topographic__elevation']
        flow_receivers = self._grid['node']['flow__receiver_node']

        # Get an array of flow-link length for each node that has a defined
        # receiver (i.e., that drains to another node).
        defined_flow_receivers = np.not_equal(self._grid['node'][
            'flow__link_to_receiver_node'], UNDEFINED_INDEX)
        if flooded_nodes is not None:
            defined_flow_receivers[flooded_nodes] = False
        flow_link_lengths = self._grid._length_of_link_with_diagonals[
            self._grid['node']['flow__link_to_receiver_node'][
                defined_flow_receivers]]

        # (Later on, add functionality for a runoff rate, or discharge, or
        # variable K)

        # Set up alpha, beta, delta arrays
        #
        #   First, compute drainage area raised to the power m.
        np.power(self._grid['node']['drainage_area'], self.m,
                 out=self.A_to_the_m)

        #   Alpha
        self.alpha[defined_flow_receivers==False] = 0.0
        self.alpha[defined_flow_receivers] = (self.K * dt
            * self.A_to_the_m[defined_flow_receivers] / flow_link_lengths)

        #   Gamma
        self.gamma[defined_flow_receivers==False] = 0.0
        self.gamma[defined_flow_receivers] = dt * self.thresholds

        #   Delta
        self.delta[defined_flow_receivers] = ((self.K
            * self.A_to_the_m[defined_flow_receivers] )
            / (self.thresholds * flow_link_lengths))

        #   Epsilon
        self.epsilon[defined_flow_receivers] = (
            (self.alpha[defined_flow_receivers]
             * z[flow_receivers[defined_flow_receivers]])
            + self.gamma[defined_flow_receivers] + z[defined_flow_receivers])

        # Iterate over nodes from downstream to upstream, using scipy's
        # 'newton' function to find new elevation at each node in turn.
        for node in upstream_order_IDs:
            if defined_flow_receivers[node]:
                z[node] = newton(new_elev, z[node],
                                 #fprime=new_elev_prime,
                                 args=(self.alpha[node],
                                       z[flow_receivers[node]],
                                       self.gamma[node],
                                       self.delta[node],
                                       self.epsilon[node]))  #,
                                 #fprime2=new_elev_prime2)

        # TODO: handle case self.thresholds = 0

#if __name__ == '__main__':
#    from landlab import RasterModelGrid
#    rg = RasterModelGrid((3, 4), 1.0)
#    z = rg.add_zeros('node', 'topographic__elevation')
#    rg.set_closed_boundaries_at_grid_edges(False, True, True, True)
#    z[5] = 2.0
#    z[6] = 1.0
#    from landlab.components import FlowRouter
#    fr = FlowRouter(rg, method='D4')
#    fr.run_one_step()
#    sp = StreamPowerSmoothThresholdEroder(rg, K_sp=1.0)
#    sp.run_one_step(dt=1.0)
#    print(z)<|endoftext|>


<commit_before>
import unittest

from dispatch import Emitter, EventData

class State(object):
    pass

class TestEvents(unittest.TestCase):
    def test_init(self):
        emitter = Emitter()

    def test_order(self):
        order = []

        emitter = Emitter()

        @emitter.on('event:order')
        def callback(evt):
            order.append(1)

        @emitter.on('event:order')
        def callback(evt):
            order.append(2)

        emitter.fire('event:order')
        self.assertEqual(order, [1, 2])

    def test_defer(self):
        order = []

        emitter = Emitter()

        @emitter.on('event:order')
        def callback(evt):
            evt.defer()
            order.append(1)

        @emitter.on('event:order')
        def callback(evt):
            order.append(2)

        emitter.fire('event:order')
        self.assertEqual(order, [2, 1])
<commit_msg>Added tests for EventData.cancel()<commit_after>
import unittest

from dispatch import Emitter, EventData

class State(object):
    pass

class TestEvents(unittest.TestCase):
    def test_init(self):
        emitter = Emitter()

    def test_order(self):
        order = []

        emitter = Emitter()

        @emitter.on('event:order')
        def callback(evt):
            order.append(1)

        @emitter.on('event:order')
        def callback(evt):
            order.append(2)

        emitter.fire('event:order')
        self.assertEqual(order, [1, 2])

    def test_defer(self):
        order = []

        emitter = Emitter()

        @emitter.on('event:order')
        def callback(evt):
            evt.defer()
            order.append(1)

        @emitter.on('event:order')
        def callback(evt):
            order.append(2)

        emitter.fire('event:order')
        self.assertEqual(order, [2, 1])

    def test_cancel(self):
        order = []

        emitter = Emitter()

        @emitter.on('event:cancel')
        def callback(evt):
            order.append(1)
            evt.cancel()
            order.append(2)

        @emitter.on('event:cancel')
        def callback(evt):
            order.append(3)

        emitter.fire('event:cancel')
        self.assertEqual(order, [1])

    def test_cancel_nested(self):
        order = []

        emitter = Emitter()

        @emitter.on('event:cancel')
        def callback(evt):
            order.append(1)

            @emitter.on('event:cancel_nested')
            def callback_nested(subevt):
                order.append(2)
                evt.cancel()
                order.append(3)

            @emitter.on('event:cancel_nested')
            def callback_nested(subevt):
                order.append(4)
            emitter.fire('event:cancel_nested')
            order.append(5)

        @emitter.on('event:cancel')
        def callback(evt):
            order.append(6)

        emitter.fire('event:cancel')
        self.assertEqual(order, [1, 2])

    def test_cancel_nested2(self):
        order = []

        emitter = Emitter()

        @emitter.on('event:cancel')
        def callback(evt):
            order.append(1)

            @emitter.on('event:cancel_nested')
            def callback_nested(subevt):
                order.append(2)
                subevt.cancel()
                order.append(3)

            @emitter.on('event:cancel_nested')
            def callback_nested(subevt):
                order.append(4)
            emitter.fire('event:cancel_nested')
            order.append(5)

        @emitter.on('event:cancel')
        def callback(evt):
            order.append(6)

        emitter.fire('event:cancel')
        self.assertEqual(order, [1, 2, 5, 6])
<|endoftext|>


<commit_before>import freemail
from django.core.management.base import BaseCommand
from django.core.paginator import Paginator
from django.db import transaction
from django.db.models import Q

from lily.accounts.models import Website, Account
from lily.contacts.models import Contact, Function
from lily.hubspot.prefetch_objects import accounts_prefetch, email_addresses_prefetch
from lily.hubspot.utils import get_accounts_without_website, get_contacts_without_email_address, _strip_website
from lily.tags.models import Tag
from lily.tenant.middleware import set_current_user
from lily.users.models import LilyUser
from lily.utils.functions import clean_website
from lily.utils.models.models import EmailAddress

def find_suitable_website_from_email_addresses(email_address_list):
    for email in email_address_list:
        email_address = email.email_address.encode('ascii', errors='ignore')
        if freemail.is_free(email_address):
            continue
        else:
            return email_address.rsplit('@')[-1]

class Command(BaseCommand):
    def add_arguments(self, parser):
        parser.add_argument('tenant_id', type=int)

    def handle(self, tenant_id, *args, **options):
        self.stdout.write(self.style.SUCCESS('>>') + '  Starting fixing. \n\n')
        set_current_user(LilyUser.objects.filter(tenant_id=tenant_id, is_active=True).first())

        # self.fix_accounts(tenant_id)
        # self.fix_contacts(tenant_id)
        # self.put_accounts_emails_in_contacts()
        self.delete_accounts_with_cold_email_tag()

        self.stdout.write(self.style.SUCCESS('>>') + '  Successfully fixed. \n\n')

    def fix_accounts(self, tenant_id):
        self.stdout.write(u'Fixing accounts:')

        account_id_list = [a[0] for a in get_accounts_without_website(tenant_id)]
        account_list = Account.objects.filter(
            id__in=account_id_list
        ).prefetch_related(
            email_addresses_prefetch
        )

        paginator = Paginator(account_list, 100)
        self.stdout.write(u'    Page: 0 / {}    ({} items)'.format(paginator.num_pages, paginator.count))
        for page_number in paginator.page_range:
            object_list = paginator.page(page_number).object_list

            self.stdout.write('    Page: {} / {}'.format(page_number, paginator.num_pages))
            for account in object_list:
                website = find_suitable_website_from_email_addresses(account.prefetched_email_addresses)

                if not website:
                    # No website was found, so try the contact email addresses.
                    contact_list = Contact.objects.filter(
                        functions__account_id=account.pk
                    ).prefetch_related(
                        email_addresses_prefetch
                    )

                    for contact in contact_list:
                        website = find_suitable_website_from_email_addresses(contact.prefetched_email_addresses)

                        if website:
                            break

                if not website or Website.objects.filter(website=clean_website(website), account=account).exists():
                    website = 'account-{}-from-import.nl'.format(account.pk)

                account.websites.add(Website.objects.create(
                    website=website,
                    account=account,
                    is_primary=True
                ))

    def fix_contacts(self, tenant_id):
        self.stdout.write(u'Fixing contacts:')

        contact_id_list = [c[0] for c in get_contacts_without_email_address(tenant_id)]
        contact_list = Contact.objects.filter(
            id__in=contact_id_list
        ).prefetch_related(
            accounts_prefetch
        )

        paginator = Paginator(contact_list, 100)
        self.stdout.write(u'    Page: 0 / {}    ({} items)'.format(paginator.num_pages, paginator.count))
        for page_number in paginator.page_range:
            object_list = paginator.page(page_number).object_list

            self.stdout.write(u'    Page: {} / {}'.format(page_number, paginator.num_pages))
            for contact in object_list:
                account = contact.prefetched_accounts[0] if contact.prefetched_accounts else None
                website = None
                if account:
                    website = account.websites.exclude(website='http://').order_by('-is_primary').first()

                if website:
                    new_email = u'{}@{}'.format(
                        contact.full_name.replace(' ', '.').lower(),
                        _strip_website(website.website)
                    ).replace(' ', '')
                else:
                    new_email = u'contact-{}@data-from-import.nl'.format(contact.pk)

                email_address = EmailAddress.objects.create(
                    email_address=new_email,
                    status=EmailAddress.PRIMARY_STATUS,
                    tenant_id=tenant_id
                )
                contact.email_addresses.add(email_address)

    def put_accounts_emails_in_contacts(self):
        self.stdout.write(u'Fixing accounts with email addresses:')

        # Get all active email addresses for accounts, where the email address is not already included in a contact.
        account_email_addresses = EmailAddress.objects.prefetch_related('account_set').filter(
            account__is_deleted=False,
        ).exclude(
            Q(status=EmailAddress.INACTIVE_STATUS) | Q(email_address__in=EmailAddress.objects.filter(
                contact__is_deleted=False
            ).values_list('email_address', flat=True).distinct())
        )

        paginator = Paginator(account_email_addresses, 100)
        self.stdout.write(u'    Page: 0 / {}    ({} items)'.format(paginator.num_pages, paginator.count))
        for page_number in paginator.page_range:
            object_list = paginator.page(page_number).object_list

            self.stdout.write(u'    Page: {} / {}'.format(page_number, paginator.num_pages))
            for email_address in object_list:
                account = email_address.account_set.first()

                with transaction.atomic():
                    # Create the new contact.
                    contact = Contact.objects.create(
                        first_name=email_address.email_address,
                    )

                    # Copy the email address to attach it to the contact.
                    email = EmailAddress.objects.create(
                        email_address=email_address.email_address,
                        status=EmailAddress.PRIMARY_STATUS,
                    )
                    contact.email_addresses.add(email)

                    # Attach the contact to the account.
                    Function.objects.create(
                        account=account,
                        contact=contact
                    )

    def delete_accounts_with_cold_email_tag(self):
        self.stdout.write(u'Deleting accounts with cold email tags:')

        account_content_type = Account().content_type
        account_ids = Tag.objects.filter(
            name='cold email',
            content_type=account_content_type
        ).values_list('object_id', flat=True)
        account_list = Account.objects.filter(id__in=account_ids, is_deleted=False)

        paginator = Paginator(account_list, 100)
        self.stdout.write(u'    Page: 0 / {}    ({} items)'.format(paginator.num_pages, paginator.count))
        for page_number in paginator.page_range:
            object_list = paginator.page(page_number).object_list

            self.stdout.write(u'    Page: {} / {}'.format(page_number, paginator.num_pages))
            for account in object_list:
                account.delete()
<commit_msg>Fixup<commit_after>import freemail
from django.core.management.base import BaseCommand
from django.core.paginator import Paginator
from django.db import transaction
from django.db.models import Q

from lily.accounts.models import Website, Account
from lily.contacts.models import Contact, Function
from lily.hubspot.prefetch_objects import accounts_prefetch, email_addresses_prefetch
from lily.hubspot.utils import get_accounts_without_website, get_contacts_without_email_address, _strip_website
from lily.tags.models import Tag
from lily.tenant.middleware import set_current_user
from lily.users.models import LilyUser
from lily.utils.functions import clean_website
from lily.utils.models.models import EmailAddress

def find_suitable_website_from_email_addresses(email_address_list):
    for email in email_address_list:
        email_address = email.email_address.encode('ascii', errors='ignore')
        if freemail.is_free(email_address):
            continue
        else:
            return email_address.rsplit('@')[-1]

class Command(BaseCommand):
    def add_arguments(self, parser):
        parser.add_argument('tenant_id', type=int)

    def handle(self, tenant_id, *args, **options):
        self.stdout.write(self.style.SUCCESS('>>') + '  Starting fixing. \n\n')
        set_current_user(LilyUser.objects.filter(tenant_id=tenant_id, is_active=True).first())

        self.fix_accounts(tenant_id)
        self.fix_contacts(tenant_id)
        self.put_accounts_emails_in_contacts()
        self.delete_accounts_with_cold_email_tag()

        self.stdout.write(self.style.SUCCESS('>>') + '  Successfully fixed. \n\n')

    def fix_accounts(self, tenant_id):
        self.stdout.write(u'Fixing accounts:')

        account_id_list = [a[0] for a in get_accounts_without_website(tenant_id)]
        account_list = Account.objects.filter(
            id__in=account_id_list
        ).prefetch_related(
            email_addresses_prefetch
        )

        paginator = Paginator(account_list, 100)
        self.stdout.write(u'    Page: 0 / {}    ({} items)'.format(paginator.num_pages, paginator.count))
        for page_number in paginator.page_range:
            object_list = paginator.page(page_number).object_list

            self.stdout.write('    Page: {} / {}'.format(page_number, paginator.num_pages))
            for account in object_list:
                website = find_suitable_website_from_email_addresses(account.prefetched_email_addresses)

                if not website:
                    # No website was found, so try the contact email addresses.
                    contact_list = Contact.objects.filter(
                        functions__account_id=account.pk
                    ).prefetch_related(
                        email_addresses_prefetch
                    )

                    for contact in contact_list:
                        website = find_suitable_website_from_email_addresses(contact.prefetched_email_addresses)

                        if website:
                            break

                if not website or Website.objects.filter(website=clean_website(website), account=account).exists():
                    website = 'account-{}-from-import.nl'.format(account.pk)

                account.websites.add(Website.objects.create(
                    website=website,
                    account=account,
                    is_primary=True
                ))

    def fix_contacts(self, tenant_id):
        self.stdout.write(u'Fixing contacts:')

        contact_id_list = [c[0] for c in get_contacts_without_email_address(tenant_id)]
        contact_list = Contact.objects.filter(
            id__in=contact_id_list
        ).prefetch_related(
            accounts_prefetch
        )

        paginator = Paginator(contact_list, 100)
        self.stdout.write(u'    Page: 0 / {}    ({} items)'.format(paginator.num_pages, paginator.count))
        for page_number in paginator.page_range:
            object_list = paginator.page(page_number).object_list

            self.stdout.write(u'    Page: {} / {}'.format(page_number, paginator.num_pages))
            for contact in object_list:
                account = contact.prefetched_accounts[0] if contact.prefetched_accounts else None
                website = None
                if account:
                    website = account.websites.exclude(website='http://').order_by('-is_primary').first()

                if website:
                    new_email = u'{}@{}'.format(
                        contact.full_name.replace(' ', '.').lower(),
                        _strip_website(website.website)
                    ).replace(' ', '')
                else:
                    new_email = u'contact-{}@data-from-import.nl'.format(contact.pk)

                email_address = EmailAddress.objects.create(
                    email_address=new_email,
                    status=EmailAddress.PRIMARY_STATUS,
                    tenant_id=tenant_id
                )
                contact.email_addresses.add(email_address)

    def put_accounts_emails_in_contacts(self):
        self.stdout.write(u'Fixing accounts with email addresses:')

        # Get all active email addresses for accounts, where the email address is not already included in a contact.
        account_email_addresses = EmailAddress.objects.prefetch_related('account_set').filter(
            account__is_deleted=False,
        ).exclude(
            Q(status=EmailAddress.INACTIVE_STATUS) | Q(email_address__in=EmailAddress.objects.filter(
                contact__is_deleted=False
            ).values_list('email_address', flat=True).distinct())
        )

        paginator = Paginator(account_email_addresses, 100)
        self.stdout.write(u'    Page: 0 / {}    ({} items)'.format(paginator.num_pages, paginator.count))
        for page_number in paginator.page_range:
            object_list = paginator.page(page_number).object_list

            self.stdout.write(u'    Page: {} / {}'.format(page_number, paginator.num_pages))
            for email_address in object_list:
                account = email_address.account_set.first()

                with transaction.atomic():
                    # Create the new contact.
                    contact = Contact.objects.create(
                        first_name=email_address.email_address,
                    )

                    # Copy the email address to attach it to the contact.
                    email = EmailAddress.objects.create(
                        email_address=email_address.email_address,
                        status=EmailAddress.PRIMARY_STATUS,
                    )
                    contact.email_addresses.add(email)

                    # Attach the contact to the account.
                    Function.objects.create(
                        account=account,
                        contact=contact
                    )

    def delete_accounts_with_cold_email_tag(self):
        self.stdout.write(u'Deleting accounts with cold email tags:')

        account_content_type = Account().content_type
        account_ids = Tag.objects.filter(
            name='cold email',
            content_type=account_content_type
        ).values_list('object_id', flat=True)
        account_list = Account.objects.filter(id__in=account_ids, is_deleted=False)

        paginator = Paginator(account_list, 100)
        self.stdout.write(u'    Page: 0 / {}    ({} items)'.format(paginator.num_pages, paginator.count))
        for page_number in paginator.page_range:
            object_list = paginator.page(page_number).object_list

            self.stdout.write(u'    Page: {} / {}'.format(page_number, paginator.num_pages))
            for account in object_list:
                account.delete()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2018 New Vector Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from twisted.internet import defer
from synapse.api.errors import AuthError, Codes

from synapse.api.filtering import DEFAULT_FILTER_COLLECTION
from synapse.handlers.sync import SyncConfig, SyncHandler
from synapse.types import UserID

import tests.unittest
import tests.utils
from tests.utils import setup_test_homeserver

class SyncTestCase(tests.unittest.TestCase):
    """ Tests Sync Handler. """

    @defer.inlineCallbacks
    def setUp(self):
        self.hs = yield setup_test_homeserver()
        self.sync_handler = SyncHandler(self.hs)
        self.store = self.hs.get_datastore()

    @defer.inlineCallbacks
    def test_wait_for_sync_for_user_auth_blocking(self):

        user_id1 = "@user1:server"
        user_id2 = "@user2:server"
        sync_config = self._generate_sync_config(user_id1)

        self.hs.config.limit_usage_by_mau = True
        self.hs.config.max_mau_value = 1

        # Check that the happy case does not throw errors
        yield self.store.upsert_monthly_active_user(user_id1)
        yield self.sync_handler.wait_for_sync_for_user(sync_config)

        # Test that global lock works
        self.hs.config.hs_disabled = True
        with self.assertRaises(AuthError) as e:
            yield self.sync_handler.wait_for_sync_for_user(sync_config)
        self.assertEquals(e.exception.errcode, Codes.HS_DISABLED)

        self.hs.config.hs_disabled = False

        sync_config = self._generate_sync_config(user_id2)

        with self.assertRaises(AuthError) as e:
            yield self.sync_handler.wait_for_sync_for_user(sync_config)
        self.assertEquals(e.exception.errcode, Codes.MAU_LIMIT_EXCEEDED)

    def _generate_sync_config(self, user_id):
        return SyncConfig(
            user=UserID(user_id.split(":")[0][1:], user_id.split(":")[1]),
            filter_collection=DEFAULT_FILTER_COLLECTION,
            is_guest=False,
            request_key="request_key",
            device_id="device_id",
        )
<commit_msg>fix imports<commit_after># -*- coding: utf-8 -*-
# Copyright 2018 New Vector Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from twisted.internet import defer

from synapse.api.errors import AuthError, Codes
from synapse.api.filtering import DEFAULT_FILTER_COLLECTION
from synapse.handlers.sync import SyncConfig, SyncHandler
from synapse.types import UserID

import tests.unittest
import tests.utils
from tests.utils import setup_test_homeserver

class SyncTestCase(tests.unittest.TestCase):
    """ Tests Sync Handler. """

    @defer.inlineCallbacks
    def setUp(self):
        self.hs = yield setup_test_homeserver()
        self.sync_handler = SyncHandler(self.hs)
        self.store = self.hs.get_datastore()

    @defer.inlineCallbacks
    def test_wait_for_sync_for_user_auth_blocking(self):

        user_id1 = "@user1:server"
        user_id2 = "@user2:server"
        sync_config = self._generate_sync_config(user_id1)

        self.hs.config.limit_usage_by_mau = True
        self.hs.config.max_mau_value = 1

        # Check that the happy case does not throw errors
        yield self.store.upsert_monthly_active_user(user_id1)
        yield self.sync_handler.wait_for_sync_for_user(sync_config)

        # Test that global lock works
        self.hs.config.hs_disabled = True
        with self.assertRaises(AuthError) as e:
            yield self.sync_handler.wait_for_sync_for_user(sync_config)
        self.assertEquals(e.exception.errcode, Codes.HS_DISABLED)

        self.hs.config.hs_disabled = False

        sync_config = self._generate_sync_config(user_id2)

        with self.assertRaises(AuthError) as e:
            yield self.sync_handler.wait_for_sync_for_user(sync_config)
        self.assertEquals(e.exception.errcode, Codes.MAU_LIMIT_EXCEEDED)

    def _generate_sync_config(self, user_id):
        return SyncConfig(
            user=UserID(user_id.split(":")[0][1:], user_id.split(":")[1]),
            filter_collection=DEFAULT_FILTER_COLLECTION,
            is_guest=False,
            request_key="request_key",
            device_id="device_id",
        )
<|endoftext|>


<commit_before>import subprocess
import os
import sys
import time
import sqlite3
import urlparse

import irc.client
import yaml

import py.test

class TestingClient(object):
	def __init__(self, server, port, nickname):
		self.irc = irc.client.IRC()
		self.c = self.irc.server()
		self.c.connect(server, port, nickname)
		self.irc.process_once(0.1)

	def send_message(self, channel, message):
		self.c.privmsg(channel, message)
		time.sleep(0.05)

class PmxbotHarness(object):
	@classmethod
	def setup_class(cls):
		"""Start a tcl IRC server, launch the bot, and
		ask it to do stuff"""
		path = os.path.dirname(os.path.abspath(__file__))
		configfile = os.path.join(path, 'testconf.yaml')
		cls.config = yaml.load(open(configfile))
		cls.dbfile = urlparse.urlparse(cls.config['database']).path
		cls.db = sqlite3.connect(cls.dbfile)
		try:
			cls.server = subprocess.Popen(['tclsh', os.path.join(path,
				'tclircd/ircd.tcl')], stdout=open(os.path.devnull, 'w'),
				stderr=open(os.path.devnull, 'w'))
		except OSError:
			py.test.skip("Unable to launch irc server (tclsh must be in the path)")
		time.sleep(0.5)
		try:
			# Launch pmxbot using Python directly (rather than through
			#  the console entry point, which can't be properly
			#  .terminate()d on Windows.
			cls.bot = subprocess.Popen([sys.executable, '-c',
				'from pmxbot.pmxbot import run; run()', configfile],)
		except OSError:
			py.test.skip("Unable to launch pmxbot (pmxbot must be installed)")
		time.sleep(2)
		cls.client = TestingClient('localhost', 6668, 'testingbot')

	def check_logs(cls, channel='', nick='', message=''):
		if channel.startswith('#'):
			channel = channel[1:]
		time.sleep(0.1)
		cursor = cls.db.cursor()
		query = "select * from logs where 1=1"
		if channel:
			query += " and channel = :channel"
		if nick:
			query += " and nick = :nick"
		if message:
			query += " and message = :message"
		cursor.execute(query, {'channel' : channel, 'nick' : nick, 'message' : message})
		res = cursor.fetchall()
		print res
		return len(res) >= 1

	@classmethod
	def teardown_class(cls):
		if hasattr(cls, 'bot'):
			cls.bot.terminate()
			cls.bot.wait()
		if hasattr(cls, 'server') and cls.server.poll() == None:
			cls.server.terminate()
			cls.server.wait()
		if hasattr(cls, 'db'):
			cls.db.rollback()
			cls.db.close()
			del cls.db
		if hasattr(cls, 'client'):
			del cls.client
		# wait up to 10 seconds for the file to be removable
		for x in range(100):
			try:
				if os.path.isfile(cls.dbfile): os.remove(cls.dbfile)
				break
			except OSError:
				time.sleep(.1)
		else:
			raise RuntimeError('Could not remove log db', cls.dbfile)

<commit_msg>delint<commit_after>import subprocess
import os
import sys
import time
import sqlite3
import urlparse

import irc.client
import yaml

import py.test

class TestingClient(object):
	def __init__(self, server, port, nickname):
		self.irc = irc.client.IRC()
		self.c = self.irc.server()
		self.c.connect(server, port, nickname)
		self.irc.process_once(0.1)

	def send_message(self, channel, message):
		self.c.privmsg(channel, message)
		time.sleep(0.05)

class PmxbotHarness(object):
	@classmethod
	def setup_class(cls):
		"""Start a tcl IRC server, launch the bot, and
		ask it to do stuff"""
		path = os.path.dirname(os.path.abspath(__file__))
		configfile = os.path.join(path, 'testconf.yaml')
		cls.config = yaml.load(open(configfile))
		cls.dbfile = urlparse.urlparse(cls.config['database']).path
		cls.db = sqlite3.connect(cls.dbfile)
		try:
			cls.server = subprocess.Popen(['tclsh', os.path.join(path,
				'tclircd/ircd.tcl')], stdout=open(os.path.devnull, 'w'),
				stderr=open(os.path.devnull, 'w'))
		except OSError:
			py.test.skip("Unable to launch irc server (tclsh must be in the path)")
		time.sleep(0.5)
		try:
			# Launch pmxbot using Python directly (rather than through
			#  the console entry point, which can't be properly
			#  .terminate()d on Windows.
			cls.bot = subprocess.Popen([sys.executable, '-c',
				'from pmxbot.pmxbot import run; run()', configfile],)
		except OSError:
			py.test.skip("Unable to launch pmxbot (pmxbot must be installed)")
		time.sleep(2)
		cls.client = TestingClient('localhost', 6668, 'testingbot')

	def check_logs(cls, channel='', nick='', message=''):
		if channel.startswith('#'):
			channel = channel[1:]
		time.sleep(0.1)
		cursor = cls.db.cursor()
		query = "select * from logs where 1=1"
		if channel:
			query += " and channel = :channel"
		if nick:
			query += " and nick = :nick"
		if message:
			query += " and message = :message"
		cursor.execute(query, dict(
			channel = channel,
			nick = nick,
			message = message,
			))
		res = cursor.fetchall()
		print res
		return len(res) >= 1

	@classmethod
	def teardown_class(cls):
		if hasattr(cls, 'bot'):
			cls.bot.terminate()
			cls.bot.wait()
		if hasattr(cls, 'server') and cls.server.poll() == None:
			cls.server.terminate()
			cls.server.wait()
		if hasattr(cls, 'db'):
			cls.db.rollback()
			cls.db.close()
			del cls.db
		if hasattr(cls, 'client'):
			del cls.client
		# wait up to 10 seconds for the file to be removable
		for x in range(100):
			try:
				if os.path.isfile(cls.dbfile): os.remove(cls.dbfile)
				break
			except OSError:
				time.sleep(.1)
		else:
			raise RuntimeError('Could not remove log db', cls.dbfile)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
version = '0.6.0'
<commit_msg>Post release.<commit_after># -*- coding: utf-8 -*-
version = '0.7.0.dev'
<|endoftext|>


<commit_before>"""
An ASCII text progress bar. See __main__ for command line use (using \r to
move the cursor back to the start of the current line is the key, on
terminals that do not support this functionality the progress bar will
not work as well).

http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/168639
"""

import sys

class ProgressBar:
    def __init__(self, minValue = 0, maxValue = 10, totalWidth=12):
        self.progBar = "[]"   # This holds the progress bar string
        self.min = minValue
        self.max = maxValue
        self.span = maxValue - minValue
        self.width = totalWidth
        self.amount = 0       # When amount == max, we are 100% done
        self.update(0)  # Build progress bar string

    def update(self, newAmount = 0):
        if newAmount < self.min: newAmount = self.min
        if newAmount > self.max: newAmount = self.max
        self.amount = newAmount

        # Figure out the new percent done, round to an integer
        diffFromMin = float(self.amount - self.min)
        percentDone = (diffFromMin / float(self.span)) * 100.0
        percentDone = round(percentDone)
        percentDone = int(percentDone)

        # Figure out how many hash bars the percentage should be
        allFull = self.width - 2
        numHashes = (percentDone / 100.0) * allFull
        numHashes = int(round(numHashes))

        # build a progress bar with hashes and spaces
        if allFull == numHashes:
            self.progBar = "[" + '='*(numHashes) + "]"
        else:
            self.progBar = "[" + '='*(numHashes-1) + '>' + ' '*(allFull-numHashes) + "]"

        # figure out where to put the percentage, roughly centered
        percentPlace = (len(self.progBar) / 2) - len(str(percentDone))
        percentString = str(percentDone) + "%"

        # slice the percentage into the bar
        self.progBar = self.progBar[0:percentPlace] + percentString + self.progBar[percentPlace+len(percentString):]

    def update_and_print( self, newAmount = 0, f = sys.stdout ):
        self.update( newAmount )
        print >> f, "\r", self,
        f.flush()

    def __str__(self):
        return str(self.progBar)

if __name__ == "__main__":
    import time

    bar = ProgressBar( 0, 1000, 80 )

    for i in range(1000):
        bar.update( i )
        print "\r", bar,
        sys.stdout.flush()

    print
<commit_msg>Automated merge with https://bitbucket.org/james_taylor/bx-python<commit_after>"""
An ASCII text progress bar. See __main__ for command line use (using \r to
move the cursor back to the start of the current line is the key, on
terminals that do not support this functionality the progress bar will
not work as well).

http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/168639
"""

import sys

class ProgressBar:
    def __init__(self, minValue = 0, maxValue = 10, totalWidth=72):
        self.progBar = "[]"   # This holds the progress bar string
        self.min = minValue
        self.max = maxValue
        self.span = maxValue - minValue
        self.width = totalWidth
        self.amount = 0       # When amount == max, we are 100% done
        self.update(0)  # Build progress bar string

    def update(self, newAmount = 0):
        if newAmount < self.min: newAmount = self.min
        if newAmount > self.max: newAmount = self.max
        self.amount = newAmount

        # Figure out the new percent done, round to an integer
        diffFromMin = float(self.amount - self.min)
        percentDone = (diffFromMin / float(self.span)) * 100.0
        percentDone = round(percentDone)
        percentDone = int(percentDone)

        # Figure out how many hash bars the percentage should be
        allFull = self.width - 2
        numHashes = (percentDone / 100.0) * allFull
        numHashes = int(round(numHashes))

        # build a progress bar with hashes and spaces
        if allFull == numHashes:
            self.progBar = "[" + '='*(numHashes) + "]"
        else:
            self.progBar = "[" + '='*(numHashes-1) + '>' + ' '*(allFull-numHashes) + "]"

        # figure out where to put the percentage, roughly centered
        percentPlace = (len(self.progBar) / 2) - len(str(percentDone))
        percentString = str(percentDone) + "%"

        # slice the percentage into the bar
        self.progBar = self.progBar[0:percentPlace] + percentString + self.progBar[percentPlace+len(percentString):]

    def update_and_print( self, newAmount = 0, f = sys.stdout ):
        self.update( newAmount )
        print >> f, "\r", self,
        f.flush()

    def __str__(self):
        return str(self.progBar)

def iterprogress( sized_iterable ):
    """
    Iterate something printing progress bar to stdout
    """
    pb = ProgressBar( 0, len( sized_iterable ) )
    for i, value in enumerate( sized_iterable ):
        yield value
        pb.update_and_print( i, sys.stderr )

if __name__ == "__main__":
    import time

    bar = ProgressBar( 0, 1000, 80 )

    for i in range(1000):
        bar.update( i )
        print "\r", bar,
        sys.stdout.flush()

    print
<|endoftext|>


<commit_before>"""This module contains functions for loading Stage information from
external XML files into StageData objects that can be used by the game
engine.

Module Constants:
    STAGE_LIST_PATH (String): The relative file path to the text file
        containing the file paths of all included Stages.
    STAGE_SCHEMA_PATH: The file path to the XML Schema document that
        will be used to validate stage files.
    FILEPATH_PREFIX (String): The relative file path prefix for the
        directory containing all Stage files.
"""
import os
from lib.custom_data.xml_ops import load_xml_doc_as_object
from lib.custom_data.text_ops import get_prefixed_lines_from_txt
from lib.custom_data.text_ops import num_of_lines_in_txt

STAGE_LIST_PATH = 'stages/stage_list.txt'
STAGE_SCHEMA_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                 'stage.xsd')
FILEPATH_PREFIX = 'stages/'

def load_all_stages():
    """Load the data from all of the XML files specified in the
    stage_list file, and return it as a tuple of StageData objects.

    If no stages could be loaded, None is returned instead.
    """
    stages = []

    for filepath_index in range(0, num_of_lines_in_txt(STAGE_LIST_PATH)):
        stage = load_stage(filepath_index)
        if stage is not None:
            stages.append(stage)

    if len(stages) > 0:
        return tuple(stages)
    else:
        return None

def load_stage(line_index):
    """Load a specific stage from the list specified in the Stage list
    text file.

    Args:
        line_index (int): The line index of the stage file's file
            path within the stage list text file.
            Note that like indexing in other parts of Python, this also
            starts at 0.

    Returns:
        The specified stage's data as a StageData object. If there was
        an error loading data, None is returned instead.
        None will also be returned if line_index exceeds the number of
        lines in the text file.
    """
    stage_paths = get_prefixed_lines_from_txt(STAGE_LIST_PATH,
        FILEPATH_PREFIX)
    if line_index > len(stage_paths) - 1:
        return None

    stage_path = stage_paths[line_index]
    stage_data = load_xml_doc_as_object(stage_path, STAGE_SCHEMA_PATH)

    if stage_data is None:
        return None
    else:
        prepend_prefix_to_filepaths(stage_data)
        return stage_data

def prepend_prefix_to_filepaths(stage_data):
    """Preprend FILEPATH_PREFIX to all file path attributes of a
    StageData object.

    Args:
        stage_data (StageData): A StageData instance.
    """
    stage_data.background.spritesheet_path = (FILEPATH_PREFIX +
        stage_data.background.spritesheet_path)
    stage_data.parallax.spritesheet_path = (FILEPATH_PREFIX +
        stage_data.parallax.spritesheet_path)
    for front_prop in stage_data.front_props:
        front_prop.spritesheet_path = (FILEPATH_PREFIX +
            front_prop.spritesheet_path)
    for back_prop in stage_data.back_props:
        back_prop.spritesheet_path = (FILEPATH_PREFIX +
            back_prop.spritesheet_path)
    if stage_data.music is not None:
        stage_data.music = FILEPATH_PREFIX + stage_data.music
<commit_msg>Fix no music check in prepend_prefix_to_filepaths<commit_after>"""This module contains functions for loading Stage information from
external XML files into StageData objects that can be used by the game
engine.

Module Constants:
    STAGE_LIST_PATH (String): The relative file path to the text file
        containing the file paths of all included Stages.
    STAGE_SCHEMA_PATH: The file path to the XML Schema document that
        will be used to validate stage files.
    FILEPATH_PREFIX (String): The relative file path prefix for the
        directory containing all Stage files.
"""
import os
from lib.custom_data.xml_ops import load_xml_doc_as_object
from lib.custom_data.text_ops import get_prefixed_lines_from_txt
from lib.custom_data.text_ops import num_of_lines_in_txt

STAGE_LIST_PATH = 'stages/stage_list.txt'
STAGE_SCHEMA_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                 'stage.xsd')
FILEPATH_PREFIX = 'stages/'

def load_all_stages():
    """Load the data from all of the XML files specified in the
    stage_list file, and return it as a tuple of StageData objects.

    If no stages could be loaded, None is returned instead.
    """
    stages = []

    for filepath_index in range(0, num_of_lines_in_txt(STAGE_LIST_PATH)):
        stage = load_stage(filepath_index)
        if stage is not None:
            stages.append(stage)

    if len(stages) > 0:
        return tuple(stages)
    else:
        return None

def load_stage(line_index):
    """Load a specific stage from the list specified in the Stage list
    text file.

    Args:
        line_index (int): The line index of the stage file's file
            path within the stage list text file.
            Note that like indexing in other parts of Python, this also
            starts at 0.

    Returns:
        The specified stage's data as a StageData object. If there was
        an error loading data, None is returned instead.
        None will also be returned if line_index exceeds the number of
        lines in the text file.
    """
    stage_paths = get_prefixed_lines_from_txt(STAGE_LIST_PATH,
        FILEPATH_PREFIX)
    if line_index > len(stage_paths) - 1:
        return None

    stage_path = stage_paths[line_index]
    stage_data = load_xml_doc_as_object(stage_path, STAGE_SCHEMA_PATH)

    if stage_data is None:
        return None
    else:
        prepend_prefix_to_filepaths(stage_data)
        return stage_data

def prepend_prefix_to_filepaths(stage_data):
    """Preprend FILEPATH_PREFIX to all file path attributes of a
    StageData object.

    Args:
        stage_data (StageData): A StageData instance.
    """
    stage_data.background.spritesheet_path = (FILEPATH_PREFIX +
        stage_data.background.spritesheet_path)
    stage_data.parallax.spritesheet_path = (FILEPATH_PREFIX +
        stage_data.parallax.spritesheet_path)
    for front_prop in stage_data.front_props:
        front_prop.spritesheet_path = (FILEPATH_PREFIX +
            front_prop.spritesheet_path)
    for back_prop in stage_data.back_props:
        back_prop.spritesheet_path = (FILEPATH_PREFIX +
            back_prop.spritesheet_path)
    if stage_data.music != '':
        stage_data.music = FILEPATH_PREFIX + stage_data.music
<|endoftext|>


<commit_before>#!/usr/bin/env python3

from netv2_base import *

from litevideo.input import HDMIIn
from litevideo.output import VideoOut
from litedram.common import LiteDRAMPort

base_cls = BaseSoC

class VideoOutSoC(base_cls):
    csr_map = {
        "hdmi_out": 26,
        "hdmi_in": 27,
        "hdmi_in_edid_mem": 28,
    }
    csr_map.update(base_cls.csr_map)

    interrupt_map = {
        "hdmi_in": 3,
    }
    interrupt_map.update(base_cls.interrupt_map)

    def __init__(self, platform, *args, **kwargs):
        base_cls.__init__(self, platform, *args, pcie_blink=False, **kwargs)

        # # #

        # hdmi in
        self.submodules.hdmi_in = HDMIIn(platform.request("hdmi_in", 0),
                                         self.sdram.crossbar.get_port(mode="write"),
                                         fifo_depth=512,
                                         device="xc7")

        # hdmi out
        hdmi_out_dram_port = self.sdram.crossbar.get_port(mode="read", dw=16, cd="hdmi_out_pix", reverse=True)
        self.submodules.hdmi_out = VideoOut(platform.device,
                                            platform.request("hdmi_out"),
                                            hdmi_out_dram_port,
                                            "ycbcr422")

        self.platform.add_period_constraint(self.crg.cd_sys.clk, 10.0)
        self.platform.add_period_constraint(self.hdmi_out.driver.clocking.cd_pix.clk, 10.0)
        self.platform.add_false_path_constraints(
            self.crg.cd_sys.clk,
            self.hdmi_out.driver.clocking.cd_pix.clk)

def main():
    parser = argparse.ArgumentParser(description="NeTV2 LiteX SoC")
    builder_args(parser)
    soc_sdram_args(parser)
    args = parser.parse_args()

    platform = netv2.Platform()
    soc = VideoOutSoC(platform, **soc_sdram_argdict(args))
    builder = Builder(soc, output_dir="build")
    vns = builder.build()

if __name__ == "__main__":
    main()
<commit_msg>netv2_video: cleanup/update<commit_after>#!/usr/bin/env python3

from netv2_base import *

from litevideo.input import HDMIIn
from litevideo.output import VideoOut

from litex.soc.cores.frequency_meter import FrequencyMeter

from litescope import LiteScopeAnalyzer

base_cls = BaseSoC

class VideoSoC(base_cls):
    csr_peripherals = {
        "hdmi_out0",
        "hdmi_in0",
        "hdmi_in0_freq",
        "hdmi_in0_edid_mem",
        "analyzer",
    }
    csr_map_update(base_cls.csr_map, csr_peripherals)

    interrupt_map = {
        "hdmi_in0": 3,
    }
    interrupt_map.update(base_cls.interrupt_map)

    def __init__(self, platform, *args, **kwargs):
        base_cls.__init__(self, platform, *args, **kwargs)

        # # #

        pix_freq = 148.50e6

        # hdmi in
        hdmi_in0_pads = platform.request("hdmi_in")
        self.submodules.hdmi_in0_freq = FrequencyMeter(period=self.clk_freq)
        self.submodules.hdmi_in0 = HDMIIn(hdmi_in0_pads,
                                         self.sdram.crossbar.get_port(mode="write"),
                                         fifo_depth=512,
                                         device="xc7")
        self.comb += self.hdmi_in0_freq.clk.eq(self.hdmi_in0.clocking.cd_pix.clk)
        self.platform.add_period_constraint(self.hdmi_in0.clocking.cd_pix.clk, period_ns(1*pix_freq))
        self.platform.add_period_constraint(self.hdmi_in0.clocking.cd_pix1p25x.clk, period_ns(1.25*pix_freq))
        self.platform.add_period_constraint(self.hdmi_in0.clocking.cd_pix5x.clk, period_ns(5*pix_freq))

        self.platform.add_false_path_constraints(
            self.crg.cd_sys.clk,
            self.hdmi_in0.clocking.cd_pix.clk,
            self.hdmi_in0.clocking.cd_pix1p25x.clk,
            self.hdmi_in0.clocking.cd_pix5x.clk)

        # hdmi out
        hdmi_out0_dram_port = self.sdram.crossbar.get_port(mode="read", dw=16, cd="hdmi_out0_pix", reverse=True)
        self.submodules.hdmi_out0 = VideoOut(platform.device,
                                            platform.request("hdmi_out"),
                                            hdmi_out0_dram_port,
                                            "ycbcr422",
                                            fifo_depth=4096)

        self.platform.add_period_constraint(self.hdmi_out0.driver.clocking.cd_pix.clk, period_ns(1*pix_freq))
        self.platform.add_period_constraint(self.hdmi_out0.driver.clocking.cd_pix5x.clk, period_ns(5*pix_freq))

        self.platform.add_false_path_constraints(
            self.crg.cd_sys.clk,
            self.hdmi_out0.driver.clocking.cd_pix.clk,
            self.hdmi_out0.driver.clocking.cd_pix5x.clk)

def main():
    parser = argparse.ArgumentParser(description="NeTV2 LiteX SoC")
    builder_args(parser)
    soc_sdram_args(parser)
    parser.add_argument("--nocompile-gateware", action="store_true")
    args = parser.parse_args()

    platform = netv2.Platform()
    soc = VideoSoC(platform, **soc_sdram_argdict(args))
    builder = Builder(soc, output_dir="build")
    vns = builder.build()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import os
import sys
import subprocess
import json
import datetime
import re #regex

ip = "127.0.0.1"
if len(sys.argv) > 1:
    ip = sys.argv[1]

print "This will create a new genesis block and make changes to your config files."
print "Continue? [y/n]"
entry = raw_input()
if entry != "y":
    sys.exit("Operation canceled")

input_log = []

input_log.append(">>> wallet_create default password")
input_log.append(">>> wallet_set_automatic_backups false ")

new_genesis = {}
with open("libraries/blockchain/genesis.json") as genesis:
    new_genesis = json.load(genesis)

new_genesis["timestamp"] = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
new_genesis["bts_sharedrop"] = []
new_genesis["names"] = []

with open("libraries/blockchain/bts-sharedrop.json") as snapshot:
    items = json.loads(snapshot.read())
    for item in items:
        new_genesis["bts_sharedrop"].append(item)

for i in range(101):
    keys = json.loads(subprocess.check_output(["./programs/utils/bts_create_key"]))
    input_log.append(">>> wallet_import_private_key " + keys["wif_private_key"])

    acct = {
        "name": "init" + str(i),
        "owner": keys["public_key"],
        "delegate_pay_rate": 1
    }
    new_genesis["names"].append(acct)

input_log.append(">>> wallet_delegate_set_block_production ALL true")
input_log.append(">>> wallet_set_transaction_scanning true")

with open("libraries/blockchain/include/bts/blockchain/config.hpp") as config:
    with open("tmp_config", "w") as tmp:
        for line in config:
            if line.startswith("#define BTS_TEST_NETWORK_VERSION"):
                n = int(re.findall(r'\d+', line)[0])
                tmp.write("#define BTS_TEST_NETWORK_VERSION                            " + str(n+1) + " // autogenerated\n")
            else:
                tmp.write(line)
os.rename("tmp_config", "libraries/blockchain/include/bts/blockchain/config.hpp")

#with open("libraries/net/include/bts/net/config.hpp") as config:
#    with open("tmp_config", "w") as tmp:
#        for line in config:
#            if line.startswith("#define BTS_NET_TEST_SEED_IP"):
#                n = int(re.findall(r'\d+', line)[0])
#                tmp.write("#define BTS_NET_TEST_SEED_IP                            \"" + ip + "\" // autogenerated\n")
#            else:
#                tmp.write(line)
#os.rename("tmp_config", "libraries/net/include/bts/net/config.hpp")

old_balances = []

with open("libraries/blockchain/genesis.json", "w") as genesis:
    genesis.write(json.dumps(new_genesis, indent=4))

with open("testnet_setup_log.txt", "w") as log:
    for line in input_log:
        log.write(line + "\n")

subprocess.call(["make", "-j4"])
subprocess.call(["./programs/client/bitshares_client", "--input-log", "testnet_setup_log.txt", "--min-delegate-connection-count", "0"])

<commit_msg>testnet setup script asks before rebuilding<commit_after>import os
import sys
import subprocess
import json
import datetime
import re #regex

ip = "127.0.0.1"
if len(sys.argv) > 1:
    ip = sys.argv[1]

print "This will create a new genesis block and make changes to your config files."
n = raw_input("Overwrite config files? [Y/n]")
if n != "n":
    sys.exit("Operation canceled")

input_log = []

input_log.append(">>> wallet_create default password")
input_log.append(">>> wallet_set_automatic_backups false ")

new_genesis = {}
with open("libraries/blockchain/genesis.json") as genesis:
    new_genesis = json.load(genesis)

new_genesis["timestamp"] = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
new_genesis["bts_sharedrop"] = []
new_genesis["names"] = []

with open("libraries/blockchain/bts-sharedrop.json") as snapshot:
    items = json.loads(snapshot.read())
    for item in items:
        new_genesis["bts_sharedrop"].append(item)

for i in range(101):
    keys = json.loads(subprocess.check_output(["./programs/utils/bts_create_key"]))
    input_log.append(">>> wallet_import_private_key " + keys["wif_private_key"])

    acct = {
        "name": "init" + str(i),
        "owner": keys["public_key"],
        "delegate_pay_rate": 1
    }
    new_genesis["names"].append(acct)

input_log.append(">>> wallet_delegate_set_block_production ALL true")
input_log.append(">>> wallet_set_transaction_scanning true")

with open("libraries/blockchain/include/bts/blockchain/config.hpp") as config:
    with open("tmp_config", "w") as tmp:
        for line in config:
            if line.startswith("#define BTS_TEST_NETWORK_VERSION"):
                n = int(re.findall(r'\d+', line)[0])
                tmp.write("#define BTS_TEST_NETWORK_VERSION                            " + str(n+1) + " // autogenerated\n")
            else:
                tmp.write(line)
os.rename("tmp_config", "libraries/blockchain/include/bts/blockchain/config.hpp")

#with open("libraries/net/include/bts/net/config.hpp") as config:
#    with open("tmp_config", "w") as tmp:
#        for line in config:
#            if line.startswith("#define BTS_NET_TEST_SEED_IP"):
#                n = int(re.findall(r'\d+', line)[0])
#                tmp.write("#define BTS_NET_TEST_SEED_IP                            \"" + ip + "\" // autogenerated\n")
#            else:
#                tmp.write(line)
#os.rename("tmp_config", "libraries/net/include/bts/net/config.hpp")

old_balances = []

with open("libraries/blockchain/genesis.json", "w") as genesis:
    genesis.write(json.dumps(new_genesis, indent=4))

with open("testnet_setup_log.txt", "w") as log:
    for line in input_log:
        log.write(line + "\n")

n = raw_input("Start build? [Y/n]")
if n != "n":
    subprocess.call(["make", "-j4"])
    subprocess.call(["./programs/client/bitshares_client", "--input-log", "testnet_setup_log.txt", "--min-delegate-connection-count", "0"])

<|endoftext|>


<commit_before>import numpy as np
from ngboost.ngboost import NGBoost
from ngboost.distns import RegressionDistn, ClassificationDistn
from ngboost.distns import Bernoulli, Normal, LogNormal
from ngboost.scores import LogScore
from ngboost.learners import default_tree_learner
from sklearn.base import BaseEstimator

class NGBRegressor(NGBoost, BaseEstimator):

    def __init__(self,
                 Dist=Normal,
                 Score=LogScore,
                 Base=default_tree_learner,
                 natural_gradient=True,
                 n_estimators=500,
                 learning_rate=0.01,
                 minibatch_frac=1.0,
                 verbose=True,
                 verbose_eval=100,
                 tol=1e-4):
        assert issubclass(Dist, RegressionDistn), f'{Dist.__name__} is not useable for regression.'
        super().__init__(Dist, Score, Base, natural_gradient, n_estimators, learning_rate,
                         minibatch_frac, verbose, verbose_eval, tol, random_state)

class NGBClassifier(NGBoost, BaseEstimator):

    def __init__(self,
                 Dist=Bernoulli,
                 Score=LogScore,
                 Base=default_tree_learner,
                 natural_gradient=True,
                 n_estimators=500,
                 learning_rate=0.01,
                 minibatch_frac=1.0,
                 verbose=True,
                 verbose_eval=100,
                 tol=1e-4):
        assert issubclass(Dist, ClassificationDistn), f'{Dist.__name__} is not useable for classification.'
        super().__init__(Dist, Score, Base, natural_gradient, n_estimators, learning_rate,
                         minibatch_frac, verbose, verbose_eval, tol, random_state)

    def predict_proba(self, X, max_iter=None):
        return self.pred_dist(X, max_iter=max_iter).class_probs()

    def staged_predict_proba(self, X, max_iter=None):
        return [dist.class_probs() for dist in self.staged_pred_dist(X, max_iter=max_iter)]

class NGBSurvival(NGBoost, BaseEstimator):

    def __init__(self,
                 Dist=LogNormal,
                 Score=LogScore,
                 Base=default_tree_learner,
                 natural_gradient=True,
                 n_estimators=500,
                 learning_rate=0.01,
                 minibatch_frac=1.0,
                 verbose=True,
                 verbose_eval=100,
                 tol=1e-4):
        # do something else here to check survival
        super().__init__(Dist, Score, Base, natural_gradient, n_estimators, learning_rate,
                         minibatch_frac, verbose, verbose_eval, tol, random_state)

<commit_msg>fix random_state<commit_after>import numpy as np
from ngboost.ngboost import NGBoost
from ngboost.distns import RegressionDistn, ClassificationDistn
from ngboost.distns import Bernoulli, Normal, LogNormal
from ngboost.scores import LogScore
from ngboost.learners import default_tree_learner
from sklearn.base import BaseEstimator

class NGBRegressor(NGBoost, BaseEstimator):

    def __init__(self,
                 Dist=Normal,
                 Score=LogScore,
                 Base=default_tree_learner,
                 natural_gradient=True,
                 n_estimators=500,
                 learning_rate=0.01,
                 minibatch_frac=1.0,
                 verbose=True,
                 verbose_eval=100,
                 tol=1e-4,
                 random_state=None):
        assert issubclass(Dist, RegressionDistn), f'{Dist.__name__} is not useable for regression.'
        super().__init__(Dist, Score, Base, natural_gradient, n_estimators, learning_rate,
                         minibatch_frac, verbose, verbose_eval, tol, random_state)

class NGBClassifier(NGBoost, BaseEstimator):

    def __init__(self,
                 Dist=Bernoulli,
                 Score=LogScore,
                 Base=default_tree_learner,
                 natural_gradient=True,
                 n_estimators=500,
                 learning_rate=0.01,
                 minibatch_frac=1.0,
                 verbose=True,
                 verbose_eval=100,
                 tol=1e-4,
                 random_state=None):
        assert issubclass(Dist, ClassificationDistn), f'{Dist.__name__} is not useable for classification.'
        super().__init__(Dist, Score, Base, natural_gradient, n_estimators, learning_rate,
                         minibatch_frac, verbose, verbose_eval, tol, random_state)

    def predict_proba(self, X, max_iter=None):
        return self.pred_dist(X, max_iter=max_iter).class_probs()

    def staged_predict_proba(self, X, max_iter=None):
        return [dist.class_probs() for dist in self.staged_pred_dist(X, max_iter=max_iter)]

class NGBSurvival(NGBoost, BaseEstimator):

    def __init__(self,
                 Dist=LogNormal,
                 Score=LogScore,
                 Base=default_tree_learner,
                 natural_gradient=True,
                 n_estimators=500,
                 learning_rate=0.01,
                 minibatch_frac=1.0,
                 verbose=True,
                 verbose_eval=100,
                 tol=1e-4):
        # do something else here to check survival
        super().__init__(Dist, Score, Base, natural_gradient, n_estimators, learning_rate,
                         minibatch_frac, verbose, verbose_eval, tol, random_state)

<|endoftext|>