<commit_before>import json
import re
from datetime import datetime
from steembase.operations import Comment_options

import steem as stm
from funcy import walk_values

from .amount import Amount
from .utils import (
    resolveIdentifier,
    constructIdentifier,
    remove_from_dict,
    parse_time,
)

class VotingInvalidOnArchivedPost(Exception):
    pass

class Post(dict):
    """ This object gets instanciated by Steem.streams and is used as an
        abstraction layer for Comments in Steem

        :param Steem steem: An instance of the Steem() object
        :param object post: The post as obtained by `get_content`
    """
    steem = None

    def __init__(
        self,
        post,
        steem_instance=None,
        lazy=False
    ):
        # Legacy API compatibility
        if isinstance(post, stm.Steem) and steem_instance:
            import warnings
            tmp = steem_instance
            steem_instance = post
            post = tmp
            warnings.warn("The Post() API has changed! Please read the documentation and update your code")

        if not steem_instance:
            steem_instance = stm.Steem()
        self.steem = steem_instance
        self.loaded = False

        if isinstance(post, str):  # From identifier
            parts = post.split("@")
            self.identifier = "@" + parts[-1]

            if not lazy:
                self.refresh()

        elif (isinstance(post, dict) and  # From dictionary
                "author" in post and
                "permlink" in post):
            # strip leading @
            if post["author"][0] == "@":
                post["author"] = post["author"][1:]
            self.identifier = constructIdentifier(
                post["author"],
                post["permlink"]
            )

            if "created" in post:
                self._store_post(post)

        else:
            raise ValueError("Post expects an identifier or a dict "
                             "with author and permlink!")

    def refresh(self):
        post_author, post_permlink = resolveIdentifier(self.identifier)
        post = self.steem.rpc.get_content(post_author, post_permlink)
        if not post["permlink"]:
            raise Exception("Post does not exist!")

        # If this 'post' comes from an operation, it might carry a patch
        if "body" in post and re.match("^@@", post["body"]):
            self._patched = True
            self._patch = post["body"]

        # Total reward
        post["total_payout_reward"] = "%.3f SBD" % (
            Amount(post.get("total_payout_value", "0 SBD")) +
            Amount(post.get("total_pending_payout_value", "0 SBD"))
        )

        # Parse Times
        parse_times = ["active",
                       "cashout_time",
                       "created",
                       "last_payout",
                       "last_update",
                       "max_cashout_time"]
        for p in parse_times:
            post[p] = parse_time(post.get(p, "1970-01-01T00:00:00"))

        # Parse Amounts
        sbd_amounts = [
            "total_payout_reward",
            "max_accepted_payout",
            "pending_payout_value",
            "curator_payout_value",
            "total_pending_payout_value",
            "promoted",
        ]
        for p in sbd_amounts:
            post[p] = Amount(post.get(p, "0.000 SBD"))

        # Try to properly format json meta data
        try:
            meta_str = post.get("json_metadata", "")
            post['json_metadata'] = json.loads(meta_str)
        except:
            post['json_metadata'] = dict()

        post["tags"] = []
        if post["depth"] == 0:
            post["tags"] = (
                [post["parent_permlink"]] +
                post["json_metadata"].get("tags", [])
            )

        # Retrieve the root comment
        self.openingPostIdentifier, self.category = self._getOpeningPost(post)

        self._store_post(post)

    def _store_post(self, post):
        # Store original values as obtained from the rpc
        for key, value in post.items():
            super(Post, self).__setitem__(key, value)

        # Set attributes as well
        for key in post:
            setattr(self, key, post[key])

        # also set identifier
        super(Post, self).__setitem__("identifier", self.identifier)

        self.loaded = True

    def __getattr__(self, key):
        if not self.loaded:
            self.refresh()
        return object.__getattribute__(self, key)

    def __getitem__(self, key):
        if not self.loaded:
            self.refresh()
        return super(Post, self).__getitem__(key)

    def __repr__(self):
        return "<Post-%s>" % self.identifier

    __str__ = __repr__

    def _getOpeningPost(self, post=None):
        if not post:
            post = self
        m = re.match("/([^/]*)/@([^/]*)/([^#]*).*",
                     post.get("url", ""))
        if not m:
            return "", ""
        else:
            category = m.group(1)
            author = m.group(2)
            permlink = m.group(3)
            return constructIdentifier(
                author, permlink
            ), category

    def get_comments(self, sort="total_payout_reward"):
        """ Return **first-level** comments of the post.
        """
        post_author, post_permlink = resolveIdentifier(self.identifier)
        posts = self.steem.rpc.get_content_replies(post_author, post_permlink)
        r = []
        for post in posts:
            r.append(Post(post, steem_instance=self.steem))
        if sort == "total_payout_value":
            r = sorted(r, key=lambda x: float(
                x["total_payout_value"]
            ), reverse=True)
        elif sort == "total_payout_reward":
            r = sorted(r, key=lambda x: float(
                x["total_payout_reward"]
            ), reverse=True)
        else:
            r = sorted(r, key=lambda x: x[sort])
        return(r)

    def reply(self, body, title="", author="", meta=None):
        """ Reply to the post

            :param str body: (required) body of the reply
            :param str title: Title of the reply
            :param str author: Author of reply
            :param json meta: JSON Meta data
        """
        return self.steem.reply(self.identifier, body, title, author, meta)

    def upvote(self, weight=+100, voter=None):
        """ Upvote the post

            :param float weight: (optional) Weight for posting (-100.0 - +100.0) defaults to +100.0
            :param str voter: (optional) Voting account
        """
        return self.vote(weight, voter=voter)

    def downvote(self, weight=-100, voter=None):
        """ Downvote the post

            :param float weight: (optional) Weight for posting (-100.0 - +100.0) defaults to -100.0
            :param str voter: (optional) Voting account
        """
        return self.vote(weight, voter=voter)

    def vote(self, weight, voter=None):
        """ Vote the post

            :param float weight: Weight for posting (-100.0 - +100.0)
            :param str voter: Voting account
        """
        # Test if post is archived, if so, voting is worthless but just
        # pollutes the blockchain and account history
        if getattr(self, "mode") == "archived":
            raise VotingInvalidOnArchivedPost
        return self.steem.vote(self.identifier, weight, voter=voter)

    @property
    def reward(self):
        """Return a float value of estimated total SBD reward.
        """
        if not self.loaded:
            self.refresh()
        return self['total_payout_reward']

    @property
    def meta(self):
        if not self.loaded:
            self.refresh()
        return self.get('json_metadata', dict())

    def time_elapsed(self):
        """Return a timedelta on how old the post is.
        """
        return datetime.utcnow() - self['created']

    def is_main_post(self):
        """ Retuns True if main post, and False if this is a comment (reply).
        """
        return self['depth'] == 0

    def is_opening_post(self):
        """ Retuns True if main post, and False if this is a comment (reply).
        """
        return self['depth'] == 0

    def is_comment(self):
        """ Retuns True if post is a comment
        """
        return self['depth'] > 0

    def curation_reward_pct(self):
        """ If post is less than 30 minutes old, it will incur a curation reward penalty.
        """
        reward = (self.time_elapsed().seconds / 1800) * 100
        if reward > 100:
            reward = 100
        return reward

    def export(self):
        """ This method returns a dictionary that is type-safe to store as JSON or in a database.
        """
        self.refresh()

        # Remove Steem instance object
        safe_dict = remove_from_dict(self, ['steem'])

        # Convert Amount class objects into pure dictionaries
        def decompose_amounts(item):
            if type(item) == Amount:
                return item.__dict__
            return item
        return walk_values(decompose_amounts, safe_dict)

    def set_comment_options(self, options):
        op = Comment_options(
            **{
                "author": self["author"],
                "permlink": self["permlink"],
                "max_accepted_payout": options.get("max_accepted_payout", "1000000.000 SBD"),
                "percent_steem_dollars": options.get("percent_steem_dollars", 100) * 100,
                "allow_votes": options.get("allow_votes", True),
                "allow_curation_rewards": options.get("allow_curation_rewards", True),
            }
        )
        return self.steem.finalizeOp(op, self["author"], "posting")
<commit_msg>[post] fix set_comment_options()<commit_after>import json
import re
from datetime import datetime
from steembase.operations import Comment_options

import steem as stm
from funcy import walk_values

from .amount import Amount
from .utils import (
    resolveIdentifier,
    constructIdentifier,
    remove_from_dict,
    parse_time,
)

class VotingInvalidOnArchivedPost(Exception):
    pass

class Post(dict):
    """ This object gets instanciated by Steem.streams and is used as an
        abstraction layer for Comments in Steem

        :param Steem steem: An instance of the Steem() object
        :param object post: The post as obtained by `get_content`
    """
    steem = None

    def __init__(
        self,
        post,
        steem_instance=None,
        lazy=False
    ):
        # Legacy API compatibility
        if isinstance(post, stm.Steem) and steem_instance:
            import warnings
            tmp = steem_instance
            steem_instance = post
            post = tmp
            warnings.warn("The Post() API has changed! Please read the documentation and update your code")

        if not steem_instance:
            steem_instance = stm.Steem()
        self.steem = steem_instance
        self.loaded = False

        if isinstance(post, str):  # From identifier
            parts = post.split("@")
            self.identifier = "@" + parts[-1]

            if not lazy:
                self.refresh()

        elif (isinstance(post, dict) and  # From dictionary
                "author" in post and
                "permlink" in post):
            # strip leading @
            if post["author"][0] == "@":
                post["author"] = post["author"][1:]
            self.identifier = constructIdentifier(
                post["author"],
                post["permlink"]
            )

            if "created" in post:
                self._store_post(post)

        else:
            raise ValueError("Post expects an identifier or a dict "
                             "with author and permlink!")

    def refresh(self):
        post_author, post_permlink = resolveIdentifier(self.identifier)
        post = self.steem.rpc.get_content(post_author, post_permlink)
        if not post["permlink"]:
            raise Exception("Post does not exist!")

        # If this 'post' comes from an operation, it might carry a patch
        if "body" in post and re.match("^@@", post["body"]):
            self._patched = True
            self._patch = post["body"]

        # Total reward
        post["total_payout_reward"] = "%.3f SBD" % (
            Amount(post.get("total_payout_value", "0 SBD")) +
            Amount(post.get("total_pending_payout_value", "0 SBD"))
        )

        # Parse Times
        parse_times = ["active",
                       "cashout_time",
                       "created",
                       "last_payout",
                       "last_update",
                       "max_cashout_time"]
        for p in parse_times:
            post[p] = parse_time(post.get(p, "1970-01-01T00:00:00"))

        # Parse Amounts
        sbd_amounts = [
            "total_payout_reward",
            "max_accepted_payout",
            "pending_payout_value",
            "curator_payout_value",
            "total_pending_payout_value",
            "promoted",
        ]
        for p in sbd_amounts:
            post[p] = Amount(post.get(p, "0.000 SBD"))

        # Try to properly format json meta data
        try:
            meta_str = post.get("json_metadata", "")
            post['json_metadata'] = json.loads(meta_str)
        except:
            post['json_metadata'] = dict()

        post["tags"] = []
        if post["depth"] == 0:
            post["tags"] = (
                [post["parent_permlink"]] +
                post["json_metadata"].get("tags", [])
            )

        # Retrieve the root comment
        self.openingPostIdentifier, self.category = self._getOpeningPost(post)

        self._store_post(post)

    def _store_post(self, post):
        # Store original values as obtained from the rpc
        for key, value in post.items():
            super(Post, self).__setitem__(key, value)

        # Set attributes as well
        for key in post:
            setattr(self, key, post[key])

        # also set identifier
        super(Post, self).__setitem__("identifier", self.identifier)

        self.loaded = True

    def __getattr__(self, key):
        if not self.loaded:
            self.refresh()
        return object.__getattribute__(self, key)

    def __getitem__(self, key):
        if not self.loaded:
            self.refresh()
        return super(Post, self).__getitem__(key)

    def __repr__(self):
        return "<Post-%s>" % self.identifier

    __str__ = __repr__

    def _getOpeningPost(self, post=None):
        if not post:
            post = self
        m = re.match("/([^/]*)/@([^/]*)/([^#]*).*",
                     post.get("url", ""))
        if not m:
            return "", ""
        else:
            category = m.group(1)
            author = m.group(2)
            permlink = m.group(3)
            return constructIdentifier(
                author, permlink
            ), category

    def get_comments(self, sort="total_payout_reward"):
        """ Return **first-level** comments of the post.
        """
        post_author, post_permlink = resolveIdentifier(self.identifier)
        posts = self.steem.rpc.get_content_replies(post_author, post_permlink)
        r = []
        for post in posts:
            r.append(Post(post, steem_instance=self.steem))
        if sort == "total_payout_value":
            r = sorted(r, key=lambda x: float(
                x["total_payout_value"]
            ), reverse=True)
        elif sort == "total_payout_reward":
            r = sorted(r, key=lambda x: float(
                x["total_payout_reward"]
            ), reverse=True)
        else:
            r = sorted(r, key=lambda x: x[sort])
        return(r)

    def reply(self, body, title="", author="", meta=None):
        """ Reply to the post

            :param str body: (required) body of the reply
            :param str title: Title of the reply
            :param str author: Author of reply
            :param json meta: JSON Meta data
        """
        return self.steem.reply(self.identifier, body, title, author, meta)

    def upvote(self, weight=+100, voter=None):
        """ Upvote the post

            :param float weight: (optional) Weight for posting (-100.0 - +100.0) defaults to +100.0
            :param str voter: (optional) Voting account
        """
        return self.vote(weight, voter=voter)

    def downvote(self, weight=-100, voter=None):
        """ Downvote the post

            :param float weight: (optional) Weight for posting (-100.0 - +100.0) defaults to -100.0
            :param str voter: (optional) Voting account
        """
        return self.vote(weight, voter=voter)

    def vote(self, weight, voter=None):
        """ Vote the post

            :param float weight: Weight for posting (-100.0 - +100.0)
            :param str voter: Voting account
        """
        # Test if post is archived, if so, voting is worthless but just
        # pollutes the blockchain and account history
        if getattr(self, "mode") == "archived":
            raise VotingInvalidOnArchivedPost
        return self.steem.vote(self.identifier, weight, voter=voter)

    @property
    def reward(self):
        """Return a float value of estimated total SBD reward.
        """
        if not self.loaded:
            self.refresh()
        return self['total_payout_reward']

    @property
    def meta(self):
        if not self.loaded:
            self.refresh()
        return self.get('json_metadata', dict())

    def time_elapsed(self):
        """Return a timedelta on how old the post is.
        """
        return datetime.utcnow() - self['created']

    def is_main_post(self):
        """ Retuns True if main post, and False if this is a comment (reply).
        """
        return self['depth'] == 0

    def is_opening_post(self):
        """ Retuns True if main post, and False if this is a comment (reply).
        """
        return self['depth'] == 0

    def is_comment(self):
        """ Retuns True if post is a comment
        """
        return self['depth'] > 0

    def curation_reward_pct(self):
        """ If post is less than 30 minutes old, it will incur a curation reward penalty.
        """
        reward = (self.time_elapsed().seconds / 1800) * 100
        if reward > 100:
            reward = 100
        return reward

    def export(self):
        """ This method returns a dictionary that is type-safe to store as JSON or in a database.
        """
        self.refresh()

        # Remove Steem instance object
        safe_dict = remove_from_dict(self, ['steem'])

        # Convert Amount class objects into pure dictionaries
        def decompose_amounts(item):
            if type(item) == Amount:
                return item.__dict__
            return item
        return walk_values(decompose_amounts, safe_dict)

    def set_comment_options(self, options):
        op = Comment_options(
            **{"author": self["author"],
               "permlink": self["permlink"],
               "max_accepted_payout":
                   options.get("max_accepted_payout", str(self["max_accepted_payout"])),
               "percent_steem_dollars": int(
                   options.get("percent_steem_dollars",
                               self["percent_steem_dollars"] / 100
                               ) * 100),
               "allow_votes":
                   options.get("allow_votes", self["allow_votes"]),
               "allow_curation_rewards":
                   options.get("allow_curation_rewards", self["allow_curation_rewards"]),
               }
        )
        return self.steem.finalizeOp(op, self["author"], "posting")
<|endoftext|>


<commit_before><commit_msg>Update results.py<commit_after><|endoftext|>


<commit_before>""" This file contains different utility functions that are not connected
in anyway to the networks presented in the tutorials, but rather help in
processing the outputs into a more understandable way.

For example ``tiled_array_image`` helps in generating a easy to grasp
image from a set of samples or weights.
"""

import os, cPickle, gzip
import numpy
import theano
import PIL.Image

def tiled_array_image(A):
    """
    Transform an array with one flattened image per row, into an image in
    which the rows are reshaped and layed out like tiles on a floor.

    This function is useful for visualizing datasets whose rows are images,
    and also columns of matrices for transforming those rows
    (such as the first layer of a neural net).

    """

    # Turn [tiles, pixels] array 'A' into
    # [X, Y] tiles each being [x,y] pixels.
    # where X*Y = tiles and x*y = pixels
    # in an [X*x, Y*y] image, plus padding.

    A = A.T
    (tiles, pixels) = A.shape
    if tiles >= pixels:
        A = A.T
        (tiles, pixels) = (pixels, tiles)
    x = int(numpy.sqrt(pixels))
    y = (pixels-1) // x + 1
    img_shape = (x, y)
    pad = 1 if max(x,y) < 4 else 2
    tile_spacing = (pad, pad)
    (x1, y1) = [d+s for (d,s) in zip(img_shape, tile_spacing)]
    X = min((300 // x1) + 1, int(numpy.sqrt(tiles))+1)
    Y = min((300 // y1) + 1, (tiles-1) // X + 1)
    tile_shape = (X, Y)

    # scale_rows_to_unit_interval
    A = A.copy()
    A -= A.min(axis=1)[:, numpy.newaxis]
    A /= (A.max(axis=1)[:, numpy.newaxis] + 1e-8)

    # pad because pixels per tile may be not square
    padded = numpy.zeros([A.shape[0], numpy.product(img_shape)], A.dtype)
    padded[:] = numpy.NaN
    padded[:, 0:A.shape[1]] = A
    A = padded

    (xp, yp) = x+pad, y+pad
    out_array = numpy.zeros([X*xp-pad, Y*yp-pad], dtype=A.dtype)
    out_array[:] = numpy.NaN

    for tile in range(min(X*Y, tiles)):
        (row, col) = divmod(tile, X)
        out_array[row*xp:row*xp+x, col*yp:col*yp+y] = A[tile].reshape(img_shape)

    image = PIL.Image.fromarray(255*numpy.nan_to_num(out_array)).convert("L")
    mask = PIL.Image.fromarray(255-255*(numpy.isnan(out_array))).convert("L")
    image.putalpha(mask)
    scale = 300 // max(image.size)
    if scale > 1:
        scale = min(scale, 10)
        image = image.resize([scale*d for d in image.size])

    return image

def load_data_from_file(dataset_file):
    ''' Loads the dataset

    :type dataset_file: string
    :param dataset_file: the path to the dataset
    '''

    if dataset_file.endswith('.npy'):
        #x = numpy.lib.format.open_memmap(dataset_file, mode='r') # or mode='c'
        x = numpy.lib.format.read_array(open(dataset_file, 'rb'))
        assert x.dtype is numpy.dtype(theano.config.floatX), (x.dtype, theano.config.floatX)
        y = None
    else:
        f = gzip.open(dataset_file, 'rb')
        data = cPickle.load(f)
        if len(data) == 3:
            (x, x_sums, y) = data
        else:
            (x, y) = data
        x = numpy.asarray(x, dtype=theano.config.floatX)
    return (x, y)

def load_data(file_prefix):
    """Try any known file formats in which the array might be stored"""
    for file_suffix in ['.npy', '_0.pkl.gz']:
        filename = file_prefix + file_suffix
        if os.path.exists(filename):
            return load_data_from_file(filename)
    raise RuntimeError('No {0} data found in {1}/'.format(part, datadir))

<commit_msg>Rows/columns were confused at one point in trace image layout<commit_after>""" This file contains different utility functions that are not connected
in anyway to the networks presented in the tutorials, but rather help in
processing the outputs into a more understandable way.

For example ``tiled_array_image`` helps in generating a easy to grasp
image from a set of samples or weights.
"""

import os, cPickle, gzip
import numpy
import theano
import PIL.Image

def tiled_array_image(A):
    """
    Transform an array with one flattened image per row, into an image in
    which the rows are reshaped and layed out like tiles on a floor.

    This function is useful for visualizing datasets whose rows are images,
    and also columns of matrices for transforming those rows
    (such as the first layer of a neural net).

    """

    # Turn [tiles, pixels] array 'A' into
    # [X, Y] tiles each being [x,y] pixels.
    # where X*Y = tiles and x*y = pixels
    # in an [X*x, Y*y] image, plus padding.

    A = A.T
    (tiles, pixels) = A.shape
    if tiles >= pixels:
        A = A.T
        (tiles, pixels) = (pixels, tiles)
    x = int(numpy.sqrt(pixels))
    y = (pixels-1) // x + 1
    img_shape = (y, x)
    pad = 1 if max(x,y) < 4 else 2
    tile_spacing = (pad, pad)
    (y1, x1) = [d+s for (d,s) in zip(img_shape, tile_spacing)]
    X = min((300 // x1) + 1, int(numpy.sqrt(tiles))+1)
    Y = min((300 // y1) + 1, (tiles-1) // X + 1)
    tile_shape = (Y, X)

    # scale_rows_to_unit_interval
    A = A.copy()
    A -= A.min(axis=1)[:, numpy.newaxis]
    A /= (A.max(axis=1)[:, numpy.newaxis] + 1e-8)

    # pad because pixels per tile may be not square
    padded = numpy.zeros([A.shape[0], numpy.product(img_shape)], A.dtype)
    padded[:] = numpy.NaN
    padded[:, 0:A.shape[1]] = A
    A = padded

    (xp, yp) = x+pad, y+pad
    out_array = numpy.zeros([Y*yp-pad, X*xp-pad], dtype=A.dtype)
    out_array[:] = numpy.NaN

    for tile in range(min(X*Y, tiles)):
        (row, col) = divmod(tile, X)
        out_array[row*yp:row*yp+y, col*xp:col*xp+x] = A[tile].reshape(img_shape)

    image = PIL.Image.fromarray(255*numpy.nan_to_num(out_array)).convert("L")
    mask = PIL.Image.fromarray(255-255*(numpy.isnan(out_array))).convert("L")
    image.putalpha(mask)
    scale = 300 // max(image.size)
    if scale > 1:
        scale = min(scale, 10)
        image = image.resize([scale*d for d in image.size])

    return image

def load_data_from_file(dataset_file):
    ''' Loads the dataset

    :type dataset_file: string
    :param dataset_file: the path to the dataset
    '''

    if dataset_file.endswith('.npy'):
        #x = numpy.lib.format.open_memmap(dataset_file, mode='r') # or mode='c'
        x = numpy.lib.format.read_array(open(dataset_file, 'rb'))
        assert x.dtype is numpy.dtype(theano.config.floatX), (x.dtype, theano.config.floatX)
        y = None
    else:
        f = gzip.open(dataset_file, 'rb')
        data = cPickle.load(f)
        if len(data) == 3:
            (x, x_sums, y) = data
        else:
            (x, y) = data
        x = numpy.asarray(x, dtype=theano.config.floatX)
    return (x, y)

def load_data(file_prefix):
    """Try any known file formats in which the array might be stored"""
    for file_suffix in ['.npy', '_0.pkl.gz']:
        filename = file_prefix + file_suffix
        if os.path.exists(filename):
            return load_data_from_file(filename)
    raise RuntimeError('No {0} data found in {1}/'.format(part, datadir))

<|endoftext|>


<commit_before><commit_msg>add comment to api models file<commit_after># here to keep django from complaining about missing file
<|endoftext|>


<commit_before>from streaming import retrieve_authentication
import re, tweepy, json

EVENT = r'(fire|flooding|winds)'
HASHTAG_PATTERN = re.compile('(#\w+' + EVENT + ')', re.IGNORECASE)

extract = lambda x: HASHTAG_PATTERN.search(x).group()
match = lambda x: HASHTAG_PATTERN.search(x) is not None

def most_common(xs, n):
    from collections import Counter
    c = Counter()
    for x in xs:
        c[x] += 1
    return [word for word, count in c.most_common(n)]

def parse(xs):
    for x in xs:
        if match(x): yield extract(x)

def calfire_hashtags(calfire_twitter_name):
    auth = retrieve_authentication()
    api = tweepy.API(auth)
    response = api.user_timeline(screen_name=calfire_twitter_name, count=100)
    statuses = [obj.text for obj in response]
    return most_common(parse(statuses), 3)

def top_trending(api, woeid):
    auth = retrieve_authentication()
    api = tweepy.API(auth)
    response = api.trends_place(woeid)[0]
    trends = [trend['name'] for trend in response['trends']]
    return trends

if __name__ == "__main__":

    for hashtag in calfire_hashtags("CAL_FIRE"):
        print(hashtag)

    '''
    with open("woeids.json") as handle:
        woeids = json.load(handle)

    for hashtag in top_trending(api, woeids['los angeles']):
        print(hashtag)
    '''
<commit_msg>refactored the code to work on top trending list<commit_after>from streaming import retrieve_authentication
import re, tweepy, json

EVENTS = '(fire|flooding|wind|earthquake)'

def most_common(xs, n):
    from collections import Counter
    c = Counter()
    for x in xs:
        c[x] += 1
    return [word for word, count in c.most_common(n)]

def parse(xs):
    r = re.compile(r'(#\w+' + EVENTS + ")", re.IGNORECASE)
    for x in xs:
        m = r.match(x)
        if m:
            yield m.group()

def calfire_hashtags(calfire_twitter_name):
    auth = retrieve_authentication()
    api = tweepy.API(auth)
    response = api.user_timeline(screen_name=calfire_twitter_name, count=100)
    statuses = [obj.text for obj in response]
    return most_common(parse(statuses), 3)

def event_hashtags(woeid):
    auth = retrieve_authentication()
    api = tweepy.API(auth)
    response = api.trends_place(woeid)[0]
    trends = [trend['name'] for trend in response['trends'] if trend['name'].startswith("#")]
    return parse(trends)

if __name__ == "__main__":

    for hashtag in calfire_hashtags("CAL_FIRE"):
        print(hashtag)

    with open("woeids.json") as handle:
        woeids = json.load(handle)

    for hashtag in event_hashtags(woeids['los angeles']):
        print(hashtag)
<|endoftext|>


<commit_before>#   Foremast - Pipeline Tooling
#
#   Copyright 2018 Gogo, LLC
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
"""Create Pipelines for Spinnaker."""
import collections
import json
import logging
import os
from pprint import pformat

import requests

from ..consts import API_URL, DEFAULT_RUN_AS_USER, EC2_PIPELINE_TYPES, GATE_CA_BUNDLE, GATE_CLIENT_CERT
from ..exceptions import SpinnakerPipelineCreationFailed
from ..utils import ami_lookup, generate_packer_filename, get_details, get_properties, get_subnets, get_template
from .clean_pipelines import clean_pipelines
from .construct_pipeline_block import construct_pipeline_block
from .renumerate_stages import renumerate_stages

class SpinnakerPipeline:
    """Manipulate Spinnaker Pipelines.

    Args:
        app (str): Application name.
        trigger_job (str): Jenkins trigger job.
        base (str): Base image name (i.e: fedora).
        prop_path (str): Path to the raw.properties.json.
        runway_dir (str): Path to local runway directory.
    """

    def __init__(self, app='', trigger_job='', prop_path='', base='', runway_dir=''):
        self.log = logging.getLogger(__name__)

        self.header = {'content-type': 'application/json'}
        self.here = os.path.dirname(os.path.realpath(__file__))

        self.runway_dir = os.path.expandvars(os.path.expanduser(runway_dir or ''))

        self.base = base
        self.trigger_job = trigger_job
        self.generated = get_details(app=app)
        self.app_name = self.generated.app_name()
        self.group_name = self.generated.project

        self.settings = get_properties(prop_path)
        self.environments = self.settings['pipeline']['env']

    def post_pipeline(self, pipeline):
        """Send Pipeline JSON to Spinnaker.

        Args:
            pipeline (json): json of the pipeline to be created in Spinnaker
        """
        url = "{0}/pipelines".format(API_URL)

        if isinstance(pipeline, str):
            pipeline_json = pipeline
        else:
            pipeline_json = json.dumps(pipeline)

        pipeline_dict = json.loads(pipeline_json)

        self.log.debug('Pipeline JSON:\n%s', pipeline_json)

        pipeline_response = requests.post(
            url, data=pipeline_json, headers=self.header, verify=GATE_CA_BUNDLE, cert=GATE_CLIENT_CERT)

        self.log.debug('Pipeline creation response:\n%s', pipeline_response.text)

        if not pipeline_response.ok:
            raise SpinnakerPipelineCreationFailed(
                'Failed to create pipeline for {0}: {1}'.format(self.app_name, pipeline_response.json()))

        self.log.info('Successfully created "%s" pipeline in application "%s".', pipeline_dict['name'],
                      pipeline_dict['application'])

    def render_wrapper(self, region='us-east-1'):
        """Generate the base Pipeline wrapper.

        This renders the non-repeatable stages in a pipeline, like jenkins, baking, tagging and notifications.

        Args:
            region (str): AWS Region.

        Returns:
            dict: Rendered Pipeline wrapper.

        """
        base = self.settings['pipeline']['base']

        if self.base:
            base = self.base

        email = self.settings['pipeline']['notifications']['email']
        slack = self.settings['pipeline']['notifications']['slack']
        baking_process = self.settings['pipeline']['image']['builder']
        provider = 'aws'
        root_volume_size = self.settings['pipeline']['image']['root_volume_size']
        bake_instance_type = self.settings['pipeline']['image']['bake_instance_type']

        ami_id = ami_lookup(name=base, region=region)

        ami_template_file = generate_packer_filename(provider, region, baking_process)

        pipeline_id = self.compare_with_existing(region=region)

        data = {
            'app': {
                'ami_id': ami_id,
                'appname': self.app_name,
                'base': base,
                'environment': 'packaging',
                'region': region,
                'triggerjob': self.trigger_job,
                'run_as_user': DEFAULT_RUN_AS_USER,
                'email': email,
                'slack': slack,
                'root_volume_size': root_volume_size,
                'bake_instance_type': bake_instance_type,
                'ami_template_file': ami_template_file,
                'pipeline': self.settings['pipeline']
            },
            'id': pipeline_id
        }

        self.log.debug('Wrapper app data:\n%s', pformat(data))

        wrapper = get_template(template_file='pipeline/pipeline_wrapper.json.j2', data=data, formats=self.generated)

        return json.loads(wrapper)

    def get_existing_pipelines(self):
        """Get existing pipeline configs for specific application.

        Returns:
            str: Pipeline config json

        """
        url = "{0}/applications/{1}/pipelineConfigs".format(API_URL, self.app_name)
        resp = requests.get(url, verify=GATE_CA_BUNDLE, cert=GATE_CLIENT_CERT)
        assert resp.ok, 'Failed to lookup pipelines for {0}: {1}'.format(self.app_name, resp.text)

        return resp.json()

    def compare_with_existing(self, region='us-east-1', onetime=False):
        """Compare desired pipeline with existing pipelines.

        Args:
            region (str): Region of desired pipeline.
            onetime (bool): Looks for different pipeline if Onetime

        Returns:
            str: pipeline_id if existing, empty string of not.

        """
        pipelines = self.get_existing_pipelines()
        pipeline_id = None
        found = False
        for pipeline in pipelines:
            correct_app_and_region = (pipeline['application'] == self.app_name) and (region in pipeline['name'])
            if onetime:
                onetime_str = "(onetime-{})".format(self.environments[0])
                if correct_app_and_region and onetime_str in pipeline['name']:
                    found = True
            elif correct_app_and_region:
                found = True

            if found:
                self.log.info('Existing pipeline found - %s', pipeline['name'])
                pipeline_id = pipeline['id']
                break
        else:
            self.log.info('No existing pipeline found')

        return pipeline_id

    def create_pipeline(self):
        """Main wrapper for pipeline creation.
        1. Runs clean_pipelines to clean up existing ones
        2. determines which environments the pipeline needs
        3. gets all subnets for template rendering
        4. Renders all of the pipeline blocks as defined in configs
        5. Runs post_pipeline to create pipeline
        """
        clean_pipelines(app=self.app_name, settings=self.settings)

        pipeline_envs = self.environments
        self.log.debug('Envs from pipeline.json: %s', pipeline_envs)

        regions_envs = collections.defaultdict(list)
        for env in pipeline_envs:
            for region in self.settings[env]['regions']:
                regions_envs[region].append(env)
        self.log.info('Environments and Regions for Pipelines:\n%s', json.dumps(regions_envs, indent=4))

        subnets = None
        pipelines = {}
        for region, envs in regions_envs.items():
            # TODO: Overrides for an environment no longer makes sense. Need to
            # provide override for entire Region possibly.
            pipelines[region] = self.render_wrapper(region=region)

            previous_env = None
            for env in envs:
                pipeline_block_data = {
                    "env": env,
                    "generated": self.generated,
                    "previous_env": previous_env,
                    "region": region,
                    "settings": self.settings[env][region],
                    "pipeline_data": self.settings['pipeline'],
                }

                if self.settings['pipeline']['type'] in EC2_PIPELINE_TYPES:
                    if not subnets:
                        subnets = get_subnets()
                    try:
                        region_subnets = {region: subnets[env][region]}
                    except KeyError:
                        self.log.info('%s is not available for %s.', env, region)
                        continue
                    pipeline_block_data['region_subnets'] = region_subnets

                block = construct_pipeline_block(**pipeline_block_data)
                pipelines[region]['stages'].extend(json.loads(block))
                previous_env = env

        self.log.debug('Assembled Pipelines:\n%s', pformat(pipelines))

        for region, pipeline in pipelines.items():
            renumerate_stages(pipeline)

            self.post_pipeline(pipeline)

        return True
<commit_msg>fix: Less redundant error message<commit_after>#   Foremast - Pipeline Tooling
#
#   Copyright 2018 Gogo, LLC
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
"""Create Pipelines for Spinnaker."""
import collections
import json
import logging
import os
from pprint import pformat

import requests

from ..consts import API_URL, DEFAULT_RUN_AS_USER, EC2_PIPELINE_TYPES, GATE_CA_BUNDLE, GATE_CLIENT_CERT
from ..exceptions import SpinnakerPipelineCreationFailed
from ..utils import ami_lookup, generate_packer_filename, get_details, get_properties, get_subnets, get_template
from .clean_pipelines import clean_pipelines
from .construct_pipeline_block import construct_pipeline_block
from .renumerate_stages import renumerate_stages

class SpinnakerPipeline:
    """Manipulate Spinnaker Pipelines.

    Args:
        app (str): Application name.
        trigger_job (str): Jenkins trigger job.
        base (str): Base image name (i.e: fedora).
        prop_path (str): Path to the raw.properties.json.
        runway_dir (str): Path to local runway directory.
    """

    def __init__(self, app='', trigger_job='', prop_path='', base='', runway_dir=''):
        self.log = logging.getLogger(__name__)

        self.header = {'content-type': 'application/json'}
        self.here = os.path.dirname(os.path.realpath(__file__))

        self.runway_dir = os.path.expandvars(os.path.expanduser(runway_dir or ''))

        self.base = base
        self.trigger_job = trigger_job
        self.generated = get_details(app=app)
        self.app_name = self.generated.app_name()
        self.group_name = self.generated.project

        self.settings = get_properties(prop_path)
        self.environments = self.settings['pipeline']['env']

    def post_pipeline(self, pipeline):
        """Send Pipeline JSON to Spinnaker.

        Args:
            pipeline (json): json of the pipeline to be created in Spinnaker
        """
        url = "{0}/pipelines".format(API_URL)

        if isinstance(pipeline, str):
            pipeline_json = pipeline
        else:
            pipeline_json = json.dumps(pipeline)

        pipeline_dict = json.loads(pipeline_json)

        self.log.debug('Pipeline JSON:\n%s', pipeline_json)

        pipeline_response = requests.post(
            url, data=pipeline_json, headers=self.header, verify=GATE_CA_BUNDLE, cert=GATE_CLIENT_CERT)

        self.log.debug('Pipeline creation response:\n%s', pipeline_response.text)

        if not pipeline_response.ok:
            raise SpinnakerPipelineCreationFailed('Pipeline for {0}: {1}'.format(self.app_name,
                                                                                pipeline_response.json()))

        self.log.info('Successfully created "%s" pipeline in application "%s".', pipeline_dict['name'],
                      pipeline_dict['application'])

    def render_wrapper(self, region='us-east-1'):
        """Generate the base Pipeline wrapper.

        This renders the non-repeatable stages in a pipeline, like jenkins, baking, tagging and notifications.

        Args:
            region (str): AWS Region.

        Returns:
            dict: Rendered Pipeline wrapper.

        """
        base = self.settings['pipeline']['base']

        if self.base:
            base = self.base

        email = self.settings['pipeline']['notifications']['email']
        slack = self.settings['pipeline']['notifications']['slack']
        baking_process = self.settings['pipeline']['image']['builder']
        provider = 'aws'
        root_volume_size = self.settings['pipeline']['image']['root_volume_size']
        bake_instance_type = self.settings['pipeline']['image']['bake_instance_type']

        ami_id = ami_lookup(name=base, region=region)

        ami_template_file = generate_packer_filename(provider, region, baking_process)

        pipeline_id = self.compare_with_existing(region=region)

        data = {
            'app': {
                'ami_id': ami_id,
                'appname': self.app_name,
                'base': base,
                'environment': 'packaging',
                'region': region,
                'triggerjob': self.trigger_job,
                'run_as_user': DEFAULT_RUN_AS_USER,
                'email': email,
                'slack': slack,
                'root_volume_size': root_volume_size,
                'bake_instance_type': bake_instance_type,
                'ami_template_file': ami_template_file,
                'pipeline': self.settings['pipeline']
            },
            'id': pipeline_id
        }

        self.log.debug('Wrapper app data:\n%s', pformat(data))

        wrapper = get_template(template_file='pipeline/pipeline_wrapper.json.j2', data=data, formats=self.generated)

        return json.loads(wrapper)

    def get_existing_pipelines(self):
        """Get existing pipeline configs for specific application.

        Returns:
            str: Pipeline config json

        """
        url = "{0}/applications/{1}/pipelineConfigs".format(API_URL, self.app_name)
        resp = requests.get(url, verify=GATE_CA_BUNDLE, cert=GATE_CLIENT_CERT)
        assert resp.ok, 'Failed to lookup pipelines for {0}: {1}'.format(self.app_name, resp.text)

        return resp.json()

    def compare_with_existing(self, region='us-east-1', onetime=False):
        """Compare desired pipeline with existing pipelines.

        Args:
            region (str): Region of desired pipeline.
            onetime (bool): Looks for different pipeline if Onetime

        Returns:
            str: pipeline_id if existing, empty string of not.

        """
        pipelines = self.get_existing_pipelines()
        pipeline_id = None
        found = False
        for pipeline in pipelines:
            correct_app_and_region = (pipeline['application'] == self.app_name) and (region in pipeline['name'])
            if onetime:
                onetime_str = "(onetime-{})".format(self.environments[0])
                if correct_app_and_region and onetime_str in pipeline['name']:
                    found = True
            elif correct_app_and_region:
                found = True

            if found:
                self.log.info('Existing pipeline found - %s', pipeline['name'])
                pipeline_id = pipeline['id']
                break
        else:
            self.log.info('No existing pipeline found')

        return pipeline_id

    def create_pipeline(self):
        """Main wrapper for pipeline creation.
        1. Runs clean_pipelines to clean up existing ones
        2. determines which environments the pipeline needs
        3. gets all subnets for template rendering
        4. Renders all of the pipeline blocks as defined in configs
        5. Runs post_pipeline to create pipeline
        """
        clean_pipelines(app=self.app_name, settings=self.settings)

        pipeline_envs = self.environments
        self.log.debug('Envs from pipeline.json: %s', pipeline_envs)

        regions_envs = collections.defaultdict(list)
        for env in pipeline_envs:
            for region in self.settings[env]['regions']:
                regions_envs[region].append(env)
        self.log.info('Environments and Regions for Pipelines:\n%s', json.dumps(regions_envs, indent=4))

        subnets = None
        pipelines = {}
        for region, envs in regions_envs.items():
            # TODO: Overrides for an environment no longer makes sense. Need to
            # provide override for entire Region possibly.
            pipelines[region] = self.render_wrapper(region=region)

            previous_env = None
            for env in envs:
                pipeline_block_data = {
                    "env": env,
                    "generated": self.generated,
                    "previous_env": previous_env,
                    "region": region,
                    "settings": self.settings[env][region],
                    "pipeline_data": self.settings['pipeline'],
                }

                if self.settings['pipeline']['type'] in EC2_PIPELINE_TYPES:
                    if not subnets:
                        subnets = get_subnets()
                    try:
                        region_subnets = {region: subnets[env][region]}
                    except KeyError:
                        self.log.info('%s is not available for %s.', env, region)
                        continue
                    pipeline_block_data['region_subnets'] = region_subnets

                block = construct_pipeline_block(**pipeline_block_data)
                pipelines[region]['stages'].extend(json.loads(block))
                previous_env = env

        self.log.debug('Assembled Pipelines:\n%s', pformat(pipelines))

        for region, pipeline in pipelines.items():
            renumerate_stages(pipeline)

            self.post_pipeline(pipeline)

        return True
<|endoftext|>


<commit_before># coding=utf-8

from __future__ import absolute_import
import os
import octoprint.plugin
import urllib2

class FreemobilenotifierPlugin(octoprint.plugin.EventHandlerPlugin,
                               octoprint.plugin.SettingsPlugin,
                               octoprint.plugin.AssetPlugin,
                               octoprint.plugin.TemplatePlugin):

	#~~ SettingsPlugin
	def get_settings_defaults(self):
		return dict(
			enabled=False,
			login="",
			pass_key="",
			message_format=dict(
				body="Job complete: {filename} done printing after {elapsed_time}"
			)
		)

	def get_settings_version(self):
		return 1

	#~~ TemplatePlugin
	def get_template_configs(self):
		return [dict(type="settings", name="FreeMobile Notifier", custom_bindings=False)]

	#~~ EventPlugin
	def on_event(self, event, payload):
		if event != "PrintDone":
			return

		if not self._settings.get(['enabled']):
			return

		filename = os.path.basename(payload["file"])

		import datetime
		import octoprint.util
		elapsed_time = octoprint.util.get_formatted_timedelta(datetime.timedelta(seconds=payload["time"]))

		tags = {'filename': filename, 'elapsed_time': elapsed_time}
		message = self._settings.get(["message_format", "body"]).format(**tags)
		login = self._settings.get(["login"])
		pass_key = self._settings.get(["pass_key"])
		url = 'https://smsapi.free-mobile.fr/sendmsg?&user='+login+'&pass='+pass_key+'&msg='+message
		request = urllib2.Request(url)

		try:
			urllib2.urlopen(request)
		except Exception as e:
			# report problem sending sms
			self._logger.exception("SMS notification error: %s" % (str(e)))
		else:
			# report notification was sent
			self._logger.info("Print notification sent to %s" % (self._settings.get(['login'])))

	##~~ Softwareupdate hook
	def get_update_information(self):
		return dict(
			freemobilenotifier=dict(
				displayName="FreeMobile Notifier",
				displayVersion=self._plugin_version,

				# version check: github repository
				type="github_release",
				user="Pinaute",
				repo="OctoPrint_FreeMobile-Notifier",
				current=self._plugin_version,

				# update method: pip
				pip="https://github.com/Pinaute/OctoPrint_FreeMobile-Notifier/archive/{target_version}.zip"
			)
		)

__plugin_name__ = "FreeMobile Notifier"

def __plugin_load__():
	global __plugin_implementation__
	__plugin_implementation__ = FreemobilenotifierPlugin()

	global __plugin_hooks__
	__plugin_hooks__ = {
		"octoprint.plugin.softwareupdate.check_config": __plugin_implementation__.get_update_information
	}<commit_msg>Update __init__.py<commit_after># coding=utf-8

from __future__ import absolute_import
import os
import octoprint.plugin
try:
	import urllib2
except ImportError:
	import urllib

class FreemobilenotifierPlugin(octoprint.plugin.EventHandlerPlugin,
                               octoprint.plugin.SettingsPlugin,
                               octoprint.plugin.AssetPlugin,
                               octoprint.plugin.TemplatePlugin):

	#~~ SettingsPlugin
	def get_settings_defaults(self):
		return dict(
			enabled=False,
			login="",
			pass_key="",
			message_format=dict(
				body="Job complete: {filename} done printing after {elapsed_time}"
			)
		)

	def get_settings_version(self):
		return 1

	#~~ TemplatePlugin
	def get_template_configs(self):
		return [dict(type="settings", name="FreeMobile Notifier", custom_bindings=False)]

	#~~ EventPlugin
	def on_event(self, event, payload):
		if event != "PrintDone":
			return

		if not self._settings.get(['enabled']):
			return

		filename = os.path.basename(payload["file"])

		import datetime
		import octoprint.util
		elapsed_time = octoprint.util.get_formatted_timedelta(datetime.timedelta(seconds=payload["time"]))

		tags = {'filename': filename, 'elapsed_time': elapsed_time}
		message = self._settings.get(["message_format", "body"]).format(**tags)
		login = self._settings.get(["login"])
		pass_key = self._settings.get(["pass_key"])
		url = 'https://smsapi.free-mobile.fr/sendmsg?&user='+login+'&pass='+pass_key+'&msg='+message
		request = urllib2.Request(url)

		try:
			urllib2.urlopen(request)
		except Exception as e:
			# report problem sending sms
			self._logger.exception("SMS notification error: %s" % (str(e)))
		else:
			# report notification was sent
			self._logger.info("Print notification sent to %s" % (self._settings.get(['login'])))

	##~~ Softwareupdate hook
	def get_update_information(self):
		return dict(
			freemobilenotifier=dict(
				displayName="FreeMobile Notifier",
				displayVersion=self._plugin_version,

				# version check: github repository
				type="github_release",
				user="Pinaute",
				repo="OctoPrint_FreeMobile-Notifier",
				current=self._plugin_version,

				# update method: pip
				pip="https://github.com/Pinaute/OctoPrint_FreeMobile-Notifier/archive/{target_version}.zip"
			)
		)

__plugin_name__ = "FreeMobile Notifier"

def __plugin_load__():
	global __plugin_implementation__
	__plugin_implementation__ = FreemobilenotifierPlugin()

	global __plugin_hooks__
	__plugin_hooks__ = {
		"octoprint.plugin.softwareupdate.check_config": __plugin_implementation__.get_update_information
	}
<|endoftext|>


<commit_before># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import dateutil.parser
from datetime import timedelta
import logging

from ocw.dataset import Bounds
from ocw.evaluation import Evaluation
import ocw.dataset_processor as dsp
import ocw.data_source.local as local
import ocw.data_source.rcmed as rcmed
import ocw.data_source.esgf as esgf
import ocw.data_source.dap as dap
import ocw.metrics as metrics

import numpy as np

logging.basicConfig()
logger = logging.getLogger(__name__)

def generate_evaluation_from_config(config_data):
    """ Generate an Evaluation object from configuration data.

    :param config_data: Dictionary of the data parsed from the supplied YAML
        configuration file.
    :type config_data: :func:`dict`

    :returns: An Evaluation object containing the data specified in the
        supplied configuration data.
    """
    # Load datasets
    reference = None
    targets = None
    if 'reference' in config_data['datasets']:
        reference = _load_dataset(config_data['datasets']['reference'])

    if 'targets' in config_data['datasets']:
        targets = [_load_dataset(t) for t in config_data['datasets']['targets']]

    reference, targets = _prepare_datasets_for_evaluation(reference,
                                                          targets,
                                                          config_data)
    # Load metrics
    eval_metrics = [_load_metric(m)() for m in config_data['metrics']]

    return Evaluation(reference, targets, eval_metrics)

def _load_dataset(dataset_config_data):
    """"""
    if dataset_config_data['data_source'] == 'local':
        if dataset_config_data['file_count'] > 1:
            logger.error(
                'Multi-file datasets are currently not supported. Cancelling load '
                'of the following dataset: {}'.format(dataset_config_data)
            )
            return None

        return local.load_file(dataset_config_data['path'],
                               dataset_config_data['variable'],
                               **dataset_config_data.get('optional_args', {}))
    elif dataset_config_data['data_source'] == 'rcmed':
        return rcmed.parameter_dataset(dataset_config_data['dataset_id'],
                                       dataset_config_data['parameter_id'],
                                       dataset_config_data['min_lat'],
                                       dataset_config_data['max_lat'],
                                       dataset_config_data['min_lon'],
                                       dataset_config_data['min_lon'],
                                       dataset_config_data['start_time'],
                                       dataset_config_data['end_time'],
                                       **dataset_config_data.get('optional_args', {}))
    elif dataset_config_data['data_source'] == 'esgf':
        return esgf.load_dataset(dataset_config_data['dataset_id'],
                                 dataset_config_data['variable'],
                                 dataset_config_data['esgf_username'],
                                 dataset_config_data['esgf_password'],
                                 **dataset_config_data.get('optional_args', {}))
    elif dataset_config_data['data_source'] == 'dap':
        return dap.load(dataset_config_data['url'],
                        dataset_config_data['variable'],
                        **dataset_config_data('optional_args', {}))

def _prepare_datasets_for_evaluation(reference, targets, config_data):
    """"""
    subset = config_data['evaluation'].get('subset', None)
    temporal_time_delta = config_data['evaluation'].get('temporal_time_delta', None)
    spatial_regrid_lats = config_data['evaluation'].get('spatial_regrid_lats', None)
    spatial_regrid_lons = config_data['evaluation'].get('spatial_regrid_lons', None)

    if subset:
        start = dateutil.parser.parse(subset[4])
        end = dateutil.parser.parse(subset[5])
        bounds = Bounds(subset[0], subset[1], subset[2], subset[3], start, end)

        if reference:
            reference = dsp.safe_subset(bounds, reference)

        if targets:
            targets = [dsp.safe_subset(bounds, t) for t in targets]

    if temporal_time_delta:
        resolution = timedelta(temporal_time_delta)

        if reference:
            reference = dsp.temporal_rebin(reference, resolution)

        if targets:
            targets = [dsp.temporal_rebin(t, resolution) for t in targets]

    if spatial_regrid_lats and spatial_regrid_lons:
        lats = np.arange(spatial_regrid_lats[0], spatial_regrid_lats[1], spatial_regrid_lats[2])
        lons = np.arange(spatial_regrid_lons[0], spatial_regrid_lons[1], spatial_regrid_lons[2])

        if reference:
            reference = dsp.spatial_regrid(reference, lats, lons)

        if targets:
            targets = [dsp.spatial_regrid(t, lats, lons) for t in targets]

    return reference, targets

def _load_metric(metric_config_data):
    """"""
    # If the dataset is user defined outside of ocw.metrics we won't currently
    # handle loading it.
    if '.' in metric_config_data:
        logger.error(
            'User-defined metrics outside of the ocw.metrics module '
            'cannot currently be loaded. If you just wanted a metric '
            'found in ocw.metrics then do not specify the full '
            'package and module names. See the documentation for examples.'
        )
        return None

    return getattr(metrics, metric_config_data)
<commit_msg>CLIMATE-639 - Add subregion handling to evaluation creation<commit_after># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import dateutil.parser
from datetime import timedelta
import logging

from ocw.dataset import Bounds
from ocw.evaluation import Evaluation
import ocw.dataset_processor as dsp
import ocw.data_source.local as local
import ocw.data_source.rcmed as rcmed
import ocw.data_source.esgf as esgf
import ocw.data_source.dap as dap
import ocw.metrics as metrics

import numpy as np

logging.basicConfig()
logger = logging.getLogger(__name__)

def generate_evaluation_from_config(config_data):
    """ Generate an Evaluation object from configuration data.

    :param config_data: Dictionary of the data parsed from the supplied YAML
        configuration file.
    :type config_data: :func:`dict`

    :returns: An Evaluation object containing the data specified in the
        supplied configuration data.
    """
    # Load datasets
    reference = None
    targets = None
    if 'reference' in config_data['datasets']:
        reference = _load_dataset(config_data['datasets']['reference'])

    if 'targets' in config_data['datasets']:
        targets = [_load_dataset(t) for t in config_data['datasets']['targets']]

    reference, targets = _prepare_datasets_for_evaluation(reference,
                                                          targets,
                                                          config_data)
    # Load metrics
    eval_metrics = [_load_metric(m)() for m in config_data['metrics']]

    # Load Subregions (if present)
    subregions = None
    if 'subregions' in config_data:
        subregions = [_load_subregion(s) for s in config_data['subregions']]

    return Evaluation(reference, targets, eval_metrics, subregions=subregions)

def _load_dataset(dataset_config_data):
    """"""
    if dataset_config_data['data_source'] == 'local':
        if dataset_config_data['file_count'] > 1:
            logger.error(
                'Multi-file datasets are currently not supported. Cancelling load '
                'of the following dataset: {}'.format(dataset_config_data)
            )
            return None

        return local.load_file(dataset_config_data['path'],
                               dataset_config_data['variable'],
                               **dataset_config_data.get('optional_args', {}))
    elif dataset_config_data['data_source'] == 'rcmed':
        return rcmed.parameter_dataset(dataset_config_data['dataset_id'],
                                       dataset_config_data['parameter_id'],
                                       dataset_config_data['min_lat'],
                                       dataset_config_data['max_lat'],
                                       dataset_config_data['min_lon'],
                                       dataset_config_data['min_lon'],
                                       dataset_config_data['start_time'],
                                       dataset_config_data['end_time'],
                                       **dataset_config_data.get('optional_args', {}))
    elif dataset_config_data['data_source'] == 'esgf':
        return esgf.load_dataset(dataset_config_data['dataset_id'],
                                 dataset_config_data['variable'],
                                 dataset_config_data['esgf_username'],
                                 dataset_config_data['esgf_password'],
                                 **dataset_config_data.get('optional_args', {}))
    elif dataset_config_data['data_source'] == 'dap':
        return dap.load(dataset_config_data['url'],
                        dataset_config_data['variable'],
                        **dataset_config_data('optional_args', {}))

def _prepare_datasets_for_evaluation(reference, targets, config_data):
    """"""
    subset = config_data['evaluation'].get('subset', None)
    temporal_time_delta = config_data['evaluation'].get('temporal_time_delta', None)
    spatial_regrid_lats = config_data['evaluation'].get('spatial_regrid_lats', None)
    spatial_regrid_lons = config_data['evaluation'].get('spatial_regrid_lons', None)

    if subset:
        start = dateutil.parser.parse(subset[4])
        end = dateutil.parser.parse(subset[5])
        bounds = Bounds(subset[0], subset[1], subset[2], subset[3], start, end)

        if reference:
            reference = dsp.safe_subset(bounds, reference)

        if targets:
            targets = [dsp.safe_subset(bounds, t) for t in targets]

    if temporal_time_delta:
        resolution = timedelta(temporal_time_delta)

        if reference:
            reference = dsp.temporal_rebin(reference, resolution)

        if targets:
            targets = [dsp.temporal_rebin(t, resolution) for t in targets]

    if spatial_regrid_lats and spatial_regrid_lons:
        lats = np.arange(spatial_regrid_lats[0], spatial_regrid_lats[1], spatial_regrid_lats[2])
        lons = np.arange(spatial_regrid_lons[0], spatial_regrid_lons[1], spatial_regrid_lons[2])

        if reference:
            reference = dsp.spatial_regrid(reference, lats, lons)

        if targets:
            targets = [dsp.spatial_regrid(t, lats, lons) for t in targets]

    return reference, targets

def _load_metric(metric_config_data):
    """"""
    # If the dataset is user defined outside of ocw.metrics we won't currently
    # handle loading it.
    if '.' in metric_config_data:
        logger.error(
            'User-defined metrics outside of the ocw.metrics module '
            'cannot currently be loaded. If you just wanted a metric '
            'found in ocw.metrics then do not specify the full '
            'package and module names. See the documentation for examples.'
        )
        return None

    return getattr(metrics, metric_config_data)

def _load_subregion(subregion_config_data):
    """"""
    return Bounds(float(subregion_config_data[0]),
                  float(subregion_config_data[1]),
                  float(subregion_config_data[2]),
                  float(subregion_config_data[3]))
<|endoftext|>


<commit_before>import uuid
from django.db import models
from base.models import ObjectBaseClass
from django.utils.translation import ugettext_lazy as _

from results.models import Result

class Discipline(ObjectBaseClass):
    FORMS = (
        ('e', _('Экзамен')),
        ('z', _('Зачет'))
        )
    title = models.CharField(_('Название дисциплины'), max_length=256, blank=False, default='')
    description = models.TextField(_("Короткое описание"), max_length=16384, blank=True, null=True)
    module = models.ForeignKey("modules.Module", blank=True, null=True)
    labor = models.PositiveIntegerField(_("зачётных единиц"), blank=False, default=0)
    period = models.IntegerField(_("Период освоения в модуле"), default=1)
    form = models.CharField(_("Форма контроля"), max_length=1, choices=FORMS, default='z')
    results = models.ManyToManyField(Result, verbose_name=_("Результаты обучения"), blank=True)
    results_text = models.TextField(_("Результаты обучения"), max_length=16384, blank=True, default="")

    uni_uid = models.CharField(max_length=256, blank=True, null=True)
    uni_discipline = models.CharField(max_length=256, blank=True, null=True)
    uni_number = models.CharField(max_length=256, blank=True, null=True)
    uni_section = models.CharField(max_length=256, blank=True, null=True)
    uni_file = models.CharField(max_length=256, blank=True, null=True)

    class Meta:
        verbose_name = 'дисциплина'
        verbose_name_plural = 'дисциплины'

    def __str__(self):
        return self.title

    def num_courses(self):
        return len(self.courses.all())

class TrainingTerms(models.Model):
    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
    title = models.CharField(_('Наименование срока обучения'), max_length=256, blank=False, default='')
    limit = models.PositiveIntegerField(_("Лимит ЗЕ в год"), default=0)

    def __str__(self):
        return self.title

    class Meta:
        verbose_name = 'срок обучения'
        verbose_name_plural = 'сроки обучения'

class Semester(models.Model):
    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
    discipline = models.ForeignKey("Discipline")
    program = models.ForeignKey("programs.Program")
    year = models.PositiveIntegerField(_("Год поступления"), default=1970)
    admission_semester = models.PositiveIntegerField(_("Семестр поступления"), default=0)
    training_semester = models.PositiveIntegerField(_("Семестр изучения"), default=0)
    term = models.ForeignKey("TrainingTerms", blank=True, null=True)

    def __str__(self):
        return f"{self.program} - {self.discipline} - {self.year}"

    class Meta:
        verbose_name = 'семестр изучения дисциплины'
        verbose_name_plural = 'семестры изучения дисциплины'

class Variant(ObjectBaseClass):
    def __str__(self):
        return self.title

    class Meta:
        verbose_name = 'вариант реализации дисциплин'
        verbose_name_plural = 'варианты реализации дисциплин'

    PARITY = (('1', 'Нечетный'), ('2', 'Четный'))

    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
    discipline = models.ForeignKey("Discipline", null=True, blank=True)
    program = models.ForeignKey("programs.Program", null=True, blank=True)
    diagram = models.ForeignKey("Diagram", null=True, blank=True)
    technology = models.ForeignKey("Technology", null=True, blank=True)
    course = models.ForeignKey("courses.Course", null=True)
    semester = models.ForeignKey("Semester", null=True)
    parity = models.CharField(_("Четность семестра дисциплины"), max_length=2, choices=PARITY, null=True, blank=True)
    link = models.CharField(_("Ссылка на страницу дисциплины"), max_length=512, blank=True, null=True)

class Diagram(ObjectBaseClass):
    def __str__(self):
        return self.title

    class Meta:
        verbose_name = 'график варианта'
        verbose_name_plural = 'графики варианта'

    title = models.CharField(_("Название графика"), max_length=512)
    # TODO: What is this?
    # TODO: add to admin site

class Technology(ObjectBaseClass):
    def __str__(self):
        return self.title

    class Meta:
        verbose_name = 'технология'
        verbose_name_plural = 'технологии'

    def get_color(self):
        return f"

"

    get_color.allow_tags = True
    get_color.short_description = _("Цвет технологии")

    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
    title = models.CharField(_("Название технологии"), max_length=512)
    description = models.TextField(_("Описание технологии"), max_length=16384, blank=True, null=True)
    contact_work_category = models.CharField(_("Категория контактной работы"), max_length=512, blank=True, null=True)
    color = models.CharField(_("Цвет технологии"), max_length=16, blank=True, null=True)
<commit_msg>Technology#11<commit_after>import uuid
from django.db import models
from base.models import ObjectBaseClass
from django.utils.translation import ugettext_lazy as _

from results.models import Result

class Discipline(ObjectBaseClass):
    FORMS = (
        ('e', _('Экзамен')),
        ('z', _('Зачет'))
        )
    title = models.CharField(_('Название дисциплины'), max_length=256, blank=False, default='')
    description = models.TextField(_("Короткое описание"), max_length=16384, blank=True, null=True)
    module = models.ForeignKey("modules.Module", blank=True, null=True)
    labor = models.PositiveIntegerField(_("зачётных единиц"), blank=False, default=0)
    period = models.IntegerField(_("Период освоения в модуле"), default=1)
    form = models.CharField(_("Форма контроля"), max_length=1, choices=FORMS, default='z')
    results = models.ManyToManyField(Result, verbose_name=_("Результаты обучения"), blank=True)
    results_text = models.TextField(_("Результаты обучения"), max_length=16384, blank=True, default="")

    uni_uid = models.CharField(max_length=256, blank=True, null=True)
    uni_discipline = models.CharField(max_length=256, blank=True, null=True)
    uni_number = models.CharField(max_length=256, blank=True, null=True)
    uni_section = models.CharField(max_length=256, blank=True, null=True)
    uni_file = models.CharField(max_length=256, blank=True, null=True)

    class Meta:
        verbose_name = 'дисциплина'
        verbose_name_plural = 'дисциплины'

    def __str__(self):
        return self.title

    def num_courses(self):
        return len(self.courses.all())

class TrainingTerms(models.Model):
    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
    title = models.CharField(_('Наименование срока обучения'), max_length=256, blank=False, default='')
    limit = models.PositiveIntegerField(_("Лимит ЗЕ в год"), default=0)

    def __str__(self):
        return self.title

    class Meta:
        verbose_name = 'срок обучения'
        verbose_name_plural = 'сроки обучения'

class Semester(models.Model):
    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
    discipline = models.ForeignKey("Discipline")
    program = models.ForeignKey("programs.Program")
    year = models.PositiveIntegerField(_("Год поступления"), default=1970)
    admission_semester = models.PositiveIntegerField(_("Семестр поступления"), default=0)
    training_semester = models.PositiveIntegerField(_("Семестр изучения"), default=0)
    term = models.ForeignKey("TrainingTerms", blank=True, null=True)

    def __str__(self):
        return f"{self.program} - {self.discipline} - {self.year}"

    class Meta:
        verbose_name = 'семестр изучения дисциплины'
        verbose_name_plural = 'семестры изучения дисциплины'

class Variant(ObjectBaseClass):
    def __str__(self):
        return self.title

    class Meta:
        verbose_name = 'вариант реализации дисциплин'
        verbose_name_plural = 'варианты реализации дисциплин'

    PARITY = (('1', 'Нечетный'), ('2', 'Четный'))

    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
    discipline = models.ForeignKey("Discipline", null=True, blank=True)
    program = models.ForeignKey("programs.Program", null=True, blank=True)
    diagram = models.ForeignKey("Diagram", null=True, blank=True)
    technology = models.ForeignKey("Technology", null=True, blank=True)
    course = models.ForeignKey("courses.Course", null=True)
    semester = models.ForeignKey("Semester", null=True)
    parity = models.CharField(_("Четность семестра дисциплины"), max_length=2, choices=PARITY, null=True, blank=True)
    link = models.CharField(_("Ссылка на страницу дисциплины"), max_length=512, blank=True, null=True)

class Diagram(ObjectBaseClass):
    def __str__(self):
        return self.title

    class Meta:
        verbose_name = 'график варианта'
        verbose_name_plural = 'графики варианта'

    title = models.CharField(_("Название графика"), max_length=512)
    # TODO: What is this?
    # TODO: add to admin site

class Technology(ObjectBaseClass):
    def __str__(self):
        return self.title

    class Meta:
        verbose_name = 'технология'
        verbose_name_plural = 'технологии'

    def get_color(self):
        return f"

"

    get_color.allow_tags = True
    get_color.short_description = _("Цвет технологии")

    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
    title = models.CharField(_("Название технологии"), max_length=512)
    description = models.TextField(_("Описание технологии"), max_length=16384, blank=True, null=True)
    contact_work_category = models.CharField(_("Категория контактной работы"), max_length=512, blank=True, null=True)
    color = models.CharField(_("Цвет технологии"), max_length=16, blank=True, null=True)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import fnmatch
import logging
import traceback
from datetime import datetime
from random import randint

from plone.dexterity.utils import createContentInContainer
from plone.jsonserializer.exceptions import DeserializationError
from plone.jsonserializer.interfaces import IDeserializeFromJson
from plone.jsonserializer.interfaces import ISerializeToJson
from plone.server import _
from plone.server.api.service import Service
from plone.server.api.service import TraversableService
from plone.server.browser import ErrorResponse
from plone.server.browser import Response
from plone.server.browser import UnauthorizedResponse
from plone.server.browser import get_physical_path
from plone.server.events import ObjectFinallyCreatedEvent
from plone.server.interfaces import IAbsoluteURL
from plone.server.registry import ICors
from plone.server.utils import DefaultRootCors
from plone.server.utils import apply_cors
from plone.server.utils import get_authenticated_user_id
from plone.server.utils import iter_parents
from zope.component import getMultiAdapter
from zope.component import queryMultiAdapter
from zope.container.interfaces import INameChooser
from zope.event import notify
from zope.security import checkPermission
from zope.security.interfaces import Unauthorized
from zope.securitypolicy.interfaces import IPrincipalPermissionMap
from zope.securitypolicy.interfaces import IPrincipalRoleManager
from zope.securitypolicy.interfaces import IPrincipalRoleMap
from zope.securitypolicy.interfaces import IRolePermissionMap

logger = logging.getLogger(__name__)

class DefaultGET(Service):
    async def __call__(self):
        serializer = getMultiAdapter(
            (self.context, self.request),
            ISerializeToJson)
        return serializer()

class DefaultPOST(Service):
    async def __call__(self):
        """To create a content. Its a copy of plone.restapi"""
        data = await self.request.json()
        type_ = data.get('@type', None)
        id_ = data.get('id', None)
        title = data.get('title', None)

        if not type_:
            return ErrorResponse(
                'RequiredParam',
                _("Property '@type' is required"))

        # Generate a temporary id if the id is not given
        if not id_ and title:
            new_id = INameChooser(self.context).chooseName(title, object())
        elif not id_:
            now = datetime.now()
            new_id = '{}.{}.{}{:04d}'.format(
                type_.lower().replace(' ', '_'),
                now.strftime('%Y-%m-%d'),
                str(now.timestamp())[7:],
                randint(0, 9999))
        else:
            new_id = id_

        # It already exists
        if new_id in self.context:
            return ErrorResponse(
                'Conflict',
                _("Id already exists"))

        user = get_authenticated_user_id(self.request)
        # Create object
        try:
            obj = createContentInContainer(
                self.context, type_, id=new_id, creators=(user,),
                contributors=(user,))
        except ValueError as e:
            return ErrorResponse(
                'CreatingObject',
                str(e),
                status=400)

        # Update fields
        deserializer = queryMultiAdapter((obj, self.request),
                                         IDeserializeFromJson)
        if deserializer is None:
            return ErrorResponse(
                'DeserializationError',
                'Cannot deserialize type {}'.format(obj.portal_type),
                status=501)

        try:
            deserializer(data, validate_all=True)
        except DeserializationError as e:
            return ErrorResponse(
                'DeserializationError',
                str(e),
                status=400)

        # Local Roles assign owner as the creator user
        roleperm = IPrincipalRoleManager(obj)
        roleperm.assignRoleToPrincipal(
            'plone.Owner',
            user)

        notify(ObjectFinallyCreatedEvent(obj))

        absolute_url = queryMultiAdapter((obj, self.request), IAbsoluteURL)

        headers = {
            'Location': absolute_url()
        }

        serializer = queryMultiAdapter(
            (obj, self.request),
            ISerializeToJson
        )
        return Response(response=serializer(), headers=headers, status=201)

class DefaultPUT(Service):
    pass

class DefaultPATCH(Service):
    async def __call__(self):
        data = await self.request.json()
        deserializer = queryMultiAdapter((self.context, self.request),
                                         IDeserializeFromJson)
        if deserializer is None:
            return ErrorResponse(
                'DeserializationError',
                'Cannot deserialize type {}'.format(obj.portal_type),
                status=501)

        try:
            deserializer(data)
        except DeserializationError as e:
            return ErrorResponse(
                'DeserializationError',
                str(e),
                status=400)

        return Response(response={}, status=204)

class SharingGET(Service):
    """ Return the list of permissions """

    async def __call__(self):
        roleperm = IRolePermissionMap(self.context)
        prinperm = IPrincipalPermissionMap(self.context)
        prinrole = IPrincipalRoleMap(self.context)
        result = {
            'local': {},
            'inherit': []
        }
        result['local']['role_permission'] = roleperm._byrow
        result['local']['principal_permission'] = prinperm._byrow
        result['local']['principal_role'] = prinrole._byrow
        for obj in iter_parents(self.context):
            roleperm = IRolePermissionMap(obj)
            prinperm = IPrincipalPermissionMap(obj)
            prinrole = IPrincipalRoleMap(obj)
            result['inherit'].append({
                '@id': IAbsoluteURL(obj, self.request)(),
                'role_permission': roleperm._byrow,
                'principal_permission': prinperm._byrow,
                'principal_role': prinrole._byrow,
            })
        return result

class SharingPOST(Service):
    data = await self.request.json()
    prinrole = IPrincipalRoleManager(obj)
    if 'prinrole' not in data:
        raise HTTPNotFound('prinrole missing')
    for user, roles in data['prinrole'].items():
        for role in roles:
            prinrole.assignRoleToPrincipal(user, role)

class DefaultDELETE(Service):

    async def __call__(self):
        content_id = self.context.id
        del self.context.__parent__[content_id]

class DefaultOPTIONS(Service):
    """Preflight view for Cors support on DX content."""

    def getRequestMethod(self):
        """Get the requested method."""
        return self.request.headers.get(
            'Access-Control-Request-Method', None)

    async def preflight(self):
        """We need to check if there is cors enabled and is valid."""
        headers = {}

        if self.settings.enabled is False:
            return {}

        origin = self.request.headers.get('Origin', None)
        if not origin:
            raise HTTPNotFound(text='Origin this header is mandatory')

        requested_method = self.getRequestMethod()
        if not requested_method:
            raise HTTPNotFound(
                text='Access-Control-Request-Method this header is mandatory')

        requested_headers = (
            self.request.headers.get('Access-Control-Request-Headers', ()))

        if requested_headers:
            requested_headers = map(str.strip, requested_headers.split(', '))

        requested_method = requested_method.upper()
        allowed_methods = self.settings.allow_methods
        if requested_method not in allowed_methods:
            raise HTTPMethodNotAllowed(
                requested_method, allowed_methods,
                text='Access-Control-Request-Method Method not allowed')

        supported_headers = self.settings.allow_headers
        if '*' not in supported_headers and requested_headers:
            supported_headers = [s.lower() for s in supported_headers]
            for h in requested_headers:
                if not h.lower() in supported_headers:
                    raise HTTPUnauthorized(
                        text='Access-Control-Request-Headers Header %s not allowed' % h)

        supported_headers = [] if supported_headers is None else supported_headers
        requested_headers = [] if requested_headers is None else requested_headers

        supported_headers = set(supported_headers) | set(requested_headers)

        headers['Access-Control-Allow-Headers'] = ','.join(
            supported_headers)
        headers['Access-Control-Allow-Methods'] = ','.join(
            self.settings.allow_methods)
        headers['Access-Control-Max-Age'] = str(self.settings.max_age)
        return headers

    async def render(self):
        """Need to be overwritten in case you implement OPTIONS."""
        return {}

    async def __call__(self):
        """Apply CORS on the OPTIONS view."""
        if hasattr(self.request, 'site_settings'):
            self.settings = self.request.site_settings.forInterface(ICors)
        else:
            # CORS method for non plone endpoints
            self.settings = DefaultRootCors()

        headers = await self.preflight()
        resp = await self.render()
        return Response(response=resp, headers=headers, status=200)
<commit_msg>missing call function<commit_after># -*- coding: utf-8 -*-
import fnmatch
import logging
import traceback
from datetime import datetime
from random import randint

from plone.dexterity.utils import createContentInContainer
from plone.jsonserializer.exceptions import DeserializationError
from plone.jsonserializer.interfaces import IDeserializeFromJson
from plone.jsonserializer.interfaces import ISerializeToJson
from plone.server import _
from plone.server.api.service import Service
from plone.server.api.service import TraversableService
from plone.server.browser import ErrorResponse
from plone.server.browser import Response
from plone.server.browser import UnauthorizedResponse
from plone.server.browser import get_physical_path
from plone.server.events import ObjectFinallyCreatedEvent
from plone.server.interfaces import IAbsoluteURL
from plone.server.registry import ICors
from plone.server.utils import DefaultRootCors
from plone.server.utils import apply_cors
from plone.server.utils import get_authenticated_user_id
from plone.server.utils import iter_parents
from zope.component import getMultiAdapter
from zope.component import queryMultiAdapter
from zope.container.interfaces import INameChooser
from zope.event import notify
from zope.security import checkPermission
from zope.security.interfaces import Unauthorized
from zope.securitypolicy.interfaces import IPrincipalPermissionMap
from zope.securitypolicy.interfaces import IPrincipalRoleManager
from zope.securitypolicy.interfaces import IPrincipalRoleMap
from zope.securitypolicy.interfaces import IRolePermissionMap

logger = logging.getLogger(__name__)

class DefaultGET(Service):
    async def __call__(self):
        serializer = getMultiAdapter(
            (self.context, self.request),
            ISerializeToJson)
        return serializer()

class DefaultPOST(Service):
    async def __call__(self):
        """To create a content. Its a copy of plone.restapi"""
        data = await self.request.json()
        type_ = data.get('@type', None)
        id_ = data.get('id', None)
        title = data.get('title', None)

        if not type_:
            return ErrorResponse(
                'RequiredParam',
                _("Property '@type' is required"))

        # Generate a temporary id if the id is not given
        if not id_ and title:
            new_id = INameChooser(self.context).chooseName(title, object())
        elif not id_:
            now = datetime.now()
            new_id = '{}.{}.{}{:04d}'.format(
                type_.lower().replace(' ', '_'),
                now.strftime('%Y-%m-%d'),
                str(now.timestamp())[7:],
                randint(0, 9999))
        else:
            new_id = id_

        # It already exists
        if new_id in self.context:
            return ErrorResponse(
                'Conflict',
                _("Id already exists"))

        user = get_authenticated_user_id(self.request)
        # Create object
        try:
            obj = createContentInContainer(
                self.context, type_, id=new_id, creators=(user,),
                contributors=(user,))
        except ValueError as e:
            return ErrorResponse(
                'CreatingObject',
                str(e),
                status=400)

        # Update fields
        deserializer = queryMultiAdapter((obj, self.request),
                                         IDeserializeFromJson)
        if deserializer is None:
            return ErrorResponse(
                'DeserializationError',
                'Cannot deserialize type {}'.format(obj.portal_type),
                status=501)

        try:
            deserializer(data, validate_all=True)
        except DeserializationError as e:
            return ErrorResponse(
                'DeserializationError',
                str(e),
                status=400)

        # Local Roles assign owner as the creator user
        roleperm = IPrincipalRoleManager(obj)
        roleperm.assignRoleToPrincipal(
            'plone.Owner',
            user)

        notify(ObjectFinallyCreatedEvent(obj))

        absolute_url = queryMultiAdapter((obj, self.request), IAbsoluteURL)

        headers = {
            'Location': absolute_url()
        }

        serializer = queryMultiAdapter(
            (obj, self.request),
            ISerializeToJson
        )
        return Response(response=serializer(), headers=headers, status=201)

class DefaultPUT(Service):
    pass

class DefaultPATCH(Service):
    async def __call__(self):
        data = await self.request.json()
        deserializer = queryMultiAdapter((self.context, self.request),
                                         IDeserializeFromJson)
        if deserializer is None:
            return ErrorResponse(
                'DeserializationError',
                'Cannot deserialize type {}'.format(obj.portal_type),
                status=501)

        try:
            deserializer(data)
        except DeserializationError as e:
            return ErrorResponse(
                'DeserializationError',
                str(e),
                status=400)

        return Response(response={}, status=204)

class SharingGET(Service):
    """ Return the list of permissions """

    async def __call__(self):
        roleperm = IRolePermissionMap(self.context)
        prinperm = IPrincipalPermissionMap(self.context)
        prinrole = IPrincipalRoleMap(self.context)
        result = {
            'local': {},
            'inherit': []
        }
        result['local']['role_permission'] = roleperm._byrow
        result['local']['principal_permission'] = prinperm._byrow
        result['local']['principal_role'] = prinrole._byrow
        for obj in iter_parents(self.context):
            roleperm = IRolePermissionMap(obj)
            prinperm = IPrincipalPermissionMap(obj)
            prinrole = IPrincipalRoleMap(obj)
            result['inherit'].append({
                '@id': IAbsoluteURL(obj, self.request)(),
                'role_permission': roleperm._byrow,
                'principal_permission': prinperm._byrow,
                'principal_role': prinrole._byrow,
            })
        return result

class SharingPOST(Service):

    async def __call__(self):
        data = await self.request.json()
        prinrole = IPrincipalRoleManager(obj)
        if 'prinrole' not in data:
            raise HTTPNotFound('prinrole missing')
        for user, roles in data['prinrole'].items():
            for role in roles:
                prinrole.assignRoleToPrincipal(user, role)

class DefaultDELETE(Service):

    async def __call__(self):
        content_id = self.context.id
        del self.context.__parent__[content_id]

class DefaultOPTIONS(Service):
    """Preflight view for Cors support on DX content."""

    def getRequestMethod(self):
        """Get the requested method."""
        return self.request.headers.get(
            'Access-Control-Request-Method', None)

    async def preflight(self):
        """We need to check if there is cors enabled and is valid."""
        headers = {}

        if self.settings.enabled is False:
            return {}

        origin = self.request.headers.get('Origin', None)
        if not origin:
            raise HTTPNotFound(text='Origin this header is mandatory')

        requested_method = self.getRequestMethod()
        if not requested_method:
            raise HTTPNotFound(
                text='Access-Control-Request-Method this header is mandatory')

        requested_headers = (
            self.request.headers.get('Access-Control-Request-Headers', ()))

        if requested_headers:
            requested_headers = map(str.strip, requested_headers.split(', '))

        requested_method = requested_method.upper()
        allowed_methods = self.settings.allow_methods
        if requested_method not in allowed_methods:
            raise HTTPMethodNotAllowed(
                requested_method, allowed_methods,
                text='Access-Control-Request-Method Method not allowed')

        supported_headers = self.settings.allow_headers
        if '*' not in supported_headers and requested_headers:
            supported_headers = [s.lower() for s in supported_headers]
            for h in requested_headers:
                if not h.lower() in supported_headers:
                    raise HTTPUnauthorized(
                        text='Access-Control-Request-Headers Header %s not allowed' % h)

        supported_headers = [] if supported_headers is None else supported_headers
        requested_headers = [] if requested_headers is None else requested_headers

        supported_headers = set(supported_headers) | set(requested_headers)

        headers['Access-Control-Allow-Headers'] = ','.join(
            supported_headers)
        headers['Access-Control-Allow-Methods'] = ','.join(
            self.settings.allow_methods)
        headers['Access-Control-Max-Age'] = str(self.settings.max_age)
        return headers

    async def render(self):
        """Need to be overwritten in case you implement OPTIONS."""
        return {}

    async def __call__(self):
        """Apply CORS on the OPTIONS view."""
        if hasattr(self.request, 'site_settings'):
            self.settings = self.request.site_settings.forInterface(ICors)
        else:
            # CORS method for non plone endpoints
            self.settings = DefaultRootCors()

        headers = await self.preflight()
        resp = await self.render()
        return Response(response=resp, headers=headers, status=200)
<|endoftext|>


<commit_before>from block import Block
from constants import *

class Gameboard:
    def __init__(self, first_tetro):
        self.block_x = 4
        self.block_y = 0
        self.tetromino = Block(first_tetro)

        self.gameboard = []
        for i in range (0,BOARD_CELL_HEIGHT):
            new = []
            for j in range (0,BOARD_CELL_WIDTH):
                new.append(0)
            self.gameboard.append(new)

    def move_block(self, dir):
        if(dir == RIGHT and self.block_x + len(self.tetromino.get_pic()[0])
        < BOARD_CELL_WIDTH):
            self.block_x += 1

        if(dir == LEFT and self.block_x > 0):
            self.block_x += dir

    def new_block(self,new_tetro):
        self.tetromino = Block(new_tetro)
        self.block_x =4
        self.block_y =0

    def block_fall(self):
        if(self.block_y + len(self.tetromino.get_pic())
            < BOARD_CELL_HEIGHT):
            self.block_y += 1
            return True
        else:
            self.add_to_board()
            return False

    def add_to_board(self):

        rel_y = -1
        block = self.tetromino.get_pic()
        for cell in block:
            rel_y += 1
            rel_x = -1
            for val in cell:
                rel_x +=1
                x = self.block_x + rel_x
                y = self.block_y + rel_y

                if val = > 0:
                    self.gameboard[y][x] = val

    def print_gameboard(self):
        for line in self.gameboard:
            print(line)
<commit_msg>Learn 2 Type > = < ?<commit_after>from block import Block
from constants import *

class Gameboard:
    def __init__(self, first_tetro):
        self.block_x = 4
        self.block_y = 0
        self.tetromino = Block(first_tetro)

        self.gameboard = []
        for i in range (0,BOARD_CELL_HEIGHT):
            new = []
            for j in range (0,BOARD_CELL_WIDTH):
                new.append(0)
            self.gameboard.append(new)

    def move_block(self, dir):
        if(dir == RIGHT and self.block_x + len(self.tetromino.get_pic()[0])
        < BOARD_CELL_WIDTH):
            self.block_x += 1

        if(dir == LEFT and self.block_x > 0):
            self.block_x += dir

    def new_block(self,new_tetro):
        self.tetromino = Block(new_tetro)
        self.block_x =4
        self.block_y =0

    def block_fall(self):
        if(self.block_y + len(self.tetromino.get_pic())
            < BOARD_CELL_HEIGHT):
            self.block_y += 1
            return True
        else:
            self.add_to_board()
            return False

    def add_to_board(self):

        rel_y = -1
        block = self.tetromino.get_pic()
        for cell in block:
            rel_y += 1
            rel_x = -1
            for val in cell:
                rel_x +=1
                x = self.block_x + rel_x
                y = self.block_y + rel_y

                if val >= 0:
                    self.gameboard[y][x] = val

    def print_gameboard(self):
        for line in self.gameboard:
            print(line)
<|endoftext|>