<commit_before># -*- coding: utf-8 -*-
from __future__ import with_statement, print_function, absolute_import
from trello.member import Member
from trello.card import Card
from trello.compat import force_str
from trello.trellolist import List
from trello.label import Label
from trello.checklist import Checklist
from dateutil import parser as dateparser

class Board(object):
    """
    Class representing a Trello board. Board attributes are stored as normal
    Python attributes; access to all sub-objects, however, is always
    an API call (Lists, Cards).
    """

    def __init__(self, client=None, board_id=None, organization=None, name=''):
        """
        :trello: Reference to a Trello object
        :board_id: ID for the board

        Alternative Constructor

        :organization: reference to the parent organization
        :board_id: ID for this board

        """
        if organization is None:
            self.client = client
        else:
            self.organization = organization
            self.client = organization.client
        self.id = board_id
        self.name = name

        self.date_last_activity = None

    @classmethod
    def from_json(cls, trello_client=None, organization=None, json_obj=None):
        """
        Deserialize the board json object to a Board object

        :trello_client: the trello client
        :json_obj: the board json object

        Alternative contrustraction:

        Deserialize the board json object to a board object

        :organization: the organization object that the board belongs to
        :json_obj: the json board object
        """
        if organization is None:
            board = Board(client=trello_client, board_id=json_obj['id'], name=json_obj['name'])
        else:
            board = Board(organization=organization, board_id=json_obj['id'], name=json_obj['name'])

        board.description = json_obj.get('desc', '')
        board.closed = json_obj['closed']
        board.url = json_obj['url']

        try:
            board.date_last_activity = dateparser.parse(json_obj['dateLastActivity'])
        except:
            pass

        return board

    def __repr__(self):
        return force_str(u'<Board %s>' % self.name)

    def fetch(self):
        """Fetch all attributes for this board"""
        json_obj = self.client.fetch_json('/boards/' + self.id)
        self.name = json_obj['name']
        self.description = json_obj.get('desc', '')
        self.closed = json_obj['closed']
        self.url = json_obj['url']
        try:
            self.date_last_activity = dateparser.parse(json_obj['dateLastActivity'])
        except:
            self.date_last_activity = None

    # Saves a Trello Board
    def save(self):
        json_obj = self.client.fetch_json(
            '/boards/',
            http_method='POST',
            post_args={'name': self.name, "desc": self.description, "defaultLists": False}, )
        # Set initial data from Trello
        self.from_json(json_obj=json_obj)
        self.id = json_obj["id"]

    def set_name(self, name):
        self.client.fetch_json(
            '/boards/{board_id}/name'.format(board_id=self.id),
            http_method='PUT',
            post_args={'value': name})
        self.name = name

    def set_description(self, desc):
        self.client.fetch_json(
            '/boards/{board_id}/desc'.format(board_id=self.id),
            http_method='PUT',
            post_args={'value': desc})
        self.description = desc

    def close(self):
        self.client.fetch_json(
            '/boards/' + self.id + '/closed',
            http_method='PUT',
            post_args={'value': 'true', }, )
        self.closed = True

    def open(self):
        self.client.fetch_json(
            '/boards/' + self.id + '/closed',
            http_method='PUT',
            post_args={'value': 'false', }, )
        self.closed = False

    def get_list(self, list_id):
        """Get list

        :rtype: List
        """
        obj = self.client.fetch_json('/lists/' + list_id)
        return List.from_json(board=self, json_obj=obj)

    def all_lists(self):
        """Returns all lists on this board

        :rtype: list of List
        """
        return self.get_lists('all')

    def open_lists(self):
        """Returns all open lists on this board

        :rtype: list of List
        """
        return self.get_lists('open')

    def closed_lists(self):
        """Returns all closed lists on this board

        :rtype: list of List
        """
        return self.get_lists('closed')

    def get_lists(self, list_filter):
        """Get lists from filter

        :rtype: list of List
        """
        # error checking
        json_obj = self.client.fetch_json(
            '/boards/' + self.id + '/lists',
            query_params={'cards': 'none', 'filter': list_filter})
        return [List.from_json(board=self, json_obj=obj) for obj in json_obj]

    def list_lists(self, list_filter='all'):
        """Get lists from filter

        :rtype: list of List
        """
        return self.get_lists(list_filter=list_filter)

    def get_labels(self, fields='all', limit=50):
        """Get label

        :rtype: list of Label
        """
        json_obj = self.client.fetch_json(
              '/boards/' + self.id + '/labels',
              query_params={'fields': fields, 'limit': limit})
        return Label.from_json_list(self, json_obj)

    def get_checklists(self, cards='all'):
        """Get checklists

        :rtype: list of Checklist
        """
        checklists = []
        json_obj = self.client.fetch_json(
              '/boards/' + self.id + '/checklists',
              query_params={'cards': cards})
        json_obj = sorted(json_obj, key=lambda checklist: checklist['pos'])
        for cl in json_obj:
            checklists.append(Checklist(self.client, cl.get('checkItemStates',[]), cl,
                                        trello_card=cl.get('idCard')))
        return checklists

    def add_list(self, name, pos=None):
        """Add a list to this board

        :name: name for the list
        :pos: position of the list: "bottom", "top" or a positive number
        :return: the list
        :rtype: List
        """
        arguments = {'name': name, 'idBoard': self.id}
        if pos:
            arguments["pos"] = pos
        obj = self.client.fetch_json(
            '/lists',
            http_method='POST',
            post_args=arguments, )
        return List.from_json(board=self, json_obj=obj)

    def add_label(self, name, color):
        """Add a label to this board

        :name: name of the label
        :color: the color, either green, yellow, orange
            red, purple, blue, sky, lime, pink, or black
        :return: the label
        :rtype: Label
        """
        obj = self.client.fetch_json(
            '/labels',
            http_method='POST',
            post_args={'name': name, 'idBoard': self.id, 'color': color},)
        return Label.from_json(board=self, json_obj=obj)

    def all_cards(self):
        """Returns all cards on this board

        :rtype: list of Card
        """
        filters = {
            'filter': 'all',
            'fields': 'all'
        }
        return self.get_cards(filters)

    def open_cards(self):
        """Returns all open cards on this board

        :rtype: list of Card
        """
        filters = {
            'filter': 'open',
            'fields': 'all'
        }
        return self.get_cards(filters)

    def closed_cards(self):
        """Returns all closed cards on this board

        :rtype: list of Card
        """
        filters = {
            'filter': 'closed',
            'fields': 'all'
        }
        return self.get_cards(filters)

    def get_cards(self, filters=None, card_filter=""):
        """
        :filters: dict containing query parameters. Eg. {'fields': 'all'}
        :card_filter: filters on card status ('open', 'closed', 'all')

        More info on card queries:
        https://trello.com/docs/api/board/index.html#get-1-boards-board-id-cards

        :rtype: list of Card
        """
        json_obj = self.client.fetch_json(
            '/boards/' + self.id + '/cards/' + card_filter,
            query_params=filters
        )

        return list([Card.from_json(self, json) for json in json_obj])

    def all_members(self):
        """Returns all members on this board

        :rtype: list of Member
        """
        filters = {
            'filter': 'all',
            'fields': 'all'
        }
        return self.get_members(filters)

    def normal_members(self):
        """Returns all normal members on this board

        :rtype: list of Member
        """
        filters = {
            'filter': 'normal',
            'fields': 'all'
        }
        return self.get_members(filters)

    def admin_members(self):
        """Returns all admin members on this board

        :rtype: list of Member
        """
        filters = {
            'filter': 'admins',
            'fields': 'all'
        }
        return self.get_members(filters)

    def owner_members(self):
        """Returns all owner members on this board

        :rtype: list of Member
        """
        filters = {
            'filter': 'owners',
            'fields': 'all'
        }
        return self.get_members(filters)

    def get_members(self, filters=None):
        """Get members with filter

        :rtype: list of Member
        """
        json_obj = self.client.fetch_json(
            '/boards/' + self.id + '/members',
            query_params=filters)
        members = list()
        for obj in json_obj:
            m = Member(self.client, obj['id'])
            m.status = obj.get('status', '')
            m.id = obj.get('id', '')
            m.bio = obj.get('bio', '')
            m.url = obj.get('url', '')
            m.username = obj['username']
            m.full_name = obj['fullName']
            m.initials = obj.get('initials', '')
            m.member_type = obj.get('memberType', '')
            members.append(m)

        return members

    # Add a member to a board
    def add_member(self, member, member_type="normal"):
        json_obj = self.client.fetch_json(
            '/boards/{0}/members/{1}'.format(self.id, member.id),
            http_method='PUT',
            post_args={'idMember': member.id, "type": member_type},
        )
        return json_obj

    # Removes an existing member of a board
    def remove_member(self, member):
        json_obj = self.client.fetch_json(
            '/boards/{0}/members/{1}'.format(self.id, member.id),
            http_method='DELETE',
            post_args={'idMember': member.id},
        )
        return json_obj

    def fetch_actions(self, action_filter, action_limit=50, before=None, since=None):
        """Returns all actions that conform to the given filters.

        :action_filter: str of possible actions separated by comma
            ie. 'createCard,updateCard'
        :action_limit: int of max items returned
        :before: datetime obj
        :since: datetime obj

        More info on action filter values:
        https://developers.trello.com/advanced-reference/board#get-1-boards-board-id-actions

        :rtype: json list of past actions
        """
        query_params = {'filter': action_filter, 'limit': action_limit}

        if since:
            query_params["since"] = since

        if before:
            query_params["before"] = before

        json_obj = self.client.fetch_json('/boards/' + self.id + '/actions',
                                          query_params=query_params)

        self.actions = json_obj
        return self.actions
<commit_msg>board get_members docstring<commit_after># -*- coding: utf-8 -*-
from __future__ import with_statement, print_function, absolute_import
from trello.member import Member
from trello.card import Card
from trello.compat import force_str
from trello.trellolist import List
from trello.label import Label
from trello.checklist import Checklist
from dateutil import parser as dateparser

class Board(object):
    """
    Class representing a Trello board. Board attributes are stored as normal
    Python attributes; access to all sub-objects, however, is always
    an API call (Lists, Cards).
    """

    def __init__(self, client=None, board_id=None, organization=None, name=''):
        """
        :trello: Reference to a Trello object
        :board_id: ID for the board

        Alternative Constructor

        :organization: reference to the parent organization
        :board_id: ID for this board

        """
        if organization is None:
            self.client = client
        else:
            self.organization = organization
            self.client = organization.client
        self.id = board_id
        self.name = name

        self.date_last_activity = None

    @classmethod
    def from_json(cls, trello_client=None, organization=None, json_obj=None):
        """
        Deserialize the board json object to a Board object

        :trello_client: the trello client
        :json_obj: the board json object

        Alternative contrustraction:

        Deserialize the board json object to a board object

        :organization: the organization object that the board belongs to
        :json_obj: the json board object
        """
        if organization is None:
            board = Board(client=trello_client, board_id=json_obj['id'], name=json_obj['name'])
        else:
            board = Board(organization=organization, board_id=json_obj['id'], name=json_obj['name'])

        board.description = json_obj.get('desc', '')
        board.closed = json_obj['closed']
        board.url = json_obj['url']

        try:
            board.date_last_activity = dateparser.parse(json_obj['dateLastActivity'])
        except:
            pass

        return board

    def __repr__(self):
        return force_str(u'<Board %s>' % self.name)

    def fetch(self):
        """Fetch all attributes for this board"""
        json_obj = self.client.fetch_json('/boards/' + self.id)
        self.name = json_obj['name']
        self.description = json_obj.get('desc', '')
        self.closed = json_obj['closed']
        self.url = json_obj['url']
        try:
            self.date_last_activity = dateparser.parse(json_obj['dateLastActivity'])
        except:
            self.date_last_activity = None

    # Saves a Trello Board
    def save(self):
        json_obj = self.client.fetch_json(
            '/boards/',
            http_method='POST',
            post_args={'name': self.name, "desc": self.description, "defaultLists": False}, )
        # Set initial data from Trello
        self.from_json(json_obj=json_obj)
        self.id = json_obj["id"]

    def set_name(self, name):
        self.client.fetch_json(
            '/boards/{board_id}/name'.format(board_id=self.id),
            http_method='PUT',
            post_args={'value': name})
        self.name = name

    def set_description(self, desc):
        self.client.fetch_json(
            '/boards/{board_id}/desc'.format(board_id=self.id),
            http_method='PUT',
            post_args={'value': desc})
        self.description = desc

    def close(self):
        self.client.fetch_json(
            '/boards/' + self.id + '/closed',
            http_method='PUT',
            post_args={'value': 'true', }, )
        self.closed = True

    def open(self):
        self.client.fetch_json(
            '/boards/' + self.id + '/closed',
            http_method='PUT',
            post_args={'value': 'false', }, )
        self.closed = False

    def get_list(self, list_id):
        """Get list

        :rtype: List
        """
        obj = self.client.fetch_json('/lists/' + list_id)
        return List.from_json(board=self, json_obj=obj)

    def all_lists(self):
        """Returns all lists on this board

        :rtype: list of List
        """
        return self.get_lists('all')

    def open_lists(self):
        """Returns all open lists on this board

        :rtype: list of List
        """
        return self.get_lists('open')

    def closed_lists(self):
        """Returns all closed lists on this board

        :rtype: list of List
        """
        return self.get_lists('closed')

    def get_lists(self, list_filter):
        """Get lists from filter

        :rtype: list of List
        """
        # error checking
        json_obj = self.client.fetch_json(
            '/boards/' + self.id + '/lists',
            query_params={'cards': 'none', 'filter': list_filter})
        return [List.from_json(board=self, json_obj=obj) for obj in json_obj]

    def list_lists(self, list_filter='all'):
        """Get lists from filter

        :rtype: list of List
        """
        return self.get_lists(list_filter=list_filter)

    def get_labels(self, fields='all', limit=50):
        """Get label

        :rtype: list of Label
        """
        json_obj = self.client.fetch_json(
              '/boards/' + self.id + '/labels',
              query_params={'fields': fields, 'limit': limit})
        return Label.from_json_list(self, json_obj)

    def get_checklists(self, cards='all'):
        """Get checklists

        :rtype: list of Checklist
        """
        checklists = []
        json_obj = self.client.fetch_json(
              '/boards/' + self.id + '/checklists',
              query_params={'cards': cards})
        json_obj = sorted(json_obj, key=lambda checklist: checklist['pos'])
        for cl in json_obj:
            checklists.append(Checklist(self.client, cl.get('checkItemStates',[]), cl,
                                        trello_card=cl.get('idCard')))
        return checklists

    def add_list(self, name, pos=None):
        """Add a list to this board

        :name: name for the list
        :pos: position of the list: "bottom", "top" or a positive number
        :return: the list
        :rtype: List
        """
        arguments = {'name': name, 'idBoard': self.id}
        if pos:
            arguments["pos"] = pos
        obj = self.client.fetch_json(
            '/lists',
            http_method='POST',
            post_args=arguments, )
        return List.from_json(board=self, json_obj=obj)

    def add_label(self, name, color):
        """Add a label to this board

        :name: name of the label
        :color: the color, either green, yellow, orange
            red, purple, blue, sky, lime, pink, or black
        :return: the label
        :rtype: Label
        """
        obj = self.client.fetch_json(
            '/labels',
            http_method='POST',
            post_args={'name': name, 'idBoard': self.id, 'color': color},)
        return Label.from_json(board=self, json_obj=obj)

    def all_cards(self):
        """Returns all cards on this board

        :rtype: list of Card
        """
        filters = {
            'filter': 'all',
            'fields': 'all'
        }
        return self.get_cards(filters)

    def open_cards(self):
        """Returns all open cards on this board

        :rtype: list of Card
        """
        filters = {
            'filter': 'open',
            'fields': 'all'
        }
        return self.get_cards(filters)

    def closed_cards(self):
        """Returns all closed cards on this board

        :rtype: list of Card
        """
        filters = {
            'filter': 'closed',
            'fields': 'all'
        }
        return self.get_cards(filters)

    def get_cards(self, filters=None, card_filter=""):
        """
        :filters: dict containing query parameters. Eg. {'fields': 'all'}
        :card_filter: filters on card status ('open', 'closed', 'all')

        More info on card queries:
        https://trello.com/docs/api/board/index.html#get-1-boards-board-id-cards

        :rtype: list of Card
        """
        json_obj = self.client.fetch_json(
            '/boards/' + self.id + '/cards/' + card_filter,
            query_params=filters
        )

        return list([Card.from_json(self, json) for json in json_obj])

    def all_members(self):
        """Returns all members on this board

        :rtype: list of Member
        """
        filters = {
            'filter': 'all',
            'fields': 'all'
        }
        return self.get_members(filters)

    def normal_members(self):
        """Returns all normal members on this board

        :rtype: list of Member
        """
        filters = {
            'filter': 'normal',
            'fields': 'all'
        }
        return self.get_members(filters)

    def admin_members(self):
        """Returns all admin members on this board

        :rtype: list of Member
        """
        filters = {
            'filter': 'admins',
            'fields': 'all'
        }
        return self.get_members(filters)

    def owner_members(self):
        """Returns all owner members on this board

        :rtype: list of Member
        """
        filters = {
            'filter': 'owners',
            'fields': 'all'
        }
        return self.get_members(filters)

    def get_members(self, filters=None):
        """Get members with filter

        :filters: dict containing query parameters.
            Eg. {'fields': 'all', 'filter': 'admins'}

        More info on possible filters:
        https://developers.trello.com/advanced-reference/board#get-1-boards-board-id-members

        :rtype: list of Member
        """
        json_obj = self.client.fetch_json(
            '/boards/' + self.id + '/members',
            query_params=filters)
        members = list()
        for obj in json_obj:
            m = Member(self.client, obj['id'])
            m.status = obj.get('status', '')
            m.id = obj.get('id', '')
            m.bio = obj.get('bio', '')
            m.url = obj.get('url', '')
            m.username = obj['username']
            m.full_name = obj['fullName']
            m.initials = obj.get('initials', '')
            m.member_type = obj.get('memberType', '')
            members.append(m)

        return members

    # Add a member to a board
    def add_member(self, member, member_type="normal"):
        json_obj = self.client.fetch_json(
            '/boards/{0}/members/{1}'.format(self.id, member.id),
            http_method='PUT',
            post_args={'idMember': member.id, "type": member_type},
        )
        return json_obj

    # Removes an existing member of a board
    def remove_member(self, member):
        json_obj = self.client.fetch_json(
            '/boards/{0}/members/{1}'.format(self.id, member.id),
            http_method='DELETE',
            post_args={'idMember': member.id},
        )
        return json_obj

    def fetch_actions(self, action_filter, action_limit=50, before=None, since=None):
        """Returns all actions that conform to the given filters.

        :action_filter: str of possible actions separated by comma
            ie. 'createCard,updateCard'
        :action_limit: int of max items returned
        :before: datetime obj
        :since: datetime obj

        More info on action filter values:
        https://developers.trello.com/advanced-reference/board#get-1-boards-board-id-actions

        :rtype: json list of past actions
        """
        query_params = {'filter': action_filter, 'limit': action_limit}

        if since:
            query_params["since"] = since

        if before:
            query_params["before"] = before

        json_obj = self.client.fetch_json('/boards/' + self.id + '/actions',
                                          query_params=query_params)

        self.actions = json_obj
        return self.actions
<|endoftext|>


<commit_before>
from TA_GP import GooglePlay
from youtubeAPI import YoutubeAPI
#from twitterDataAcquisition import TwitterDataAcquisition
from twitterScreenScrape import TwitterScreenScrape
import json
import pymongo

def youtube_vid_info(apps):
    youtube = YoutubeAPI()
    vid_dict = youtube.get_all_info(apps)
    return vid_dict

def google_app_info():
    google = GooglePlay()
    app_list = google.getGPInfo()
    return app_list

def twitter_info(app_list):
    #twitter = TwitterDataAcquisition(app_list)
    twitter = TwitterScreenScrape()
    return twitter.get_twitter_data(app_list)

def main():
    try:
        conn = pymongo.MongoClient()
        print "Successful Connection"
    except pymongo.errors.ConnectionFailure, e:
        print "Error During Connection"

    db = conn['trendingapps']

    apps, gpInfo = google_app_info()
    yt = youtube_vid_info(apps)
    tw = twitter_info(apps)

    ycollec = db['youtube']
    tcollec = db['twitter']
    gcollec = db['googleplay']

    for (ykey, yvalue) in yt['youtube'].items():
        ycollec.insert( {ykey:yvalue} )

    for (tkey, tvalue) in tw['twitter'].items():
        tcollec.insert( {tkey:tvalue} )

    for (gkey, gvalue) in gpInfo['googlePlay'].items():
        gcollec.insert( {gkey:gvalue} )

    print db.youtube.find_one()
    print
    print db.twitter.find_one()
    print
    print db.googleplay.find_one()

if __name__ == "__main__":
    main()<commit_msg>changes to youtube loop<commit_after>
from TA_GP import GooglePlay
from youtubeAPI import YoutubeAPI
#from twitterDataAcquisition import TwitterDataAcquisition
from twitterScreenScrape import TwitterScreenScrape
import json
import pymongo

def youtube_vid_info(apps):
    youtube = YoutubeAPI()
    vid_dict = youtube.get_all_info(apps)
    return vid_dict

def google_app_info():
    google = GooglePlay()
    app_list = google.getGPInfo()
    return app_list

def twitter_info(app_list):
    #twitter = TwitterDataAcquisition(app_list)
    twitter = TwitterScreenScrape()
    return twitter.get_twitter_data(app_list)

def main():
    try:
        conn = pymongo.MongoClient()
        print "Successful Connection"
    except pymongo.errors.ConnectionFailure, e:
        print "Error During Connection"

    db = conn['trendingapps']

    apps, gpInfo = google_app_info()
    yt = youtube_vid_info(apps)
    tw = twitter_info(apps)

    ycollec = db['youtube']
    tcollec = db['twitter']
    gcollec = db['googleplay']

    for docu in yt:
        ycollec.insert(docu)

    for (tkey, tvalue) in tw['twitter'].items():
        tcollec.insert( {tkey:tvalue} )

    for (gkey, gvalue) in gpInfo['googlePlay'].items():
        gcollec.insert( {gkey:gvalue} )

    print db.youtube.find_one()
    print
    print db.twitter.find_one()
    print
    print db.googleplay.find_one()

if __name__ == "__main__":
    main()<|endoftext|>


<commit_before>#!/usr/bin/env python2.7

import argparse
from cStringIO import StringIO
import itertools
import logging
import os
import re
import sys
import urllib2

try:
    import Image
    import ImageDraw
except ImportError:
    try:
        import PIL.Image as Image
        import PIL.ImageDraw as ImageDraw
    except ImportError:
        if __name__ == '__main__':
            print >> sys.stderr, 'Could not import Python Imaging Library'
            sys.exit(1)
        else:
            raise

def triangulize(image, tile_size):
    """Processes the given image by breaking it down into tiles of the given
    size and applying a triangular effect to each tile. Returns the processed
    image.

    If tile_size is 0, the tile size will be guessed based on the image
    size. It will also be adjusted to be divisible by 2 if it is not already.
    """
    assert isinstance(image, Image.Image), type(image)
    assert isinstance(tile_size, int)

    # Make sure we have a usable tile size, by guessing based on image size
    # and making sure it's a multiple of two.
    if tile_size == 0:
        tile_size = guess_tile_size(image)
    if tile_size % 2 != 0:
        tile_size = (tile_size / 2) * 2

    logging.info('Input image size: %r', image.size)
    logging.info('Tile size: %r', tile_size)

    # Preprocess image to make sure it's at a size we can handle
    image = prep_image(image, tile_size)
    logging.info('Prepped image size: %r', image.size)

    # Get pixmap (for direct pixel access) and draw objects for the image.
    pix = image.load()
    draw = ImageDraw.Draw(image)

    # Process the image, tile by tile
    for x, y in iter_tiles(image, tile_size):
        process_tile(x, y, tile_size, pix, draw, image)
    return image

def process_tile(tile_x, tile_y, tile_size, pix, draw, image):
    """Process a tile whose top left corner is at the given x and y
    coordinates.
    """
    logging.debug('Processing tile (%d, %d)', tile_x, tile_y)

    # Calculate average color for each "triangle" in the given tile
    n, e, s, w = triangle_colors(tile_x, tile_y, tile_size, pix)

    # Calculate distance between triangle pairs
    d_ne = get_color_dist(n, e)
    d_nw = get_color_dist(n, w)
    d_se = get_color_dist(s, e)
    d_sw = get_color_dist(s, w)

    # Figure out which pair is the closest, which will determine the direction
    # we'll split this tile into triangles. A 'right' split runs from top left
    # to bottom right. A 'left' split runs bottom left to top right.
    closest = sorted([d_ne, d_nw, d_se, d_sw])[0]
    if closest in (d_ne, d_sw):
        split = 'right'
    elif closest in (d_nw, d_se):
        split = 'left'

    # Figure out the average color for each side of the "split"
    if split == 'right':
        top_color = get_average_color([n, e])
        bottom_color = get_average_color([s, w])
    else:
        top_color = get_average_color([n, w])
        bottom_color = get_average_color([s, e])

    draw_triangles(tile_x, tile_y, tile_size, split, top_color, bottom_color,
                   draw)

def triangle_colors(tile_x, tile_y, tile_size, pix):
    """Extracts the average color for each triangle in the given tile. Returns
    a 4-tuple of colors for the triangles in this order: North, East, South,
    West (clockwise).
    """
    quad_size = tile_size / 2

    north = []
    for y in xrange(tile_y, tile_y + quad_size):
        x_off = y - tile_y
        for x in xrange(tile_x + x_off, tile_x + tile_size - x_off):
            north.append(pix[x,y])

    south = []
    for y in xrange(tile_y + quad_size, tile_y + tile_size):
        x_off = tile_y + tile_size - y
        for x in xrange(tile_x + x_off, tile_x + tile_size - x_off):
            south.append(pix[x,y])

    east = []
    for x in xrange(tile_x, tile_x + quad_size):
        y_off = x - tile_x
        for y in xrange(tile_y + y_off, tile_y + tile_size - y_off):
            east.append(pix[x, y])

    west = []
    for x in xrange(tile_x + quad_size, tile_x + tile_size):
        y_off = tile_x + tile_size - x
        for y in xrange(tile_y + y_off, tile_y + tile_size - y_off):
            west.append(pix[x, y])

    return map(get_average_color, [north, east, south, west])

def draw_triangles(tile_x, tile_y, tile_size, split, top_color, bottom_color,
                   draw):
    """Draws a triangle on each half of the tile with the given coordinates
    and size.
    """
    assert split in ('right', 'left')

    # The four corners of this tile
    nw = (tile_x, tile_y)
    ne = (tile_x + tile_size - 1, tile_y)
    se = (tile_x + tile_size - 1, tile_y + tile_size)
    sw = (tile_x, tile_y + tile_size)

    if split == 'left':
        # top right triangle
        draw_triangle(nw, ne, se, top_color, draw)
        # bottom left triangle
        draw_triangle(nw, sw, se, bottom_color, draw)
    else:
        # top left triangle
        draw_triangle(sw, nw, ne, top_color, draw)
        # bottom right triangle
        draw_triangle(sw, se, ne, bottom_color, draw)

def draw_triangle(a, b, c, color, draw):
    """Draws a triangle with the given vertices in the given color."""
    draw.polygon([a, b, c], fill=color)

def get_average_color(colors):
    """Calculate the average color from the list of colors, where each color
    is a 3-tuple of (r, g, b) values.
    """
    c = reduce(color_reducer, colors)
    total = len(colors)
    return tuple(v/total for v in c)

def color_reducer(c1, c2):
    """Helper function used to add two colors together when averaging."""
    return tuple(v1 + v2 for v1, v2 in itertools.izip(c1, c2))

def get_color_dist(c1, c2):
    """Calculates the "distance" between two colors, where the distance is
    another color whose components are the absolute values of the difference
    between each component of the input colors.
    """
    return tuple(abs(v1-v2) for v1, v2 in itertools.izip(c1, c2))

def prep_image(image, tile_size):
    """Takes an image and a tile size and returns a possibly cropped version
    of the image that is evenly divisible in both dimensions by the tile size.
    """
    w, h = image.size
    x_tiles = w / tile_size # floor division
    y_tiles = h / tile_size
    new_w = x_tiles * tile_size
    new_h = y_tiles * tile_size
    if new_w == w and new_h == h:
        return image
    else:
        crop_bounds = (0, 0, new_w, new_h)
        return image.crop(crop_bounds)

def iter_tiles(image, tile_size):
    """Yields (x, y) coordinate pairs for the top left corner of each tile in
    the given image, based on the given tile size.
    """
    w, h = image.size
    for y in xrange(0, h, tile_size):
        for x in xrange(0, w, tile_size):
            yield x, y

def guess_tile_size(image):
    """Try to pick an appropriate tile size based on the image's size."""
    # Formula: 5% of the largest dimension of the image
    return int(max(image.size) * 0.05)

if __name__ == '__main__':

    def path_or_url(x):
        if re.match(r'^https?://', x):
            try:
                return urllib2.urlopen(x)
            except urllib2.URLError, e:
                raise argparse.ArgumentTypeError(str(e))
        elif os.path.isfile(x):
            return open(x, 'rb')
        else:
            msg = '%r not found' % x
            raise argparse.ArgumentTypeError(msg)

    arg_parser = argparse.ArgumentParser(
        description='Applies a "triangular pixel" effect to an image.')
    arg_parser.add_argument(
        'infile', nargs='?', default=sys.stdin, type=path_or_url,
        help='Image to process (path or URL; defaults to STDIN)')
    arg_parser.add_argument(
        'outfile', nargs='?', default=sys.stdout,
        type=argparse.FileType('wb'),
        help='Output file (defaults to STDOUT)')
    arg_parser.add_argument(
        '-t', '--tile-size', type=int, default=0,
        help='Tile size (should be divisible by 2)')
    arg_parser.add_argument(
        '-v', '--verbose', default=False, action='store_const', const=True,
        help='Verbose output')
    arg_parser.add_argument(
        '-vv', default=False, action='store_const', const=True,
        help='Very verbose output')
    arg_parser.add_argument(
        '-s', '--show', default=False, action='store_const', const=True,
        help='Immediately display image instead of writing to OUTFILE.')

    args = arg_parser.parse_args()

    if args.verbose or args.vv:
        logger = logging.getLogger()
        logger.setLevel(logging.DEBUG if args.vv else logging.INFO)

    # We need to buffer the input because neither sys.stdin nor urllib2
    # response objects support seek()
    inbuffer = StringIO(args.infile.read())
    try:
        image = triangulize(Image.open(inbuffer), args.tile_size)
    except IOError, e:
        logging.error('Unable to open image: %s', e)
    except KeyboardInterrupt:
        logging.info('Interrupted by user, exiting...')
        sys.exit(1)
    else:
        if args.show:
            image.show()
        else:
            image.save(args.outfile, 'png')
<commit_msg>Don't require an Image object.<commit_after>#!/usr/bin/env python2.7

import argparse
from cStringIO import StringIO
import itertools
import logging
import os
import re
import sys
import urllib2

try:
    import Image
    import ImageDraw
except ImportError:
    try:
        import PIL.Image as Image
        import PIL.ImageDraw as ImageDraw
    except ImportError:
        if __name__ == '__main__':
            print >> sys.stderr, 'Could not import Python Imaging Library'
            sys.exit(1)
        else:
            raise

def triangulize(image, tile_size):
    """Processes the given image by breaking it down into tiles of the given
    size and applying a triangular effect to each tile. Returns the processed
    image as a PIL Image object.

    The image can be given as anything suitable for passing to `Image.open`
    (ie, the path to an image or as a file-like object containing image data).

    If tile_size is 0, the tile size will be guessed based on the image
    size. It will also be adjusted to be divisible by 2 if it is not already.
    """
    if not isinstance(image, Image.Image):
        image = Image.open(image)
    assert isinstance(tile_size, int)

    # Make sure we have a usable tile size, by guessing based on image size
    # and making sure it's a multiple of two.
    if tile_size == 0:
        tile_size = guess_tile_size(image)
    if tile_size % 2 != 0:
        tile_size = (tile_size / 2) * 2

    logging.info('Input image size: %r', image.size)
    logging.info('Tile size: %r', tile_size)

    # Preprocess image to make sure it's at a size we can handle
    image = prep_image(image, tile_size)
    logging.info('Prepped image size: %r', image.size)

    # Get pixmap (for direct pixel access) and draw objects for the image.
    pix = image.load()
    draw = ImageDraw.Draw(image)

    # Process the image, tile by tile
    for x, y in iter_tiles(image, tile_size):
        process_tile(x, y, tile_size, pix, draw, image)
    return image

def process_tile(tile_x, tile_y, tile_size, pix, draw, image):
    """Process a tile whose top left corner is at the given x and y
    coordinates.
    """
    logging.debug('Processing tile (%d, %d)', tile_x, tile_y)

    # Calculate average color for each "triangle" in the given tile
    n, e, s, w = triangle_colors(tile_x, tile_y, tile_size, pix)

    # Calculate distance between triangle pairs
    d_ne = get_color_dist(n, e)
    d_nw = get_color_dist(n, w)
    d_se = get_color_dist(s, e)
    d_sw = get_color_dist(s, w)

    # Figure out which pair is the closest, which will determine the direction
    # we'll split this tile into triangles. A 'right' split runs from top left
    # to bottom right. A 'left' split runs bottom left to top right.
    closest = sorted([d_ne, d_nw, d_se, d_sw])[0]
    if closest in (d_ne, d_sw):
        split = 'right'
    elif closest in (d_nw, d_se):
        split = 'left'

    # Figure out the average color for each side of the "split"
    if split == 'right':
        top_color = get_average_color([n, e])
        bottom_color = get_average_color([s, w])
    else:
        top_color = get_average_color([n, w])
        bottom_color = get_average_color([s, e])

    draw_triangles(tile_x, tile_y, tile_size, split, top_color, bottom_color,
                   draw)

def triangle_colors(tile_x, tile_y, tile_size, pix):
    """Extracts the average color for each triangle in the given tile. Returns
    a 4-tuple of colors for the triangles in this order: North, East, South,
    West (clockwise).
    """
    quad_size = tile_size / 2

    north = []
    for y in xrange(tile_y, tile_y + quad_size):
        x_off = y - tile_y
        for x in xrange(tile_x + x_off, tile_x + tile_size - x_off):
            north.append(pix[x,y])

    south = []
    for y in xrange(tile_y + quad_size, tile_y + tile_size):
        x_off = tile_y + tile_size - y
        for x in xrange(tile_x + x_off, tile_x + tile_size - x_off):
            south.append(pix[x,y])

    east = []
    for x in xrange(tile_x, tile_x + quad_size):
        y_off = x - tile_x
        for y in xrange(tile_y + y_off, tile_y + tile_size - y_off):
            east.append(pix[x, y])

    west = []
    for x in xrange(tile_x + quad_size, tile_x + tile_size):
        y_off = tile_x + tile_size - x
        for y in xrange(tile_y + y_off, tile_y + tile_size - y_off):
            west.append(pix[x, y])

    return map(get_average_color, [north, east, south, west])

def draw_triangles(tile_x, tile_y, tile_size, split, top_color, bottom_color,
                   draw):
    """Draws a triangle on each half of the tile with the given coordinates
    and size.
    """
    assert split in ('right', 'left')

    # The four corners of this tile
    nw = (tile_x, tile_y)
    ne = (tile_x + tile_size - 1, tile_y)
    se = (tile_x + tile_size - 1, tile_y + tile_size)
    sw = (tile_x, tile_y + tile_size)

    if split == 'left':
        # top right triangle
        draw_triangle(nw, ne, se, top_color, draw)
        # bottom left triangle
        draw_triangle(nw, sw, se, bottom_color, draw)
    else:
        # top left triangle
        draw_triangle(sw, nw, ne, top_color, draw)
        # bottom right triangle
        draw_triangle(sw, se, ne, bottom_color, draw)

def draw_triangle(a, b, c, color, draw):
    """Draws a triangle with the given vertices in the given color."""
    draw.polygon([a, b, c], fill=color)

def get_average_color(colors):
    """Calculate the average color from the list of colors, where each color
    is a 3-tuple of (r, g, b) values.
    """
    c = reduce(color_reducer, colors)
    total = len(colors)
    return tuple(v/total for v in c)

def color_reducer(c1, c2):
    """Helper function used to add two colors together when averaging."""
    return tuple(v1 + v2 for v1, v2 in itertools.izip(c1, c2))

def get_color_dist(c1, c2):
    """Calculates the "distance" between two colors, where the distance is
    another color whose components are the absolute values of the difference
    between each component of the input colors.
    """
    return tuple(abs(v1-v2) for v1, v2 in itertools.izip(c1, c2))

def prep_image(image, tile_size):
    """Takes an image and a tile size and returns a possibly cropped version
    of the image that is evenly divisible in both dimensions by the tile size.
    """
    w, h = image.size
    x_tiles = w / tile_size # floor division
    y_tiles = h / tile_size
    new_w = x_tiles * tile_size
    new_h = y_tiles * tile_size
    if new_w == w and new_h == h:
        return image
    else:
        crop_bounds = (0, 0, new_w, new_h)
        return image.crop(crop_bounds)

def iter_tiles(image, tile_size):
    """Yields (x, y) coordinate pairs for the top left corner of each tile in
    the given image, based on the given tile size.
    """
    w, h = image.size
    for y in xrange(0, h, tile_size):
        for x in xrange(0, w, tile_size):
            yield x, y

def guess_tile_size(image):
    """Try to pick an appropriate tile size based on the image's size."""
    # Formula: 5% of the largest dimension of the image
    return int(max(image.size) * 0.05)

if __name__ == '__main__':

    def path_or_url(x):
        if re.match(r'^https?://', x):
            try:
                return urllib2.urlopen(x)
            except urllib2.URLError, e:
                raise argparse.ArgumentTypeError(str(e))
        elif os.path.isfile(x):
            return open(x, 'rb')
        else:
            msg = '%r not found' % x
            raise argparse.ArgumentTypeError(msg)

    arg_parser = argparse.ArgumentParser(
        description='Applies a "triangular pixel" effect to an image.')
    arg_parser.add_argument(
        'infile', nargs='?', default=sys.stdin, type=path_or_url,
        help='Image to process (path or URL; defaults to STDIN)')
    arg_parser.add_argument(
        'outfile', nargs='?', default=sys.stdout,
        type=argparse.FileType('wb'),
        help='Output file (defaults to STDOUT)')
    arg_parser.add_argument(
        '-t', '--tile-size', type=int, default=0,
        help='Tile size (should be divisible by 2)')
    arg_parser.add_argument(
        '-v', '--verbose', default=False, action='store_const', const=True,
        help='Verbose output')
    arg_parser.add_argument(
        '-vv', default=False, action='store_const', const=True,
        help='Very verbose output')
    arg_parser.add_argument(
        '-s', '--show', default=False, action='store_const', const=True,
        help='Immediately display image instead of writing to OUTFILE.')

    args = arg_parser.parse_args()

    if args.verbose or args.vv:
        logger = logging.getLogger()
        logger.setLevel(logging.DEBUG if args.vv else logging.INFO)

    # We need to buffer the input because neither sys.stdin nor urllib2
    # response objects support seek()
    inbuffer = StringIO(args.infile.read())
    try:
        image = triangulize(inbuffer, args.tile_size)
    except IOError, e:
        logging.error('Unable to open image: %s', e)
    except KeyboardInterrupt:
        logging.info('Interrupted by user, exiting...')
        sys.exit(1)
    else:
        if args.show:
            image.show()
        else:
            image.save(args.outfile, 'png')
<|endoftext|>


<commit_before># Author: Alfredo Deza
# Email: alfredodeza [at] gmail dot com
# License: MIT
# Copyright 2009-2010 Alfredo Deza
#
"""SSH connections and file transfers with limited options like non
standard ports. This is a simple wrapper to suit Pacha."""

from subprocess import call, PIPE
import os
import sys
from time import strftime
import log
import confparser
import host
import log

class Hg(object):
    """Does local commits and pushed to a central Pacha Master location"""

    def __init__(self,
            port = 22,
            host = None,
            user = None,
            dir = None
            ):
        self.port = port
        self.host = host
        self.user = user
        self.dir = dir

    def commit(self):
        """hg local commits that are needed before a push to a centralized
        server"""
        #TODO:
        # Automatically detect if a user modified a file to commit and then
        # push the changes
        # For now, the user has to manually commit

    def push(self):
        """Pushes the repository to the centralized Pacha Master server"""
        #command = "scp -P%s ~/.cuy/keys/%s_%s.keys %s@%s:~/.ssh/authorized_keys" % (
        #        self.port, self.user, self.host, self.user, self.host)
        #try:
        #    call(command, shell=True, stdout=PIPE, stderr=PIPE)
        #except Exception:
        #    print Exception

    def hgrc(self, path):
        """An option to write the default path in hgrc for pushing
        via hg"""
        if self.validate(path):
            conf = '/opt/pacha/conf/pacha.conf'
            parse = confparser.Parse(conf)
            parse.options() # get all the options in the config file
            log.append(module='hg', line="parsed options from config file")
            norm_path = os.path.normpath(path)
            base_path = os.path.basename(path)
            machine = host.hostname()
            try:
                hgrc = open(norm_path+'/.hg/hgrc', 'w')
                hgrc.write('[paths]\n')
                ssh_line = "default = ssh://%s@%s/%s/%s/%s" % (parse.user, parse.host,
                           parse.path, machine, base_path)
                hgrc.write(ssh_line)
                hgrc.close()
                log.append(module='hg', line="wrote hgrc in %s" % path)

            except Exception, e:
                log.append(module='hg', type='ERROR', line=e)

        else:
            sys.stderr.write("No repository found here: %s" % path)
                    line="no repository found: %s" % path)

    def validate(self, path):
        """Validates a working HG path"""
        #normalize the path:
        norm_path = os.path.normpath(path)
        hg_dir = norm_path+'/.hg'
        log.append(module='hg', line="validating repository at %s" % path)
        if os.path.exists(hg_dir):
            log.append(module='hg', line="hg repository found at %s" % path)
            return True
        else:
            log.append(module='hg', type='ERROR',  line="hg repository not found at %s" % path)
            return False
<commit_msg>fixed unindented line<commit_after># Author: Alfredo Deza
# Email: alfredodeza [at] gmail dot com
# License: MIT
# Copyright 2009-2010 Alfredo Deza
#
"""SSH connections and file transfers with limited options like non
standard ports. This is a simple wrapper to suit Pacha."""

from subprocess import call, PIPE
import os
import sys
from time import strftime
import log
import confparser
import host
import log

class Hg(object):
    """Does local commits and pushed to a central Pacha Master location"""

    def __init__(self,
            port = 22,
            host = None,
            user = None,
            dir = None
            ):
        self.port = port
        self.host = host
        self.user = user
        self.dir = dir

    def commit(self):
        """hg local commits that are needed before a push to a centralized
        server"""
        #TODO:
        # Automatically detect if a user modified a file to commit and then
        # push the changes
        # For now, the user has to manually commit

    def push(self):
        """Pushes the repository to the centralized Pacha Master server"""
        #command = "scp -P%s ~/.cuy/keys/%s_%s.keys %s@%s:~/.ssh/authorized_keys" % (
        #        self.port, self.user, self.host, self.user, self.host)
        #try:
        #    call(command, shell=True, stdout=PIPE, stderr=PIPE)
        #except Exception:
        #    print Exception

    def hgrc(self, path):
        """An option to write the default path in hgrc for pushing
        via hg"""
        if self.validate(path):
            conf = '/opt/pacha/conf/pacha.conf'
            parse = confparser.Parse(conf)
            parse.options() # get all the options in the config file
            log.append(module='hg', line="parsed options from config file")
            norm_path = os.path.normpath(path)
            base_path = os.path.basename(path)
            machine = host.hostname()
            try:
                hgrc = open(norm_path+'/.hg/hgrc', 'w')
                hgrc.write('[paths]\n')
                ssh_line = "default = ssh://%s@%s/%s/%s/%s" % (parse.user, parse.host,
                           parse.path, machine, base_path)
                hgrc.write(ssh_line)
                hgrc.close()
                log.append(module='hg', line="wrote hgrc in %s" % path)

            except Exception, e:
                log.append(module='hg', type='ERROR', line=e)

        else:
            sys.stderr.write("No repository found here: %s" % path)

    def validate(self, path):
        """Validates a working HG path"""
        #normalize the path:
        norm_path = os.path.normpath(path)
        hg_dir = norm_path+'/.hg'
        log.append(module='hg', line="validating repository at %s" % path)
        if os.path.exists(hg_dir):
            log.append(module='hg', line="hg repository found at %s" % path)
            return True
        else:
            log.append(module='hg', type='ERROR',  line="hg repository not found at %s" % path)
            return False
<|endoftext|>


<commit_before>"""
Tukio Workflow Engine
"""
import asyncio
import weakref
import logging

from tukio.workflow import WorkflowTemplate, OverrunPolicy, new_workflow
from tukio.broker import get_broker
from tukio.task import tukio_factory

log = logging.getLogger(__name__)

class DuplicateWorkflowError(Exception):
    pass

class Engine(asyncio.Future):
    def __init__(self, *, loop=None):
        super().__init__(loop=loop)
        # use the custom asyncio task factory
        self._loop.set_task_factory(tukio_factory)
        self._templates = dict()
        self._running = dict()
        self._broker = get_broker(self._loop)
        self._lock = asyncio.Lock()
        self._running_by_id = weakref.WeakValueDictionary()
        self._must_stop = False

    @property
    def templates(self):
        return self._templates

    def _add_wflow(self, wflow):
        """
        Adds a new entry into the dict of running workflows and updates the
        weak value dict to index it by its execution ID.
        """
        try:
            self._running[wflow.template_id].append(wflow)
        except KeyError:
            self._running[wflow.template_id] = [wflow]
        self._running_by_id[wflow.uid] = wflow
        log.debug('new workflow added to the running list: {}'.format(wflow))

    def _remove_wflow(self, wflow):
        """
        Removes a worflow instance from the dict of running workflows.
        """
        self._running[wflow.template_id].remove(wflow)
        # Cleanup the running dict if no more running instance of that template
        if len(self._running[wflow.template_id]) == 0:
            del self._running[wflow.template_id]
        del self._running_by_id[wflow.uid]
        log.debug('workflow removed from the running list: {}'.format(wflow))
        if self._must_stop and not self._running and not self.done():
            self.set_result(None)
            log.debug('no more workflow running, engine stopped')

    def stop(self, force=False):
        """
        Cancels all workflows and prevent new instances from being run.
        """
        self._must_stop = True
        if not self._running and not self.done():
            self.set_result(None)
        elif force:
            self.cancel_all()
        return self

    def _run_in_task(self, callback, *args, **kwargs):
        """
        Wrap a regular function into a coroutine and run it in a task.
        This is intended to wrap time consuming functions into a task so as to
        prevent slow operations from blocking the whole loop.
        """
        async def coro():
            return callback(*args, **kwargs)
        return asyncio.ensure_future(coro(), loop=self._loop)

    def _load(self, tmpl_dict):
        """
        Loads a workflow template into the engine. Each workflow may be
        triggered as soon as it is loaded.
        Duplicates or invalid descriptions raise an exception.
        This operation does not affect workflow executions in progress.
        """
        wf_tmpl = WorkflowTemplate.from_dict(tmpl_dict)
        wf_tmpl.validate()
        # A workflow template is uniquely defined by the tuple
        # (template ID, version)
        duplicate = None
        try:
            duplicate = self._templates[wf_tmpl.uid]
        except KeyError:
            self._templates[wf_tmpl.uid] = wf_tmpl
        else:
            if duplicate.version == wf_tmpl.version:
                raise DuplicateWorkflowError
            else:
                self._templates[wf_tmpl.uid] = wf_tmpl
        log.debug("new workflow template added '{}'".format(wf_tmpl.title))
        return wf_tmpl

    async def load(self, tmpl_dict):
        """
        A coroutine that loads a new workflow template while preventing other
        coroutines from updating the dict of loaded templates in the mean time.
        """
        with await self._lock:
            wf_tmpl = await self._run_in_task(self._load, tmpl_dict)
        return wf_tmpl

    async def reload(self, templates):
        """
        Replaces the current list of loaded workflow templates by a new one.
        This operation does not affect workflow executions in progress.
        """
        with await self._lock:
            self._templates = dict()
            for tmpl_dict in templates:
                await self._run_in_task(self._load, tmpl_dict)

    async def unload(self, template_id):
        """
        Unloads a workflow template from the engine. Returns True if the
        template was found and actually unloaded, else, returns False.
        """
        with await self._lock:
            try:
                template = self._templates.pop(template_id)
            except KeyError:
                return None
        return template

    async def data_received(self, data):
        """
        This method should be called to pass an event to the workflow engine
        which in turn will disptach this event to the right running workflows
        and may trigger new workflow executions.
        """
        self._broker.dispatch(data)
        # Don't start new workflow instances if `stop()` was called.
        if self._must_stop:
            return
        with await self._lock:
            # Try to trigger new workflows from the current dict of workflow
            # templates at all times!
            for tmpl_id in self._templates:
                await self._run_in_task(self._try_run, tmpl_id, data)

    def _try_run(self, tmpl_id, data):
        """
        Try to run a new instance of workflow defined by `tmpl_id` according to
        the instances already running and the overrun policy.
        """
        running = self._running.get(tmpl_id)
        try:
            wf_tmpl = self._templates[tmpl_id]
        except KeyError:
            # the workflow template is no longer loaded in the engine; nothing
            # to run!
            return
        # Always apply the policy of the current workflow template (workflow
        # instances may run with an old version of the template)
        wflow = new_workflow(wf_tmpl, running=running, loop=self._loop)
        if wflow:
            wflow.add_done_callback(self._remove_wflow)
            if wf_tmpl.policy == OverrunPolicy.abort_running and running:
                def cb():
                    wflow.run(data)
                asyncio.ensure_future(self._wait_abort(running, cb))
            else:
                wflow.run(data)
            self._add_wflow(wflow)

    async def _wait_abort(self, running, callback):
        """
        Wait for the end of a list of aborted (cancelled) workflows before
        starting a new one when the policy is 'abort-running'.
        """
        await asyncio.wait(running)
        callback()

    def run(self, tmpl_id, inputs):
        """
        Starts a new execution of the workflow template identified by `tmpl_id`
        regardless of the overrun policy and already running workflows.
        """
        if self._must_stop:
            return None
        wf_tmpl = self._templates[tmpl_id]
        wflow = new_workflow(wf_tmpl, loop=self._loop)
        wflow.add_done_callback(self._remove_wflow)
        wflow.run(inputs)
        self._add_wflow(wflow)
        return wflow

    def cancel(self, exec_id):
        """
        Cancels an execution of workflow identified by its execution ID.
        The cancelled workflow instance (a future object) is returned.
        If the workflow could not be found, returns None.
        """
        wflow = self._running_by_id.get(exec_id)
        if wflow:
            wflow.cancel()
            log.debug('cancelled workflow {}'.format(wflow))
        return wflow

    def cancel_all(self):
        """
        Cancels all the running workflows.
        """
        cancelled = 0
        for wf_list in self._running.values():
            for wflow in wf_list:
                is_cancelled = wflow.cancel()
                if is_cancelled:
                    cancelled += 1
        log.debug('cancelled {} workflows'.format(cancelled))
        return cancelled
<commit_msg>now load 'WorkflowTemplate' objects only in the engine<commit_after>"""
Tukio Workflow Engine
"""
import asyncio
import weakref
import logging

from tukio.workflow import WorkflowTemplate, OverrunPolicy, new_workflow
from tukio.broker import get_broker
from tukio.task import tukio_factory

log = logging.getLogger(__name__)

class DuplicateWorkflowError(Exception):
    pass

class Engine(asyncio.Future):
    def __init__(self, *, loop=None):
        super().__init__(loop=loop)
        # use the custom asyncio task factory
        self._loop.set_task_factory(tukio_factory)
        self._templates = dict()
        self._running = dict()
        self._broker = get_broker(self._loop)
        self._lock = asyncio.Lock()
        self._running_by_id = weakref.WeakValueDictionary()
        self._must_stop = False

    @property
    def templates(self):
        return self._templates

    def _add_wflow(self, wflow):
        """
        Adds a new entry into the dict of running workflows and updates the
        weak value dict to index it by its execution ID.
        """
        try:
            self._running[wflow.template_id].append(wflow)
        except KeyError:
            self._running[wflow.template_id] = [wflow]
        self._running_by_id[wflow.uid] = wflow
        log.debug('new workflow added to the running list: {}'.format(wflow))

    def _remove_wflow(self, wflow):
        """
        Removes a worflow instance from the dict of running workflows.
        """
        self._running[wflow.template_id].remove(wflow)
        # Cleanup the running dict if no more running instance of that template
        if len(self._running[wflow.template_id]) == 0:
            del self._running[wflow.template_id]
        del self._running_by_id[wflow.uid]
        log.debug('workflow removed from the running list: {}'.format(wflow))
        if self._must_stop and not self._running and not self.done():
            self.set_result(None)
            log.debug('no more workflow running, engine stopped')

    def stop(self, force=False):
        """
        Cancels all workflows and prevent new instances from being run.
        """
        self._must_stop = True
        if not self._running and not self.done():
            self.set_result(None)
        elif force:
            self.cancel_all()
        return self

    def _run_in_task(self, callback, *args, **kwargs):
        """
        Wrap a regular function into a coroutine and run it in a task.
        This is intended to wrap time consuming functions into a task so as to
        prevent slow operations from blocking the whole loop.
        """
        async def coro():
            return callback(*args, **kwargs)
        return asyncio.ensure_future(coro(), loop=self._loop)

    def _load(self, template):
        """
        Loads a workflow template into the engine. Each workflow may be
        triggered as soon as it is loaded.
        Duplicates or invalid descriptions raise an exception.
        This operation does not affect workflow executions in progress.
        """
        template.validate()
        # A workflow template is uniquely defined by the tuple
        # (template ID, version)
        duplicate = None
        try:
            duplicate = self._templates[template.uid]
        except KeyError:
            self._templates[template.uid] = template
        else:
            if duplicate.version == template.version:
                raise DuplicateWorkflowError
            else:
                self._templates[template.uid] = template
        log.debug("new workflow template added '{}'".format(template.title))

    async def load(self, template):
        """
        A coroutine that loads a new workflow template while preventing other
        coroutines from updating the dict of loaded templates in the mean time.
        """
        with await self._lock:
            await self._run_in_task(self._load, template)

    async def reload(self, templates):
        """
        Replaces the current list of loaded workflow templates by a new one.
        This operation does not affect workflow executions in progress.
        """
        with await self._lock:
            self._templates = dict()
            for tmpl in templates:
                await self._run_in_task(self._load, tmpl)

    async def unload(self, template_id):
        """
        Unloads a workflow template from the engine. Returns True if the
        template was found and actually unloaded, else, returns False.
        """
        with await self._lock:
            try:
                template = self._templates.pop(template_id)
            except KeyError:
                return None
        return template

    async def data_received(self, data):
        """
        This method should be called to pass an event to the workflow engine
        which in turn will disptach this event to the right running workflows
        and may trigger new workflow executions.
        """
        self._broker.dispatch(data)
        # Don't start new workflow instances if `stop()` was called.
        if self._must_stop:
            return
        with await self._lock:
            # Try to trigger new workflows from the current dict of workflow
            # templates at all times!
            for tmpl_id in self._templates:
                await self._run_in_task(self._try_run, tmpl_id, data)

    def _try_run(self, tmpl_id, data):
        """
        Try to run a new instance of workflow defined by `tmpl_id` according to
        the instances already running and the overrun policy.
        """
        running = self._running.get(tmpl_id)
        try:
            wf_tmpl = self._templates[tmpl_id]
        except KeyError:
            # the workflow template is no longer loaded in the engine; nothing
            # to run!
            return
        # Always apply the policy of the current workflow template (workflow
        # instances may run with an old version of the template)
        wflow = new_workflow(wf_tmpl, running=running, loop=self._loop)
        if wflow:
            wflow.add_done_callback(self._remove_wflow)
            if wf_tmpl.policy == OverrunPolicy.abort_running and running:
                def cb():
                    wflow.run(data)
                asyncio.ensure_future(self._wait_abort(running, cb))
            else:
                wflow.run(data)
            self._add_wflow(wflow)

    async def _wait_abort(self, running, callback):
        """
        Wait for the end of a list of aborted (cancelled) workflows before
        starting a new one when the policy is 'abort-running'.
        """
        await asyncio.wait(running)
        callback()

    def run(self, tmpl_id, inputs):
        """
        Starts a new execution of the workflow template identified by `tmpl_id`
        regardless of the overrun policy and already running workflows.
        """
        if self._must_stop:
            return None
        wf_tmpl = self._templates[tmpl_id]
        wflow = new_workflow(wf_tmpl, loop=self._loop)
        wflow.add_done_callback(self._remove_wflow)
        wflow.run(inputs)
        self._add_wflow(wflow)
        return wflow

    def cancel(self, exec_id):
        """
        Cancels an execution of workflow identified by its execution ID.
        The cancelled workflow instance (a future object) is returned.
        If the workflow could not be found, returns None.
        """
        wflow = self._running_by_id.get(exec_id)
        if wflow:
            wflow.cancel()
            log.debug('cancelled workflow {}'.format(wflow))
        return wflow

    def cancel_all(self):
        """
        Cancels all the running workflows.
        """
        cancelled = 0
        for wf_list in self._running.values():
            for wflow in wf_list:
                is_cancelled = wflow.cancel()
                if is_cancelled:
                    cancelled += 1
        log.debug('cancelled {} workflows'.format(cancelled))
        return cancelled
<|endoftext|>


<commit_before>from collections import defaultdict

from redash.handlers.base import BaseResource, get_object_or_404
from redash.models import AccessPermission, Query, Dashboard, User
from redash.permissions import require_admin_or_owner, ACCESS_TYPES
from flask import request
from flask_restful import abort

model_to_types = {
    'queries': Query,
    'dashboards': Dashboard
}

def get_model_from_type(type):
    model = model_to_types.get(type)
    if model is None:
        abort(404)
    return model

class ObjectPermissionsListResource(BaseResource):
    def get(self, object_type, object_id):
        model = get_model_from_type(object_type)
        obj = get_object_or_404(model.get_by_id_and_org, object_id, self.current_org)

        # TODO: include grantees in search to avoid N+1 queries
        permissions = AccessPermission.find(obj)

        result = defaultdict(list)

        for perm in permissions:
            result[perm.access_type].append(perm.grantee.to_dict())

        return result

    def post(self, object_type, object_id):
        model = get_model_from_type(object_type)
        obj = get_object_or_404(model.get_by_id_and_org, object_id, self.current_org)

        require_admin_or_owner(obj.user_id)

        req = request.get_json(True)

        access_type = req['access_type']

        if access_type not in ACCESS_TYPES:
            abort(400, message='Unknown access type.')

        try:
            grantee = User.get_by_id_and_org(req['user_id'], self.current_org)
        except User.DoesNotExist:
            abort(400, message='User not found.')

        permission = AccessPermission.grant(obj, access_type, grantee, self.current_user)

        return permission.to_dict()

    def delete(self, object_type, object_id):
        model = get_model_from_type(object_type)
        obj = get_object_or_404(model.get_by_id_and_org, object_id, self.current_org)

        require_admin_or_owner(obj.user_id)

        req = request.get_json(True)
        grantee = req['user_id']
        access_type = req['access_type']

        AccessPermission.revoke(obj, grantee, access_type)

class CheckPermissionResource(BaseResource):
    def get(self, object_type, object_id, access_type):
        model = get_model_from_type(object_type)
        obj = get_object_or_404(model.get_by_id_and_org, object_id, self.current_org)

        has_access = AccessPermission.exists(obj, access_type, self.current_user)

        return {'response': has_access}
<commit_msg>Record events for permission changes<commit_after>from collections import defaultdict

from redash.handlers.base import BaseResource, get_object_or_404
from redash.models import AccessPermission, Query, Dashboard, User
from redash.permissions import require_admin_or_owner, ACCESS_TYPES
from flask import request
from flask_restful import abort

model_to_types = {
    'queries': Query,
    'dashboards': Dashboard
}

def get_model_from_type(type):
    model = model_to_types.get(type)
    if model is None:
        abort(404)
    return model

class ObjectPermissionsListResource(BaseResource):
    def get(self, object_type, object_id):
        model = get_model_from_type(object_type)
        obj = get_object_or_404(model.get_by_id_and_org, object_id, self.current_org)

        # TODO: include grantees in search to avoid N+1 queries
        permissions = AccessPermission.find(obj)

        result = defaultdict(list)

        for perm in permissions:
            result[perm.access_type].append(perm.grantee.to_dict())

        return result

    def post(self, object_type, object_id):
        model = get_model_from_type(object_type)
        obj = get_object_or_404(model.get_by_id_and_org, object_id, self.current_org)

        require_admin_or_owner(obj.user_id)

        req = request.get_json(True)

        access_type = req['access_type']

        if access_type not in ACCESS_TYPES:
            abort(400, message='Unknown access type.')

        try:
            grantee = User.get_by_id_and_org(req['user_id'], self.current_org)
        except User.DoesNotExist:
            abort(400, message='User not found.')

        permission = AccessPermission.grant(obj, access_type, grantee, self.current_user)

        self.record_event({
            'action': 'grant_permission',
            'object_id': object_id,
            'object_type': object_type,
            'access_type': access_type,
            'grantee': grantee.id
        })

        return permission.to_dict()

    def delete(self, object_type, object_id):
        model = get_model_from_type(object_type)
        obj = get_object_or_404(model.get_by_id_and_org, object_id, self.current_org)

        require_admin_or_owner(obj.user_id)

        req = request.get_json(True)
        grantee = req['user_id']
        access_type = req['access_type']

        AccessPermission.revoke(obj, grantee, access_type)

        self.record_event({
            'action': 'revoke_permission',
            'object_id': object_id,
            'object_type': object_type,
            'access_type': access_type,
            'grantee': grantee.id
        })

class CheckPermissionResource(BaseResource):
    def get(self, object_type, object_id, access_type):
        model = get_model_from_type(object_type)
        obj = get_object_or_404(model.get_by_id_and_org, object_id, self.current_org)

        has_access = AccessPermission.exists(obj, access_type, self.current_user)

        return {'response': has_access}
<|endoftext|>