<commit_before>"""Vim specific logic"""
import os
import time

import vim

from common import msg, shared as G, utils
import protocol

# Foreground: background
COLORS = (
    ('white', 'red'),
    ('white', 'orange'),
    ('white', 'yellow'),
    ('white', 'blue'),
)
HL_RULES = ['ctermfg=%s ctermbg=%s guifg=%s guibg=%s' % (fg, bg, fg, bg) for fg, bg in COLORS]

class View(object):
    """editors representation of the buffer"""
    highlight_regions = set()

    def __init__(self, vim_buf, buf):
        self.vim_buf = vim_buf
        self.buf = buf

    def __repr__(self):
        return '%s %s %s' % (self.native_id, self.buf['id'], self.buf['path'].encode('utf-8'))

    def __str__(self):
        return repr(self)

    def _offset_to_vim(self, offset):
        current_offset = 0
        for line_num, line in enumerate(self.vim_buf):
            next_offset = len(line) + 1
            if current_offset + next_offset > offset:
                break
            current_offset += next_offset
        col = offset - current_offset
        msg.debug('offset %s is line %s column %s' % (offset, line_num + 1, col + 1))
        return line_num + 1, col + 1

    @property
    def native_id(self):
        return self.vim_buf.number

    def is_loading(self):
        return False

    def get_text(self):
        text = '\n'.join(self.vim_buf[:])
        return text.decode('utf-8')

    def set_text(self, text):
        msg.debug('\n\nabout to patch %s %s' % (str(self), self.vim_buf.name))
        try:
            msg.debug("now buf is loadedish? %s" % vim.eval('bufloaded(%s)' % self.native_id))
            self.vim_buf[:] = text.encode('utf-8').split('\n')
        except Exception as e:
            msg.error("couldn't apply patches because: %s!\nThe unencoded text was: %s" % (str(e), text))
            raise

    def apply_patches(self, buf, patches):
        cursor_offset = self.get_cursor_offset()
        msg.debug('cursor offset is %s bytes' % cursor_offset)
        self.set_text(patches[0])

        for patch in patches[2]:
            offset = patch[0]
            length = patch[1]
            patch_text = patch[2]
            if cursor_offset > offset:
                new_offset = len(patch_text) - length
                cursor_offset += new_offset

        self.set_cursor_position(cursor_offset)

    def focus(self):
        vim.command(':edit! %s' % self.vim_buf.name)
        vim.command(':filetype detect')

    def set_cursor_position(self, offset):
        line_num, col = self._offset_to_vim(offset)
        command = 'setpos(".", [%s, %s, %s, %s])' % (self.native_id, line_num, col, 0)
        msg.debug("setting pos: %s" % command)
        rv = int(vim.eval(command))
        if rv != 0:
            msg.debug('SHIIIIIIIIT %s %s' % (rv, command))

    def get_cursor_position(self):
        """ [bufnum, lnum, col, off] """
        return vim.eval('getpos(".")')

    def get_cursor_offset(self):
        return int(vim.eval('line2byte(line("."))+col(".")')) - 2

    def get_selections(self):
        cursor = self.get_cursor_offset()
        return [[cursor, cursor]]

    def clear_highlight(self, user_id):
        region = "floobitsuser%s" % str(user_id)
        if region not in self.highlight_regions:
            return
        msg.debug('clearing selections for view %s' % self.vim_buf.name)
        self.highlight_regions.remove(region)
        vim.command(":syntax clear %s" % region)

    def highlight(self, ranges, user_id):
        msg.debug('highlighting ranges %s' % (ranges))
        if vim.current.buffer.number != self.vim_buf.number:
            return

        region = "floobitsuser%s" % str(user_id)
        if region in self.highlight_regions:
            vim.command(":syntax clear %s" % region)

        hl_rule = HL_RULES[user_id % len(HL_RULES)]
        vim.command(":highlight %s %s" % (region, hl_rule))
        self.highlight_regions.add(region)

        for _range in ranges:
            start_row, start_col = self._offset_to_vim(_range[0])
            end_row, end_col = self._offset_to_vim(_range[1])
            if start_row == end_row and start_col == end_col:
                end_col += 1
            vim_region = ":syntax region {region} start=/\%{start_col}v\%{start_row}l/ end=/\%{end_col}v\%{end_row}l/".\
                format(region=region, start_col=start_col, start_row=start_row, end_col=end_col, end_row=end_row)
            # print("highlight command: %s" % vim_region)
            vim.command(vim_region)

    def rename(self, name):
        msg.debug('renaming %s to %s' % (self.vim_buf.name, name))
        current = vim.current.buffer
        text = self.get_text()
        old_name = self.vim_buf.name
        old_number = self.native_id
        with open(name, 'wb') as fd:
            fd.write(text.encode('utf-8'))
        vim.command('edit! %s' % name)
        self.vim_buf = vim.current.buffer
        vim.command('edit! %s' % current.name)
        vim.command('bdelete! %s' % old_number)
        try:
            utils.rm(old_name)
        except Exception as e:
            msg.debug("couldn't delete %s... maybe thats OK?" % str(e))

    def save(self):
        vim.command(':s!')

class Protocol(protocol.BaseProtocol):
    """understands vim"""
    CLIENT = 'VIM'

    def on_room_info(self, workspace_info):
        super(Protocol, self).on_room_info(workspace_info)
        vim.command(':Explore %s | redraw' % G.PROJECT_PATH)

    def maybe_selection_changed(self, vim_buf, is_ping):
        buf = self.get_buf(vim_buf)
        if not buf:
            msg.debug('no buffer found for view %s' % vim_buf.number)
            return
        view = self.get_view(buf['id'])
        msg.debug("selection changed: %s %s %s" % (vim_buf.number, buf['id'], view))
        self.SELECTION_CHANGED.append([view, is_ping])

    def maybe_buffer_changed(self, vim_buf):
        text = vim_buf[:]
        buf = self.get_buf(vim_buf)
        if not buf or 'buf' not in buf:
            return
        if buf['buf'] != text:
            self.BUFS_CHANGED.append(buf['id'])

    def get_vim_buf_by_path(self, p):
        for vim_buf in vim.buffers:
            if vim_buf.name and p == utils.to_rel_path(vim_buf.name):
                return vim_buf
        return None

    def get_view(self, buf_id):
        buf = self.FLOO_BUFS.get(buf_id)
        if not buf:
            return None

        vb = self.get_vim_buf_by_path(buf['path'])
        if not vb:
            return None

        if vim.eval('bufloaded(%s)' % vb.number) == '0':
            return None

        return View(vb, buf)

    def create_view(self, buf):
        path = self.save_buf(buf)
        vb = self.get_vim_buf_by_path(buf['path'])
        if vb:
            return View(vb, buf)

        vim.command(':edit! %s' % path)
        vb = self.get_vim_buf_by_path(buf['path'])
        if vb is None:
            msg.debug('vim buffer is none even though we tried to open it: %s' % path)
            return
        return View(vb, buf)

    def get_buf(self, vim_buf):
        """None- no sharing, False- should be but isn't """
        if vim_buf.name is None or vim_buf.name == "":
            msg.debug('get:buf buffer has no filename')
            return None

        if not utils.is_shared(vim_buf.name):
            msg.debug('get_buf: %s is not shared' % vim_buf.name)
            return None

        buf = self.get_buf_by_path(vim_buf.name)
        if buf:
            return buf

        msg.debug('get_buf: no buf has path %s' % vim_buf.name)
        return False

    def save_buf(self, buf):
        path = utils.get_full_path(buf['path'])
        utils.mkdir(os.path.split(path)[0])
        with open(path, 'wb') as fd:
            if buf['encoding'] == 'utf8':
                fd.write(buf['buf'].encode('utf-8'))
            else:
                fd.write(buf['buf'])
        return path

    def chat(self, username, timestamp, message, self_msg=False):
        pass
        # envelope = msg.MSG(message, timestamp, username)
        # if not self_msg:
        #     self.chat_deck.appendleft(envelope)
        # envelope.display()

    def on_msg(self, data):
        timestamp = data.get('time') or time.time()
        msg.log('[%s] <%s> %s' % (time.ctime(timestamp), data.get('username', ''), data.get('data', '')))

    def update_view(self, buf, view=None):
        msg.debug('updating view for buf %s' % buf['id'])
        view = view or self.get_view(buf['id'])
        if not view:
            msg.log('view for buf %s not found. not updating' % buf['id'])
            return
        self.MODIFIED_EVENTS.put(1)
        view.set_text(buf['buf'])

    def on_part(self, data):
        msg.log('%s left the workspace' % data['username'])
        user_id = data['user_id']
        highlight = self.user_highlights.get(user_id)
        if not highlight:
            return
        view = self.get_view(highlight['id'])
        if not view:
            return
        if vim.current.buffer.number != view.native_id:
            return
        view.clear_highlight(user_id)
        del self.user_highlights[user_id]
<commit_msg>Fix highlighting when end of highlight is at the end of a line.<commit_after>"""Vim specific logic"""
import os
import time

import vim

from common import msg, shared as G, utils
import protocol

# Foreground: background
COLORS = (
    ('white', 'red'),
    ('white', 'orange'),
    ('black', 'yellow'),
    ('black', 'green'),
    ('white', 'blue'),
)
HL_RULES = ['ctermfg=%s ctermbg=%s guifg=%s guibg=%s' % (fg, bg, fg, bg) for fg, bg in COLORS]

class View(object):
    """editors representation of the buffer"""
    highlight_regions = set()

    def __init__(self, vim_buf, buf):
        self.vim_buf = vim_buf
        self.buf = buf

    def __repr__(self):
        return '%s %s %s' % (self.native_id, self.buf['id'], self.buf['path'].encode('utf-8'))

    def __str__(self):
        return repr(self)

    def _offset_to_vim(self, offset):
        current_offset = 0
        for line_num, line in enumerate(self.vim_buf):
            next_offset = len(line) + 1
            if current_offset + next_offset > offset:
                break
            current_offset += next_offset
        col = offset - current_offset
        msg.debug('offset %s is line %s column %s' % (offset, line_num + 1, col + 1))
        return line_num + 1, col + 1

    @property
    def native_id(self):
        return self.vim_buf.number

    def is_loading(self):
        return False

    def get_text(self):
        text = '\n'.join(self.vim_buf[:])
        return text.decode('utf-8')

    def set_text(self, text):
        msg.debug('\n\nabout to patch %s %s' % (str(self), self.vim_buf.name))
        try:
            msg.debug("now buf is loadedish? %s" % vim.eval('bufloaded(%s)' % self.native_id))
            self.vim_buf[:] = text.encode('utf-8').split('\n')
        except Exception as e:
            msg.error("couldn't apply patches because: %s!\nThe unencoded text was: %s" % (str(e), text))
            raise

    def apply_patches(self, buf, patches):
        cursor_offset = self.get_cursor_offset()
        msg.debug('cursor offset is %s bytes' % cursor_offset)
        self.set_text(patches[0])

        for patch in patches[2]:
            offset = patch[0]
            length = patch[1]
            patch_text = patch[2]
            if cursor_offset > offset:
                new_offset = len(patch_text) - length
                cursor_offset += new_offset

        self.set_cursor_position(cursor_offset)

    def focus(self):
        vim.command(':edit! %s' % self.vim_buf.name)
        vim.command(':filetype detect')

    def set_cursor_position(self, offset):
        line_num, col = self._offset_to_vim(offset)
        command = 'setpos(".", [%s, %s, %s, %s])' % (self.native_id, line_num, col, 0)
        msg.debug("setting pos: %s" % command)
        rv = int(vim.eval(command))
        if rv != 0:
            msg.debug('SHIIIIIIIIT %s %s' % (rv, command))

    def get_cursor_position(self):
        """ [bufnum, lnum, col, off] """
        return vim.eval('getpos(".")')

    def get_cursor_offset(self):
        return int(vim.eval('line2byte(line("."))+col(".")')) - 2

    def get_selections(self):
        cursor = self.get_cursor_offset()
        return [[cursor, cursor]]

    def clear_highlight(self, user_id):
        region = "floobitsuser%s" % str(user_id)
        if region not in self.highlight_regions:
            return
        msg.debug('clearing selections for view %s' % self.vim_buf.name)
        self.highlight_regions.remove(region)
        vim.command(":syntax clear %s" % region)

    def highlight(self, ranges, user_id):
        msg.debug('highlighting ranges %s' % (ranges))
        if vim.current.buffer.number != self.vim_buf.number:
            return

        region = "floobitsuser%s" % str(user_id)
        if region in self.highlight_regions:
            vim.command(":syntax clear %s" % region)

        hl_rule = HL_RULES[user_id % len(HL_RULES)]
        vim.command(":highlight %s %s" % (region, hl_rule))
        self.highlight_regions.add(region)

        for _range in ranges:
            start_row, start_col = self._offset_to_vim(_range[0])
            end_row, end_col = self._offset_to_vim(_range[1])
            if start_row == end_row and start_col == end_col:
                if end_col >= len(self.vim_buf[end_row - 1]):
                    end_row += 1
                    end_col = 1
                else:
                    end_col += 1
            vim_region = ":syntax region {region} start=/\%{start_col}v\%{start_row}l/ end=/\%{end_col}v\%{end_row}l/".\
                format(region=region, start_col=start_col, start_row=start_row, end_col=end_col, end_row=end_row)
            # print("highlight command: %s" % vim_region)
            vim.command(vim_region)

    def rename(self, name):
        msg.debug('renaming %s to %s' % (self.vim_buf.name, name))
        current = vim.current.buffer
        text = self.get_text()
        old_name = self.vim_buf.name
        old_number = self.native_id
        with open(name, 'wb') as fd:
            fd.write(text.encode('utf-8'))
        vim.command('edit! %s' % name)
        self.vim_buf = vim.current.buffer
        vim.command('edit! %s' % current.name)
        vim.command('bdelete! %s' % old_number)
        try:
            utils.rm(old_name)
        except Exception as e:
            msg.debug("couldn't delete %s... maybe thats OK?" % str(e))

    def save(self):
        vim.command(':s!')

class Protocol(protocol.BaseProtocol):
    """understands vim"""
    CLIENT = 'VIM'

    def on_room_info(self, workspace_info):
        super(Protocol, self).on_room_info(workspace_info)
        vim.command(':Explore %s | redraw' % G.PROJECT_PATH)

    def maybe_selection_changed(self, vim_buf, is_ping):
        buf = self.get_buf(vim_buf)
        if not buf:
            msg.debug('no buffer found for view %s' % vim_buf.number)
            return
        view = self.get_view(buf['id'])
        msg.debug("selection changed: %s %s %s" % (vim_buf.number, buf['id'], view))
        self.SELECTION_CHANGED.append([view, is_ping])

    def maybe_buffer_changed(self, vim_buf):
        text = vim_buf[:]
        buf = self.get_buf(vim_buf)
        if not buf or 'buf' not in buf:
            return
        if buf['buf'] != text:
            self.BUFS_CHANGED.append(buf['id'])

    def get_vim_buf_by_path(self, p):
        for vim_buf in vim.buffers:
            if vim_buf.name and p == utils.to_rel_path(vim_buf.name):
                return vim_buf
        return None

    def get_view(self, buf_id):
        buf = self.FLOO_BUFS.get(buf_id)
        if not buf:
            return None

        vb = self.get_vim_buf_by_path(buf['path'])
        if not vb:
            return None

        if vim.eval('bufloaded(%s)' % vb.number) == '0':
            return None

        return View(vb, buf)

    def create_view(self, buf):
        path = self.save_buf(buf)
        vb = self.get_vim_buf_by_path(buf['path'])
        if vb:
            return View(vb, buf)

        vim.command(':edit! %s' % path)
        vb = self.get_vim_buf_by_path(buf['path'])
        if vb is None:
            msg.debug('vim buffer is none even though we tried to open it: %s' % path)
            return
        return View(vb, buf)

    def get_buf(self, vim_buf):
        """None- no sharing, False- should be but isn't """
        if vim_buf.name is None or vim_buf.name == "":
            msg.debug('get:buf buffer has no filename')
            return None

        if not utils.is_shared(vim_buf.name):
            msg.debug('get_buf: %s is not shared' % vim_buf.name)
            return None

        buf = self.get_buf_by_path(vim_buf.name)
        if buf:
            return buf

        msg.debug('get_buf: no buf has path %s' % vim_buf.name)
        return False

    def save_buf(self, buf):
        path = utils.get_full_path(buf['path'])
        utils.mkdir(os.path.split(path)[0])
        with open(path, 'wb') as fd:
            if buf['encoding'] == 'utf8':
                fd.write(buf['buf'].encode('utf-8'))
            else:
                fd.write(buf['buf'])
        return path

    def chat(self, username, timestamp, message, self_msg=False):
        pass
        # envelope = msg.MSG(message, timestamp, username)
        # if not self_msg:
        #     self.chat_deck.appendleft(envelope)
        # envelope.display()

    def on_msg(self, data):
        timestamp = data.get('time') or time.time()
        msg.log('[%s] <%s> %s' % (time.ctime(timestamp), data.get('username', ''), data.get('data', '')))

    def update_view(self, buf, view=None):
        msg.debug('updating view for buf %s' % buf['id'])
        view = view or self.get_view(buf['id'])
        if not view:
            msg.log('view for buf %s not found. not updating' % buf['id'])
            return
        self.MODIFIED_EVENTS.put(1)
        view.set_text(buf['buf'])

    def on_part(self, data):
        msg.log('%s left the workspace' % data['username'])
        user_id = data['user_id']
        highlight = self.user_highlights.get(user_id)
        if not highlight:
            return
        view = self.get_view(highlight['id'])
        if not view:
            return
        if vim.current.buffer.number != view.native_id:
            return
        view.clear_highlight(user_id)
        del self.user_highlights[user_id]
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @Author: KevinMidboe
# @Date:   2017-03-05 13:52:45
# @Last Modified by:   KevinMidboe
# @Last Modified time: 2017-03-05 15:53:04

import sqlite3, json, os
from re import sub

dbPath = 'shows.db'

def unpackEpisodes():
	conn = sqlite3.connect(dbPath)
	c = conn.cursor()

	cursor = c.execute('SELECT * FROM stray_episodes WHERE verified = 1 AND moved = 0')
	episodeList = []
	for row in c.fetchall():
		columnNames = [description[0] for description in cursor.description]

		episodeDict = dict.fromkeys(columnNames)

		for i, key in enumerate(episodeDict.keys()):
			episodeDict[key] = row[i]

		episodeList.append(episodeDict)

	conn.close()

	return episodeList

def createFolders(episode):
	showDir = '/Volumes/media/tv/%s/'% episode['name']
	episodeFormat = '%s S%sE%s/'% (episode['name'], episode['season'], episode['episode'])
	seasonFormat = '%s Season %s/'% (episode['name'], episode['season'])

	if not os.path.isdir(showDir + seasonFormat):
		os.makedirs(showDir + seasonFormat)

	if not os.path.isdir(showDir + seasonFormat + episodeFormat):
		os.makedirs(showDir + seasonFormat + episodeFormat)

def newnameMediaitems(media_items):
	# media_items = [['New.Girl.S06E18.720p.HDTV.x264-EZTV.srt', '-EZTV', 'nl'], ['New.Girl.S06E18.720p.HDTV.x264-FLEET.srt', '-FLEET', 'en']]
	media_items = json.loads(media_items)

	returnList = []
	for item in media_items:
		returnList.append([item[0], item[0].replace(item[1], '')])

	return returnList

def newnameSubtitles(subtitles):
	subtitles = json.loads(subtitles)

	returnList = []
	for item in subtitles:
		returnList.append([item[0], item[0].replace(item[1], '.' + item[2])])

	return returnList

def updateMovedStatus(episodeDict):
	conn = sqlite3.connect(dbPath)
	c = conn.cursor()

	c.execute('UPDATE stray_episodes SET moved = 1 WHERE original is "' + episodeDict['original'] + '"')

	conn.commit()
	conn.close()

def moveFiles(episode):
	showDir = '/Volumes/media/tv/'
	episodeFormat = '%s S%sE%s/'% (episode['name'], episode['season'], episode['episode'])
	seasonFormat = '%s/%s Season %s/'% (episode['name'], episode['name'], episode['season'])

	newMediaitems = newnameMediaitems(episode['media_items'])
	for item in newMediaitems:
		print(showDir + episode['original'] + '/' + item[0])
		print(showDir + seasonFormat + episodeFormat + item[1] + '\n')

	if episode['subtitles']:
		newSubtitles = newnameSubtitles(episode['subtitles'])
		for item in newSubtitles:
			print(showDir + episode['original'] + '/' + item[0])
			print(showDir + seasonFormat + episodeFormat + item[1] + '\n')

	updateMovedStatus(episode)

def findVerified():
	episodes = unpackEpisodes()
	if episodes:
		for episode in episodes:
			createFolders(episode)
			moveFiles(episode)

if __name__ == '__main__':
	findVerified()
	<commit_msg>Corrected show directory<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @Author: KevinMidboe
# @Date:   2017-03-05 13:52:45
# @Last Modified by:   KevinMidboe
# @Last Modified time: 2017-03-05 15:53:04

import sqlite3, json, os
from re import sub

dbPath = 'shows.db'

def unpackEpisodes():
	conn = sqlite3.connect(dbPath)
	c = conn.cursor()

	cursor = c.execute('SELECT * FROM stray_episodes WHERE verified = 1 AND moved = 0')
	episodeList = []
	for row in c.fetchall():
		columnNames = [description[0] for description in cursor.description]

		episodeDict = dict.fromkeys(columnNames)

		for i, key in enumerate(episodeDict.keys()):
			episodeDict[key] = row[i]

		episodeList.append(episodeDict)

	conn.close()

	return episodeList

def createFolders(episode):
	showDir = '/media/hdd1/tv/%s/'% episode['name']
	episodeFormat = '%s S%sE%s/'% (episode['name'], episode['season'], episode['episode'])
	seasonFormat = '%s Season %s/'% (episode['name'], episode['season'])

	if not os.path.isdir(showDir + seasonFormat):
		os.makedirs(showDir + seasonFormat)

	if not os.path.isdir(showDir + seasonFormat + episodeFormat):
		os.makedirs(showDir + seasonFormat + episodeFormat)

def newnameMediaitems(media_items):
	# media_items = [['New.Girl.S06E18.720p.HDTV.x264-EZTV.srt', '-EZTV', 'nl'], ['New.Girl.S06E18.720p.HDTV.x264-FLEET.srt', '-FLEET', 'en']]
	media_items = json.loads(media_items)

	returnList = []
	for item in media_items:
		returnList.append([item[0], item[0].replace(item[1], '')])

	return returnList

def newnameSubtitles(subtitles):
	subtitles = json.loads(subtitles)

	returnList = []
	for item in subtitles:
		returnList.append([item[0], item[0].replace(item[1], '.' + item[2])])

	return returnList

def updateMovedStatus(episodeDict):
	conn = sqlite3.connect(dbPath)
	c = conn.cursor()

	c.execute('UPDATE stray_episodes SET moved = 1 WHERE original is "' + episodeDict['original'] + '"')

	conn.commit()
	conn.close()

def moveFiles(episode):
	showDir = '/media/hdd1/tv/'
	episodeFormat = '%s S%sE%s/'% (episode['name'], episode['season'], episode['episode'])
	seasonFormat = '%s/%s Season %s/'% (episode['name'], episode['name'], episode['season'])

	newMediaitems = newnameMediaitems(episode['media_items'])
	for item in newMediaitems:
		print(showDir + episode['original'] + '/' + item[0])
		print(showDir + seasonFormat + episodeFormat + item[1] + '\n')

	if episode['subtitles']:
		newSubtitles = newnameSubtitles(episode['subtitles'])
		for item in newSubtitles:
			print(showDir + episode['original'] + '/' + item[0])
			print(showDir + seasonFormat + episodeFormat + item[1] + '\n')

	updateMovedStatus(episode)

def findVerified():
	episodes = unpackEpisodes()
	if episodes:
		for episode in episodes:
			createFolders(episode)
			moveFiles(episode)

if __name__ == '__main__':
	findVerified()

<|endoftext|>


<commit_before>"""Allow you to run tests in an isolated container for an app or a lib

Usage:
  tests create_image <app_or_lib_name> [--recreate]

Options:
  --recreate  ensures that the testing image will be recreated

"""

from docopt import docopt

from ..commands.test import run_app_or_lib_tests

def main(argv):
    args = docopt(__doc__, argv)
    run_app_or_lib_tests(args['<app_or_lib_name>'], force_recreate=args['--recreate'])
<commit_msg>change to test<commit_after>"""Allow you to run tests in an isolated container for an app or a lib

Usage:
  test create_image <app_or_lib_name> [--recreate]

Options:
  --recreate  ensures that the testing image will be recreated

"""

from docopt import docopt

from ..commands.test import run_app_or_lib_tests

def main(argv):
    args = docopt(__doc__, argv)
    run_app_or_lib_tests(args['<app_or_lib_name>'], force_recreate=args['--recreate'])
<|endoftext|>


<commit_before>from django.conf import settings

import cronjobs

from celery.task.sets import TaskSet
from elasticutils.contrib.django import get_es

from mozillians.users.tasks import index_objects
from mozillians.users.models import UserProfile, UserProfileMappingType

@cronjobs.register
def index_all_profiles():
    # Get an es object, delete index and re-create it
    es = get_es(timeout=settings.ES_INDEXING_TIMEOUT)
    mappings = {'mappings':
                {UserProfileMappingType.get_mapping_type_name():
                 UserProfileMappingType.get_mapping()}}

    def _recreate_index(index):
        es.indices.delete(index=index, ignore=[400, 404])
        es.indices.create(index, body=mappings)

    _recreate_index(settings.ES_INDEXES['default'])
    _recreate_index(settings.ES_INDEXES['public'])

    # mozillians index
    ids = UserProfile.objects.complete().values_list('id', flat=True)
    ts = [index_objects.subtask(kwargs={'mapping_type': UserProfileMappingType,
                                        'ids': ids,
                                        'chunk_size': 150,
                                        'public_index': False})]

    # public index
    ts += [index_objects.subtask(kwargs={'mapping_type': UserProfileMappingType,
                                         'ids': ids,
                                         'chunk_size': 150,
                                         'public_index': True})]

    TaskSet(ts).apply_async()
<commit_msg>Index profiles in chunks and not altogether.<commit_after>from django.conf import settings

import cronjobs

from celery.task.sets import TaskSet
from celeryutils import chunked
from elasticutils.contrib.django import get_es

from mozillians.users.tasks import index_objects
from mozillians.users.models import UserProfile, UserProfileMappingType

@cronjobs.register
def index_all_profiles():
    # Get an es object, delete index and re-create it
    es = get_es(timeout=settings.ES_INDEXING_TIMEOUT)
    mappings = {'mappings':
                {UserProfileMappingType.get_mapping_type_name():
                 UserProfileMappingType.get_mapping()}}

    def _recreate_index(index):
        es.indices.delete(index=index, ignore=[400, 404])
        es.indices.create(index, body=mappings)

    _recreate_index(settings.ES_INDEXES['default'])
    _recreate_index(settings.ES_INDEXES['public'])

    # mozillians index
    ids = UserProfile.objects.complete().values_list('id', flat=True)
    ts = [index_objects.subtask(args=[UserProfileMappingType, chunk, 150, False])
          for chunk in chunked(sorted(list(ids)), 150)]

    # public index
    ts += [index_objects.subtask(args=[UserProfileMappingType, chunk, 150, True])
           for chunk in chunked(sorted(list(ids)), 150)]

    TaskSet(ts).apply_async()
<|endoftext|>


<commit_before><commit_msg><commit_after><|endoftext|>


<commit_before># coding: utf-8
from fbads.managers.base import Manager
from fbads.resources.group import GroupResource, BidType

class GroupManager(Manager):
    resource_class = GroupResource
    resource_name = 'adgroup'

    def _get_api_path(self, object_id):
        return '{0}'.format(object_id)

    def add(self, name, bid_type, bid_info, campaign_id, creative_id,
            targeting_specs, tracking_specs=None, conversion_specs=None,
            view_tags=[], redownload=False):
        assert not redownload  # if redownload is True, must instantiate a resource_class object... later

        if bid_type in (BidType.ABSOLUTE_OCPM, BidType.CPA):
            assert conversion_specs, u'provide arg conversion_specs'

        payload = {
            'name': name,
            'bid_type': bid_type,
            'campaign_id': campaign_id,
            'creative': {
                'creative_id': creative_id,
            },
            'targeting_specs': targeting_specs.get(),
        }

        if tracking_specs:
            payload.update({
                'tracking_specs': tracking_specs.get(),
            })

        if conversion_specs:
            payload.update({
                'conversion_specs': conversion_specs,
            })

        if view_tags:
            payload.update({
                'view_tags': view_tags,
            })

        return super(GroupManager, self).add(
            payload=payload
        )
<commit_msg>fixed resource_name for ad groups<commit_after># coding: utf-8
from fbads.managers.base import Manager
from fbads.resources.group import GroupResource, BidType

class GroupManager(Manager):
    resource_class = GroupResource
    resource_name = 'adgroups'

    def _get_api_path(self, object_id):
        return '{0}'.format(object_id)

    def add(self, name, bid_type, bid_info, campaign_id, creative_id,
            targeting_specs, tracking_specs=None, conversion_specs=None,
            view_tags=[], redownload=False):
        assert not redownload  # if redownload is True, must instantiate a resource_class object... later

        if bid_type in (BidType.ABSOLUTE_OCPM, BidType.CPA):
            assert conversion_specs, u'provide arg conversion_specs'

        payload = {
            'name': name,
            'bid_type': bid_type,
            'campaign_id': campaign_id,
            'creative': {
                'creative_id': creative_id,
            },
            'targeting_specs': targeting_specs.get(),
        }

        if tracking_specs:
            payload.update({
                'tracking_specs': tracking_specs.get(),
            })

        if conversion_specs:
            payload.update({
                'conversion_specs': conversion_specs,
            })

        if view_tags:
            payload.update({
                'view_tags': view_tags,
            })

        return super(GroupManager, self).add(
            payload=payload
        )
<|endoftext|>


<commit_before>"""
In a city laid out as a perfect grid, we want to walk from point
A at some intersection to point B at another intersection. The
problem is to figure out how many ways are there to get from A to
B without going out of your way.

The distances from A to B along each dimension of the grid are
given by x and y.
"""
def number_of_paths(x,y):
    ## We'll fill in each position in the grid with the number of ways
    ## to get from the start to that position.
    grid = [[None for j in range(y)] for i in range(x)]

    ## The start will look something like this:
    ## 1-1-1-1- ...
    ## | | | |
    ## 1-2-3-4- ...
    ## | | | |
    ## 1-3-6-10-...
    ## ...which is just Pascal's Triangle.

    ## along the edges of the grid, there's only 1
    for i in range(x):
        grid[i][0] = 1
    for j in range(y):
        grid[0][j] = 1

    ## any position in the grid is the sum of the two positions
    ## to the left and up from the current one.
    for i in range(1,x):
        for j in range(1,y):
            grid[i][j] = grid[i-1][j] + grid[i][j-1]

    ## print out the grid, just for laughs
    # for r in grid:
    #     print(r)

    ## return the resulting count
    return grid[x-1][y-1]

print("number_of_paths(9,6) =", number_of_paths(9,6))
assert number_of_paths(3,3) == 6
assert number_of_paths(9,6) == 1287
assert number_of_paths(10,5) == 715
assert number_of_paths(7,6) == 462
assert number_of_paths(8,7) == 1716
assert number_of_paths(11,5) == 1001

## Since the entries in Pascal's triangle can be computed
## directly by n-choose-k or:
##     n!
## ----------
##  k!(n-k)!
##

from functools import reduce
from fractions import Fraction
import operator

def prod(factors):
    return reduce(operator.mul, factors, 1)

def n_choose_k(n,k):
    return int(prod(Fraction(n-i,i+1) for i in range(k)))

def number_of_paths(x,y):
    return n_choose_k(x+y-2,y-1)

assert number_of_paths(3,3) == 6
assert number_of_paths(9,6) == 1287
assert number_of_paths(10,5) == 715
assert number_of_paths(7,6) == 462
assert number_of_paths(8,7) == 1716
assert number_of_paths(11,5) == 1001

x,y = 9,6
print(x,y,n_choose_k(x+y-2,y-1))

<commit_msg>3 implementations of counting paths through a grid<commit_after>"""
In a city laid out as a perfect grid, we want to walk from point
A at some intersection to point B at another intersection. The
problem is to figure out how many ways are there to get from A to
B without going out of your way.

The distances from A to B along each dimension of the grid are
given by x and y.
"""
from functools import lru_cache
from functools import reduce
from fractions import Fraction
import operator

@lru_cache(maxsize=1024)
def npaths(x, y):
    """
    Count paths recursively. Memoizing makes this efficient.
    """

    if x>0 and y>0:
        return npaths(x-1, y) + npaths(x, y-1)
    if x>0:
        return npaths(x-1, y)
    if y>0:
        return npaths(x, y-1)
    return 1

def npaths_dp(x,y):
    """
    Count number of paths by constructing Pascal's triangle,
    which is dynamic programming.
    """

    ## We'll fill in each position in the grid with the number of ways
    ## to get from the start to that position.
    grid = [[None for j in range(y+1)] for i in range(x+1)]

    ## The grid will look something like this:
    ## 1-1-1-1- ...
    ## | | | |
    ## 1-2-3-4- ...
    ## | | | |
    ## 1-3-6-10-...
    ## ...which is just Pascal's Triangle.

    ## along the edges, there's only 1 path
    for i in range(x+1):
        grid[i][0] = 1
    for j in range(y+1):
        grid[0][j] = 1

    ## any position in the grid is the sum of the two positions
    ## to the left and up from the current one.
    for i in range(1, x+1):
        for j in range(1, y+1):
            grid[i][j] = grid[i-1][j] + grid[i][j-1]

    ## print out the grid, just for laughs
    # for r in grid:
    #     print(r)

    ## return the resulting count
    return grid[x][y]

def prod(factors):
    """
    return the product of a sequence of factors
    """
    return reduce(operator.mul, factors, 1)

def n_choose_k(n,k):
    """
        n!
    ----------
     k!(n-k)!
    """
    return int(prod(Fraction(n-i,i+1) for i in range(k)))

def npaths_direct(x,y):
    """
    Entries in Pascal's triangle can be computed directly
    by n-choose-k
    """
    return n_choose_k(x+y, y)

def tests(number_of_paths):
    assert number_of_paths(0, 0) == 1
    assert number_of_paths(1, 0) == 1
    assert number_of_paths(0, 1) == 1
    assert number_of_paths(1, 1) == 2
    assert number_of_paths(2, 1) == 3
    assert number_of_paths(2, 2) == 6
    assert number_of_paths(3, 1) == 4
    assert number_of_paths(3, 2) == 10
    assert number_of_paths(3, 3) == 20
    assert number_of_paths(9, 4) == 715
    assert number_of_paths(6, 5) == 462
    assert number_of_paths(7, 6) == 1716
    assert number_of_paths(10, 4) == 1001
    assert number_of_paths(8, 5) == 1287
    assert number_of_paths(9, 6) == 5005

## Test eacb implementation
for f in (npaths, npaths_dp, npaths_direct):
    tests(f)

print("number_of_paths(9,6) =", npaths_direct(9, 6))
print("number_of_paths(15,15) =", npaths_direct(15, 15))

<|endoftext|>


<commit_before>#!/usr/bin/env python2
# -*- coding: utf-8 -*-
#
# publish.py - part of the FDroid server tools
# Copyright (C) 2010-2014, Ciaran Gultnieks, ciaran@ciarang.com
# Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import sys
import os
import shutil
import md5
import glob
from optparse import OptionParser
import logging

import common
import metadata
from common import FDroidPopen, BuildException

config = None
options = None

def main():

    global config, options

    # Parse command line...
    parser = OptionParser(usage="Usage: %prog [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
    parser.add_option("-v", "--verbose", action="store_true", default=False,
                      help="Spew out even more information than normal")
    parser.add_option("-q", "--quiet", action="store_true", default=False,
                      help="Restrict output to warnings and errors")
    (options, args) = parser.parse_args()

    config = common.read_config(options)

    output_dir = 'repo'
    if not os.path.isdir(output_dir):
        logging.error("Missing output directory")
        sys.exit(1)

    # Process any apks that are waiting to be signed...
    for apkfile in sorted(glob.glob(os.path.join(output_dir, '*.apk'))):

        apkfilename = os.path.basename(apkfile)
        sigfilename = apkfilename + ".txt"
        sigpath = os.path.join(output_dir, sigfilename)

        if not os.path.exists(sigpath):
            p = FDroidPopen(['gpg', '-a',
                             '--output', sigpath,
                             '--detach-sig',
                             os.path.join(output_dir, apkfilename)])
            if p.returncode != 0:
                logging.error("Signing failed.")
                sys.exit(1)

            logging.info('Signed ' + apkfilename)

if __name__ == "__main__":
    main()
<commit_msg>Remove unused imports<commit_after>#!/usr/bin/env python2
# -*- coding: utf-8 -*-
#
# publish.py - part of the FDroid server tools
# Copyright (C) 2010-2014, Ciaran Gultnieks, ciaran@ciarang.com
# Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import sys
import os
import glob
from optparse import OptionParser
import logging

import common
from common import FDroidPopen

config = None
options = None

def main():

    global config, options

    # Parse command line...
    parser = OptionParser(usage="Usage: %prog [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
    parser.add_option("-v", "--verbose", action="store_true", default=False,
                      help="Spew out even more information than normal")
    parser.add_option("-q", "--quiet", action="store_true", default=False,
                      help="Restrict output to warnings and errors")
    (options, args) = parser.parse_args()

    config = common.read_config(options)

    output_dir = 'repo'
    if not os.path.isdir(output_dir):
        logging.error("Missing output directory")
        sys.exit(1)

    # Process any apks that are waiting to be signed...
    for apkfile in sorted(glob.glob(os.path.join(output_dir, '*.apk'))):

        apkfilename = os.path.basename(apkfile)
        sigfilename = apkfilename + ".txt"
        sigpath = os.path.join(output_dir, sigfilename)

        if not os.path.exists(sigpath):
            p = FDroidPopen(['gpg', '-a',
                             '--output', sigpath,
                             '--detach-sig',
                             os.path.join(output_dir, apkfilename)])
            if p.returncode != 0:
                logging.error("Signing failed.")
                sys.exit(1)

            logging.info('Signed ' + apkfilename)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#
# Copyright 2018 Analytics Zoo Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from py4j.protocol import Py4JError

from zoo.orca.data.utils import *
from zoo.common.nncontext import init_nncontext
import os
from pyspark.context import SparkContext

class XShards(object):
    """
    A collection of data which can be pre-processed parallelly.
    """

    def transform_shard(self, func, *args):
        """
        Transform each shard in the XShards using func
        :param func: pre-processing function
        :param args: arguments for the pre-processing function
        :return: DataShard
        """
        pass

    def collect(self):
        """
        Returns a list that contains all of the elements in this XShards
        :return: list of elements
        """
        pass

    def num_partitions(self):
        """
        return the number of partitions in this XShards
        :return: an int
        """
        pass

class RayXShards(XShards):
    """
    A collection of data which can be pre-processed in parallel on Ray
    """
    def __init__(self, partitions):
        self.partitions = partitions
        self.shard_list = flatten([partition.shard_list for partition in partitions])

    def transform_shard(self, func, *args):
        """
        Transform each shard in the XShards using func
        :param func: pre-processing function.
        In the function, the element object should be the first argument
        :param args: rest arguments for the pre-processing function
        :return: this DataShard
        """
        import ray
        done_ids, undone_ids = ray.wait([shard.transform.remote(func, *args)
                                         for shard in self.shard_list],
                                        num_returns=len(self.shard_list))
        assert len(undone_ids) == 0
        return self

    def collect(self):
        """
        Returns a list that contains all of the elements in this XShards
        :return: list of elements
        """
        import ray
        return ray.get([shard.get_data.remote() for shard in self.shard_list])

    def num_partitions(self):
        return len(self.partitions)

    def repartition(self, num_partitions):
        """
        Repartition XShards.
        :param num_partitions: number of partitions
        :return: this XShards
        """
        shards_partitions = list(chunk(self.shard_list, num_partitions))
        self.partitions = [RayPartition(shards) for shards in shards_partitions]
        return self

    def get_partitions(self):
        """
        Return partition list of the XShards
        :return: partition list
        """
        return self.partitions

    def colocate_actors(self, actors):
        """
        Sort Ray actors and RayPartitions by node_ip so that each actor is colocated
        with the data partition on the same node.
        """
        if self.partitions[0].node_ip:
            # Assume that the partitions are already sorted by node_ip
            import ray
            actor_ips = ray.get([actor.get_node_ip.remote() for actor in actors])
            actor_zip_ips = list(zip(actors, actor_ips))
            actor_zip_ips.sort(key=lambda x: x[1])
            for i in range(len(actors)):
                actor_ip = actor_zip_ips[i][1]
                partition_ip = self.partitions[i].node_ip
                assert actor_ip == partition_ip
            return [actor_ip[0] for actor_ip in actor_zip_ips]
        else:
            return actors

class RayPartition(object):
    """
    Partition of RayXShards
    """
    def __init__(self, shard_list, node_ip=None, object_store_address=None):
        self.shard_list = shard_list
        self.node_ip = node_ip
        self.object_store_address = object_store_address

    def get_data(self):
        # For partitions read by Ray, shard_list is a list of Ray actors.
        # Each Ray actor contains a partition of data.
        if isinstance(self.shard_list, list):
            import ray
            return ray.get([shard.get_data.remote() for shard in self.shard_list])
        # For partitions transfromed from Spark, shard_list is a single plasma ObjectID.
        # The ObjectID would contain a list of data.
        else:
            import pyarrow.plasma as plasma
            client = plasma.connect(self.object_store_address)
            return client.get(self.shard_list)

def get_eager_mode():
    is_eager = True
    if os.getenv("EAGER_EXECUTION"):
        eager_execution = os.getenv("EAGER_EXECUTION").lower()
        if eager_execution == "false":
            is_eager = False
    return is_eager

class SparkXShards(XShards):
    def __init__(self, rdd):
        self.rdd = rdd
        self.user_cached = False
        self.eager = get_eager_mode()
        self.rdd.cache()
        if self.eager:
            self.compute()

    def transform_shard(self, func, *args):
        transformed_shard = SparkXShards(self.rdd.map(lambda data: func(data, *args)))
        self._uncache()
        return transformed_shard

    def collect(self):
        return self.rdd.collect()

    def cache(self):
        self.user_cached = True
        self.rdd.cache()
        return self

    def uncache(self):
        self.user_cached = False
        if self.is_cached():
            try:
                self.rdd.unpersist()
            except Py4JError:
                print("Try to unpersist an uncached rdd")
        return self

    def _uncache(self):
        if not self.user_cached:
            self.uncache()

    def is_cached(self):
        return self.rdd.is_cached

    def compute(self):
        self.rdd.count()
        return self

    def num_partitions(self):
        return self.rdd.getNumPartitions()

    def repartition(self, num_partitions):
        repartitioned_shard = SparkXShards(self.rdd.repartition(num_partitions))
        self._uncache()
        return repartitioned_shard

    def partition_by(self, cols, num_partitions=None):
        import pandas as pd
        elem_class, columns = self.rdd.map(
            lambda data: (type(data), data.columns) if isinstance(data, pd.DataFrame)
            else (type(data), None)).first()
        if issubclass(elem_class, pd.DataFrame):
            # if partition by a column
            if isinstance(cols, str):
                if cols not in columns:
                    raise Exception("The partition column is not in the DataFrame")
                # change data to key value pairs
                rdd = self.rdd.flatMap(
                    lambda df: df.apply(lambda row: (row[cols], row.values.tolist()), axis=1)
                    .values.tolist())

                partition_num = self.rdd.getNumPartitions() if not num_partitions \
                    else num_partitions
                # partition with key
                partitioned_rdd = rdd.partitionBy(partition_num)
            else:
                raise Exception("Only support partition by a column name")

            def merge(iterator):
                data = [value[1] for value in list(iterator)]
                if data:
                    df = pd.DataFrame(data=data, columns=columns)
                    return [df]
                else:
                    # no data in this partition
                    return []
            # merge records to df in each partition
            partitioned_shard = SparkXShards(partitioned_rdd.mapPartitions(merge))
            self._uncache()
            return partitioned_shard
        else:
            raise Exception("Currently only support partition by for XShards"
                            " of Pandas DataFrame")

    def unique(self, key):
        import pandas as pd
        elem_class, columns = self.rdd.map(
            lambda data: (type(data), data.columns) if isinstance(data, pd.DataFrame)
            else (type(data), None)).first()
        if issubclass(elem_class, pd.DataFrame):
            if key is None:
                raise Exception("Cannot apply unique operation on XShards of Pandas Dataframe"
                                " without column name")
            if key in columns:
                rdd = self.rdd.map(lambda df: df[key].unique())
                import numpy as np
                result = rdd.reduce(lambda list1, list2: pd.unique(np.concatenate((list1, list2),
                                                                                axis=0)))
                return result
            else:
                raise Exception("The select key is not in the DataFrame in this XShards")
        else:
            # we may support numpy or other types later
            raise Exception("Currently only support unique() on XShards of Pandas DataFrame")

    def split(self):
        """
        Split SparkXShards into multiple SparkXShards.
        Each element in the SparkXShards needs be a list or tuple with same length.
        :return: Splits of SparkXShards. If element in the input SparkDataShard is not
                list or tuple, return list of input SparkDataShards.
        """
        # get number of splits
        list_split_length = self.rdd.map(lambda data: len(data) if isinstance(data, list) or
                                         isinstance(data, tuple) else 1).collect()
        # check if each element has same splits
        if list_split_length.count(list_split_length[0]) != len(list_split_length):
            raise Exception("Cannot split this XShards because its partitions "
                            "have different split length")
        else:
            if list_split_length[0] > 1:
                def get_data(order):
                    def transform(data):
                        return data[order]
                    return transform
                split_shard_list = [SparkXShards(self.rdd.map(get_data(i)))
                                    for i in range(list_split_length[0])]
                self._uncache()
                return split_shard_list
            else:
                return [self]

    def len(self, key=None):
        if key is None:
            return self.rdd.map(lambda data: len(data) if hasattr(data, '__len__') else 1)\
                .reduce(lambda l1, l2: l1 + l2)
        else:

            def get_len(data):
                assert hasattr(data, '__getitem__'), \
                    "No selection operation available for this XShards"
                try:
                    value = data[key]
                except:
                    raise Exception("Invalid key for this XShards")
                return len(value) if hasattr(value, '__len__') else 1
            return self.rdd.map(get_len).reduce(lambda l1, l2: l1 + l2)

    def save_pickle(self, path, batchSize=10):
        self.rdd.saveAsPickleFile(path, batchSize)
        return self

    @classmethod
    def load_pickle(cls, path, minPartitions=None):
        sc = init_nncontext()
        return SparkXShards(sc.pickleFile(path, minPartitions))

    def __del__(self):
        self.uncache()

    def to_ray(self):
        import random
        import string
        from zoo.ray import RayContext
        ray_ctx = RayContext.get()
        object_store_address = ray_ctx.address_info["object_store_address"]

        # TODO: Handle failure when doing this?
        # TODO: delete the data in the plasma?
        def put_to_plasma(seed):
            def f(index, iterator):
                import pyarrow.plasma as plasma
                from zoo.orca.data.utils import get_node_ip
                # mapPartition would set the same random seed for each partition?
                # Here use the partition index to override the random seed so that there won't be
                # identical object_ids in plasma.
                random.seed(seed+str(index))
                res = list(iterator)
                client = plasma.connect(object_store_address)
                object_id = client.put(res)
                yield object_id, get_node_ip()
            return f

        # Generate a random string here to make sure that when this method is called twice, the
        # seeds to generate plasma ObjectID are different.
        random_str = ''.join(
            [random.choice(string.ascii_letters + string.digits) for i in range(32)])
        object_id_node_ips = self.rdd.mapPartitionsWithIndex(put_to_plasma(random_str)).collect()
        self.uncache()
        # Sort the data according to the node_ips.
        object_id_node_ips.sort(key=lambda x: x[1])
        partitions = [RayPartition(shard_list=id_ip[0], node_ip=id_ip[1],
                                   object_store_address=object_store_address)
                      for id_ip in object_id_node_ips]
        return RayXShards(partitions)

class SharedValue(object):
    def __init__(self, data):
        sc = init_nncontext()
        self.broadcast_data = sc.broadcast(data)
        self._value = None

    @property
    def value(self):
        self._value = self.broadcast_data.value
        return self._value

    def unpersist(self):
        self.broadcast_data.unpersist()
<commit_msg>Add pyarrow version 0.17.0 (#2461)<commit_after>#
# Copyright 2018 Analytics Zoo Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from py4j.protocol import Py4JError

from zoo.orca.data.utils import *
from zoo.common.nncontext import init_nncontext
import os
from pyspark.context import SparkContext

class XShards(object):
    """
    A collection of data which can be pre-processed parallelly.
    """

    def transform_shard(self, func, *args):
        """
        Transform each shard in the XShards using func
        :param func: pre-processing function
        :param args: arguments for the pre-processing function
        :return: DataShard
        """
        pass

    def collect(self):
        """
        Returns a list that contains all of the elements in this XShards
        :return: list of elements
        """
        pass

    def num_partitions(self):
        """
        return the number of partitions in this XShards
        :return: an int
        """
        pass

class RayXShards(XShards):
    """
    A collection of data which can be pre-processed in parallel on Ray
    """
    def __init__(self, partitions):
        self.partitions = partitions
        self.shard_list = flatten([partition.shard_list for partition in partitions])

    def transform_shard(self, func, *args):
        """
        Transform each shard in the XShards using func
        :param func: pre-processing function.
        In the function, the element object should be the first argument
        :param args: rest arguments for the pre-processing function
        :return: this DataShard
        """
        import ray
        done_ids, undone_ids = ray.wait([shard.transform.remote(func, *args)
                                         for shard in self.shard_list],
                                        num_returns=len(self.shard_list))
        assert len(undone_ids) == 0
        return self

    def collect(self):
        """
        Returns a list that contains all of the elements in this XShards
        :return: list of elements
        """
        import ray
        return ray.get([shard.get_data.remote() for shard in self.shard_list])

    def num_partitions(self):
        return len(self.partitions)

    def repartition(self, num_partitions):
        """
        Repartition XShards.
        :param num_partitions: number of partitions
        :return: this XShards
        """
        shards_partitions = list(chunk(self.shard_list, num_partitions))
        self.partitions = [RayPartition(shards) for shards in shards_partitions]
        return self

    def get_partitions(self):
        """
        Return partition list of the XShards
        :return: partition list
        """
        return self.partitions

    def colocate_actors(self, actors):
        """
        Sort Ray actors and RayPartitions by node_ip so that each actor is colocated
        with the data partition on the same node.
        """
        if self.partitions[0].node_ip:
            # Assume that the partitions are already sorted by node_ip
            import ray
            actor_ips = ray.get([actor.get_node_ip.remote() for actor in actors])
            actor_zip_ips = list(zip(actors, actor_ips))
            actor_zip_ips.sort(key=lambda x: x[1])
            for i in range(len(actors)):
                actor_ip = actor_zip_ips[i][1]
                partition_ip = self.partitions[i].node_ip
                assert actor_ip == partition_ip
            return [actor_ip[0] for actor_ip in actor_zip_ips]
        else:
            return actors

class RayPartition(object):
    """
    Partition of RayXShards
    """
    def __init__(self, shard_list, node_ip=None, object_store_address=None):
        self.shard_list = shard_list
        self.node_ip = node_ip
        self.object_store_address = object_store_address

    def get_data(self):
        # For partitions read by Ray, shard_list is a list of Ray actors.
        # Each Ray actor contains a partition of data.
        if isinstance(self.shard_list, list):
            import ray
            return ray.get([shard.get_data.remote() for shard in self.shard_list])
        # For partitions transfromed from Spark, shard_list is a single plasma ObjectID.
        # The ObjectID would contain a list of data.
        else:
            import pyarrow.plasma as plasma
            client = plasma.connect(self.object_store_address)
            return client.get(self.shard_list)

def get_eager_mode():
    is_eager = True
    if os.getenv("EAGER_EXECUTION"):
        eager_execution = os.getenv("EAGER_EXECUTION").lower()
        if eager_execution == "false":
            is_eager = False
    return is_eager

class SparkXShards(XShards):
    def __init__(self, rdd):
        self.rdd = rdd
        self.user_cached = False
        self.eager = get_eager_mode()
        self.rdd.cache()
        if self.eager:
            self.compute()

    def transform_shard(self, func, *args):
        transformed_shard = SparkXShards(self.rdd.map(lambda data: func(data, *args)))
        self._uncache()
        return transformed_shard

    def collect(self):
        return self.rdd.collect()

    def cache(self):
        self.user_cached = True
        self.rdd.cache()
        return self

    def uncache(self):
        self.user_cached = False
        if self.is_cached():
            try:
                self.rdd.unpersist()
            except Py4JError:
                print("Try to unpersist an uncached rdd")
        return self

    def _uncache(self):
        if not self.user_cached:
            self.uncache()

    def is_cached(self):
        return self.rdd.is_cached

    def compute(self):
        self.rdd.count()
        return self

    def num_partitions(self):
        return self.rdd.getNumPartitions()

    def repartition(self, num_partitions):
        repartitioned_shard = SparkXShards(self.rdd.repartition(num_partitions))
        self._uncache()
        return repartitioned_shard

    def partition_by(self, cols, num_partitions=None):
        import pandas as pd
        elem_class, columns = self.rdd.map(
            lambda data: (type(data), data.columns) if isinstance(data, pd.DataFrame)
            else (type(data), None)).first()
        if issubclass(elem_class, pd.DataFrame):
            # if partition by a column
            if isinstance(cols, str):
                if cols not in columns:
                    raise Exception("The partition column is not in the DataFrame")
                # change data to key value pairs
                rdd = self.rdd.flatMap(
                    lambda df: df.apply(lambda row: (row[cols], row.values.tolist()), axis=1)
                    .values.tolist())

                partition_num = self.rdd.getNumPartitions() if not num_partitions \
                    else num_partitions
                # partition with key
                partitioned_rdd = rdd.partitionBy(partition_num)
            else:
                raise Exception("Only support partition by a column name")

            def merge(iterator):
                data = [value[1] for value in list(iterator)]
                if data:
                    df = pd.DataFrame(data=data, columns=columns)
                    return [df]
                else:
                    # no data in this partition
                    return []
            # merge records to df in each partition
            partitioned_shard = SparkXShards(partitioned_rdd.mapPartitions(merge))
            self._uncache()
            return partitioned_shard
        else:
            raise Exception("Currently only support partition by for XShards"
                            " of Pandas DataFrame")

    def unique(self, key):
        import pandas as pd
        elem_class, columns = self.rdd.map(
            lambda data: (type(data), data.columns) if isinstance(data, pd.DataFrame)
            else (type(data), None)).first()
        if issubclass(elem_class, pd.DataFrame):
            if key is None:
                raise Exception("Cannot apply unique operation on XShards of Pandas Dataframe"
                                " without column name")
            if key in columns:
                rdd = self.rdd.map(lambda df: df[key].unique())
                import numpy as np
                result = rdd.reduce(lambda list1, list2: pd.unique(np.concatenate((list1, list2),
                                                                                axis=0)))
                return result
            else:
                raise Exception("The select key is not in the DataFrame in this XShards")
        else:
            # we may support numpy or other types later
            raise Exception("Currently only support unique() on XShards of Pandas DataFrame")

    def split(self):
        """
        Split SparkXShards into multiple SparkXShards.
        Each element in the SparkXShards needs be a list or tuple with same length.
        :return: Splits of SparkXShards. If element in the input SparkDataShard is not
                list or tuple, return list of input SparkDataShards.
        """
        # get number of splits
        list_split_length = self.rdd.map(lambda data: len(data) if isinstance(data, list) or
                                         isinstance(data, tuple) else 1).collect()
        # check if each element has same splits
        if list_split_length.count(list_split_length[0]) != len(list_split_length):
            raise Exception("Cannot split this XShards because its partitions "
                            "have different split length")
        else:
            if list_split_length[0] > 1:
                def get_data(order):
                    def transform(data):
                        return data[order]
                    return transform
                split_shard_list = [SparkXShards(self.rdd.map(get_data(i)))
                                    for i in range(list_split_length[0])]
                self._uncache()
                return split_shard_list
            else:
                return [self]

    def len(self, key=None):
        if key is None:
            return self.rdd.map(lambda data: len(data) if hasattr(data, '__len__') else 1)\
                .reduce(lambda l1, l2: l1 + l2)
        else:

            def get_len(data):
                assert hasattr(data, '__getitem__'), \
                    "No selection operation available for this XShards"
                try:
                    value = data[key]
                except:
                    raise Exception("Invalid key for this XShards")
                return len(value) if hasattr(value, '__len__') else 1
            return self.rdd.map(get_len).reduce(lambda l1, l2: l1 + l2)

    def save_pickle(self, path, batchSize=10):
        self.rdd.saveAsPickleFile(path, batchSize)
        return self

    @classmethod
    def load_pickle(cls, path, minPartitions=None):
        sc = init_nncontext()
        return SparkXShards(sc.pickleFile(path, minPartitions))

    def __del__(self):
        self.uncache()

    # Tested on pyarrow 0.17.0; 0.16.0 would get errors.
    def to_ray(self):
        import random
        import string
        from zoo.ray import RayContext
        ray_ctx = RayContext.get()
        object_store_address = ray_ctx.address_info["object_store_address"]

        # TODO: Handle failure when doing this?
        # TODO: delete the data in the plasma?
        def put_to_plasma(seed):
            def f(index, iterator):
                import pyarrow.plasma as plasma
                from zoo.orca.data.utils import get_node_ip
                # mapPartition would set the same random seed for each partition?
                # Here use the partition index to override the random seed so that there won't be
                # identical object_ids in plasma.
                random.seed(seed+str(index))
                res = list(iterator)
                client = plasma.connect(object_store_address)
                object_id = client.put(res)
                yield object_id, get_node_ip()
            return f

        # Generate a random string here to make sure that when this method is called twice, the
        # seeds to generate plasma ObjectID are different.
        random_str = ''.join(
            [random.choice(string.ascii_letters + string.digits) for i in range(32)])
        object_id_node_ips = self.rdd.mapPartitionsWithIndex(put_to_plasma(random_str)).collect()
        self.uncache()
        # Sort the data according to the node_ips.
        object_id_node_ips.sort(key=lambda x: x[1])
        partitions = [RayPartition(shard_list=id_ip[0], node_ip=id_ip[1],
                                   object_store_address=object_store_address)
                      for id_ip in object_id_node_ips]
        return RayXShards(partitions)

class SharedValue(object):
    def __init__(self, data):
        sc = init_nncontext()
        self.broadcast_data = sc.broadcast(data)
        self._value = None

    @property
    def value(self):
        self._value = self.broadcast_data.value
        return self._value

    def unpersist(self):
        self.broadcast_data.unpersist()
<|endoftext|>


<commit_before>#!/usr/bin/python3
#
#
# mpp_info_pub.py
#
# script to query MPP Solar PIP-4048MS inverter/charger
# - inverter connected to computer via serial
#      (USB to Serial converter used for testing)
# - posts results to MQTT broker
# - uses mpputils.py / mppcommands.py to abstract PIP communications
#
import paho.mqtt.publish as publish

from .mpputils import mppUtils
grab_settings = False

def main():
    from argparse import ArgumentParser
    parser = ArgumentParser(description='MPP Solar Inverter Info Utility')
    parser.add_argument('-s', '--grabsettings', action='store_true', help='Also get the inverter settings')
    parser.add_argument('-t', '--getstatus', action='store_true', help='Use the getstatus "helper"')
    parser.add_argument('-c', '--command', type=str, help='Command to execute [comma separated]', default=None)
    parser.add_argument('-d', '--device', type=str, help='Serial device(s) to communicate with [comma separated]', default='/dev/ttyUSB0')
    parser.add_argument('-M', '--model', type=str, help='Specifies the inverter model to select commands for, defaults to "standard", currently supports LV5048', default='standard')
    parser.add_argument('-b', '--baud', type=int, help='Baud rate for serial communications', default=2400)
    parser.add_argument('-q', '--broker', type=str, help='MQTT Broker hostname', default='mqtt_broker')
    parser.add_argument('-i', '--influx', action='store_true', help='Use Influx Line Protocol for the messgae format')
    parser.add_argument('-I', '--influx2', action='store_true', help='Use Influx Line Protocol II for the messgae format')
    parser.add_argument('--tag', type=str, help='Influx tag to use for all commands in this execution', default=None)
    args = parser.parse_args()

    # Process / loop through all supplied devices
    for usb_port in args.device.split(','):
        mp = mppUtils(usb_port, args.baud, args.model)
        serial_number = mp.getSerialNumber()

        # Collect Inverter Settings and publish
        if args.grabsettings:
            msgs = []
            settings = mp.getSettings()

            for setting in settings:
                for i in ['value', 'default', 'unit']:
                    topic = '{}/settings/{}/{}'.format(serial_number, setting, i)
                    msg = {'topic': topic, 'payload': '{}'.format(settings[setting][i])}
                    msgs.append(msg)
            # publish.multiple(msgs, hostname=args.broker)
            print(msgs)
            print(args.broker)

        if args.command:
            if args.influx:
                for _command in args.command.split(','):
                    msgs = []
                    _data = mp.getInfluxLineProtocol(_command)
                    for _item in _data:
                        # print(_item)
                        # _item = setting=total_ac_output_apparent_power value=1577.0,unit="VA"
                        # weather,location=us-midwest temperature=82 1465839830100400200
                        # |    -------------------- --------------  |
                        # |             |             |             |
                        # |             |             |             |
                        # +-----------+--------+-+---------+-+---------+
                        # |measurement|,tag_set| |field_set| |timestamp|
                        # +-----------+--------+-+---------+-+---------+
                        if args.tag:
                            tag = args.tag
                        else:
                            tag = _command
                        payload = '{},{}'.format(tag, _item)
                        msg = {'topic': tag, 'payload': payload}
                        msgs.append(msg)
                    publish.multiple(msgs, hostname=args.broker)
            elif args.influx2:
                for _command in args.command.split(','):
                    msgs = []
                    _data = mp.getInfluxLineProtocol2(_command)
                    if args.tag:
                        tag = args.tag
                    else:
                        tag = _command
                    for _item in _data:
                        # print(_item)
                        # _item = setting=total_ac_output_apparent_power value=1577.0,unit="VA"
                        # weather,location=us-midwest temperature=82 1465839830100400200
                        # |    -------------------- --------------  |
                        # |             |             |             |
                        # |             |             |             |
                        # +-----------+--------+-+---------+-+---------+
                        # |measurement|,tag_set| |field_set| |timestamp|
                        # +-----------+--------+-+---------+-+---------+
                        payload = 'mpp-solar,command={} {}'.format(tag, _item)
                        msg = {'topic': 'mpp-solar', 'payload': payload}
                        msgs.append(msg)
                    publish.multiple(msgs, hostname=args.broker)
            else:
                for _command in args.command.split(','):
                    msgs = []
                    _data = mp.getResponseDict(_command)
                    # {'serial_number': [u'9293333010501', u'']}
                    for _item in _data:
                        # 92931509101901/status/total_output_active_power/value 1250
                        # 92931509101901/status/total_output_active_power/unit W
                        # topic = '{}/status/{}/value'.format(serial_number, _item)
                        topic = '{}/{}/value'.format(_command, _item)
                        msg = {'topic': topic, 'payload': '{}'.format(_data[_item][0])}
                        msgs.append(msg)
                        topic = '{}/{}/unit'.format(_command, _item)
                        msg = {'topic': topic, 'payload': '{}'.format(_data[_item][1])}
                        msgs.append(msg)
                    publish.multiple(msgs, hostname=args.broker)
        # Collect Inverter Status data and publish
        if args.getstatus:
            msgs = []
            status_data = mp.getFullStatus()
            for status_line in status_data:
                for i in ['value', 'unit']:
                    # 92931509101901/status/total_output_active_power/value 1250
                    # 92931509101901/status/total_output_active_power/unit W
                    topic = '{}/status/{}/{}'.format(serial_number, status_line, i)
                    msg = {'topic': topic, 'payload': '{}'.format(status_data[status_line][i])}
                    msgs.append(msg)
            publish.multiple(msgs, hostname=args.broker)
<commit_msg>uncomment publish for default output in mpp-info-pub<commit_after>#!/usr/bin/python3
#
#
# mpp_info_pub.py
#
# script to query MPP Solar PIP-4048MS inverter/charger
# - inverter connected to computer via serial
#      (USB to Serial converter used for testing)
# - posts results to MQTT broker
# - uses mpputils.py / mppcommands.py to abstract PIP communications
#
import paho.mqtt.publish as publish

from .mpputils import mppUtils
grab_settings = False

def main():
    from argparse import ArgumentParser
    parser = ArgumentParser(description='MPP Solar Inverter Info Utility')
    parser.add_argument('-s', '--grabsettings', action='store_true', help='Also get the inverter settings')
    parser.add_argument('-t', '--getstatus', action='store_true', help='Use the getstatus "helper"')
    parser.add_argument('-c', '--command', type=str, help='Command to execute [comma separated]', default=None)
    parser.add_argument('-d', '--device', type=str, help='Serial device(s) to communicate with [comma separated]', default='/dev/ttyUSB0')
    parser.add_argument('-M', '--model', type=str, help='Specifies the inverter model to select commands for, defaults to "standard", currently supports LV5048', default='standard')
    parser.add_argument('-b', '--baud', type=int, help='Baud rate for serial communications', default=2400)
    parser.add_argument('-q', '--broker', type=str, help='MQTT Broker hostname', default='mqtt_broker')
    parser.add_argument('-i', '--influx', action='store_true', help='Use Influx Line Protocol for the messgae format')
    parser.add_argument('-I', '--influx2', action='store_true', help='Use Influx Line Protocol II for the messgae format')
    parser.add_argument('--tag', type=str, help='Influx tag to use for all commands in this execution', default=None)
    args = parser.parse_args()

    # Process / loop through all supplied devices
    for usb_port in args.device.split(','):
        mp = mppUtils(usb_port, args.baud, args.model)
        serial_number = mp.getSerialNumber()

        # Collect Inverter Settings and publish
        if args.grabsettings:
            msgs = []
            settings = mp.getSettings()

            for setting in settings:
                for i in ['value', 'default', 'unit']:
                    topic = '{}/settings/{}/{}'.format(serial_number, setting, i)
                    msg = {'topic': topic, 'payload': '{}'.format(settings[setting][i])}
                    msgs.append(msg)
            publish.multiple(msgs, hostname=args.broker)
            print(msgs)
            print(args.broker)

        if args.command:
            if args.influx:
                for _command in args.command.split(','):
                    msgs = []
                    _data = mp.getInfluxLineProtocol(_command)
                    for _item in _data:
                        # print(_item)
                        # _item = setting=total_ac_output_apparent_power value=1577.0,unit="VA"
                        # weather,location=us-midwest temperature=82 1465839830100400200
                        # |    -------------------- --------------  |
                        # |             |             |             |
                        # |             |             |             |
                        # +-----------+--------+-+---------+-+---------+
                        # |measurement|,tag_set| |field_set| |timestamp|
                        # +-----------+--------+-+---------+-+---------+
                        if args.tag:
                            tag = args.tag
                        else:
                            tag = _command
                        payload = '{},{}'.format(tag, _item)
                        msg = {'topic': tag, 'payload': payload}
                        msgs.append(msg)
                    publish.multiple(msgs, hostname=args.broker)
            elif args.influx2:
                for _command in args.command.split(','):
                    msgs = []
                    _data = mp.getInfluxLineProtocol2(_command)
                    if args.tag:
                        tag = args.tag
                    else:
                        tag = _command
                    for _item in _data:
                        # print(_item)
                        # _item = setting=total_ac_output_apparent_power value=1577.0,unit="VA"
                        # weather,location=us-midwest temperature=82 1465839830100400200
                        # |    -------------------- --------------  |
                        # |             |             |             |
                        # |             |             |             |
                        # +-----------+--------+-+---------+-+---------+
                        # |measurement|,tag_set| |field_set| |timestamp|
                        # +-----------+--------+-+---------+-+---------+
                        payload = 'mpp-solar,command={} {}'.format(tag, _item)
                        msg = {'topic': 'mpp-solar', 'payload': payload}
                        msgs.append(msg)
                    publish.multiple(msgs, hostname=args.broker)
            else:
                for _command in args.command.split(','):
                    msgs = []
                    _data = mp.getResponseDict(_command)
                    # {'serial_number': [u'9293333010501', u'']}
                    for _item in _data:
                        # 92931509101901/status/total_output_active_power/value 1250
                        # 92931509101901/status/total_output_active_power/unit W
                        # topic = '{}/status/{}/value'.format(serial_number, _item)
                        topic = '{}/{}/value'.format(_command, _item)
                        msg = {'topic': topic, 'payload': '{}'.format(_data[_item][0])}
                        msgs.append(msg)
                        topic = '{}/{}/unit'.format(_command, _item)
                        msg = {'topic': topic, 'payload': '{}'.format(_data[_item][1])}
                        msgs.append(msg)
                    publish.multiple(msgs, hostname=args.broker)
        # Collect Inverter Status data and publish
        if args.getstatus:
            msgs = []
            status_data = mp.getFullStatus()
            for status_line in status_data:
                for i in ['value', 'unit']:
                    # 92931509101901/status/total_output_active_power/value 1250
                    # 92931509101901/status/total_output_active_power/unit W
                    topic = '{}/status/{}/{}'.format(serial_number, status_line, i)
                    msg = {'topic': topic, 'payload': '{}'.format(status_data[status_line][i])}
                    msgs.append(msg)
            publish.multiple(msgs, hostname=args.broker)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
flask_store.providers.s3
========================

AWS Simple Storage Service file Store.

Example
-------
.. sourcecode:: python

    from flask import Flask, request
    from flask.ext.Store import Provider, Store
    from wtforms import Form
    from wtforms.fields import FileField

    class FooForm(Form):
        foo = FileField('foo')

    app = Flask(__app__)
    app.config['STORE_PROVIDER'] = 'flask_store.providers.s3.S3Provider'
    app.config['STORE_S3_ACCESS_KEY'] = 'foo'
    app.config['STORE_S3_SECRET_KEY'] = 'bar'
    app.config['STORE_S3_REDUCED_REDUNDANCY'] = False
    app.config['STORE_S3_HEADERS'] = {
                                        'Expires': 'Thu, 15 Apr 2010 20:00:00 GMT',
                                        'Cache-Control': 'max-age=86400',
                                    }

    store = Store(app)

    @app,route('/upload')
    def upload():
        form = FooForm()
        form.validate_on_submit()

        provider = Provider(form.files.get('foo'))
        provider.save()
"""

# import gevent before anything else
try:
    import gevent.monkey
    gevent.monkey.patch_all()
    GEVENT_INSTALLED = True
except ImportError:
    GEVENT_INSTALLED = False

try:
    import boto
    BOTO_INSTALLED = True
except ImportError:
    BOTO_INSTALLED = False

# Standard Libs
import io
import mimetypes
import os
try:
    from StringIO import StringIO
except ImportError:
    from io import StringIO

# Third Party Libs
from flask import copy_current_request_context, current_app
from flask_store.exceptions import NotConfiguredError
from flask_store.providers import Provider
from flask_store.providers.temp import TemporaryStore
from werkzeug.datastructures import FileStorage

class S3Provider(Provider):
    """ Amazon Simple Storage Service Store (S3). Allows files to be stored in
    an AWS S3 bucket.
    """

    #: Required application configuration variables
    REQUIRED_CONFIGURATION = [
        'STORE_S3_ACCESS_KEY',
        'STORE_S3_SECRET_KEY',
        'STORE_S3_BUCKET',
        'STORE_S3_REGION',
        'STORE_S3_REDUCED_REDUNDANCY',
        'STORE_S3_HEADERS']
    policy = 'public-read'

    _bucket = None

    def __init__(self, *args, **kwargs):
        super(S3Provider, self).__init__(*args, **kwargs)
        self._bucket = current_app.config.get('STORE_S3_BUCKET')

    @staticmethod
    def app_defaults(app):
        """ Sets sensible application configuration settings for this
        provider.

        Arguments
        ---------
        app : flask.app.Flask
            Flask application at init
        """

        # For S3 by default the STORE_PATH is the root of the bucket
        app.config.setdefault('STORE_PATH', '/')

        # For S3 the STORE_PATH makes up part of the key and therefore
        # doubles up as the STORE_URL_PREFIX
        app.config.setdefault('STORE_URL_PREFIX', app.config['STORE_PATH'])

        app.config.setdefault('STORE_S3_REDUCED_REDUNDANCY', False)
        app.config.setdefault('STORE_S3_HEADERS', {})

        # Default ACL
        # http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
        app.config.setdefault('STORE_S3_ACL', 'private')

        if not BOTO_INSTALLED:
            raise ImportError(
                'boto must be installed to use the S3Provider or the '
                'S3GeventProvider')

    def connect(self):
        """ Returns an S3 connection instance.
        """

        if not hasattr(self, '_s3connection'):
            try:
                s3connection = boto.s3.connect_to_region(
                    current_app.config['STORE_S3_REGION'],
                    aws_access_key_id=current_app.config['STORE_S3_ACCESS_KEY'],
                    aws_secret_access_key=current_app.config['STORE_S3_SECRET_KEY'])
                setattr(self, '_s3connection', s3connection)
            except S3ResponseError:
                raise
        return getattr(self, '_s3connection')

    @property
    def bucket(self):
        """make bucket a property so user can change the bucket on runtime"""
        return self._bucket
    @bucket.setter
    def bucket(self, value):
        self._bucket = value

    def get_bucket(self, s3connection):
        """ Returns an S3 bucket instance"""

        return s3connection.get_bucket(self.bucket)

    def join(self, *parts):
        """ Joins paths into a url.

        Arguments
        ---------
        \*parts : list
            List of arbitrary paths to join together

        Returns
        -------
        str
            S3 save joined paths
        """

        return self.url_join(*parts)

    def exists(self, filename):
        """ Checks if the file already exists in the bucket using Boto.

        Arguments
        ---------
        name : str
            Filename to check its existence

        Returns
        -------
        bool
            Whether the file exists on the file system
        """

        s3connection = self.connect()
        bucket = self.get_bucket(s3connection)
        path = self.join(self.store_path, filename)

        key = boto.s3.key.Key(name=path, bucket=bucket)

        return key.exists()

    def save(self):
        """ Takes the uploaded file and uploads it to S3.

        Note
        ----
        This is a blocking call and therefore will increase the time for your
        application to respond to the client and may cause request timeouts.
        """

        fp = self.fp
        s3connection = self.connect()
        bucket = self.get_bucket(s3connection)
        filename = self.safe_filename(self.filename)
        path = self.join(self.store_path, filename)
        mimetype, encoding = mimetypes.guess_type(filename)

        fp.seek(0)

        key = bucket.new_key(path)
        key.set_metadata('Content-Type', mimetype)

        for header, value in current_app.config['STORE_S3_HEADERS'].iteritems():
            key.set_metadata(header, value)
        if isinstance(fp,StringIO):
            key.set_contents_from_string(fp.getvalue(), reduced_redundancy=current_app.config['STORE_S3_REDUCED_REDUNDANCY'])
        else:
            key.set_contents_from_file(fp, reduced_redundancy=current_app.config['STORE_S3_REDUCED_REDUNDANCY'])

        key.set_acl(self.policy)

        # Update the filename - it may have changes
        self.filename = filename

    def open(self):
        """ Opens an S3 key and returns an open File Like object pointer.

        Returns
        -------
        _io.BytesIO
            In memory file data
        """

        s3connection = self.connect()
        bucket = self.get_bucket(s3connection)
        key = bucket.get_key(self.relative_path)

        if not key:
            raise IOError('File does not exist: {0}'.format(self.relative_path))

        return io.BytesIO(key.read())  # In memory

class S3GeventProvider(S3Provider):
    """ A Gevent Support for :class:`.S3Provider`. Calling :meth:`.save`
    here will spawn a greenlet which will handle the actual upload process.
    """

    def __init__(self, *args, **kwargs):
        """
        """

        if not GEVENT_INSTALLED:
            raise NotConfiguredError(
                'You must have gevent installed to use the S3GeventProvider')

        super(S3GeventProvider, self).__init__(*args, **kwargs)

    def save(self):
        """ Acts as a proxy to the actual save method in the parent class. The
        save method will be called in a ``greenlet`` so ``gevent`` must be
        installed.

        Since the origional request will close the file object we write the
        file to a temporary location on disk and create a new
        :class:`werkzeug.datastructures.FileStorage` instance with the stram
        being the temporary file.
        """

        fp = self.fp
        temp = TemporaryStore(fp)
        path = temp.save()
        filename = self.safe_filename(fp.filename)

        @copy_current_request_context
        def _save():
            self.fp = FileStorage(
                stream=open(path, 'rb'),
                filename=filename,
                name=fp.name,
                content_type=fp.content_type,
                content_length=fp.content_length,
                headers=fp.headers)

            super(S3GeventProvider, self).save()

            # Cleanup - Delete the temp file
            os.unlink(path)

        gevent.spawn(_save)

        self.filename = filename
<commit_msg>items, not iteritems<commit_after># -*- coding: utf-8 -*-

"""
flask_store.providers.s3
========================

AWS Simple Storage Service file Store.

Example
-------
.. sourcecode:: python

    from flask import Flask, request
    from flask.ext.Store import Provider, Store
    from wtforms import Form
    from wtforms.fields import FileField

    class FooForm(Form):
        foo = FileField('foo')

    app = Flask(__app__)
    app.config['STORE_PROVIDER'] = 'flask_store.providers.s3.S3Provider'
    app.config['STORE_S3_ACCESS_KEY'] = 'foo'
    app.config['STORE_S3_SECRET_KEY'] = 'bar'
    app.config['STORE_S3_REDUCED_REDUNDANCY'] = False
    app.config['STORE_S3_HEADERS'] = {
                                        'Expires': 'Thu, 15 Apr 2010 20:00:00 GMT',
                                        'Cache-Control': 'max-age=86400',
                                    }

    store = Store(app)

    @app,route('/upload')
    def upload():
        form = FooForm()
        form.validate_on_submit()

        provider = Provider(form.files.get('foo'))
        provider.save()
"""

# import gevent before anything else
try:
    import gevent.monkey
    gevent.monkey.patch_all()
    GEVENT_INSTALLED = True
except ImportError:
    GEVENT_INSTALLED = False

try:
    import boto
    BOTO_INSTALLED = True
except ImportError:
    BOTO_INSTALLED = False

# Standard Libs
import io
import mimetypes
import os
try:
    from StringIO import StringIO
except ImportError:
    from io import StringIO

# Third Party Libs
from flask import copy_current_request_context, current_app
from flask_store.exceptions import NotConfiguredError
from flask_store.providers import Provider
from flask_store.providers.temp import TemporaryStore
from werkzeug.datastructures import FileStorage

class S3Provider(Provider):
    """ Amazon Simple Storage Service Store (S3). Allows files to be stored in
    an AWS S3 bucket.
    """

    #: Required application configuration variables
    REQUIRED_CONFIGURATION = [
        'STORE_S3_ACCESS_KEY',
        'STORE_S3_SECRET_KEY',
        'STORE_S3_BUCKET',
        'STORE_S3_REGION',
        'STORE_S3_REDUCED_REDUNDANCY',
        'STORE_S3_HEADERS']
    policy = 'public-read'

    _bucket = None

    def __init__(self, *args, **kwargs):
        super(S3Provider, self).__init__(*args, **kwargs)
        self._bucket = current_app.config.get('STORE_S3_BUCKET')

    @staticmethod
    def app_defaults(app):
        """ Sets sensible application configuration settings for this
        provider.

        Arguments
        ---------
        app : flask.app.Flask
            Flask application at init
        """

        # For S3 by default the STORE_PATH is the root of the bucket
        app.config.setdefault('STORE_PATH', '/')

        # For S3 the STORE_PATH makes up part of the key and therefore
        # doubles up as the STORE_URL_PREFIX
        app.config.setdefault('STORE_URL_PREFIX', app.config['STORE_PATH'])

        app.config.setdefault('STORE_S3_REDUCED_REDUNDANCY', False)
        app.config.setdefault('STORE_S3_HEADERS', {})

        # Default ACL
        # http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
        app.config.setdefault('STORE_S3_ACL', 'private')

        if not BOTO_INSTALLED:
            raise ImportError(
                'boto must be installed to use the S3Provider or the '
                'S3GeventProvider')

    def connect(self):
        """ Returns an S3 connection instance.
        """

        if not hasattr(self, '_s3connection'):
            try:
                s3connection = boto.s3.connect_to_region(
                    current_app.config['STORE_S3_REGION'],
                    aws_access_key_id=current_app.config['STORE_S3_ACCESS_KEY'],
                    aws_secret_access_key=current_app.config['STORE_S3_SECRET_KEY'])
                setattr(self, '_s3connection', s3connection)
            except S3ResponseError:
                raise
        return getattr(self, '_s3connection')

    @property
    def bucket(self):
        """make bucket a property so user can change the bucket on runtime"""
        return self._bucket
    @bucket.setter
    def bucket(self, value):
        self._bucket = value

    def get_bucket(self, s3connection):
        """ Returns an S3 bucket instance"""

        return s3connection.get_bucket(self.bucket)

    def join(self, *parts):
        """ Joins paths into a url.

        Arguments
        ---------
        \*parts : list
            List of arbitrary paths to join together

        Returns
        -------
        str
            S3 save joined paths
        """

        return self.url_join(*parts)

    def exists(self, filename):
        """ Checks if the file already exists in the bucket using Boto.

        Arguments
        ---------
        name : str
            Filename to check its existence

        Returns
        -------
        bool
            Whether the file exists on the file system
        """

        s3connection = self.connect()
        bucket = self.get_bucket(s3connection)
        path = self.join(self.store_path, filename)

        key = boto.s3.key.Key(name=path, bucket=bucket)

        return key.exists()

    def save(self):
        """ Takes the uploaded file and uploads it to S3.

        Note
        ----
        This is a blocking call and therefore will increase the time for your
        application to respond to the client and may cause request timeouts.
        """

        fp = self.fp
        s3connection = self.connect()
        bucket = self.get_bucket(s3connection)
        filename = self.safe_filename(self.filename)
        path = self.join(self.store_path, filename)
        mimetype, encoding = mimetypes.guess_type(filename)

        fp.seek(0)

        key = bucket.new_key(path)
        key.set_metadata('Content-Type', mimetype)

        for header, value in current_app.config['STORE_S3_HEADERS'].items():
            key.set_metadata(header, value)
        if isinstance(fp,StringIO):
            key.set_contents_from_string(fp.getvalue(), reduced_redundancy=current_app.config['STORE_S3_REDUCED_REDUNDANCY'])
        else:
            key.set_contents_from_file(fp, reduced_redundancy=current_app.config['STORE_S3_REDUCED_REDUNDANCY'])

        key.set_acl(self.policy)

        # Update the filename - it may have changes
        self.filename = filename

    def open(self):
        """ Opens an S3 key and returns an open File Like object pointer.

        Returns
        -------
        _io.BytesIO
            In memory file data
        """

        s3connection = self.connect()
        bucket = self.get_bucket(s3connection)
        key = bucket.get_key(self.relative_path)

        if not key:
            raise IOError('File does not exist: {0}'.format(self.relative_path))

        return io.BytesIO(key.read())  # In memory

class S3GeventProvider(S3Provider):
    """ A Gevent Support for :class:`.S3Provider`. Calling :meth:`.save`
    here will spawn a greenlet which will handle the actual upload process.
    """

    def __init__(self, *args, **kwargs):
        """
        """

        if not GEVENT_INSTALLED:
            raise NotConfiguredError(
                'You must have gevent installed to use the S3GeventProvider')

        super(S3GeventProvider, self).__init__(*args, **kwargs)

    def save(self):
        """ Acts as a proxy to the actual save method in the parent class. The
        save method will be called in a ``greenlet`` so ``gevent`` must be
        installed.

        Since the origional request will close the file object we write the
        file to a temporary location on disk and create a new
        :class:`werkzeug.datastructures.FileStorage` instance with the stram
        being the temporary file.
        """

        fp = self.fp
        temp = TemporaryStore(fp)
        path = temp.save()
        filename = self.safe_filename(fp.filename)

        @copy_current_request_context
        def _save():
            self.fp = FileStorage(
                stream=open(path, 'rb'),
                filename=filename,
                name=fp.name,
                content_type=fp.content_type,
                content_length=fp.content_length,
                headers=fp.headers)

            super(S3GeventProvider, self).save()

            # Cleanup - Delete the temp file
            os.unlink(path)

        gevent.spawn(_save)

        self.filename = filename
<|endoftext|>