<commit_before>from PyQt5.QtCore import *
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *
from Windows.manualAddDialog import *

from Windows.centralWindow import *
from Windows import favoriteWindow_ui
from Util import constants, scroller
from Util.enums import *
from Util.SqlTableModel import *

class CategoryTab():
    __slots__ = ['widget', 'horizontalLayout', 'listView', 'listModel']
    def __init__(self, widget = None, horizontalLayout = None, listView = None, listModel = None):
        self.widget = widget
        self.horizontalLayout = horizontalLayout
        self.listView = listView
        self.listModel = listModel

class FavoriteWindow(QWidget, favoriteWindow_ui.Ui_FavoriteWindow):
    def __init__(self, centralWindow, config, dbManager, parent=None):
        super(FavoriteWindow, self).__init__(parent)
        self.setupUi(self)

        self.centralWindow = centralWindow
        self.config = config
        self.dbManager = dbManager

        self.favoritesTabModel = QStandardItemModel(4, 5, self)

        self.favoritesTabModel.setItem(0, 0, QStandardItem("Rawr"))
        self.favoritesTabModel.setItem(0, 1, QStandardItem("Really Long String of Words"))
        self.favoritesTabModel.setItem(0, 2, QStandardItem("Really Long"))
        self.favoritesTabModel.setItem(0, 3, QStandardItem("POPTARTS STRAWBERRY"))
        self.favoritesTabModel.setItem(1, 0, QStandardItem("Rawr"))
        self.favoritesTabModel.setItem(1, 1, QStandardItem("OOOOOOOOOO OOOOOOOOOO"))
        self.favoritesTabModel.setItem(1, 2, QStandardItem("OOOOOOOOOO OOOOOOOOO"))
        self.favoritesTabModel.setItem(1, 3, QStandardItem("OOOOOOOOO OOOOOOOO"))
        self.favoritesTabModel.setItem(1, 4, QStandardItem("OOOOOOOO OOOOOOO"))

        self.favoritesTableView.setModel(self.favoritesTabModel)
        scroller.setupScrolling(self.favoritesTableView)

        # Create an empty tabDict which will contain a dictionary for each tab in the window, query all categories and
        # add each tab to the window
        self.tabDict = {}
        self.categories = self.dbManager.GetCategories(True)
        for category in self.categories:
            self.addTab(category['id'], category['name'])

    def addTab(self, id, name):
        newTab = CategoryTab(QWidget(self.categoryTabWidget))
        newTab.horizontalLayout = QHBoxLayout(newTab.widget)
        newTab.horizontalLayout.setContentsMargins(0, 0, 0, 0)

        # Create list view for category and with the appropiate settings
        newTab.listView = QListView(newTab.widget)
        font = QFont()
        font.setPointSize(21)
        newTab.listView.setFont(font)
        newTab.listView.setFrameShape(QFrame.NoFrame)
        newTab.listView.setFrameShadow(QFrame.Plain)
        newTab.listView.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn)
        newTab.listView.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff)
        newTab.listView.setEditTriggers(QAbstractItemView.NoEditTriggers)
        newTab.listView.setDragDropMode(QAbstractItemView.DragDrop)
        newTab.listView.setDefaultDropAction(Qt.ActionMask)
        newTab.listView.setAlternatingRowColors(True)
        newTab.listView.setSelectionMode(QAbstractItemView.MultiSelection)
        newTab.listView.setVerticalScrollMode(QAbstractItemView.ScrollPerPixel)
        newTab.listView.setHorizontalScrollMode(QAbstractItemView.ScrollPerPixel)
        newTab.horizontalLayout.addWidget(newTab.listView)

        newTab.listModel = SqlTableModel(self.dbManager.connection, 'inventory', 'name', Qt.AscendingOrder, 'category=%s',
                                         (id,), ('name',))
        newTab.listView.setModel(newTab.listModel)

        # Add tab to tab widget as well as the tabDict variable
        self.categoryTabWidget.addTab(newTab.widget, name)
        self.tabDict[id] = newTab

    @pyqtSlot()
    def showEvent(self, event):
        self.centralWindow.primaryScanner.barcodeReceived.connect(self.primaryScanner_barcodeReceived)
        self.centralWindow.secondaryScanner.barcodeReceived.connect(self.secondaryScanner_barcodeReceived)

    @pyqtSlot()
    def hideEvent(self, event):
        self.centralWindow.primaryScanner.barcodeReceived.disconnect(self.primaryScanner_barcodeReceived)
        self.centralWindow.secondaryScanner.barcodeReceived.disconnect(self.secondaryScanner_barcodeReceived)

    @pyqtSlot()
    def on_backBtn_clicked(self):
        self.parent().setCurrentIndex(WindowType.Main)

    @pyqtSlot()
    def on_homeBtn_clicked(self):
        self.parent().setCurrentIndex(WindowType.Main)

    @pyqtSlot()
    def on_listAddBtn_clicked(self):
        # Create a ManualAddDialog, pass the configuration variables such as config, dbManager and categories
        # The category combo box in the dialog will be set to the currently selected tab in the Favorite's Menu. However,
        # if the currently selected tab is the favorite's, then it will default to the first category
        dialog = ManualAddDialog(self.config, self.dbManager, self.categories,
                                 max(self.categoryTabWidget.currentIndex() - 1, 0), self)

        # Run the dialog, if successfully completed, then add new item to category
        if dialog.exec():
            item = {}

            catComboInd = dialog.categoryComboBox.currentIndex()
            item['name'] = dialog.nameEdit.text()
            item['category'] = dialog.categories[catComboInd]['id']

            if dialog.favoritesCheckbox.isChecked():
                index = self.dbManager.GetFavoritesCount()
                if index is None: # If the values are all NULL, set to 0
                    index = 0

                index += 1 # Increment index by 1
                item['favoritesIndex'] = index

            self.dbManager.AddItemToInventory(item)

            # If the category currently exists (check tabDict for the id), then refresh the list model by calling select
            # Otherwise, the category will be added later on and it will automatically be updated on creation
            if item['category'] in self.tabDict:
                self.tabDict[item['category']].listModel.select()

        # If the categories list that was sent to the dialog is not the same when finishing, that means a new category
        # was added and the new category list was queried. Update the categories variable in FavoriteWindow and add the
        # appropiate tabs
        if self.categories is not dialog.categories:
            self.categories = dialog.categories

            for category in self.categories:
                if not category['id'] in self.tabDict:
                    self.addTab(category['id'], category['name'])

    @pyqtSlot(str)
    def primaryScanner_barcodeReceived(self, barcode):
        print("Fav: Primary barcode scanner got: %s" % barcode)
        # TODO Send the barcode scanner information to be processed

    @pyqtSlot(str)
    def secondaryScanner_barcodeReceived(self, barcode):
        print("Fav: Secondary barcode scanner got: %s" % barcode)
        # TODO Send the barcode scanner information to be processed
<commit_msg>Add/Remove Button on Favorite's Menu is Disabled/Enabled based on Highlighted Items<commit_after>from PyQt5.QtCore import *
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *
from Windows.manualAddDialog import *

from Windows.centralWindow import *
from Windows import favoriteWindow_ui
from Util import constants, scroller
from Util.enums import *
from Util.SqlTableModel import *

class CategoryTab():
    __slots__ = ['widget', 'horizontalLayout', 'listView', 'listModel']
    def __init__(self, widget = None, horizontalLayout = None, listView = None, listModel = None):
        self.widget = widget
        self.horizontalLayout = horizontalLayout
        self.listView = listView
        self.listModel = listModel

class FavoriteWindow(QWidget, favoriteWindow_ui.Ui_FavoriteWindow):
    def __init__(self, centralWindow, config, dbManager, parent=None):
        super(FavoriteWindow, self).__init__(parent)
        self.setupUi(self)

        self.centralWindow = centralWindow
        self.config = config
        self.dbManager = dbManager

        self.favoritesTabModel = QStandardItemModel(4, 5, self)

        self.favoritesTabModel.setItem(0, 0, QStandardItem("Rawr"))
        self.favoritesTabModel.setItem(0, 1, QStandardItem("Really Long String of Words"))
        self.favoritesTabModel.setItem(0, 2, QStandardItem("Really Long"))
        self.favoritesTabModel.setItem(0, 3, QStandardItem("POPTARTS STRAWBERRY"))
        self.favoritesTabModel.setItem(1, 0, QStandardItem("Rawr"))
        self.favoritesTabModel.setItem(1, 1, QStandardItem("OOOOOOOOOO OOOOOOOOOO"))
        self.favoritesTabModel.setItem(1, 2, QStandardItem("OOOOOOOOOO OOOOOOOOO"))
        self.favoritesTabModel.setItem(1, 3, QStandardItem("OOOOOOOOO OOOOOOOO"))
        self.favoritesTabModel.setItem(1, 4, QStandardItem("OOOOOOOO OOOOOOO"))

        # Set Favorite's View to have model
        self.favoritesTableView.setModel(self.favoritesTabModel)
        # Setup kinetic scrolling on favorite's table view (can use touchscreen to flick and scroll like on phones)
        scroller.setupScrolling(self.favoritesTableView)
        # Connect selectItem function to the selectionChanged signal that is triggered when an item is deselected or
        # selected
        self.favoritesTableView.selectionModel().selectionChanged.connect(self.selectItem)

        # Create an empty tabDict which will contain a dictionary for each tab in the window, query all categories and
        # add each tab to the window
        self.tabDict = {}
        self.categories = self.dbManager.GetCategories(True)
        for category in self.categories:
            self.addTab(category['id'], category['name'])

        # Enable the add and remove button because they are only shown when items are selected on the current tab
        self.addBtn.setEnabled(False)
        self.removeBtn.setEnabled(False)

    def addTab(self, id, name):
        newTab = CategoryTab(QWidget(self.categoryTabWidget))
        newTab.horizontalLayout = QHBoxLayout(newTab.widget)
        newTab.horizontalLayout.setContentsMargins(0, 0, 0, 0)

        # Create list view for category and with the appropiate settings
        newTab.listView = QListView(newTab.widget)
        font = QFont()
        font.setPointSize(21)
        newTab.listView.setFont(font)
        newTab.listView.setFrameShape(QFrame.NoFrame)
        newTab.listView.setFrameShadow(QFrame.Plain)
        newTab.listView.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn)
        newTab.listView.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff)
        newTab.listView.setEditTriggers(QAbstractItemView.NoEditTriggers)
        newTab.listView.setDragDropMode(QAbstractItemView.DragDrop)
        newTab.listView.setDefaultDropAction(Qt.ActionMask)
        newTab.listView.setAlternatingRowColors(True)
        newTab.listView.setSelectionMode(QAbstractItemView.MultiSelection)
        newTab.listView.setVerticalScrollMode(QAbstractItemView.ScrollPerPixel)
        newTab.listView.setHorizontalScrollMode(QAbstractItemView.ScrollPerPixel)
        newTab.horizontalLayout.addWidget(newTab.listView)

        newTab.listModel = SqlTableModel(self.dbManager.connection, 'inventory', 'name', Qt.AscendingOrder, 'category=%s',
                                         (id,), ('name',))
        newTab.listView.setModel(newTab.listModel)
        newTab.listView.selectionModel().selectionChanged.connect(self.selectItem)

        # Add tab to tab widget as well as the tabDict variable
        self.categoryTabWidget.addTab(newTab.widget, name)
        self.tabDict[id] = newTab

    @pyqtSlot()
    def selectItem(self):
        hasSelection = not self.sender().selection().isEmpty()
        self.addBtn.setEnabled(hasSelection)
        self.removeBtn.setEnabled(hasSelection)

    @pyqtSlot(int)
    def on_categoryTabWidget_currentChanged(self, index):
        # If the index is zero, this get the selection model for the favorite's tab, otherwise, get the selection model
        # for the indexed tab.
        if index == 0:
            selectionModel = self.favoritesTableView.selectionModel()
        else:
            # Get category ID by looking up index in categories variable
            categoryID = self.categories[index - 1]['id']

            # Get selection model based on category ID
            selectionModel = self.tabDict[categoryID].listView.selectionModel()

        hasSelection = not selectionModel.selection().isEmpty()
        self.addBtn.setEnabled(hasSelection)
        self.removeBtn.setEnabled(hasSelection)

    @pyqtSlot()
    def showEvent(self, event):
        self.centralWindow.primaryScanner.barcodeReceived.connect(self.primaryScanner_barcodeReceived)
        self.centralWindow.secondaryScanner.barcodeReceived.connect(self.secondaryScanner_barcodeReceived)

    @pyqtSlot()
    def hideEvent(self, event):
        self.centralWindow.primaryScanner.barcodeReceived.disconnect(self.primaryScanner_barcodeReceived)
        self.centralWindow.secondaryScanner.barcodeReceived.disconnect(self.secondaryScanner_barcodeReceived)

    @pyqtSlot()
    def on_backBtn_clicked(self):
        self.parent().setCurrentIndex(WindowType.Main)

    @pyqtSlot()
    def on_homeBtn_clicked(self):
        self.parent().setCurrentIndex(WindowType.Main)

    @pyqtSlot()
    def on_listAddBtn_clicked(self):
        # Create a ManualAddDialog, pass the configuration variables such as config, dbManager and categories
        # The category combo box in the dialog will be set to the currently selected tab in the Favorite's Menu. However,
        # if the currently selected tab is the favorite's, then it will default to the first category
        dialog = ManualAddDialog(self.config, self.dbManager, self.categories,
                                 max(self.categoryTabWidget.currentIndex() - 1, 0), self)

        # Run the dialog, if successfully completed, then add new item to category
        if dialog.exec():
            item = {}

            catComboInd = dialog.categoryComboBox.currentIndex()
            item['name'] = dialog.nameEdit.text()
            item['category'] = dialog.categories[catComboInd]['id']

            if dialog.favoritesCheckbox.isChecked():
                index = self.dbManager.GetFavoritesCount()
                if index is None: # If the values are all NULL, set to 0
                    index = 0

                index += 1 # Increment index by 1
                item['favoritesIndex'] = index

            self.dbManager.AddItemToInventory(item)

            # If the category currently exists (check tabDict for the id), then refresh the list model by calling select
            # Otherwise, the category will be added later on and it will automatically be updated on creation
            if item['category'] in self.tabDict:
                self.tabDict[item['category']].listModel.select()

        # If the categories list that was sent to the dialog is not the same when finishing, that means a new category
        # was added and the new category list was queried. Update the categories variable in FavoriteWindow and add the
        # appropiate tabs
        if self.categories is not dialog.categories:
            self.categories = dialog.categories

            for category in self.categories:
                if not category['id'] in self.tabDict:
                    self.addTab(category['id'], category['name'])

    @pyqtSlot(str)
    def primaryScanner_barcodeReceived(self, barcode):
        print("Fav: Primary barcode scanner got: %s" % barcode)
        # TODO Send the barcode scanner information to be processed

    @pyqtSlot(str)
    def secondaryScanner_barcodeReceived(self, barcode):
        print("Fav: Secondary barcode scanner got: %s" % barcode)
        # TODO Send the barcode scanner information to be processed
<|endoftext|>


<commit_before>import datetime
import math
import time
from collections import namedtuple

import dateutil
import numpy as np
import piston
from dateutil import parser
from steemtools.helpers import read_asset, parse_payout, time_diff, simple_cache
from steemtools.node import Node
from tqdm import tqdm

from werkzeug.contrib.cache import SimpleCache

base_cache = SimpleCache()  # is this threadsafe?

class Account(object):
    def __init__(self, account_name, steem=None):
        if not steem:
            steem = Node().default()
        self.steem = steem

        self.name = account_name

        self.converter = Converter(steem=self.steem)

        # caches
        self._blog = None
        self._props = None

    def get_props(self):
        if self._props is None:
            self._props = self.steem.rpc.get_account(self.name)
        return self._props

    def get_blog(self):
        if self._blog is None:
            self._blog = self.steem.get_blog(self.name)
        return self._blog

    @property
    def sp(self):
        return self.get_sp()

    @property
    def rep(self):
        return self.reputation()

    def get_sp(self):
        vests = int(parse_payout(self.get_props()['vesting_shares']))
        return self.converter.vests_to_sp(vests)

    def get_balances(self):
        my_account_balances = self.steem.get_balances(self.name)
        return {
            "steem": parse_payout(my_account_balances["balance"]),
            "sbd": parse_payout(my_account_balances["sbd_balance"]),
            "vests": parse_payout(my_account_balances["vesting_shares"]),
        }

    def reputation(self):
        rep = int(self.get_props()['reputation'])
        if rep < 0:
            return -1
        if rep == 0:
            return 25

        score = (math.log10(abs(rep)) - 9) * 9 + 25
        return float("%.2f" % score)

    def voting_power(self):
        return self.get_props()['voting_power'] / 100

    def number_of_winning_posts(self, skip=1, payout_requirement=300, max_posts=10):
        winning_posts = 0
        blog = self.get_blog()[skip:max_posts + skip]
        for post in blog:
            total_payout = parse_payout(post['total_payout_reward'])
            if total_payout >= payout_requirement:
                winning_posts += 1

        nt = namedtuple('WinningPosts', ['winners', 'blog_posts'])
        return nt(winning_posts, len(blog))

    def avg_payout_per_post(self, skip=1, max_posts=10):
        total_payout = 0
        blog = self.get_blog()[skip:max_posts + skip]
        for post in blog:
            total_payout += parse_payout(post['total_payout_reward'])

        if len(blog) == 0:
            return 0

        return total_payout / len(blog)

    def time_to_whale(self, verbose=False, whale_sp=1e5, skip=1, max_posts=10, mean_of_recent=3):
        blog = self.get_blog()[skip:max_posts + skip]

        max_rshares = self.converter.sp_to_rshares(whale_sp)
        time_to_whale = []

        for post in blog:
            votes = []
            rshares_sum = 0

            for vote in post['active_votes']:
                vote['time_elapsed'] = int(time_diff(post['created'], vote['time']))
                votes.append(vote)

            # note: this function will already filter out posts without votes
            for vote in sorted(votes, key=lambda k: k['time_elapsed']):
                rshares_sum += int(vote['rshares'])
                if rshares_sum >= max_rshares:
                    ttw = time_diff(post['created'], vote['time'])
                    if verbose:
                        print('%s on %s' % (ttw, post['permlink']))
                    time_to_whale.append(ttw)
                    break

        if len(time_to_whale) == 0:
            return None
        return np.mean(time_to_whale[:mean_of_recent])

    def get_followers(self):
        return [x['follower'] for x in self._get_followers(direction="follower")]

    def get_following(self):
        return [x['following'] for x in self._get_followers(direction="following")]

    def _get_followers(self, direction="follower", last_user=""):
        if direction == "follower":
            followers = self.steem.rpc.get_followers(self.name, last_user, "blog", 100, api="follow")
        elif direction == "following":
            followers = self.steem.rpc.get_following(self.name, last_user, "blog", 100, api="follow")
        if len(followers) == 100:
            followers += self._get_followers(direction=direction, last_user=followers[-1][direction])[1:]
        return followers

    def check_if_already_voted(self, post):
        for v in self.history2(filter_by="vote"):
            vote = v['op']
            if vote['permlink'] == post['permlink']:
                return True

        return False

    def curation_stats(self):
        trailing_24hr_t = time.time() - datetime.timedelta(hours=24).total_seconds()
        trailing_7d_t = time.time() - datetime.timedelta(days=7).total_seconds()

        reward_24h = 0.0
        reward_7d = 0.0

        for event in self.history2(filter_by="curation_reward", take=2000):

            if parser.parse(event['timestamp'] + "UTC").timestamp() > trailing_7d_t:
                reward_7d += parse_payout(event['op']['reward'])

            if parser.parse(event['timestamp'] + "UTC").timestamp() > trailing_24hr_t:
                reward_24h += parse_payout(event['op']['reward'])

        reward_7d = self.converter.vests_to_sp(reward_7d)
        reward_24h = self.converter.vests_to_sp(reward_24h)
        return {
            "24hr": reward_24h,
            "7d": reward_7d,
            "avg": reward_7d / 7,
        }

    def get_features(self, max_posts=10, payout_requirement=300):
        num_winning_posts, post_count = self.number_of_winning_posts(payout_requirement=payout_requirement,
                                                                     max_posts=max_posts)
        return {
            "name": self.name,
            "settings": {
                "max_posts": max_posts,
                "payout_requirement": payout_requirement,
            },
            "author": {
                "post_count": post_count,
                "winners": num_winning_posts,
                "sp": int(self.get_sp()),
                "rep": self.reputation(),
                "followers": len(self.get_followers()),
                "ttw": self.time_to_whale(max_posts=max_posts),
                "ppp": self.avg_payout_per_post(max_posts=max_posts),
            },
        }

    def virtual_op_count(self):
        return self.steem.rpc.get_account_history(self.name, -1, 0)[0][0]

    def history(self, filter_by=None, start=0):
        """
        Take all elements from start to last from history, oldest first.
        """
        batch_size = 1000
        max_index = self.virtual_op_count()

        start_index = start + batch_size
        i = start_index
        while True:
            if i == start_index:
                limit = batch_size
            else:
                limit = batch_size - 1
            history = self.steem.rpc.get_account_history(self.name, i, limit)
            for item in history:
                index = item[0]
                if index >= max_index:
                    return

                op_type = item[1]['op'][0]
                op = item[1]['op'][1]
                timestamp = item[1]['timestamp']
                trx_id = item[1]['trx_id']

                def construct_op():
                    return {
                        "index": index,
                        "trx_id": trx_id,
                        "timestamp": timestamp,
                        "op_type": op_type,
                        "op": op,
                    }

                if filter_by is None:
                    yield construct_op()
                else:
                    if type(filter_by) is list:
                        if op_type in filter_by:
                            yield construct_op()

                    if type(filter_by) is str:
                        if op_type == filter_by:
                            yield construct_op()
            i += batch_size

    def history2(self, filter_by=None, take=1000):
        """
        Take X elements from most recent history, oldest first.
        """
        max_index = self.virtual_op_count()
        start_index = max_index - take
        if start_index < 0:
            start_index = 0

        return self.history(filter_by, start=start_index)

    def get_account_votes(self):
        return self.steem.rpc.get_account_votes(self.name)

    def get_withdraw_routes(self):
        return self.steem.rpc.get_withdraw_routes(self.name, 'all')

    def get_conversion_requests(self):
        return self.steem.rpc.get_conversion_requests(self.name)

    @staticmethod
    def filter_by_date(items, start_time, end_time=None):
        start_time = dateutil.parser.parse(start_time + "UTC").timestamp()
        if end_time:
            end_time = dateutil.parser.parse(end_time + "UTC").timestamp()
        else:
            end_time = time.time()

        filtered_items = []
        for item in items:
            if 'time' in item:
                item_time = item['time']
            elif 'timestamp' in item:
                item_time = item['timestamp']
            timestamp = dateutil.parser.parse(item_time + "UTC").timestamp()
            if end_time > timestamp > start_time:
                filtered_items.append(item)

        return filtered_items

class Post(piston.steem.Post):
    def __init__(self, post, steem=None):
        if not steem:
            steem = Node().default()
        if isinstance(post, piston.steem.Post):
            post = post.identifier
        super(Post, self).__init__(steem, post)

    def is_comment(self):
        if len(self['title']) == 0:
            return True

        if self['depth'] > 0:
            return True

        if len(self['parent_author']) > 0:
            return True

        return False

    def get_votes(self, from_account=None):
        votes = []
        for vote in self['active_votes']:
            vote['time_elapsed'] = int(time_diff(self['created'], vote['time']))
            if from_account and vote['voter'] == from_account:
                return vote
            votes.append(vote)
        return votes

    def get_metadata(self):
        rshares = int(self["vote_rshares"])
        weight = int(self["total_vote_weight"])

        if int(self["total_vote_weight"]) == 0 and self.time_elapsed() > 3600:
            weight = 0
            rshares = 0
            for vote in self['active_votes']:
                weight += int(vote['weight'])
                rshares += int(vote['rshares'])

        return {
            "rshares": rshares,
            "weight": weight,
            "time_elapsed": self.time_elapsed(),
        }

    def contains_tags(self, filter_by=('spam', 'test', 'nsfw')):
        for tag in filter_by:
            if tag in self['_tags']:
                return True

        return False

    def get_url(self):
        return "https://steemit.com/%s/%s" % (self.category, self.identifier)

    def time_elapsed(self):
        created_at = parser.parse(self['created'] + "UTC").timestamp()
        now_adjusted = time.time()
        return now_adjusted - created_at

    def payout(self):
        return parse_payout(self['total_payout_reward'])

    def calc_reward_pct(self):
        reward = (self.time_elapsed() / 1800) * 100
        if reward > 100:
            reward = 100
        return reward

class Converter(object):
    def __init__(self, steem=None):
        if not steem:
            steem = Node().default()
        self.steem = steem
        self.CONTENT_CONSTANT = 2000000000000

    @simple_cache(base_cache, timeout=5 * 60)
    def sbd_median_price(self):
        return read_asset(self.steem.rpc.get_feed_history()['current_median_history']['base'])['value']

    @simple_cache(base_cache, timeout=5 * 60)
    def steem_per_mvests(self):
        info = self.steem.rpc.get_dynamic_global_properties()
        return (
            parse_payout(info["total_vesting_fund_steem"]) /
            (parse_payout(info["total_vesting_shares"]) / 1e6)
        )

    def vests_to_sp(self, vests):
        return vests * self.steem_per_mvests() / 1e6

    def sp_to_vests(self, sp):
        return sp * 1e6 / self.steem_per_mvests()

    def sp_to_rshares(self, sp, voting_power=10000, vote_pct=10000):
        # calculate our account voting shares (from vests), mine is 6.08b
        vesting_shares = int(self.sp_to_vests(sp) * 1e6)

        # calculate vote rshares
        power = (((voting_power * vote_pct) / 10000) / 200) + 1
        rshares = (power * vesting_shares) / 10000

        return rshares

    def steem_to_sbd(self, amount_steem):
        return self.sbd_median_price() * amount_steem

    def sbd_to_steem(self, amount_sbd):
        return amount_sbd / self.sbd_median_price()

    def sbd_to_shares(self, sbd_payout):
        steem_payout = self.sbd_to_steem(sbd_payout)

        props = self.steem.rpc.get_dynamic_global_properties()
        total_reward_fund_steem = read_asset(props['total_reward_fund_steem'])['value']
        total_reward_shares2 = int(props['total_reward_shares2'])

        post_rshares2 = (steem_payout / total_reward_fund_steem) * total_reward_shares2

        rshares = math.sqrt(self.CONTENT_CONSTANT ** 2 + post_rshares2) - self.CONTENT_CONSTANT
        return rshares

    def rshares_2_weight(self, rshares):
        _max = 2 ** 64 - 1
        return (_max * rshares) / (2 * self.CONTENT_CONSTANT + rshares)

<commit_msg>fix virtual_op_count for empty accounts, increase take range for Account curation_rewards<commit_after>import datetime
import math
import time
from collections import namedtuple

import dateutil
import numpy as np
import piston
from dateutil import parser
from steemtools.helpers import read_asset, parse_payout, time_diff, simple_cache
from steemtools.node import Node
from tqdm import tqdm

from werkzeug.contrib.cache import SimpleCache

base_cache = SimpleCache()  # is this threadsafe?

class Account(object):
    def __init__(self, account_name, steem=None):
        if not steem:
            steem = Node().default()
        self.steem = steem

        self.name = account_name

        self.converter = Converter(steem=self.steem)

        # caches
        self._blog = None
        self._props = None

    def get_props(self):
        if self._props is None:
            self._props = self.steem.rpc.get_account(self.name)
        return self._props

    def get_blog(self):
        if self._blog is None:
            self._blog = self.steem.get_blog(self.name)
        return self._blog

    @property
    def sp(self):
        return self.get_sp()

    @property
    def rep(self):
        return self.reputation()

    def get_sp(self):
        vests = int(parse_payout(self.get_props()['vesting_shares']))
        return self.converter.vests_to_sp(vests)

    def get_balances(self):
        my_account_balances = self.steem.get_balances(self.name)
        return {
            "steem": parse_payout(my_account_balances["balance"]),
            "sbd": parse_payout(my_account_balances["sbd_balance"]),
            "vests": parse_payout(my_account_balances["vesting_shares"]),
        }

    def reputation(self):
        rep = int(self.get_props()['reputation'])
        if rep < 0:
            return -1
        if rep == 0:
            return 25

        score = (math.log10(abs(rep)) - 9) * 9 + 25
        return float("%.2f" % score)

    def voting_power(self):
        return self.get_props()['voting_power'] / 100

    def number_of_winning_posts(self, skip=1, payout_requirement=300, max_posts=10):
        winning_posts = 0
        blog = self.get_blog()[skip:max_posts + skip]
        for post in blog:
            total_payout = parse_payout(post['total_payout_reward'])
            if total_payout >= payout_requirement:
                winning_posts += 1

        nt = namedtuple('WinningPosts', ['winners', 'blog_posts'])
        return nt(winning_posts, len(blog))

    def avg_payout_per_post(self, skip=1, max_posts=10):
        total_payout = 0
        blog = self.get_blog()[skip:max_posts + skip]
        for post in blog:
            total_payout += parse_payout(post['total_payout_reward'])

        if len(blog) == 0:
            return 0

        return total_payout / len(blog)

    def time_to_whale(self, verbose=False, whale_sp=1e5, skip=1, max_posts=10, mean_of_recent=3):
        blog = self.get_blog()[skip:max_posts + skip]

        max_rshares = self.converter.sp_to_rshares(whale_sp)
        time_to_whale = []

        for post in blog:
            votes = []
            rshares_sum = 0

            for vote in post['active_votes']:
                vote['time_elapsed'] = int(time_diff(post['created'], vote['time']))
                votes.append(vote)

            # note: this function will already filter out posts without votes
            for vote in sorted(votes, key=lambda k: k['time_elapsed']):
                rshares_sum += int(vote['rshares'])
                if rshares_sum >= max_rshares:
                    ttw = time_diff(post['created'], vote['time'])
                    if verbose:
                        print('%s on %s' % (ttw, post['permlink']))
                    time_to_whale.append(ttw)
                    break

        if len(time_to_whale) == 0:
            return None
        return np.mean(time_to_whale[:mean_of_recent])

    def get_followers(self):
        return [x['follower'] for x in self._get_followers(direction="follower")]

    def get_following(self):
        return [x['following'] for x in self._get_followers(direction="following")]

    def _get_followers(self, direction="follower", last_user=""):
        if direction == "follower":
            followers = self.steem.rpc.get_followers(self.name, last_user, "blog", 100, api="follow")
        elif direction == "following":
            followers = self.steem.rpc.get_following(self.name, last_user, "blog", 100, api="follow")
        if len(followers) == 100:
            followers += self._get_followers(direction=direction, last_user=followers[-1][direction])[1:]
        return followers

    def check_if_already_voted(self, post):
        for v in self.history2(filter_by="vote"):
            vote = v['op']
            if vote['permlink'] == post['permlink']:
                return True

        return False

    def curation_stats(self):
        trailing_24hr_t = time.time() - datetime.timedelta(hours=24).total_seconds()
        trailing_7d_t = time.time() - datetime.timedelta(days=7).total_seconds()

        reward_24h = 0.0
        reward_7d = 0.0

        for event in self.history2(filter_by="curation_reward", take=10000):

            if parser.parse(event['timestamp'] + "UTC").timestamp() > trailing_7d_t:
                reward_7d += parse_payout(event['op']['reward'])

            if parser.parse(event['timestamp'] + "UTC").timestamp() > trailing_24hr_t:
                reward_24h += parse_payout(event['op']['reward'])

        reward_7d = self.converter.vests_to_sp(reward_7d)
        reward_24h = self.converter.vests_to_sp(reward_24h)
        return {
            "24hr": reward_24h,
            "7d": reward_7d,
            "avg": reward_7d / 7,
        }

    def get_features(self, max_posts=10, payout_requirement=300):
        num_winning_posts, post_count = self.number_of_winning_posts(payout_requirement=payout_requirement,
                                                                     max_posts=max_posts)
        return {
            "name": self.name,
            "settings": {
                "max_posts": max_posts,
                "payout_requirement": payout_requirement,
            },
            "author": {
                "post_count": post_count,
                "winners": num_winning_posts,
                "sp": int(self.get_sp()),
                "rep": self.reputation(),
                "followers": len(self.get_followers()),
                "ttw": self.time_to_whale(max_posts=max_posts),
                "ppp": self.avg_payout_per_post(max_posts=max_posts),
            },
        }

    def virtual_op_count(self):
        last_item = self.steem.rpc.get_account_history(self.name, -1, 0)
        if not last_item:
            return 0
        return last_item[0][0]

    def history(self, filter_by=None, start=0):
        """
        Take all elements from start to last from history, oldest first.
        """
        batch_size = 1000
        max_index = self.virtual_op_count()

        start_index = start + batch_size
        i = start_index
        while True:
            if i == start_index:
                limit = batch_size
            else:
                limit = batch_size - 1
            history = self.steem.rpc.get_account_history(self.name, i, limit)
            for item in history:
                index = item[0]
                if index >= max_index:
                    return

                op_type = item[1]['op'][0]
                op = item[1]['op'][1]
                timestamp = item[1]['timestamp']
                trx_id = item[1]['trx_id']

                def construct_op():
                    return {
                        "index": index,
                        "trx_id": trx_id,
                        "timestamp": timestamp,
                        "op_type": op_type,
                        "op": op,
                    }

                if filter_by is None:
                    yield construct_op()
                else:
                    if type(filter_by) is list:
                        if op_type in filter_by:
                            yield construct_op()

                    if type(filter_by) is str:
                        if op_type == filter_by:
                            yield construct_op()
            i += batch_size

    def history2(self, filter_by=None, take=1000):
        """
        Take X elements from most recent history, oldest first.
        """
        max_index = self.virtual_op_count()
        start_index = max_index - take
        if start_index < 0:
            start_index = 0

        return self.history(filter_by, start=start_index)

    def get_account_votes(self):
        return self.steem.rpc.get_account_votes(self.name)

    def get_withdraw_routes(self):
        return self.steem.rpc.get_withdraw_routes(self.name, 'all')

    def get_conversion_requests(self):
        return self.steem.rpc.get_conversion_requests(self.name)

    @staticmethod
    def filter_by_date(items, start_time, end_time=None):
        start_time = dateutil.parser.parse(start_time + "UTC").timestamp()
        if end_time:
            end_time = dateutil.parser.parse(end_time + "UTC").timestamp()
        else:
            end_time = time.time()

        filtered_items = []
        for item in items:
            if 'time' in item:
                item_time = item['time']
            elif 'timestamp' in item:
                item_time = item['timestamp']
            timestamp = dateutil.parser.parse(item_time + "UTC").timestamp()
            if end_time > timestamp > start_time:
                filtered_items.append(item)

        return filtered_items

class Post(piston.steem.Post):
    def __init__(self, post, steem=None):
        if not steem:
            steem = Node().default()
        if isinstance(post, piston.steem.Post):
            post = post.identifier
        super(Post, self).__init__(steem, post)

    def is_comment(self):
        if len(self['title']) == 0:
            return True

        if self['depth'] > 0:
            return True

        if len(self['parent_author']) > 0:
            return True

        return False

    def get_votes(self, from_account=None):
        votes = []
        for vote in self['active_votes']:
            vote['time_elapsed'] = int(time_diff(self['created'], vote['time']))
            if from_account and vote['voter'] == from_account:
                return vote
            votes.append(vote)
        return votes

    def get_metadata(self):
        rshares = int(self["vote_rshares"])
        weight = int(self["total_vote_weight"])

        if int(self["total_vote_weight"]) == 0 and self.time_elapsed() > 3600:
            weight = 0
            rshares = 0
            for vote in self['active_votes']:
                weight += int(vote['weight'])
                rshares += int(vote['rshares'])

        return {
            "rshares": rshares,
            "weight": weight,
            "time_elapsed": self.time_elapsed(),
        }

    def contains_tags(self, filter_by=('spam', 'test', 'nsfw')):
        for tag in filter_by:
            if tag in self['_tags']:
                return True

        return False

    def get_url(self):
        return "https://steemit.com/%s/%s" % (self.category, self.identifier)

    def time_elapsed(self):
        created_at = parser.parse(self['created'] + "UTC").timestamp()
        now_adjusted = time.time()
        return now_adjusted - created_at

    def payout(self):
        return parse_payout(self['total_payout_reward'])

    def calc_reward_pct(self):
        reward = (self.time_elapsed() / 1800) * 100
        if reward > 100:
            reward = 100
        return reward

class Converter(object):
    def __init__(self, steem=None):
        if not steem:
            steem = Node().default()
        self.steem = steem
        self.CONTENT_CONSTANT = 2000000000000

    @simple_cache(base_cache, timeout=5 * 60)
    def sbd_median_price(self):
        return read_asset(self.steem.rpc.get_feed_history()['current_median_history']['base'])['value']

    @simple_cache(base_cache, timeout=5 * 60)
    def steem_per_mvests(self):
        info = self.steem.rpc.get_dynamic_global_properties()
        return (
            parse_payout(info["total_vesting_fund_steem"]) /
            (parse_payout(info["total_vesting_shares"]) / 1e6)
        )

    def vests_to_sp(self, vests):
        return vests * self.steem_per_mvests() / 1e6

    def sp_to_vests(self, sp):
        return sp * 1e6 / self.steem_per_mvests()

    def sp_to_rshares(self, sp, voting_power=10000, vote_pct=10000):
        # calculate our account voting shares (from vests), mine is 6.08b
        vesting_shares = int(self.sp_to_vests(sp) * 1e6)

        # calculate vote rshares
        power = (((voting_power * vote_pct) / 10000) / 200) + 1
        rshares = (power * vesting_shares) / 10000

        return rshares

    def steem_to_sbd(self, amount_steem):
        return self.sbd_median_price() * amount_steem

    def sbd_to_steem(self, amount_sbd):
        return amount_sbd / self.sbd_median_price()

    def sbd_to_shares(self, sbd_payout):
        steem_payout = self.sbd_to_steem(sbd_payout)

        props = self.steem.rpc.get_dynamic_global_properties()
        total_reward_fund_steem = read_asset(props['total_reward_fund_steem'])['value']
        total_reward_shares2 = int(props['total_reward_shares2'])

        post_rshares2 = (steem_payout / total_reward_fund_steem) * total_reward_shares2

        rshares = math.sqrt(self.CONTENT_CONSTANT ** 2 + post_rshares2) - self.CONTENT_CONSTANT
        return rshares

    def rshares_2_weight(self, rshares):
        _max = 2 ** 64 - 1
        return (_max * rshares) / (2 * self.CONTENT_CONSTANT + rshares)

<|endoftext|>


<commit_before>from sqlalchemy import (
    Column,
    DateTime,
    ForeignKey,
    Boolean,
    Integer,
    String,
    func,
    )
from sqlalchemy.orm import relationship
from wtforms import StringField, validators, HiddenField, BooleanField, RadioField

from .support import db
from .with_offsets import WithOffsets
from ..forms import Form, SelectField

class DocumentSource(db.Model, WithOffsets):
    """
    A source is a source of information for an article.
    A source can be one of:

     - a named person (linked via person_id)
     - an unnamed person (in which case gender_id and race_id are meaningful)
     - a non-person secondary source (only name is applicable)
    """
    __tablename__ = "document_sources"

    id        = Column(Integer, primary_key=True)
    doc_id    = Column(Integer, ForeignKey('documents.id', ondelete='CASCADE'), index=True, nullable=False)

    # The type of this source. This impacts which other fields are valid and how they're interpreted.
    #
    # person:
    #   - if unnamed is False, then
    #     - person_id is valid and links to the person
    #   - if unnamed is True, then
    #     - person_id is not valid
    #     - unnamed_* are all valid
    #   - source_function and affiliation are valid
    #   - quoted is valid
    #   - everything else (role, age, etc.) are ignored
    #
    # child:
    #   - unnamed is valid and, if False, 'name' has the source name
    #   - race and gender are valid
    #   - age and role are valid
    #   - quoted is alid
    #   - affiliation and function are ignored
    #
    # secondary:
    #   - only name, affiliation and function are valid
    source_type = Column(String(50), nullable=False, default='person')

    person_id = Column(Integer, ForeignKey('people.id', ondelete='CASCADE'), index=True)

    # if this is True, then person_id is ignored and this is an anonymous source
    unnamed           = Column(Boolean, default=False)
    unnamed_gender_id = Column(Integer, ForeignKey('genders.id'))
    unnamed_race_id   = Column(Integer, ForeignKey('races.id'))

    # if unnamed is False and person_id is null, then this is a secondary, named source
    name         = Column(String(100))

    source_function_id = Column(Integer, ForeignKey('source_functions.id', ondelete='SET NULL'))
    source_role_id     = Column(Integer, ForeignKey('source_roles.id', ondelete='SET NULL'))
    source_age_id      = Column(Integer, ForeignKey('source_ages.id', ondelete='SET NULL'))

    quoted       = Column(Boolean)
    photographed = Column(Boolean)

    # was this source added manually or was it inferred by machine learning?
    manual       = Column(Boolean, default=False, nullable=False)

    # who is the person affiliated with?
    affiliation_id = Column(Integer, ForeignKey('affiliations.id', ondelete='SET NULL'), index=True)

    created_at   = Column(DateTime(timezone=True), index=True, unique=False, nullable=False, server_default=func.now())
    updated_at   = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.current_timestamp())

    # Associations
    person      = relationship("Person", foreign_keys=[person_id], lazy=False)
    function    = relationship("SourceFunction", lazy=False)
    affiliation = relationship("Affiliation", lazy=False)
    role        = relationship("SourceRole", lazy=False)
    age         = relationship("SourceAge", lazy=False)
    unnamed_gender = relationship("Gender", lazy=False)
    unnamed_race   = relationship("Race", lazy=False)

    def document_entities(self):
        """ List of DocumentEntity instances that matches this source for this document. May be empty. """
        if not self.person:
            return []
        return [de for de in self.document.entities if de.entity.person == self.person]

    def utterances(self):
        """ A potentially empty list of Utterances from this source in this document. """
        return sorted([u for u in self.document.utterances if self.person and u.entity.person == self.person],
                key=lambda u: u.offset)

    def friendly_name(self):
        if self.person:
            return self.person.name

        if self.unnamed:
            return 'Unnamed'

        return self.name or '(none)'

    def sort_key(self):
        return [not self.manual, self.friendly_name()]

    def gender(self):
        if self.person:
            return self.person.gender
        return self.unnamed_gender

    def race(self):
        if self.person:
            return self.person.race
        return self.unnamed_race

    @property
    def offset_list(self):
        """ String of offset:length pairs of places in this document the entity
        is mentioned or quoted, may be empty. """
        offsets = ['%d:%d' % (u.offset, u.length) for u in self.utterances() if u.offset]
        for de in self.document_entities():
            if de.offset_list:
                offsets.append(de.offset_list)

        return ' '.join(offsets)

    def __repr__(self):
        return "<DocumentSource doc=%s, person=%s, unnamed=%s, name='%s'>" % (self.document, self.person, self.unnamed, self.name.encode('utf-8'))

class SourceFunction(db.Model):
    """
    In what role/function was the source for a document accessed?
    """
    __tablename__ = "source_functions"

    id        = Column(Integer, primary_key=True)
    name      = Column(String(50), index=True, nullable=False, unique=True)

    def __repr__(self):
        return "<SourceFunction name='%s'>" % (self.name)

    @classmethod
    def create_defaults(self):
        text = """
        Do not know
        Subject
        Expert
        Personal Experience
        Eye Witness
        Popular Opinion
        Secondary Sources
        Representative/Spokesperson
        Other
        """

        functions = []
        for s in text.strip().split("\n"):
            g = SourceFunction()
            g.name = s.strip()
            functions.append(g)

        return functions

def none_coerce(v):
    from wtforms.compat import text_type
    return text_type('' if v is None else v)

class DocumentSourceForm(Form):
    name              = StringField('Name', [validators.Length(max=100)])
    named             = BooleanField('The source is named', default=True)
    unnamed_gender_id = SelectField('Gender', [validators.Optional()], default='', coerce=none_coerce)
    unnamed_race_id   = SelectField('Race', [validators.Optional()], default='', coerce=none_coerce)

    source_type       = RadioField('Type', default='person', choices=[['person', 'Adult Person'], ['child', 'Child'], ['secondary', 'Secondary (not a person)']])

    quoted            = BooleanField('Quoted', default=False)
    photographed      = BooleanField('Photographed', default=False)

    source_function_id  = SelectField('Function', default='')
    source_role_id      = SelectField('Role', default='')
    source_age_id       = SelectField('Age', default='')
    affiliation_id      = SelectField('Affiliation', default='')

    deleted           = HiddenField('deleted', default='0')

    def __init__(self, *args, **kwargs):
        super(DocumentSourceForm, self).__init__(*args, **kwargs)

        from . import SourceAge

        if 'nature' in kwargs:
            nature = kwargs['nature']
        elif self.source:
            nature = self.source.document.analysis_nature
        else:
            raise ArgumentError("Missing analysis nature. Either pass in obj or nature")

        self.source_function_id.choices = [['', '(none)']] + [[str(s.id), s.name] for s in SourceFunction.query.order_by(SourceFunction.name).all()]
        self.source_role_id.choices = [['', '(none)']] + [[str(s.id), s.name] for s in nature.roles]
        self.source_age_id.choices = [['', '(none)']] + [[str(s.id), s.name] for s in SourceAge.query.order_by(SourceAge.id).all()]

        from . import Gender, Race
        self.unnamed_gender_id.choices = [['', '(unknown gender)']] + [[str(g.id), g.name] for g in Gender.query.order_by(Gender.name).all()]
        self.unnamed_race_id.choices = [['', '(unknown race)']] + [[str(r.id), r.name] for r in Race.query.order_by(Race.name).all()]

        # because this list is heirarchical, we class 'organisations' as
        # this with only 0 or two dots
        from . import Affiliation
        orgs = [i for i in Affiliation.query.all() if i.code.count('.') <= 1]
        orgs.sort(key=Affiliation.sort_key)
        self.affiliation_id.choices = [['', '(none)']] + [[str(s.id), s.full_name()] for s in orgs]

        if self.source:
            self.named.data = not self.source.unnamed

    def validate(self):
        # ignore some data, based on the source type
        if not self.named.data:
            # it's anonymous, so ignore the name field
            self.name.data = ''

        if self.source_type.data == 'person':
            self.source_role_id.data = ''
            self.source_age_id.data = ''
            if self.named.data:
                # it's not an anonymous source, so ignore these
                # settings
                self.unnamed_gender_id.data = ''
                self.unnamed_race_id.data = ''

        elif self.source_type.data == 'child':
            self.source_function_id.data = ''
            self.affiliation_id.data = ''

        elif self.source_type.data == 'secondary':
            self.source_role_id.data = ''
            self.unnamed_gender_id.data = ''
            self.unnamed_race_id.data = ''
            self.source_role_id.data = ''
            self.source_age_id.data = ''

        return super(DocumentSourceForm, self).validate()

    def populate_obj(self, obj):
        super(DocumentSourceForm, self).populate_obj(obj)
        obj.unnamed = not self.named.data

    @property
    def source(self):
        """ the associated source object, if any """
        return self._obj

    def is_new(self):
        return self.source is None

    def create_or_update(self, document):
        if self.deleted.data == '1':
            document.sources.remove(self.source)
        elif self.is_new():
            return self.create_source(document)
        else:
            self.populate_obj(self.source)

        return None

    def create_source(self, document):
        from . import Person

        src = DocumentSource()
        src.document = document

        self.populate_obj(src)
        src.manual = True

        # link to person if they chose that option
        if self.source_type.data == 'person' and self.named.data:
            src.person = Person.get_or_create(self.name.data)

            # override the 'quoted' attribute if we know this entity has utterances in
            # this document
            if any(src.person == u.entity.person for u in document.utterances):
                src.quoted = True

        return src
<commit_msg>Change Adult Person to Adult<commit_after>from sqlalchemy import (
    Column,
    DateTime,
    ForeignKey,
    Boolean,
    Integer,
    String,
    func,
    )
from sqlalchemy.orm import relationship
from wtforms import StringField, validators, HiddenField, BooleanField, RadioField

from .support import db
from .with_offsets import WithOffsets
from ..forms import Form, SelectField

class DocumentSource(db.Model, WithOffsets):
    """
    A source is a source of information for an article.
    A source can be one of:

     - a named person (linked via person_id)
     - an unnamed person (in which case gender_id and race_id are meaningful)
     - a non-person secondary source (only name is applicable)
    """
    __tablename__ = "document_sources"

    id        = Column(Integer, primary_key=True)
    doc_id    = Column(Integer, ForeignKey('documents.id', ondelete='CASCADE'), index=True, nullable=False)

    # The type of this source. This impacts which other fields are valid and how they're interpreted.
    #
    # person:
    #   - if unnamed is False, then
    #     - person_id is valid and links to the person
    #   - if unnamed is True, then
    #     - person_id is not valid
    #     - unnamed_* are all valid
    #   - source_function and affiliation are valid
    #   - quoted is valid
    #   - everything else (role, age, etc.) are ignored
    #
    # child:
    #   - unnamed is valid and, if False, 'name' has the source name
    #   - race and gender are valid
    #   - age and role are valid
    #   - quoted is alid
    #   - affiliation and function are ignored
    #
    # secondary:
    #   - only name, affiliation and function are valid
    source_type = Column(String(50), nullable=False, default='person')

    person_id = Column(Integer, ForeignKey('people.id', ondelete='CASCADE'), index=True)

    # if this is True, then person_id is ignored and this is an anonymous source
    unnamed           = Column(Boolean, default=False)
    unnamed_gender_id = Column(Integer, ForeignKey('genders.id'))
    unnamed_race_id   = Column(Integer, ForeignKey('races.id'))

    # if unnamed is False and person_id is null, then this is a secondary, named source
    name         = Column(String(100))

    source_function_id = Column(Integer, ForeignKey('source_functions.id', ondelete='SET NULL'))
    source_role_id     = Column(Integer, ForeignKey('source_roles.id', ondelete='SET NULL'))
    source_age_id      = Column(Integer, ForeignKey('source_ages.id', ondelete='SET NULL'))

    quoted       = Column(Boolean)
    photographed = Column(Boolean)

    # was this source added manually or was it inferred by machine learning?
    manual       = Column(Boolean, default=False, nullable=False)

    # who is the person affiliated with?
    affiliation_id = Column(Integer, ForeignKey('affiliations.id', ondelete='SET NULL'), index=True)

    created_at   = Column(DateTime(timezone=True), index=True, unique=False, nullable=False, server_default=func.now())
    updated_at   = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.current_timestamp())

    # Associations
    person      = relationship("Person", foreign_keys=[person_id], lazy=False)
    function    = relationship("SourceFunction", lazy=False)
    affiliation = relationship("Affiliation", lazy=False)
    role        = relationship("SourceRole", lazy=False)
    age         = relationship("SourceAge", lazy=False)
    unnamed_gender = relationship("Gender", lazy=False)
    unnamed_race   = relationship("Race", lazy=False)

    def document_entities(self):
        """ List of DocumentEntity instances that matches this source for this document. May be empty. """
        if not self.person:
            return []
        return [de for de in self.document.entities if de.entity.person == self.person]

    def utterances(self):
        """ A potentially empty list of Utterances from this source in this document. """
        return sorted([u for u in self.document.utterances if self.person and u.entity.person == self.person],
                key=lambda u: u.offset)

    def friendly_name(self):
        if self.person:
            return self.person.name

        if self.unnamed:
            return 'Unnamed'

        return self.name or '(none)'

    def sort_key(self):
        return [not self.manual, self.friendly_name()]

    def gender(self):
        if self.person:
            return self.person.gender
        return self.unnamed_gender

    def race(self):
        if self.person:
            return self.person.race
        return self.unnamed_race

    @property
    def offset_list(self):
        """ String of offset:length pairs of places in this document the entity
        is mentioned or quoted, may be empty. """
        offsets = ['%d:%d' % (u.offset, u.length) for u in self.utterances() if u.offset]
        for de in self.document_entities():
            if de.offset_list:
                offsets.append(de.offset_list)

        return ' '.join(offsets)

    def __repr__(self):
        return "<DocumentSource doc=%s, person=%s, unnamed=%s, name='%s'>" % (self.document, self.person, self.unnamed, self.name.encode('utf-8'))

class SourceFunction(db.Model):
    """
    In what role/function was the source for a document accessed?
    """
    __tablename__ = "source_functions"

    id        = Column(Integer, primary_key=True)
    name      = Column(String(50), index=True, nullable=False, unique=True)

    def __repr__(self):
        return "<SourceFunction name='%s'>" % (self.name)

    @classmethod
    def create_defaults(self):
        text = """
        Do not know
        Subject
        Expert
        Personal Experience
        Eye Witness
        Popular Opinion
        Secondary Sources
        Representative/Spokesperson
        Other
        """

        functions = []
        for s in text.strip().split("\n"):
            g = SourceFunction()
            g.name = s.strip()
            functions.append(g)

        return functions

def none_coerce(v):
    from wtforms.compat import text_type
    return text_type('' if v is None else v)

class DocumentSourceForm(Form):
    name              = StringField('Name', [validators.Length(max=100)])
    named             = BooleanField('The source is named', default=True)
    unnamed_gender_id = SelectField('Gender', [validators.Optional()], default='', coerce=none_coerce)
    unnamed_race_id   = SelectField('Race', [validators.Optional()], default='', coerce=none_coerce)

    source_type       = RadioField('Type', default='person', choices=[['person', 'Adult'], ['child', 'Child'], ['secondary', 'Secondary (not a person)']])

    quoted            = BooleanField('Quoted', default=False)
    photographed      = BooleanField('Photographed', default=False)

    source_function_id  = SelectField('Function', default='')
    source_role_id      = SelectField('Role', default='')
    source_age_id       = SelectField('Age', default='')
    affiliation_id      = SelectField('Affiliation', default='')

    deleted           = HiddenField('deleted', default='0')

    def __init__(self, *args, **kwargs):
        super(DocumentSourceForm, self).__init__(*args, **kwargs)

        from . import SourceAge

        if 'nature' in kwargs:
            nature = kwargs['nature']
        elif self.source:
            nature = self.source.document.analysis_nature
        else:
            raise ArgumentError("Missing analysis nature. Either pass in obj or nature")

        self.source_function_id.choices = [['', '(none)']] + [[str(s.id), s.name] for s in SourceFunction.query.order_by(SourceFunction.name).all()]
        self.source_role_id.choices = [['', '(none)']] + [[str(s.id), s.name] for s in nature.roles]
        self.source_age_id.choices = [['', '(none)']] + [[str(s.id), s.name] for s in SourceAge.query.order_by(SourceAge.id).all()]

        from . import Gender, Race
        self.unnamed_gender_id.choices = [['', '(unknown gender)']] + [[str(g.id), g.name] for g in Gender.query.order_by(Gender.name).all()]
        self.unnamed_race_id.choices = [['', '(unknown race)']] + [[str(r.id), r.name] for r in Race.query.order_by(Race.name).all()]

        # because this list is heirarchical, we class 'organisations' as
        # this with only 0 or two dots
        from . import Affiliation
        orgs = [i for i in Affiliation.query.all() if i.code.count('.') <= 1]
        orgs.sort(key=Affiliation.sort_key)
        self.affiliation_id.choices = [['', '(none)']] + [[str(s.id), s.full_name()] for s in orgs]

        if self.source:
            self.named.data = not self.source.unnamed

    def validate(self):
        # ignore some data, based on the source type
        if not self.named.data:
            # it's anonymous, so ignore the name field
            self.name.data = ''

        if self.source_type.data == 'person':
            self.source_role_id.data = ''
            self.source_age_id.data = ''
            if self.named.data:
                # it's not an anonymous source, so ignore these
                # settings
                self.unnamed_gender_id.data = ''
                self.unnamed_race_id.data = ''

        elif self.source_type.data == 'child':
            self.source_function_id.data = ''
            self.affiliation_id.data = ''

        elif self.source_type.data == 'secondary':
            self.source_role_id.data = ''
            self.unnamed_gender_id.data = ''
            self.unnamed_race_id.data = ''
            self.source_role_id.data = ''
            self.source_age_id.data = ''

        return super(DocumentSourceForm, self).validate()

    def populate_obj(self, obj):
        super(DocumentSourceForm, self).populate_obj(obj)
        obj.unnamed = not self.named.data

    @property
    def source(self):
        """ the associated source object, if any """
        return self._obj

    def is_new(self):
        return self.source is None

    def create_or_update(self, document):
        if self.deleted.data == '1':
            document.sources.remove(self.source)
        elif self.is_new():
            return self.create_source(document)
        else:
            self.populate_obj(self.source)

        return None

    def create_source(self, document):
        from . import Person

        src = DocumentSource()
        src.document = document

        self.populate_obj(src)
        src.manual = True

        # link to person if they chose that option
        if self.source_type.data == 'person' and self.named.data:
            src.person = Person.get_or_create(self.name.data)

            # override the 'quoted' attribute if we know this entity has utterances in
            # this document
            if any(src.person == u.entity.person for u in document.utterances):
                src.quoted = True

        return src
<|endoftext|>


<commit_before>import logging
import requests
import time

from reprounzip.parameters import _bundled_parameters
from reprounzip.utils import itervalues, iteritems

logger = logging.getLogger(__name__)

def _vagrant_req(method, url, json):
    headers = {'User-Agent': 'reprozip testsuite'}
    if json:
        headers['Accept'] = 'application/json'
    for _ in range(5):
        res = requests.request(
            method,
            url,
            headers=headers,
            allow_redirects=True,
        )
        if res.status_code == 429:
            logger.info("(got 429, sleeping)")
            time.sleep(60)
        else:
            return res

def check_vagrant():
    error = False

    # Get all Vagrant boxes from bundled parameters
    boxes = set()
    for distribution in itervalues(
        _bundled_parameters['vagrant_boxes']['boxes'],
    ):
        for version in distribution['versions']:
            for image in itervalues(version['architectures']):
                boxes.add(image)

    # Check that they exist
    for box in boxes:
        # Get metadata
        url = 'https://vagrantcloud.com/' + box
        metadata = _vagrant_req(
            'GET',
            url,
            True,
        )
        if metadata.status_code != 200:
            logger.error(
                "Vagrant box not found: %d %s",
                metadata.status_code, url,
            )
            error = True
            continue
        metadata = metadata.json()

        # Find most recent version
        versions = [
            v for v in metadata.get('versions', ())
            if v.get('status') == 'active'
        ]
        if not versions:
            logger.error("No versions for Vagrant box %s", box)
            error = True
            continue
        max_version = max(metadata['versions'], key=lambda v: v['version'])

        # Go over each provider
        for provider in max_version['providers']:
            url = provider['url']
            res = _vagrant_req(
                'HEAD',
                url,
                False,
            )
            # Status should be 200
            if res.status_code != 200:
                logger.error(
                    "Got %d getting Vagrant box %s: %s",
                    res.status_code, box, url,
                )
                error = True
            # Content-Type should not start with "text/" (but can be unset)
            elif res.headers.get('Content-Type', '').startswith('text/'):
                logger.error(
                    "Got type %s getting Vagrant box %s: %s",
                    res.headers['Content-Type'], box, url,
                )
                error = True
            # Size should be at least 10 MB
            elif int(res.headers.get('Content-Length', 1E8)) < 1E7:
                logger.error(
                    "Got file size %s getting Vagrant box %s: %s",
                    res.headers['Content-Length'], box, url,
                )
                error = True
            else:
                logger.info("Vagrant box ok: %s (%s)", box, provider['name'])

    if error:
        raise AssertionError("Missing Vagrant boxes")

def check_docker():
    error = False

    # Get all Docker images from bundled parameters
    images = set()
    for distribution in itervalues(
        _bundled_parameters['docker_images']['images'],
    ):
        for version in distribution['versions']:
            images.add(version['image'])

    # Rewrite images in canonical format, organize by repository
    repositories = {}
    for image in images:
        image = image.split('/')
        if ':' in image[-1]:
            image[-1], tag = image[-1].split(':', 1)
        else:
            tag = 'latest'
        if len(image) == 1:
            image = ['docker.io', 'library'] + image
        elif len(image) == 2:
            image = ['docker.io'] + image
        repositories.setdefault(tuple(image[:3]), set()).add(tag)

    # Check that each repository has the required tags
    for repository, tags in iteritems(repositories):
        registry = repository[0]
        if registry == 'docker.io':
            registry = 'hub.docker.com'
        else:
            raise AssertionError("Registry unsupported: %s" % registry)
        url = (
            'https://' + registry
            + '/v1/repositories/' + repository[1]
            + '/' + repository[2]
            + '/tags'
        )
        res = requests.get(url)
        if res.status_code != 200:
            logger.error(
                "Docker repository not found: %d %s",
                res.status_code,
                url,
            )
            error = True
            continue
        actual_tags = res.json()
        actual_tags = set(entry['name'] for entry in actual_tags)
        for tag in tags:
            if tag not in actual_tags:
                logger.error(
                    "Docker repository %s missing tag %s",
                    '/'.join(repository),
                    tag,
                )
            else:
                logger.info(
                    "Docker image ok: %s:%s",
                    '/'.join(repository),
                    tag,
                )

    if error:
        raise AssertionError("Missing Docker boxes")
<commit_msg>Implement checking Docker images with API v2<commit_after>import logging
import re
import requests
import time

from reprounzip.parameters import _bundled_parameters
from reprounzip.utils import itervalues, iteritems, PY3

if PY3:
    from urllib.parse import urlencode
else:
    from urllib import urlencode

logger = logging.getLogger(__name__)

def _vagrant_req(method, url, json):
    headers = {'User-Agent': 'reprozip testsuite'}
    if json:
        headers['Accept'] = 'application/json'
    for _ in range(5):
        res = requests.request(
            method,
            url,
            headers=headers,
            allow_redirects=True,
        )
        if res.status_code == 429:
            logger.info("(got 429, sleeping)")
            time.sleep(60)
        else:
            return res

def check_vagrant():
    error = False

    # Get all Vagrant boxes from bundled parameters
    boxes = set()
    for distribution in itervalues(
        _bundled_parameters['vagrant_boxes']['boxes'],
    ):
        for version in distribution['versions']:
            for image in itervalues(version['architectures']):
                boxes.add(image)

    # Check that they exist
    for box in boxes:
        # Get metadata
        url = 'https://vagrantcloud.com/' + box
        metadata = _vagrant_req(
            'GET',
            url,
            True,
        )
        if metadata.status_code != 200:
            logger.error(
                "Vagrant box not found: %d %s",
                metadata.status_code, url,
            )
            error = True
            continue
        metadata = metadata.json()

        # Find most recent version
        versions = [
            v for v in metadata.get('versions', ())
            if v.get('status') == 'active'
        ]
        if not versions:
            logger.error("No versions for Vagrant box %s", box)
            error = True
            continue
        max_version = max(metadata['versions'], key=lambda v: v['version'])

        # Go over each provider
        for provider in max_version['providers']:
            url = provider['url']
            res = _vagrant_req(
                'HEAD',
                url,
                False,
            )
            # Status should be 200
            if res.status_code != 200:
                logger.error(
                    "Got %d getting Vagrant box %s: %s",
                    res.status_code, box, url,
                )
                error = True
            # Content-Type should not start with "text/" (but can be unset)
            elif res.headers.get('Content-Type', '').startswith('text/'):
                logger.error(
                    "Got type %s getting Vagrant box %s: %s",
                    res.headers['Content-Type'], box, url,
                )
                error = True
            # Size should be at least 10 MB
            elif int(res.headers.get('Content-Length', 1E8)) < 1E7:
                logger.error(
                    "Got file size %s getting Vagrant box %s: %s",
                    res.headers['Content-Length'], box, url,
                )
                error = True
            else:
                logger.info("Vagrant box ok: %s (%s)", box, provider['name'])

    if error:
        raise AssertionError("Missing Vagrant boxes")

def list_docker_tags(repository, token=None):
    headers = {}
    if token is not None:
        headers['Authorization'] = 'Bearer %s' % token
    res = requests.get(
        'https://%s/v2/%s/%s/tags/list' % (
            repository[0], repository[1], repository[2],
        ),
        headers=headers,
    )
    if token is None and res.status_code == 401:
        # Authenticate
        m = re.match(
            r'Bearer realm="([^"]+)",service="([^"]+)"',
            res.headers['www-authenticate'],
        )
        if m is None:
            res.raise_for_status()
        scope = 'repository:%s/%s:pull' % (repository[1], repository[2])
        res = requests.get(
            m.group(1) + '?' + urlencode({
                'service': m.group(2),
                'scope': scope,
            }),
        )
        res.raise_for_status()
        token = res.json()['token']
        # Try again with token
        return list_docker_tags(repository, token)

    res.raise_for_status()

    return res.json()['tags']

def check_docker():
    error = False

    # Get all Docker images from bundled parameters
    images = set()
    for distribution in itervalues(
        _bundled_parameters['docker_images']['images'],
    ):
        for version in distribution['versions']:
            images.add(version['image'])

    # Rewrite images in canonical format, organize by repository
    repositories = {}
    for image in images:
        image = image.split('/')
        if ':' in image[-1]:
            image[-1], tag = image[-1].split(':', 1)
        else:
            tag = 'latest'
        if len(image) == 1:
            image = ['index.docker.io', 'library'] + image
        elif len(image) == 2:
            image = ['index.docker.io'] + image
        repositories.setdefault(tuple(image[:3]), set()).add(tag)

    # Check that each repository has the required tags
    for repository, tags in iteritems(repositories):
        try:
            actual_tags = list_docker_tags(repository)
        except requests.HTTPError as e:
            logger.error(
                "Docker repository not found: %d %s",
                e.response.status_code,
                e.request.url,
            )
            error = True
            continue
        actual_tags = set(actual_tags)
        for tag in tags:
            if tag not in actual_tags:
                logger.error(
                    "Docker repository %s missing tag %s",
                    '/'.join(repository),
                    tag,
                )
            else:
                logger.info(
                    "Docker image ok: %s:%s",
                    '/'.join(repository),
                    tag,
                )

    if error:
        raise AssertionError("Missing Docker boxes")
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os.path
import subprocess

def get_usable_ffmpeg(cmd):
    try:
        p = subprocess.Popen([cmd, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        out, err = p.communicate()
        vers = str(out, 'utf-8').split('\n')[0].split()
        assert (vers[0] == 'ffmpeg' and vers[2][0] > '0') or (vers[0] == 'avconv')
        #if the version is strange like 'N-1234-gd1111', set version to 2.0
        try:
            version = [int(i) for i in vers[2].split('.')]
        except:
            version = [1, 0]
        return cmd, version
    except:
        return None

FFMPEG, FFMPEG_VERSION = get_usable_ffmpeg('ffmpeg') or get_usable_ffmpeg('avconv') or (None, None)

def has_ffmpeg_installed():
    return FFMPEG is not None

def ffmpeg_convert_ts_to_mkv(files, output='output.mkv'):
    for file in files:
        if os.path.isfile(file):
            params = [FFMPEG, '-i']
            params.append(file)
            params.append(output)
            subprocess.call(params)

    return

def ffmpeg_concat_mp4_to_mpg(files, output='output.mpg'):
    # Use concat demuxer on FFmpeg >= 1.1
    if FFMPEG == 'ffmpeg' and (FFMPEG_VERSION[0] >= 2 or (FFMPEG_VERSION[0] == 1 and FFMPEG_VERSION[1] >= 1)):
        concat_list = open(output + '.txt', 'w')
        for file in files:
            if os.path.isfile(file):
                concat_list.write("file '%s'\n" % file)
        concat_list.close()

        params = [FFMPEG, '-f', 'concat', '-i']
        params.append(output + '.txt')
        params += ['-c', 'copy', output]

        if subprocess.call(params) == 0:
            os.remove(output + '.txt')
            return True
        else:
            raise

    for file in files:
        if os.path.isfile(file):
            params = [FFMPEG, '-i']
            params.append(file)
            params.append(file + '.mpg')
            subprocess.call(params)

    inputs = [open(file + '.mpg', 'rb') for file in files]
    with open(output + '.mpg', 'wb') as o:
        for input in inputs:
            o.write(input.read())

    params = [FFMPEG, '-i']
    params.append(output + '.mpg')
    params += ['-vcodec', 'copy', '-acodec', 'copy']
    params.append(output)
    subprocess.call(params)

    if subprocess.call(params) == 0:
        for file in files:
            os.remove(file + '.mpg')
        os.remove(output + '.mpg')
        return True
    else:
        raise

def ffmpeg_concat_ts_to_mkv(files, output='output.mkv'):
    params = [FFMPEG, '-isync', '-i']
    params.append('concat:')
    for file in files:
        if os.path.isfile(file):
            params[-1] += file + '|'
    params += ['-f', 'matroska', '-c', 'copy', output]

    try:
        if subprocess.call(params) == 0:
            return True
        else:
            return False
    except:
        return False

def ffmpeg_concat_flv_to_mp4(files, output='output.mp4'):
    # Use concat demuxer on FFmpeg >= 1.1
    if FFMPEG == 'ffmpeg' and (FFMPEG_VERSION[0] >= 2 or (FFMPEG_VERSION[0] == 1 and FFMPEG_VERSION[1] >= 1)):
        concat_list = open(output + '.txt', 'w')
        for file in files:
            if os.path.isfile(file):
                concat_list.write("file '%s'\n" % file)
        concat_list.close()

        params = [FFMPEG, '-f', 'concat', '-i']
        params.append(output + '.txt')
        params += ['-c', 'copy', output]

        if subprocess.call(params) == 0:
            os.remove(output + '.txt')
            return True
        else:
            raise

    for file in files:
        if os.path.isfile(file):
            params = [FFMPEG, '-i']
            params.append(file)
            params += ['-map', '0', '-c', 'copy', '-f', 'mpegts', '-bsf:v', 'h264_mp4toannexb']
            params.append(file + '.ts')

            subprocess.call(params)

    params = [FFMPEG, '-i']
    params.append('concat:')
    for file in files:
        f = file + '.ts'
        if os.path.isfile(f):
            params[-1] += f + '|'
    if FFMPEG == 'avconv':
        params += ['-c', 'copy', output]
    else:
        params += ['-c', 'copy', '-absf', 'aac_adtstoasc', output]

    if subprocess.call(params) == 0:
        for file in files:
            os.remove(file + '.ts')
        return True
    else:
        raise

def ffmpeg_concat_mp4_to_mp4(files, output='output.mp4'):
    # Use concat demuxer on FFmpeg >= 1.1
    if FFMPEG == 'ffmpeg' and (FFMPEG_VERSION[0] >= 2 or (FFMPEG_VERSION[0] == 1 and FFMPEG_VERSION[1] >= 1)):
        concat_list = open(output + '.txt', 'w')
        for file in files:
            if os.path.isfile(file):
                concat_list.write("file '%s'\n" % file)
        concat_list.close()

        params = [FFMPEG, '-f', 'concat', '-i']
        params.append(output + '.txt')
        params += ['-c', 'copy', output]

        if subprocess.call(params) == 0:
            os.remove(output + '.txt')
            return True
        else:
            raise

    for file in files:
        if os.path.isfile(file):
            params = [FFMPEG, '-i']
            params.append(file)
            params += ['-c', 'copy', '-f', 'mpegts', '-bsf:v', 'h264_mp4toannexb']
            params.append(file + '.ts')

            subprocess.call(params)

    params = [FFMPEG, '-i']
    params.append('concat:')
    for file in files:
        f = file + '.ts'
        if os.path.isfile(f):
            params[-1] += f + '|'
    if FFMPEG == 'avconv':
        params += ['-c', 'copy', output]
    else:
        params += ['-c', 'copy', '-absf', 'aac_adtstoasc', output]

    if subprocess.call(params) == 0:
        for file in files:
            os.remove(file + '.ts')
        return True
    else:
        raise
<commit_msg>force override when concat files<commit_after>#!/usr/bin/env python

import os.path
import subprocess

def get_usable_ffmpeg(cmd):
    try:
        p = subprocess.Popen([cmd, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        out, err = p.communicate()
        vers = str(out, 'utf-8').split('\n')[0].split()
        assert (vers[0] == 'ffmpeg' and vers[2][0] > '0') or (vers[0] == 'avconv')
        #if the version is strange like 'N-1234-gd1111', set version to 2.0
        try:
            version = [int(i) for i in vers[2].split('.')]
        except:
            version = [1, 0]
        return cmd, version
    except:
        return None

FFMPEG, FFMPEG_VERSION = get_usable_ffmpeg('ffmpeg') or get_usable_ffmpeg('avconv') or (None, None)

def has_ffmpeg_installed():
    return FFMPEG is not None

def ffmpeg_convert_ts_to_mkv(files, output='output.mkv'):
    for file in files:
        if os.path.isfile(file):
            params = [FFMPEG, '-y', '-i']
            params.append(file)
            params.append(output)
            subprocess.call(params)

    return

def ffmpeg_concat_mp4_to_mpg(files, output='output.mpg'):
    # Use concat demuxer on FFmpeg >= 1.1
    if FFMPEG == 'ffmpeg' and (FFMPEG_VERSION[0] >= 2 or (FFMPEG_VERSION[0] == 1 and FFMPEG_VERSION[1] >= 1)):
        concat_list = open(output + '.txt', 'w')
        for file in files:
            if os.path.isfile(file):
                concat_list.write("file '%s'\n" % file)
        concat_list.close()

        params = [FFMPEG, '-f', 'concat', '-y', '-i']
        params.append(output + '.txt')
        params += ['-c', 'copy', output]

        if subprocess.call(params) == 0:
            os.remove(output + '.txt')
            return True
        else:
            raise

    for file in files:
        if os.path.isfile(file):
            params = [FFMPEG, '-y', '-i']
            params.append(file)
            params.append(file + '.mpg')
            subprocess.call(params)

    inputs = [open(file + '.mpg', 'rb') for file in files]
    with open(output + '.mpg', 'wb') as o:
        for input in inputs:
            o.write(input.read())

    params = [FFMPEG, '-y', '-i']
    params.append(output + '.mpg')
    params += ['-vcodec', 'copy', '-acodec', 'copy']
    params.append(output)
    subprocess.call(params)

    if subprocess.call(params) == 0:
        for file in files:
            os.remove(file + '.mpg')
        os.remove(output + '.mpg')
        return True
    else:
        raise

def ffmpeg_concat_ts_to_mkv(files, output='output.mkv'):
    params = [FFMPEG, '-isync', '-y', '-i']
    params.append('concat:')
    for file in files:
        if os.path.isfile(file):
            params[-1] += file + '|'
    params += ['-f', 'matroska', '-c', 'copy', output]

    try:
        if subprocess.call(params) == 0:
            return True
        else:
            return False
    except:
        return False

def ffmpeg_concat_flv_to_mp4(files, output='output.mp4'):
    # Use concat demuxer on FFmpeg >= 1.1
    if FFMPEG == 'ffmpeg' and (FFMPEG_VERSION[0] >= 2 or (FFMPEG_VERSION[0] == 1 and FFMPEG_VERSION[1] >= 1)):
        concat_list = open(output + '.txt', 'w')
        for file in files:
            if os.path.isfile(file):
                concat_list.write("file '%s'\n" % file)
        concat_list.close()

        params = [FFMPEG, '-f', 'concat', '-y', '-i']
        params.append(output + '.txt')
        params += ['-c', 'copy', output]

        if subprocess.call(params) == 0:
            os.remove(output + '.txt')
            return True
        else:
            raise

    for file in files:
        if os.path.isfile(file):
            params = [FFMPEG, '-y', '-i']
            params.append(file)
            params += ['-map', '0', '-c', 'copy', '-f', 'mpegts', '-bsf:v', 'h264_mp4toannexb']
            params.append(file + '.ts')

            subprocess.call(params)

    params = [FFMPEG, '-y', '-i']
    params.append('concat:')
    for file in files:
        f = file + '.ts'
        if os.path.isfile(f):
            params[-1] += f + '|'
    if FFMPEG == 'avconv':
        params += ['-c', 'copy', output]
    else:
        params += ['-c', 'copy', '-absf', 'aac_adtstoasc', output]

    if subprocess.call(params) == 0:
        for file in files:
            os.remove(file + '.ts')
        return True
    else:
        raise

def ffmpeg_concat_mp4_to_mp4(files, output='output.mp4'):
    # Use concat demuxer on FFmpeg >= 1.1
    if FFMPEG == 'ffmpeg' and (FFMPEG_VERSION[0] >= 2 or (FFMPEG_VERSION[0] == 1 and FFMPEG_VERSION[1] >= 1)):
        concat_list = open(output + '.txt', 'w')
        for file in files:
            if os.path.isfile(file):
                concat_list.write("file '%s'\n" % file)
        concat_list.close()

        params = [FFMPEG, '-f', 'concat', '-y', '-i']
        params.append(output + '.txt')
        params += ['-c', 'copy', output]

        if subprocess.call(params) == 0:
            os.remove(output + '.txt')
            return True
        else:
            raise

    for file in files:
        if os.path.isfile(file):
            params = [FFMPEG, '-y', '-i']
            params.append(file)
            params += ['-c', 'copy', '-f', 'mpegts', '-bsf:v', 'h264_mp4toannexb']
            params.append(file + '.ts')

            subprocess.call(params)

    params = [FFMPEG, '-y', '-i']
    params.append('concat:')
    for file in files:
        f = file + '.ts'
        if os.path.isfile(f):
            params[-1] += f + '|'
    if FFMPEG == 'avconv':
        params += ['-c', 'copy', output]
    else:
        params += ['-c', 'copy', '-absf', 'aac_adtstoasc', output]

    if subprocess.call(params) == 0:
        for file in files:
            os.remove(file + '.ts')
        return True
    else:
        raise
<|endoftext|>