<commit_before>import discord
from discord.ext import commands
from .utils.dataIO import dataIO, fileIO
from __main__ import send_cmd_help
import os
import clashroyale
import itertools

settings_path = "data/trade/settings.json"
cards_path = "data/trade/cards.json"

creditIcon = "https://i.imgur.com/TP8GXZb.png"
credits = "Bot by GR8 | Titan"

class Trade:
    """Clash Royale Trading Helper"""

    def __init__(self, bot):
        self.bot = bot
        self.settings = dataIO.load_json(settings_path)
        self.cards = dataIO.load_json(cards_path)
        self.auth = self.bot.get_cog('crtools').auth
        self.tags = self.bot.get_cog('crtools').tags
        self.constants = self.bot.get_cog('crtools').constants
        self.clash = clashroyale.OfficialAPI(self.auth.getOfficialToken(), is_async=True)

        # init card data
        self.cards_abbrev = {}

        for k, v in self.cards.items():
            for value in v:
                self.cards_abbrev[value] = k
            self.cards_abbrev[k] = k

    def emoji(self, name):
        """Emoji by name."""
        for emoji in self.bot.get_all_emojis():
            if emoji.name == name.replace(" ", "").replace("-", "").replace(".", ""):
                return '<:{}:{}>'.format(emoji.name, emoji.id)
        return ''

    def grouper(self, iterable, n):
        args = [iter(iterable)] * n
        return itertools.zip_longest(*args)

    async def defualt_settings(self, member):
        self.settings[member.id] = {}
        self.settings[member.id]["Legendary"] = []
        self.settings[member.id]["Epic"] = []
        self.settings[member.id]["Rare"] = []
        self.settings[member.id]["Common"] = []
        self.settings[member.id]["Tokens"] = {}
        self.settings[member.id]["Tokens"]["legendary"] = False
        self.settings[member.id]["Tokens"]["epic"] = False
        self.settings[member.id]["Tokens"]["rare"] = False
        self.settings[member.id]["Tokens"]["common"] = False

    async def saveCard(self, member, card):
        rarity = await self.constants.card_to_rarity(card)

        if member.id not in self.settings:
            await self.defualt_settings(member)

        if card not in self.settings[member.id][rarity]:
            self.settings[member.id][rarity].append(card)
        dataIO.save_json(settings_path, self.settings)

    async def removeCard(self, member, card):
        rarity = await self.constants.card_to_rarity(card)

        if member.id not in self.settings:
            await self.defualt_settings(member)

        if card in self.settings[member.id][rarity]:
            self.settings[member.id][rarity].remove(card)
        dataIO.save_json(settings_path, self.settings)

    async def saveToken(self, member, token):
        if member.id not in self.settings:
            await self.defualt_settings(member)

        self.settings[member.id]["Tokens"][token] = True
        dataIO.save_json(settings_path, self.settings)

    async def removeToken(self, member, token):
        if member.id not in self.settings:
            await self.defualt_settings(member)

        self.settings[member.id]["Tokens"][token] = False
        dataIO.save_json(settings_path, self.settings)

    async def tradeCards(self, cards):
        trades = {
            "Legendary": [],
            "Epic": [],
            "Rare": [],
            "Common": []
        }

        for card in cards:
            if card.max_level == 13:
                if (card.count == 250 and card.level != 1) or (card.count > 250):
                    trades["Common"].append(card.name)
            elif card.max_level == 11:
                if (card.count == 50 and card.level != 1) or (card.count > 50):
                    trades["Rare"].append(card.name)
            elif card.max_level == 8:
                if (card.count == 10 and card.level != 1) or (card.count > 10):
                    trades["Epic"].append(card.name)
            elif card.max_level == 5:
                if (card.count == 1 and card.level != 1) or (card.count > 1):
                    trades["Legendary"].append(card.name)
        return trades

    @commands.group(pass_context=True, no_pm=True)
    async def trade(self, ctx):
        """Clash Royale trade commands"""
        if ctx.invoked_subcommand is None:
            await send_cmd_help(ctx)

    @trade.command(pass_context=True, no_pm=True)
    async def show(self, ctx, member: discord.Member=None):
        """Show cards you can trade"""
        member = member or ctx.message.author
        giveset = False

        await self.bot.type()

        try:
            profiletag = await self.tags.getTag(member.id)
            profiledata = await self.clash.get_player(profiletag)
            rarities = await self.tradeCards(profiledata.cards)
        except clashroyale.RequestError:
            return await self.bot.say("Error: cannot reach Clash Royale Servers. Please try again later.")
        except KeyError:
            return await self.bot.say("You need to first save your profile using ``{}save #GAMETAG``".format(ctx.prefix))

        embed = discord.Embed(color=0xFAA61A, description="Cards available for trade.")
        embed.set_author(name=profiledata.name + " ("+profiledata.tag+")",
                         icon_url=await self.constants.get_clan_image(profiledata),
                         url="https://royaleapi.com/player/"+profiledata.tag.strip("#"))
        embed.set_footer(text=credits, icon_url=creditIcon)

        for rarity in rarities.keys():
            f_title = rarity
            groups = self.grouper(rarities[rarity], 30)
            for index, cards in enumerate(groups):
                value = "Want: "
                if member.id in self.settings:
                    giveset = True
                    for cardern in self.settings[member.id][rarity]:
                        if cardern is not None:
                            value += self.emoji(cardern)
                else:
                    value += "Type ``{}trade add`` to add cards here.".format(ctx.prefix)
                value += "\nGiving: " if index == 0 else ""
                for card in cards:
                    if card is not None:
                        if giveset:
                            if card not in self.settings[member.id][rarity]:
                                value += self.emoji(card)
                        else:
                            value += self.emoji(card)
                embed.add_field(name=f_title if index == 0 else '\u200b', value=value, inline=False)

        tokenText = ""
        if member.id in self.settings:
            for token in self.settings[member.id]["Tokens"]:
                if self.settings[member.id]["Tokens"][token]:
                    tokenText += self.emoji("Token" + token.capitalize())

        if not tokenText:
            tokenText = "*No Tokens*\nType ``{}trade token add`` to add tokens here.".format(ctx.prefix)
        embed.add_field(name="My Tokens", value=tokenText, inline=False)

        await self.bot.say(embed=embed)

    @trade.command(pass_context=True, no_pm=True)
    async def add(self, ctx, *, card):
        """Add a card you need for trading"""
        author = ctx.message.author
        try:
            card = self.cards_abbrev[card]
        except KeyError:
            return await self.bot.say("Error, Invalid Card")

        await self.saveCard(author, card)
        await self.bot.say("You are now looking for {}".format(self.emoji(card)))

    @trade.command(pass_context=True, no_pm=True)
    async def remove(self, ctx, *, card):
        """Remove a card you dont need for trading"""
        author = ctx.message.author
        try:
            card = self.cards_abbrev[card]
        except KeyError:
            return await self.bot.say("Error, Invalid Card")

        await self.removeCard(author, card)
        await self.bot.say("You are no longer looking for {}".format(self.emoji(card)))

    @trade.group(pass_context=True, no_pm=True)
    async def token(self, ctx):
        """Add/Remove trade tokens"""
        if ctx.invoked_subcommand is None:
            await send_cmd_help(ctx)

    @token.command(pass_context=True, no_pm=True, name="add")
    async def add_token(self, ctx, *, token):
        """Add a trade token"""
        author = ctx.message.author
        token = token.lower()
        try:
            await self.saveToken(author, token)
        except KeyError:
            return await self.bot.say("Error, Invalid Token")

        await self.bot.say("You now have a {}".format(self.emoji("Token" + token.capitalize())))

    @token.command(pass_context=True, no_pm=True, name="remove")
    async def remove_token(self, ctx, *, token):
        """Remove a trade token"""
        author = ctx.message.author
        token = token.lower()
        try:
            await self.removeToken(author, token)
        except KeyError:
            return await self.bot.say("Error, Invalid Token")

        await self.bot.say("You no longer have a {}".format(self.emoji("Token" + token.capitalize())))

def check_folders():
    if not os.path.exists("data/trade"):
        print("Creating data/trade folder...")
        os.makedirs("data/trade")

def check_files():
    f = settings_path
    if not fileIO(f, "check"):
        print("Creating trade settings.json...")
        fileIO(f, "save", {})

def setup(bot):
    check_folders()
    check_files()
    bot.add_cog(Trade(bot))

<commit_msg>Search tradeers<commit_after>import discord
from discord.ext import commands
from .utils.dataIO import dataIO, fileIO
from __main__ import send_cmd_help
import os
import clashroyale
import itertools

settings_path = "data/trade/settings.json"
cards_path = "data/trade/cards.json"

creditIcon = "https://i.imgur.com/TP8GXZb.png"
credits = "Bot by GR8 | Titan"

class Trade:
    """Clash Royale Trading Helper"""

    def __init__(self, bot):
        self.bot = bot
        self.settings = dataIO.load_json(settings_path)
        self.cards = dataIO.load_json(cards_path)
        self.auth = self.bot.get_cog('crtools').auth
        self.tags = self.bot.get_cog('crtools').tags
        self.constants = self.bot.get_cog('crtools').constants
        self.clash = clashroyale.OfficialAPI(self.auth.getOfficialToken(), is_async=True)

        # init card data
        self.cards_abbrev = {}

        for k, v in self.cards.items():
            for value in v:
                self.cards_abbrev[value] = k
            self.cards_abbrev[k] = k

    def emoji(self, name):
        """Emoji by name."""
        for emoji in self.bot.get_all_emojis():
            if emoji.name == name.replace(" ", "").replace("-", "").replace(".", ""):
                return '<:{}:{}>'.format(emoji.name, emoji.id)
        return ''

    def grouper(self, iterable, n):
        args = [iter(iterable)] * n
        return itertools.zip_longest(*args)

    async def defualt_settings(self, member):
        self.settings[member.id] = {}
        self.settings[member.id]["want"] = []
        self.settings[member.id]["give"] = []
        self.settings[member.id]["token"] = []

        self.settings[member.id]["want"] = {}
        self.settings[member.id]["want"]["legendary"] = []
        self.settings[member.id]["want"]["epic"] = []
        self.settings[member.id]["want"]["rare"] = []
        self.settings[member.id]["want"]["common"] = []

        self.settings[member.id]["give"] = {}
        self.settings[member.id]["give"]["legendary"] = []
        self.settings[member.id]["give"]["epic"] = []
        self.settings[member.id]["give"]["rare"] = []
        self.settings[member.id]["give"]["common"] = []

        self.settings[member.id]["token"] = {}
        self.settings[member.id]["token"]["legendary"] = False
        self.settings[member.id]["token"]["epic"] = False
        self.settings[member.id]["token"]["rare"] = False
        self.settings[member.id]["token"]["common"] = False

    async def saveCardWant(self, member, card):
        rarity = await self.constants.card_to_rarity(card)
        rarity = rarity.lower()

        if member.id not in self.settings:
            await self.defualt_settings(member)

        if card not in self.settings[member.id]['want'][rarity]:
            self.settings[member.id]['want'][rarity].append(card)
        dataIO.save_json(settings_path, self.settings)

    async def saveCardsGive(self, member, rarity, cards):
        if member.id not in self.settings:
            await self.defualt_settings(member)

        self.settings[member.id]['give'][rarity] = cards
        dataIO.save_json(settings_path, self.settings)

    async def removeCardWant(self, member, card):
        rarity = await self.constants.card_to_rarity(card)
        rarity = rarity.lower()

        if member.id not in self.settings:
            await self.defualt_settings(member)

        if card in self.settings[member.id]['want'][rarity]:
            self.settings[member.id]['want'][rarity].remove(card)
        dataIO.save_json(settings_path, self.settings)

    async def saveToken(self, member, token):
        if member.id not in self.settings:
            await self.defualt_settings(member)

        self.settings[member.id]["token"][token] = True
        dataIO.save_json(settings_path, self.settings)

    async def removeToken(self, member, token):
        if member.id not in self.settings:
            await self.defualt_settings(member)

        self.settings[member.id]["token"][token] = False
        dataIO.save_json(settings_path, self.settings)

    async def tradeCards(self, cards):
        trades = {
            "legendary": [],
            "epic": [],
            "rare": [],
            "common": []
        }

        for card in cards:
            if card.max_level == 13:
                if (card.count == 250 and card.level != 1) or (card.count > 250):
                    trades["common"].append(card.name)
            elif card.max_level == 11:
                if (card.count == 50 and card.level != 1) or (card.count > 50):
                    trades["rare"].append(card.name)
            elif card.max_level == 8:
                if (card.count == 10 and card.level != 1) or (card.count > 10):
                    trades["epic"].append(card.name)
            elif card.max_level == 5:
                if (card.count == 1 and card.level != 1) or (card.count > 1):
                    trades["legendary"].append(card.name)
        return trades

    async def searchTrades(self, card):
        rarity = await self.constants.card_to_rarity(card)
        rarity = rarity.lower()

        trades = {}
        for player in self.settings:
            trades[player] = [False, False, False]
            if card in self.settings[player]["give"][rarity]:
                trades[player][0] = True
            if card in self.settings[player]["want"][rarity]:
                trades[player][1] = True
            if self.settings[player]["token"][rarity]:
                trades[player][2] = True

        return trades

    @commands.group(pass_context=True, no_pm=True)
    async def trade(self, ctx):
        """Clash Royale trade commands"""
        if ctx.invoked_subcommand is None:
            await send_cmd_help(ctx)

    @trade.command(pass_context=True, no_pm=True)
    async def show(self, ctx, member: discord.Member=None):
        """Show cards you can trade"""
        member = member or ctx.message.author
        giveset = False

        await self.bot.type()

        try:
            profiletag = await self.tags.getTag(member.id)
            profiledata = await self.clash.get_player(profiletag)
            rarities = await self.tradeCards(profiledata.cards)
        except clashroyale.RequestError:
            return await self.bot.say("Error: cannot reach Clash Royale Servers. Please try again later.")
        except KeyError:
            return await self.bot.say("You need to first save your profile using ``{}save #GAMETAG``".format(ctx.prefix))

        embed = discord.Embed(color=0xFAA61A, description="Cards available for trade.")
        embed.set_author(name=profiledata.name + " ("+profiledata.tag+")",
                         icon_url=await self.constants.get_clan_image(profiledata),
                         url="https://royaleapi.com/player/"+profiledata.tag.strip("#"))
        embed.set_footer(text=credits, icon_url=creditIcon)

        for rarity in rarities.keys():
            f_title = rarity.capitalize()
            groups = self.grouper(rarities[rarity], 30)
            for index, cards in enumerate(groups):
                value = "Want: "
                give = []
                if member.id in self.settings:
                    giveset = True
                    for cardern in self.settings[member.id]['want'][rarity]:
                        if cardern is not None:
                            value += self.emoji(cardern)
                else:
                    value += "Type ``{}trade add`` to add cards here.".format(ctx.prefix)
                value += "\nGiving: " if index == 0 else ""
                for card in cards:
                    if card is not None:
                        if giveset:
                            if card not in self.settings[member.id]['want'][rarity]:
                                value += self.emoji(card)
                                give.append(card)
                        else:
                            value += self.emoji(card)
                            give.append(card)
                embed.add_field(name=f_title if index == 0 else '\u200b', value=value, inline=False)
                await self.saveCardsGive(member, rarity, give)

        tokenText = ""
        if member.id in self.settings:
            for token in self.settings[member.id]["token"]:
                if self.settings[member.id]["token"][token]:
                    tokenText += self.emoji("Token" + token.capitalize())

        if not tokenText:
            tokenText = "*No Tokens*\nType ``{}trade token add`` to add tokens here.".format(ctx.prefix)
        embed.add_field(name="My Tokens", value=tokenText, inline=False)

        await self.bot.say(embed=embed)

    @trade.command(pass_context=True, no_pm=True)
    async def add(self, ctx, *, card):
        """Add a card you need for trading"""
        author = ctx.message.author
        try:
            card = self.cards_abbrev[card]
        except KeyError:
            return await self.bot.say("Error, Invalid Card")

        await self.saveCardWant(author, card)
        await self.bot.say("You are now looking for {}".format(self.emoji(card)))

    @trade.command(pass_context=True, no_pm=True)
    async def remove(self, ctx, *, card):
        """Remove a card you dont need for trading"""
        author = ctx.message.author
        try:
            card = self.cards_abbrev[card]
        except KeyError:
            return await self.bot.say("Error, Invalid Card")

        await self.removeCardWant(author, card)
        await self.bot.say("You are no longer looking for {}".format(self.emoji(card)))

    @trade.group(pass_context=True, no_pm=True)
    async def token(self, ctx):
        """Add/Remove trade tokens"""
        if ctx.invoked_subcommand is None:
            await send_cmd_help(ctx)

    @trade.command(pass_context=True, no_pm=True)
    async def search(self, ctx, *, card):
        """Search Trades"""
        server = ctx.message.server
        try:
            card = self.cards_abbrev[card]
        except KeyError:
            return await self.bot.say("Error, Invalid Card")

        trades = await self.searchTrades(card)
        embed = discord.Embed(color=0xFAA61A, description="We found these members who match your card search.")
        embed.set_author(name="{} Traders".format(card),
                         icon_url="https://i.imgur.com/dtSMITE.jpg")
        embed.set_footer(text=credits, icon_url=creditIcon)

        givers = "\u200b"
        wanters = "\u200b"
        for player in trades:
            if trades[player][0]:
                member = server.get_member(player)
                if trades[player][2]:
                    givers += self.emoji("Token" + await self.constants.card_to_rarity(card))
                givers += member.mention
            if trades[player][1]:
                member = server.get_member(player)
                if trades[player][2]:
                    wanters += self.emoji("Token" + await self.constants.card_to_rarity(card))
                wanters += member.mention
        embed.add_field(name="Giving {}".format(card), value=givers, inline=False)
        embed.add_field(name="Want {}".format(card), value=wanters, inline=False)

        await self.bot.say(embed=embed)

    @token.command(pass_context=True, no_pm=True, name="add")
    async def add_token(self, ctx, *, token):
        """Add a trade token"""
        author = ctx.message.author
        token = token.lower()
        try:
            await self.saveToken(author, token)
        except KeyError:
            return await self.bot.say("Error, Invalid Token")

        await self.bot.say("You now have a {}".format(self.emoji("Token" + token.capitalize())))

    @token.command(pass_context=True, no_pm=True, name="remove")
    async def remove_token(self, ctx, *, token):
        """Remove a trade token"""
        author = ctx.message.author
        token = token.lower()
        try:
            await self.removeToken(author, token)
        except KeyError:
            return await self.bot.say("Error, Invalid Token")

        await self.bot.say("You no longer have a {}".format(self.emoji("Token" + token.capitalize())))

def check_folders():
    if not os.path.exists("data/trade"):
        print("Creating data/trade folder...")
        os.makedirs("data/trade")

def check_files():
    f = settings_path
    if not fileIO(f, "check"):
        print("Creating trade settings.json...")
        fileIO(f, "save", {})

def setup(bot):
    check_folders()
    check_files()
    bot.add_cog(Trade(bot))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding=utf-8
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import functools
import os, sys
import time
import numpy as np
import tensorflow as tf
import tensorflow.contrib.slim as slim
from time import gmtime, strftime

sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
import libs.configs.config_v1 as cfg
import libs.datasets.dataset_factory as datasets
import libs.nets.nets_factory as network

import libs.preprocessings.coco_v1 as coco_preprocess
import libs.nets.pyramid_network as pyramid_network
import libs.nets.resnet_v1 as resnet_v1

from train.train_utils import _configure_learning_rate, _configure_optimizer, \
  _get_variables_to_train, _get_init_fn, get_var_list_to_restore

resnet50 = resnet_v1.resnet_v1_50
FLAGS = tf.app.flags.FLAGS

def solve(global_step):
    """add solver to losses"""
    # learning reate
    lr = _configure_learning_rate(82783, global_step)
    optimizer = _configure_optimizer(lr)
    tf.summary.scalar('learning_rate', lr)

    # compute and apply gradient
    losses = tf.get_collection(tf.GraphKeys.LOSSES)
    regular_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
    regular_loss = tf.add_n(regular_losses)
    out_loss = tf.add_n(losses)
    total_loss = tf.add_n(losses + regular_losses)

    tf.summary.scalar('total_loss', total_loss)
    tf.summary.scalar('out_loss', out_loss)
    tf.summary.scalar('regular_loss', regular_loss)

    update_ops = []
    variables_to_train = _get_variables_to_train()
    # update_op = optimizer.minimize(total_loss)
    gradients = optimizer.compute_gradients(total_loss, var_list=variables_to_train)
    grad_updates = optimizer.apply_gradients(gradients,
            global_step=global_step)
    update_ops.append(grad_updates)

    # update moving mean and variance
    if FLAGS.update_bn:
        update_bns = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
        update_bn = tf.group(*update_bns)
        update_ops.append(update_bn)

    return tf.group(*update_ops)

def restore(sess):
     """choose which param to restore"""
     if FLAGS.restore_previous_if_exists:
        try:
            checkpoint_path = tf.train.latest_checkpoint(FLAGS.train_dir)
            restorer = tf.train.Saver()
            restorer.restore(sess, checkpoint_path)
            print ('restored previous model %s from %s'\
                    %(checkpoint_path, FLAGS.train_dir))
            time.sleep(2)
            return
        except:
            print ('--restore_previous_if_exists is set, but failed to restore in %s %s'\
                    % (FLAGS.train_dir, checkpoint_path))
            time.sleep(2)

     if FLAGS.pretrained_model:
        if tf.gfile.IsDirectory(FLAGS.pretrained_model):
            checkpoint_path = tf.train.latest_checkpoint(FLAGS.pretrained_model)
        else:
            checkpoint_path = FLAGS.pretrained_model

        if FLAGS.checkpoint_exclude_scopes is None:
            FLAGS.checkpoint_exclude_scopes='pyramid'
        if FLAGS.checkpoint_include_scopes is None:
            FLAGS.checkpoint_include_scopes='resnet_v1_50'

        vars_to_restore = get_var_list_to_restore()
        for var in vars_to_restore:
            print ('restoring ', var.name)

        try:
           restorer = tf.train.Saver(vars_to_restore)
           restorer.restore(sess, checkpoint_path)
           print ('Restored %d(%d) vars from %s' %(
               len(vars_to_restore), len(tf.global_variables()),
               checkpoint_path ))
        except:
           print ('Checking your params %s' %(checkpoint_path))
           raise

def train():
    """The main function that runs training"""

    ## data
    image, ih, iw, gt_boxes, gt_masks, num_instances, img_id = \
        datasets.get_dataset(FLAGS.dataset_name,
                             FLAGS.dataset_split_name,
                             FLAGS.dataset_dir,
                             FLAGS.im_batch,
                             is_training=True)

    data_queue = tf.RandomShuffleQueue(capacity=32, min_after_dequeue=16,
            dtypes=(
                image.dtype, ih.dtype, iw.dtype,
                gt_boxes.dtype, gt_masks.dtype,
                num_instances.dtype, img_id.dtype))
    enqueue_op = data_queue.enqueue((image, ih, iw, gt_boxes, gt_masks, num_instances, img_id))
    data_queue_runner = tf.train.QueueRunner(data_queue, [enqueue_op] * 4)
    tf.add_to_collection(tf.GraphKeys.QUEUE_RUNNERS, data_queue_runner)
    (image, ih, iw, gt_boxes, gt_masks, num_instances, img_id) =  data_queue.dequeue()
    im_shape = tf.shape(image)
    image = tf.reshape(image, (im_shape[0], im_shape[1], im_shape[2], 3))

    ## network
    logits, end_points, pyramid_map = network.get_network(FLAGS.network, image,
            weight_decay=FLAGS.weight_decay)
    outputs = pyramid_network.build(end_points, ih, iw, pyramid_map,
            num_classes=81,
            base_anchors=9,
            is_training=True,
            gt_boxes=gt_boxes, gt_masks=gt_masks,
            loss_weights=[0.2, 0.2, 1.0, 0.2, 1.0])

    total_loss = outputs['total_loss']
    losses  = outputs['losses']
    batch_info = outputs['batch_info']
    regular_loss = tf.add_n(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))

    ## solvers
    global_step = slim.create_global_step()
    update_op = solve(global_step)

    cropped_rois = tf.get_collection('__CROPPED__')[0]
    transposed = tf.get_collection('__TRANSPOSED__')[0]

    gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.8)
    sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
    init_op = tf.group(
            tf.global_variables_initializer(),
            tf.local_variables_initializer()
            )
    sess.run(init_op)

    summary_op = tf.summary.merge_all()
    logdir = os.path.join(FLAGS.train_dir, strftime('%Y%m%d%H%M%S', gmtime()))
    if not os.path.exists(logdir):
        os.makedirs(logdir)
    summary_writer = tf.summary.FileWriter(logdir, graph=sess.graph)

    ## restore
    restore(sess)

    ## main loop
    coord = tf.train.Coordinator()
    threads = []
    # print (tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS))
    for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
        threads.extend(qr.create_threads(sess, coord=coord, daemon=True,
                                         start=True))

    tf.train.start_queue_runners(sess=sess, coord=coord)
    saver = tf.train.Saver(max_to_keep=20)

    for step in range(FLAGS.max_iters):

        start_time = time.time()

        s_, tot_loss, reg_lossnp, img_id_str, \
        rpn_box_loss, rpn_cls_loss, refined_box_loss, refined_cls_loss, mask_loss, \
        gt_boxesnp, \
        rpn_batch_pos, rpn_batch, refine_batch_pos, refine_batch, mask_batch_pos, mask_batch = \
                     sess.run([update_op, total_loss, regular_loss, img_id] +
                              losses +
                              [gt_boxes] +
                              batch_info )

        duration_time = time.time() - start_time
        if step % 1 == 0:
            print ( """iter %d: image-id:%07d, time:%.3f(sec), regular_loss: %.6f, """
                    """total-loss %.4f(%.4f, %.4f, %.6f, %.4f, %.4f), """
                    """instances: %d, """
                    """batch:(%d|%d, %d|%d, %d|%d)"""
                   % (step, img_id_str, duration_time, reg_lossnp,
                      tot_loss, rpn_box_loss, rpn_cls_loss, refined_box_loss, refined_cls_loss, mask_loss,
                      gt_boxesnp.shape[0],
                      rpn_batch_pos, rpn_batch, refine_batch_pos, refine_batch, mask_batch_pos, mask_batch))

            if np.isnan(tot_loss) or np.isinf(tot_loss):
                print (gt_boxesnp)
                raise

        if step % 100 == 0:
            summary_str = sess.run(summary_op)
            summary_writer.add_summary(summary_str, step)
            summary_writer.flush()

        if (step % 10000 == 0 or step + 1 == FLAGS.max_iters) and step != 0:
            checkpoint_path = os.path.join(FLAGS.train_dir,
                                           FLAGS.dataset_name + '_' + FLAGS.network + '_model.ckpt')
            saver.save(sess, checkpoint_path, global_step=step)

        if coord.should_stop():
            coord.request_stop()
            coord.join(threads)

if __name__ == '__main__':
    train()
<commit_msg>Passed height and width params should be of the prepeocessed image. issue #48<commit_after>#!/usr/bin/env python
# coding=utf-8
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import functools
import os, sys
import time
import numpy as np
import tensorflow as tf
import tensorflow.contrib.slim as slim
from time import gmtime, strftime

sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
import libs.configs.config_v1 as cfg
import libs.datasets.dataset_factory as datasets
import libs.nets.nets_factory as network

import libs.preprocessings.coco_v1 as coco_preprocess
import libs.nets.pyramid_network as pyramid_network
import libs.nets.resnet_v1 as resnet_v1

from train.train_utils import _configure_learning_rate, _configure_optimizer, \
  _get_variables_to_train, _get_init_fn, get_var_list_to_restore

resnet50 = resnet_v1.resnet_v1_50
FLAGS = tf.app.flags.FLAGS

def solve(global_step):
    """add solver to losses"""
    # learning reate
    lr = _configure_learning_rate(82783, global_step)
    optimizer = _configure_optimizer(lr)
    tf.summary.scalar('learning_rate', lr)

    # compute and apply gradient
    losses = tf.get_collection(tf.GraphKeys.LOSSES)
    regular_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
    regular_loss = tf.add_n(regular_losses)
    out_loss = tf.add_n(losses)
    total_loss = tf.add_n(losses + regular_losses)

    tf.summary.scalar('total_loss', total_loss)
    tf.summary.scalar('out_loss', out_loss)
    tf.summary.scalar('regular_loss', regular_loss)

    update_ops = []
    variables_to_train = _get_variables_to_train()
    # update_op = optimizer.minimize(total_loss)
    gradients = optimizer.compute_gradients(total_loss, var_list=variables_to_train)
    grad_updates = optimizer.apply_gradients(gradients,
            global_step=global_step)
    update_ops.append(grad_updates)

    # update moving mean and variance
    if FLAGS.update_bn:
        update_bns = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
        update_bn = tf.group(*update_bns)
        update_ops.append(update_bn)

    return tf.group(*update_ops)

def restore(sess):
     """choose which param to restore"""
     if FLAGS.restore_previous_if_exists:
        try:
            checkpoint_path = tf.train.latest_checkpoint(FLAGS.train_dir)
            restorer = tf.train.Saver()
            restorer.restore(sess, checkpoint_path)
            print ('restored previous model %s from %s'\
                    %(checkpoint_path, FLAGS.train_dir))
            time.sleep(2)
            return
        except:
            print ('--restore_previous_if_exists is set, but failed to restore in %s %s'\
                    % (FLAGS.train_dir, checkpoint_path))
            time.sleep(2)

     if FLAGS.pretrained_model:
        if tf.gfile.IsDirectory(FLAGS.pretrained_model):
            checkpoint_path = tf.train.latest_checkpoint(FLAGS.pretrained_model)
        else:
            checkpoint_path = FLAGS.pretrained_model

        if FLAGS.checkpoint_exclude_scopes is None:
            FLAGS.checkpoint_exclude_scopes='pyramid'
        if FLAGS.checkpoint_include_scopes is None:
            FLAGS.checkpoint_include_scopes='resnet_v1_50'

        vars_to_restore = get_var_list_to_restore()
        for var in vars_to_restore:
            print ('restoring ', var.name)

        try:
           restorer = tf.train.Saver(vars_to_restore)
           restorer.restore(sess, checkpoint_path)
           print ('Restored %d(%d) vars from %s' %(
               len(vars_to_restore), len(tf.global_variables()),
               checkpoint_path ))
        except:
           print ('Checking your params %s' %(checkpoint_path))
           raise

def train():
    """The main function that runs training"""

    ## data
    image, ih, iw, gt_boxes, gt_masks, num_instances, img_id = \
        datasets.get_dataset(FLAGS.dataset_name,
                             FLAGS.dataset_split_name,
                             FLAGS.dataset_dir,
                             FLAGS.im_batch,
                             is_training=True)

    data_queue = tf.RandomShuffleQueue(capacity=32, min_after_dequeue=16,
            dtypes=(
                image.dtype, ih.dtype, iw.dtype,
                gt_boxes.dtype, gt_masks.dtype,
                num_instances.dtype, img_id.dtype))
    enqueue_op = data_queue.enqueue((image, ih, iw, gt_boxes, gt_masks, num_instances, img_id))
    data_queue_runner = tf.train.QueueRunner(data_queue, [enqueue_op] * 4)
    tf.add_to_collection(tf.GraphKeys.QUEUE_RUNNERS, data_queue_runner)
    (image, ih, iw, gt_boxes, gt_masks, num_instances, img_id) =  data_queue.dequeue()
    im_shape = tf.shape(image)
    image = tf.reshape(image, (im_shape[0], im_shape[1], im_shape[2], 3))

    ## network
    logits, end_points, pyramid_map = network.get_network(FLAGS.network, image,
            weight_decay=FLAGS.weight_decay)
    outputs = pyramid_network.build(end_points, im_shape[1], im_shape[2], pyramid_map,
            num_classes=81,
            base_anchors=9,
            is_training=True,
            gt_boxes=gt_boxes, gt_masks=gt_masks,
            loss_weights=[0.2, 0.2, 1.0, 0.2, 1.0])

    total_loss = outputs['total_loss']
    losses  = outputs['losses']
    batch_info = outputs['batch_info']
    regular_loss = tf.add_n(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))

    ## solvers
    global_step = slim.create_global_step()
    update_op = solve(global_step)

    cropped_rois = tf.get_collection('__CROPPED__')[0]
    transposed = tf.get_collection('__TRANSPOSED__')[0]

    gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.8)
    sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
    init_op = tf.group(
            tf.global_variables_initializer(),
            tf.local_variables_initializer()
            )
    sess.run(init_op)

    summary_op = tf.summary.merge_all()
    logdir = os.path.join(FLAGS.train_dir, strftime('%Y%m%d%H%M%S', gmtime()))
    if not os.path.exists(logdir):
        os.makedirs(logdir)
    summary_writer = tf.summary.FileWriter(logdir, graph=sess.graph)

    ## restore
    restore(sess)

    ## main loop
    coord = tf.train.Coordinator()
    threads = []
    # print (tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS))
    for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
        threads.extend(qr.create_threads(sess, coord=coord, daemon=True,
                                         start=True))

    tf.train.start_queue_runners(sess=sess, coord=coord)
    saver = tf.train.Saver(max_to_keep=20)

    for step in range(FLAGS.max_iters):

        start_time = time.time()

        s_, tot_loss, reg_lossnp, img_id_str, \
        rpn_box_loss, rpn_cls_loss, refined_box_loss, refined_cls_loss, mask_loss, \
        gt_boxesnp, \
        rpn_batch_pos, rpn_batch, refine_batch_pos, refine_batch, mask_batch_pos, mask_batch = \
                     sess.run([update_op, total_loss, regular_loss, img_id] +
                              losses +
                              [gt_boxes] +
                              batch_info )

        duration_time = time.time() - start_time
        if step % 1 == 0:
            print ( """iter %d: image-id:%07d, time:%.3f(sec), regular_loss: %.6f, """
                    """total-loss %.4f(%.4f, %.4f, %.6f, %.4f, %.4f), """
                    """instances: %d, """
                    """batch:(%d|%d, %d|%d, %d|%d)"""
                   % (step, img_id_str, duration_time, reg_lossnp,
                      tot_loss, rpn_box_loss, rpn_cls_loss, refined_box_loss, refined_cls_loss, mask_loss,
                      gt_boxesnp.shape[0],
                      rpn_batch_pos, rpn_batch, refine_batch_pos, refine_batch, mask_batch_pos, mask_batch))

            if np.isnan(tot_loss) or np.isinf(tot_loss):
                print (gt_boxesnp)
                raise

        if step % 100 == 0:
            summary_str = sess.run(summary_op)
            summary_writer.add_summary(summary_str, step)
            summary_writer.flush()

        if (step % 10000 == 0 or step + 1 == FLAGS.max_iters) and step != 0:
            checkpoint_path = os.path.join(FLAGS.train_dir,
                                           FLAGS.dataset_name + '_' + FLAGS.network + '_model.ckpt')
            saver.save(sess, checkpoint_path, global_step=step)

        if coord.should_stop():
            coord.request_stop()
            coord.join(threads)

if __name__ == '__main__':
    train()
<|endoftext|>


<commit_before>import argparse

from blocks.algorithms import GradientDescent, Adam, CompositeRule, StepClipping
from blocks.extensions import Timing, Printing, FinishAfter, ProgressBar
from blocks.extensions.monitoring import TrainingDataMonitoring
from blocks.extensions.training import SharedVariableModifier
from blocks.main_loop import MainLoop
from blocks.monitoring import aggregation
from blocks.monitoring.evaluators import AggregationBuffer
from fuel.datasets.hdf5 import H5PYDataset
from fuel.schemes import SequentialScheme, ShuffledScheme
from fuel.streams import DataStream
from numpy import load, array
from theano import function

from custom_blocks import PadAndAddMasks, EarlyStopping
from network import *

from prepare_lk import ALPHABET_FILE, OUT_FILE_NAME as DATA_FILE_LOC
from util import StateComputer

parser = argparse.ArgumentParser()
parser.add_argument("-c", "--clipping", default=1.0, type=float, help="Step clipping threshold (default: 1)")
parser.add_argument("-t", "--type", default="simple", help="Type of RNN ("+NetworkType.SIMPLE_RNN+", " +
                                                           NetworkType.GRU+" or "+NetworkType.LSTM+")")
parser.add_argument("-e", "--epochs", default=0, type=int, help="Stop after this many epochs (default: 0 -- means it will" +
                                                                " run forever until you cancel manually)")
parser.add_argument("-d", "--dimensions", default="512,512,512", type=str, help="Configure number of layers and dimensions by" +
                                                                                " just enumerating dimensions (e.g. 100, 100 for" +
                                                                                " a two-layered network with dims 100)")
args = parser.parse_args()

dimensions = [int(d) for d in args.dimensions.split(',')]
print('Dimensions:', dimensions)

nkwargs = {'network_type': args.type}
if dimensions:
    nkwargs['hidden_dims'] = dimensions

network = Network(**nkwargs)

cross_ent = network.cost
generator = network.generator
cost_model = network.cost_model

# do init_state stuff
initial_states = network.initial_states

# The thing that I feed into the parameters argument I copied from some blocks-examples thing. the good old computation
# graph and then cg.parameters should work as well.
# On the step rule: This means gradient clipping (threshold passed in as a command line argument) followed by Adam
# performed on the clipped gradient.
# If you don't know what gradient clipping is: Define threshold T; if the length of the gradient is > T, scale it such
# that its length is equal to T. This serves to make exploding gradients less severe.
algorithm = GradientDescent(cost=cross_ent, parameters=cost_model.parameters,
                            step_rule=CompositeRule(components=[StepClipping(threshold=args.clipping), Adam()]),
                            on_unused_sources="ignore")

# data
train_data = H5PYDataset(DATA_FILE_LOC, which_sets=("train",), load_in_memory=True)
valid_data = H5PYDataset(DATA_FILE_LOC, which_sets=("valid",), load_in_memory=True)

# see custom_blocks for the transformer
data_stream = PadAndAddMasks(DataStream.default_stream(dataset=train_data, iteration_scheme=ShuffledScheme(train_data.num_examples,
                                                                                batch_size=1)), produces_examples=False)
data_stream_valid = PadAndAddMasks(DataStream.default_stream(dataset=valid_data, iteration_scheme=SequentialScheme(valid_data.num_examples,
                                                                                batch_size=1)), produces_examples=False)

# monitor:
# - training cost every 200 batches (computed along the way, so cheap to do), as well as gradient and step lengths to
#   detect exploding gradient problems
# - validation cost once every epoch

char2ix = load(ALPHABET_FILE).item()
ix2char = {}
for k, v in char2ix.items():
    ix2char[v] = k

sc = StateComputer(network.cost_model, ix2char)
state_to_compare = list(filter(lambda x: x.name == 'sequencegenerator_cost_matrix_states#2', sc.state_variables))[0]  # notice: python2 filter seems to return a list, but anyway

def modifier_function(iterations_done):
    aggr = AggregationBuffer(state_to_compare, use_take_last=True)
    values = aggr.get_aggregated_values()
    print('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>', values[state_to_compare.name][0][-1][0])
    return values[state_to_compare.name][0][-1][0]

init_state_modifier = SharedVariableModifier(initial_states[2], num_args=1, function=modifier_function)

monitor_grad = TrainingDataMonitoring(variables=[cross_ent, aggregation.mean(algorithm.total_gradient_norm),
                                                 aggregation.mean(algorithm.total_step_norm), initial_states[2], state_to_compare], after_epoch=True,
                                      prefix="training")
#monitor_init_states = TrainingDataMonitoring(variables=[network.initial_states[2]], after_epoch=True, prefix='training')

early_stopping = EarlyStopping(variables=[cross_ent], data_stream=data_stream_valid,
                               path="seqgen_" + args.type + "_" + "_".join([str(d) for d in network.hidden_dims]) + ".pkl",
                               tolerance=4, prefix="validation")

main_loop = MainLoop(algorithm=algorithm, data_stream=data_stream, model=cost_model,
                     extensions=[init_state_modifier, early_stopping, monitor_grad, FinishAfter(after_n_epochs=args.epochs), ProgressBar(),
                                 Timing(), Printing()])

main_loop.run()
<commit_msg>try to fix error cause<commit_after>import argparse

from blocks.algorithms import GradientDescent, Adam, CompositeRule, StepClipping
from blocks.extensions import Timing, Printing, FinishAfter, ProgressBar
from blocks.extensions.monitoring import TrainingDataMonitoring
from blocks.extensions.training import SharedVariableModifier
from blocks.main_loop import MainLoop
from blocks.monitoring import aggregation
from blocks.monitoring.evaluators import AggregationBuffer
from fuel.datasets.hdf5 import H5PYDataset
from fuel.schemes import SequentialScheme, ShuffledScheme
from fuel.streams import DataStream
from numpy import load, array
from theano import function

from custom_blocks import PadAndAddMasks, EarlyStopping
from network import *

from prepare_lk import ALPHABET_FILE, OUT_FILE_NAME as DATA_FILE_LOC
from util import StateComputer

parser = argparse.ArgumentParser()
parser.add_argument("-c", "--clipping", default=1.0, type=float, help="Step clipping threshold (default: 1)")
parser.add_argument("-t", "--type", default="simple", help="Type of RNN ("+NetworkType.SIMPLE_RNN+", " +
                                                           NetworkType.GRU+" or "+NetworkType.LSTM+")")
parser.add_argument("-e", "--epochs", default=0, type=int, help="Stop after this many epochs (default: 0 -- means it will" +
                                                                " run forever until you cancel manually)")
parser.add_argument("-d", "--dimensions", default="512,512,512", type=str, help="Configure number of layers and dimensions by" +
                                                                                " just enumerating dimensions (e.g. 100, 100 for" +
                                                                                " a two-layered network with dims 100)")
args = parser.parse_args()

dimensions = [int(d) for d in args.dimensions.split(',')]
print('Dimensions:', dimensions)

nkwargs = {'network_type': args.type}
if dimensions:
    nkwargs['hidden_dims'] = dimensions

network = Network(**nkwargs)

cross_ent = network.cost
generator = network.generator
cost_model = network.cost_model

# do init_state stuff
initial_states = network.initial_states

# The thing that I feed into the parameters argument I copied from some blocks-examples thing. the good old computation
# graph and then cg.parameters should work as well.
# On the step rule: This means gradient clipping (threshold passed in as a command line argument) followed by Adam
# performed on the clipped gradient.
# If you don't know what gradient clipping is: Define threshold T; if the length of the gradient is > T, scale it such
# that its length is equal to T. This serves to make exploding gradients less severe.
algorithm = GradientDescent(cost=cross_ent, parameters=cost_model.parameters,
                            step_rule=CompositeRule(components=[StepClipping(threshold=args.clipping), Adam()]),
                            on_unused_sources="ignore")

# data
train_data = H5PYDataset(DATA_FILE_LOC, which_sets=("train",), load_in_memory=True)
valid_data = H5PYDataset(DATA_FILE_LOC, which_sets=("valid",), load_in_memory=True)

# see custom_blocks for the transformer
data_stream = PadAndAddMasks(DataStream.default_stream(dataset=train_data, iteration_scheme=ShuffledScheme(train_data.num_examples,
                                                                                batch_size=1)), produces_examples=False)
data_stream_valid = PadAndAddMasks(DataStream.default_stream(dataset=valid_data, iteration_scheme=SequentialScheme(valid_data.num_examples,
                                                                                batch_size=1)), produces_examples=False)

# monitor:
# - training cost every 200 batches (computed along the way, so cheap to do), as well as gradient and step lengths to
#   detect exploding gradient problems
# - validation cost once every epoch

char2ix = load(ALPHABET_FILE).item()
ix2char = {}
for k, v in char2ix.items():
    ix2char[v] = k

sc = StateComputer(network.cost_model, ix2char)
state_to_compare = list(filter(lambda x: x.name == 'sequencegenerator_cost_matrix_states#2', sc.state_variables))[0]  # notice: python2 filter seems to return a list, but anyway

def modifier_function(iterations_done):
    aggr = AggregationBuffer(state_to_compare, use_take_last=True)
    values = aggr.get_aggregated_values()
    print('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>', values[state_to_compare.name])
    return values[state_to_compare.name][0][-1][0]

init_state_modifier = SharedVariableModifier(initial_states[2], num_args=1, function=modifier_function)

monitor_grad = TrainingDataMonitoring(variables=[cross_ent, aggregation.mean(algorithm.total_gradient_norm),
                                                 aggregation.mean(algorithm.total_step_norm), initial_states[2], state_to_compare], after_epoch=True,
                                      prefix="training")
#monitor_init_states = TrainingDataMonitoring(variables=[network.initial_states[2]], after_epoch=True, prefix='training')

early_stopping = EarlyStopping(variables=[cross_ent], data_stream=data_stream_valid,
                               path="seqgen_" + args.type + "_" + "_".join([str(d) for d in network.hidden_dims]) + ".pkl",
                               tolerance=4, prefix="validation")

main_loop = MainLoop(algorithm=algorithm, data_stream=data_stream, model=cost_model,
                     extensions=[init_state_modifier, early_stopping, monitor_grad, FinishAfter(after_n_epochs=args.epochs), ProgressBar(),
                                 Timing(), Printing()])

main_loop.run()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Exceptions for vcspull.

vcspull.exc
~~~~~~~~~~~

"""

from __future__ import absolute_import, division, print_function, \
    with_statement, unicode_literals

class VCSPullException(Exception):

    """Standard VCSPullException."""

    pass

class NoConfigsFound(VCSPullException):
    message = (
        'No config file found. Create a .vcspull.yaml or .vcspull.json'
        ' in your $HOME directory. http://vcspull.rtfd.org for a'
        ' quickstart.'
    )

class MultipleRootConfigs(VCSPullException):
    message = (
        'Multiple configs found in home directory use only one.'
        ' .yaml, .json.'
    )
<commit_msg>rm newline<commit_after># -*- coding: utf-8 -*-
"""Exceptions for vcspull.

vcspull.exc
~~~~~~~~~~~

"""

from __future__ import absolute_import, division, print_function, \
    with_statement, unicode_literals

class VCSPullException(Exception):

    """Standard VCSPullException."""

    pass

class NoConfigsFound(VCSPullException):
    message = (
        'No config file found. Create a .vcspull.yaml or .vcspull.json'
        ' in your $HOME directory. http://vcspull.rtfd.org for a'
        ' quickstart.'
    )

class MultipleRootConfigs(VCSPullException):
    message = (
        'Multiple configs found in home directory use only one.'
        ' .yaml, .json.'
    )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import itertools
import logging
import re
from codecs import open
from collections import namedtuple
from glob import glob
from os.path import basename

import pendulum

from tsstats import events
from tsstats.client import Clients

re_log_filename = re.compile(r'ts3server_(?P<date>\d{4}-\d\d-\d\d)'
                             '__(?P<time>\d\d_\d\d_\d\d.\d+)_(?P<sid>\d).log')
re_log_entry = re.compile('(?P<timestamp>\d{4}-\d\d-\d\d\ \d\d:\d\d:\d\d.\d+)'
                          '\|\ *(?P<level>\w+)\ *\|\ *(?P<component>\w+)\ *'
                          '\|\ *(?P<sid>\d+)\ *\|\ *(?P<message>.*)')
re_dis_connect = re.compile(
    r"client (?P<action>(dis)?connected) '(?P<nick>.*)'\(id:(?P<clid>\d+)\)")
re_disconnect_invoker = re.compile(
    r'invokername=(.*)\ invokeruid=(.*)\ reasonmsg'
)

TimedLog = namedtuple('TimedLog', ['path', 'timestamp'])
Server = namedtuple('Server', ['sid', 'clients'])

logger = logging.getLogger('tsstats')

def _bundle_logs(logs):
    '''
    bundle `logs` by virtualserver-id
    and sort by timestamp from filename (if exists)

    :param logs: list of paths to logfiles

    :type logs: list

    :return: `logs` bundled by virtualserver-id and sorted by timestamp
    :rtype: dict{str: [TimedLog]}
    '''
    vserver_logfiles = {}  # sid: [/path/to/log1, ..., /path/to/logn]
    for log in logs:
        # try to get date and sid from filename
        match = re_log_filename.match(basename(log))
        if match:
            match = match.groupdict()
            timestamp = pendulum.parse('{0} {1}'.format(
                match['date'], match['time'].replace('_', ':'))
            )
            tl = TimedLog(log, timestamp)
            sid = match['sid']
            if sid in vserver_logfiles:
                # if already exists, keep list sorted by timestamp
                vserver_logfiles[sid].append(tl)
                vserver_logfiles[sid] =\
                    sorted(vserver_logfiles[sid],
                           key=lambda tl: tl.timestamp)
            else:
                # if not exists, just create a list
                vserver_logfiles[sid] = [tl]
        else:
            # fallback to plain sorting
            vserver_logfiles.setdefault('', [])\
                .append(TimedLog(log, None))
            vserver_logfiles[''] =\
                sorted(vserver_logfiles[''],
                       key=lambda tl: tl.path)
    return vserver_logfiles

def _parse_line(line):
    parsed_events = []
    match = re_log_entry.match(line)
    if not match:
        logger.debug('No match: "%s"', line)
        return []
    match = match.groupdict()
    logdatetime = pendulum.parse(match['timestamp'])
    message = match['message']
    if message.startswith('client'):
        match = re_dis_connect.match(message)
        if not match:
            logger.debug('Unsupported client action: "%s"', message)
            return []
        nick, clid = match.group('nick'), match.group('clid')

        parsed_events.append(events.nick(logdatetime, clid, nick))

        action = match.group('action')
        if action == 'connected':
            parsed_events.append(events.connect(logdatetime, clid))
        elif action == 'disconnected':
            parsed_events.append(events.disconnect(logdatetime, clid))
            if 'invokeruid' in message:
                re_disconnect_data = re_disconnect_invoker.findall(
                    message)
                invokernick, invokeruid = re_disconnect_data[0]
                parsed_events.append(
                    events.nick(logdatetime, invokeruid, invokernick)
                )
                if 'bantime' in message:
                    parsed_events.append(
                        events.ban(logdatetime, invokeruid, clid)
                    )
                else:
                    parsed_events.append(
                        events.kick(logdatetime, invokeruid, clid)
                    )
    return parsed_events

def parse_logs(log_glob, ident_map=None, online_dc=True, *args, **kwargs):
    '''
    parse logs from `log_glob`

    :param log_glob: path to server-logs (supports globbing)
    :param ident_map: identmap used for Client-initializations

    :type log_glob: str
    :type ident_map: dict
    '''
    for virtualserver_id, logs in _bundle_logs(glob(log_glob)).items():
        clients = Clients(ident_map)
        for index, log in enumerate(logs):
            with open(log.path, encoding='utf-8') as f:
                # parse logfile line by line and filter lines without events
                events = filter(None, map(_parse_line, f))
                # chain apply events to Client-obj
                clients.apply_events(itertools.chain.from_iterable(events))

                # find connected clients
                online_clients = list(filter(lambda c: c.connected, clients))

                logger.debug(
                    'Some clients are still connected: %s', online_clients
                )
                if index == len(logs) - 1 and online_dc:
                    logger.debug('Last log => disconnecting online clients')
                    # last iteration => disconnect online clients if desired
                    for online_client in online_clients:
                        online_client.disconnect(pendulum.utcnow())
                        online_client.connected += 1
        if len(clients) >= 1:
            # assemble Server-obj and yield
            yield Server(virtualserver_id, clients)
<commit_msg>Log start/end of parsing logfile<commit_after># -*- coding: utf-8 -*-
import itertools
import logging
import re
from codecs import open
from collections import namedtuple
from glob import glob
from os.path import basename

import pendulum

from tsstats import events
from tsstats.client import Clients

re_log_filename = re.compile(r'ts3server_(?P<date>\d{4}-\d\d-\d\d)'
                             '__(?P<time>\d\d_\d\d_\d\d.\d+)_(?P<sid>\d).log')
re_log_entry = re.compile('(?P<timestamp>\d{4}-\d\d-\d\d\ \d\d:\d\d:\d\d.\d+)'
                          '\|\ *(?P<level>\w+)\ *\|\ *(?P<component>\w+)\ *'
                          '\|\ *(?P<sid>\d+)\ *\|\ *(?P<message>.*)')
re_dis_connect = re.compile(
    r"client (?P<action>(dis)?connected) '(?P<nick>.*)'\(id:(?P<clid>\d+)\)")
re_disconnect_invoker = re.compile(
    r'invokername=(.*)\ invokeruid=(.*)\ reasonmsg'
)

TimedLog = namedtuple('TimedLog', ['path', 'timestamp'])
Server = namedtuple('Server', ['sid', 'clients'])

logger = logging.getLogger('tsstats')

def _bundle_logs(logs):
    '''
    bundle `logs` by virtualserver-id
    and sort by timestamp from filename (if exists)

    :param logs: list of paths to logfiles

    :type logs: list

    :return: `logs` bundled by virtualserver-id and sorted by timestamp
    :rtype: dict{str: [TimedLog]}
    '''
    vserver_logfiles = {}  # sid: [/path/to/log1, ..., /path/to/logn]
    for log in logs:
        # try to get date and sid from filename
        match = re_log_filename.match(basename(log))
        if match:
            match = match.groupdict()
            timestamp = pendulum.parse('{0} {1}'.format(
                match['date'], match['time'].replace('_', ':'))
            )
            tl = TimedLog(log, timestamp)
            sid = match['sid']
            if sid in vserver_logfiles:
                # if already exists, keep list sorted by timestamp
                vserver_logfiles[sid].append(tl)
                vserver_logfiles[sid] =\
                    sorted(vserver_logfiles[sid],
                           key=lambda tl: tl.timestamp)
            else:
                # if not exists, just create a list
                vserver_logfiles[sid] = [tl]
        else:
            # fallback to plain sorting
            vserver_logfiles.setdefault('', [])\
                .append(TimedLog(log, None))
            vserver_logfiles[''] =\
                sorted(vserver_logfiles[''],
                       key=lambda tl: tl.path)
    return vserver_logfiles

def _parse_line(line):
    parsed_events = []
    match = re_log_entry.match(line)
    if not match:
        logger.debug('No match: "%s"', line)
        return []
    match = match.groupdict()
    logdatetime = pendulum.parse(match['timestamp'])
    message = match['message']
    if message.startswith('client'):
        match = re_dis_connect.match(message)
        if not match:
            logger.debug('Unsupported client action: "%s"', message)
            return []
        nick, clid = match.group('nick'), match.group('clid')

        parsed_events.append(events.nick(logdatetime, clid, nick))

        action = match.group('action')
        if action == 'connected':
            parsed_events.append(events.connect(logdatetime, clid))
        elif action == 'disconnected':
            parsed_events.append(events.disconnect(logdatetime, clid))
            if 'invokeruid' in message:
                re_disconnect_data = re_disconnect_invoker.findall(
                    message)
                invokernick, invokeruid = re_disconnect_data[0]
                parsed_events.append(
                    events.nick(logdatetime, invokeruid, invokernick)
                )
                if 'bantime' in message:
                    parsed_events.append(
                        events.ban(logdatetime, invokeruid, clid)
                    )
                else:
                    parsed_events.append(
                        events.kick(logdatetime, invokeruid, clid)
                    )
    return parsed_events

def parse_logs(log_glob, ident_map=None, online_dc=True, *args, **kwargs):
    '''
    parse logs from `log_glob`

    :param log_glob: path to server-logs (supports globbing)
    :param ident_map: identmap used for Client-initializations

    :type log_glob: str
    :type ident_map: dict
    '''
    for virtualserver_id, logs in _bundle_logs(glob(log_glob)).items():
        clients = Clients(ident_map)
        for index, log in enumerate(logs):
            with open(log.path, encoding='utf-8') as f:
                logger.debug('Started parsing of %s', f.name)
                # parse logfile line by line and filter lines without events
                events = filter(None, map(_parse_line, f))
                # chain apply events to Client-obj
                clients.apply_events(itertools.chain.from_iterable(events))

                # find connected clients
                online_clients = list(filter(lambda c: c.connected, clients))

                logger.debug(
                    'Some clients are still connected: %s', online_clients
                )
                if index == len(logs) - 1 and online_dc:
                    logger.debug('Last log => disconnecting online clients')
                    # last iteration => disconnect online clients if desired
                    for online_client in online_clients:
                        online_client.disconnect(pendulum.utcnow())
                        online_client.connected += 1

                logger.debug('Finished parsing of %s', f.name)
        if len(clients) >= 1:
            # assemble Server-obj and yield
            yield Server(virtualserver_id, clients)
<|endoftext|>


<commit_before>"""
This module contains gopher-related stuff.
"""
import getpass
import os

from . import util

GOPHER_PROMPT = """
Would you like to publish your feels to gopher?

gopher is a pre-web technology that is text-oriented and primarily used to
share folders of your files with the world.

If you don't know what it is or don't want it that is totally ok!

You can always change this later.""".lstrip()

GOPHERMAP_HEADER = """
 welcome to {user}'s feels on gopher.

     .::                     .::
   .:                        .::
 .:.: .:   .::       .::     .:: .::::
   .::   .:   .::  .:   .::  .::.::
   .::  .::::: .::.::::: .:: .::  .:::
   .::  .:        .:         .::    .::
   .::    .::::     .::::   .:::.:: .::

 this file was created on their behalf by ttbp.

"""

def select_gopher():
    return util.input_yn(GOPHER_PROMPT)

def publish_gopher(gopher_path, entry_filenames):
    """This function (re)generates a user's list of feels posts in their gopher
    directory and their gophermap."""
    entry_filenames = entry_filenames[:]  # force a copy since this might be shared state in core.py
    entry_filenames.reverse()
    ttbp_gopher = os.path.join(
        os.path.expanduser('~/public_gopher'),
        gopher_path)

    if not os.path.isdir(ttbp_gopher):
        print('\n\tERROR: something is wrong. your gopher directory is missing. re-enable gopher publishing.')
        return

    with open(os.path.join(ttbp_gopher, 'gophermap'), 'w') as gophermap:
        gophermap.write(GOPHERMAP_HEADER.format(
                        user=getpass.getuser()))
        for entry_filename in entry_filenames:
            filename = os.path.basename(entry_filename)

            with open(os.path.join(ttbp_gopher, filename), 'w') as gopher_entry:
                with open(entry_filename, 'r') as source_entry:
                    gopher_entry.write(source_entry.read())
            gophermap.write('0{file_label}\t{filename}'.format(
                file_label=os.path.basename(entry_filename),
                filename=filename))

def setup_gopher(gopher_path):
    """Given a path relative to ~/public_gopher, this function:

    - creates a directory under public_gopher
    - creates a landing page

    It doesn't create a gophermap as that is left to the publish_gopher
    function.
    """
    public_gopher = os.path.expanduser('~/public_gopher')
    if not os.path.isdir(public_gopher):
        print("\n\tERROR: you don't seem to have gopher set up (no public_gopher directory)")
        return

    ttbp_gopher = os.path.join(public_gopher, gopher_path)
    if os.path.isdir(ttbp_gopher):
        print("\n\tERROR: gopher path is already set up. quitting so we don't overwrite anything.")
        return

    os.makedirs(ttbp_gopher)
<commit_msg>sprucing up gophermap printing<commit_after>"""
This module contains gopher-related stuff.
"""
import getpass
import os
import time

from . import util
from .core import parse_date

GOPHER_PROMPT = """
Would you like to publish your feels to gopher?

gopher is a pre-web technology that is text-oriented and primarily used to
share folders of your files with the world.

If you don't know what it is or don't want it that is totally ok!

You can always change this later.""".lstrip()

GOPHERMAP_HEADER = """
 welcome to {user}'s feels on gopher.

     .::                     .::
   .:                        .::
 .:.: .:   .::       .::     .:: .::::
   .::   .:   .::  .:   .::  .::.::
   .::  .::::: .::.::::: .:: .::  .:::
   .::  .:        .:         .::    .::
   .::    .::::     .::::   .:::.:: .::

 this file was created on their behalf by ttbp.

0(about ttbp)\t/~endorphant/ttbp.txt\ttilde.town\t70
1(back to user's home)\t/~{user}

 entries:

"""

def select_gopher():
    return util.input_yn(GOPHER_PROMPT)

def publish_gopher(gopher_path, entry_filenames):
    """This function (re)generates a user's list of feels posts in their gopher
    directory and their gophermap."""
    entry_filenames = entry_filenames[:]  # force a copy since this might be shared state in core.py
    #entry_filenames.reverse()
    ttbp_gopher = os.path.join(
        os.path.expanduser('~/public_gopher'),
        gopher_path)

    if not os.path.isdir(ttbp_gopher):
        print('\n\tERROR: something is wrong. your gopher directory is missing. re-enable gopher publishing.')
        return

    with open(os.path.join(ttbp_gopher, 'gophermap'), 'w') as gophermap:
        gophermap.write(GOPHERMAP_HEADER.format(
                        user=getpass.getuser()))
        for entry_filename in entry_filenames:
            filename = os.path.basename(entry_filename)

            with open(os.path.join(ttbp_gopher, filename), 'w') as gopher_entry:
                with open(entry_filename, 'r') as source_entry:
                    gopher_entry.write(source_entry.read())

            #label = time.strftime("%Y-%m-%d at %H:%M", time.localtime(os.path.getmtime(entry_filename)))
            label = "-".join(parse_date(entry_filename))
            gophermap.write('0{file_label}\t{filename}\n'.format(
                file_label=label,
                filename=filename))

def setup_gopher(gopher_path):
    """Given a path relative to ~/public_gopher, this function:

    - creates a directory under public_gopher
    - creates a landing page

    It doesn't create a gophermap as that is left to the publish_gopher
    function.
    """
    public_gopher = os.path.expanduser('~/public_gopher')
    if not os.path.isdir(public_gopher):
        print("\n\tERROR: you don't seem to have gopher set up (no public_gopher directory)")
        return

    ttbp_gopher = os.path.join(public_gopher, gopher_path)
    if os.path.isdir(ttbp_gopher):
        print("\n\tERROR: gopher path is already set up. quitting so we don't overwrite anything.")
        return

    os.makedirs(ttbp_gopher)
<|endoftext|>


<commit_before>import requests
from requests_oauthlib import OAuth1

from . import __version__
from .compat import json, urlencode, parse_qsl
from .exceptions import TumblpyError, TumblpyAuthError
from .helpers import _split_params_and_files

class Tumblpy(object):
    def __init__(self, app_key=None, app_secret=None, oauth_token=None,
                 oauth_token_secret=None, headers=None, proxies=None):

        # Define some API URLs real quick
        self.base_api_url = 'https://api.tumblr.com'
        self.api_version = 'v2'
        self.api_url = '%s/%s/' % (self.base_api_url, self.api_version)

        # Authentication URLs
        self.request_token_url = 'https://www.tumblr.com/oauth/request_token'
        self.access_token_url = 'https://www.tumblr.com/oauth/access_token'
        self.authorize_url = 'https://www.tumblr.com/oauth/authorize'
        self.authenticate_url = 'https://www.tumblr.com/oauth/authorize'

        self.default_params = {'api_key': app_key}

        req_headers = {'User-Agent': 'Tumblpy v' + __version__}
        if headers:
            req_headers.update(headers)

        self.app_key = app_key
        self.app_secret = app_secret
        self.oauth_token = oauth_token
        self.oauth_token_secret = oauth_token_secret

        auth = None
        if self.app_key and self.app_secret:
            if not self.oauth_token and not self.oauth_token_secret:
                auth = OAuth1(self.app_key, self.app_secret)
            else:
                auth = OAuth1(self.app_key, self.app_secret,
                              self.oauth_token, self.oauth_token_secret)

        self.client = requests.Session()
        self.client.proxies = proxies
        self.client.headers = req_headers
        self.client.auth = auth

    def get_authentication_tokens(self, callback_url=None):
        """Returns a dict including an authorization URL (auth_url) to direct a user to

            :param callback_url: (optional) Url the user is returned to after they authorize your app (web clients only)
        """

        request_args = {}
        if callback_url:
            request_args['oauth_callback'] = callback_url

        response = self.client.get(self.request_token_url, params=request_args)

        if response.status_code != 200:
            raise TumblpyAuthError('Seems something couldn\'t be verified with your OAuth junk. Error: %s, Message: %s' % (response.status_code, response.content))

        res = response.content
        if isinstance( response.content, bytes ):
            res = res.decode()

        request_tokens = dict(parse_qsl(res))

        if not request_tokens:
            raise TumblpyError('Unable to decode request tokens.')

        auth_url_params = {
            'oauth_token': request_tokens['oauth_token'],
        }
        if callback_url:
            auth_url_params['oauth_callback'] = callback_url

        request_tokens['auth_url'] = self.authenticate_url + '?' + urlencode(auth_url_params)

        return request_tokens

    def get_authorized_tokens(self, oauth_verifier):
        """Returns authorized tokens after they go through the auth_url phase.
        """
        response = self.client.get(self.access_token_url,
                                   params={'oauth_verifier': oauth_verifier})

        res = response.content
        if isinstance( response.content, bytes ):
            res = res.decode()

        authorized_tokens = dict(parse_qsl(res))
        if not authorized_tokens:
            raise TumblpyError('Unable to decode authorized tokens.')

        return authorized_tokens

    def request(self, endpoint, method='GET', blog_url=None,
                extra_endpoints=None, params=None):
        params = params or {}
        method = method.lower()

        if not method in ('get', 'post'):
            raise TumblpyError('Method must be of GET or POST')

        url = self.api_url  # http://api.tumblr.com/v2/

        if blog_url is not None:
            # http://api.tumblr.com/v2/blog/blogname.tumblr.com/
            blog_url = blog_url.rstrip('/')
            if blog_url.startswith('http://'):
                blog_url = blog_url[7:]

            url = '%sblog/%s/' % (self.api_url, blog_url)

        url = '%s%s' % (url, endpoint)
        if extra_endpoints is not None:
            # In cases like:
            # http://api.tumblr.com/v2/blog/blogname.tumblr.com/posts/type/
            # 'type' is extra in the url & thought this was the best way
            # Docs: http://www.tumblr.com/docs/en/api/v2#posts

            url = '%s/%s' % (url, '/'.join(extra_endpoints))

        params, files = _split_params_and_files(params)
        params.update(self.default_params)

        func = getattr(self.client, method)
        try:
            if method == 'get':
                response = func(url, params=params, allow_redirects=False)
            else:
                kwargs = {'data': params, 'files': files, 'allow_redirects': False}
                if files:
                    kwargs['params'] = params
                response = func(url, **kwargs)
        except requests.exceptions.RequestException:
            raise TumblpyError('An unknown error occurred.')

        if response.status_code == 401:
            raise TumblpyAuthError('Error: %s, Message: %s' % (response.status_code, response.content))

        content = response.content.decode('utf-8')
        try:
            if endpoint == 'avatar':
                content = {
                    'response': {
                        'url': response.headers.get('location')
                    }
                }
            else:
                content = json.loads(content)
        except ValueError:
            raise TumblpyError('Unable to parse response, invalid JSON.')

        try:
            content = content.get('response', {})
        except AttributeError:
            raise TumblpyError('Unable to parse response, invalid content returned: %s' % content)

        if response.status_code < 200 or response.status_code > 301:
            error_message = ''
            if content and (content.get('errors') or content.get('error')):
                if 'errors' in content:
                    for error in content['errors']:
                        error_message = '%s ' % error
                elif 'error' in content:
                    error_message = content['error']

            error_message = (error_message or
                             'There was an error making your request.')
            raise TumblpyError(error_message, error_code=response.status_code)

        return content

    def get(self, endpoint, blog_url=None, extra_endpoints=None, params=None):
        return self.request(endpoint, blog_url=blog_url,
                            extra_endpoints=extra_endpoints, params=params)

    def post(self, endpoint, blog_url=None, extra_endpoints=None, params=None):
        return self.request(endpoint, method='POST', blog_url=blog_url,
                            extra_endpoints=extra_endpoints, params=params)

    def get_avatar_url(self, blog_url, size=64):
        size = [str(size)] or ['64']
        return self.get('avatar', blog_url=blog_url, extra_endpoints=size)

    def following(self, kwargs=None):
        """
        Gets the blogs that the current user is following.
        :param limit: an int, the number of likes you want returned
        :param offset: an int, the blog you want to start at, for pagination.

            # Start at the 20th blog and get 20 more blogs.
            client.following({'offset': 20, 'limit': 20})

        :returns: A dict created from the JSON response
        """
        return self.get('user/following', params=kwargs)

    def dashboard(self, kwargs=None):
        """
        Gets the dashboard of the current user
        example: dashboard = client.dashboard({'limit': '3'})

        :param limit: an int, the number of posts you want returned
        :param offset: an int, the posts you want to start at, for pagination.
        :param type:   the type of post you want to return
        :param since_id:  return only posts that have appeared after this ID
        :param reblog_info: return reblog information about posts
        :param notes_info:  return notes information about the posts

        :returns: A dict created from the JSON response
        """
        return self.get('user/dashboard', params=kwargs)

    def __repr__(self):
        return u'<TumblrAPI: %s>' % self.app_key
<commit_msg>add function of get posts<commit_after>import requests
from requests_oauthlib import OAuth1

from . import __version__
from .compat import json, urlencode, parse_qsl
from .exceptions import TumblpyError, TumblpyAuthError
from .helpers import _split_params_and_files

class Tumblpy(object):
    def __init__(self, app_key=None, app_secret=None, oauth_token=None,
                 oauth_token_secret=None, headers=None, proxies=None):

        # Define some API URLs real quick
        self.base_api_url = 'https://api.tumblr.com'
        self.api_version = 'v2'
        self.api_url = '%s/%s/' % (self.base_api_url, self.api_version)

        # Authentication URLs
        self.request_token_url = 'https://www.tumblr.com/oauth/request_token'
        self.access_token_url = 'https://www.tumblr.com/oauth/access_token'
        self.authorize_url = 'https://www.tumblr.com/oauth/authorize'
        self.authenticate_url = 'https://www.tumblr.com/oauth/authorize'

        self.default_params = {'api_key': app_key}

        req_headers = {'User-Agent': 'Tumblpy v' + __version__}
        if headers:
            req_headers.update(headers)

        self.app_key = app_key
        self.app_secret = app_secret
        self.oauth_token = oauth_token
        self.oauth_token_secret = oauth_token_secret

        auth = None
        if self.app_key and self.app_secret:
            if not self.oauth_token and not self.oauth_token_secret:
                auth = OAuth1(self.app_key, self.app_secret)
            else:
                auth = OAuth1(self.app_key, self.app_secret,
                              self.oauth_token, self.oauth_token_secret)

        self.client = requests.Session()
        self.client.proxies = proxies
        self.client.headers = req_headers
        self.client.auth = auth

    def get_authentication_tokens(self, callback_url=None):
        """Returns a dict including an authorization URL (auth_url) to direct a user to

            :param callback_url: (optional) Url the user is returned to after they authorize your app (web clients only)
        """

        request_args = {}
        if callback_url:
            request_args['oauth_callback'] = callback_url

        response = self.client.get(self.request_token_url, params=request_args)

        if response.status_code != 200:
            raise TumblpyAuthError('Seems something couldn\'t be verified with your OAuth junk. Error: %s, Message: %s' % (response.status_code, response.content))

        res = response.content
        if isinstance( response.content, bytes ):
            res = res.decode()

        request_tokens = dict(parse_qsl(res))

        if not request_tokens:
            raise TumblpyError('Unable to decode request tokens.')

        auth_url_params = {
            'oauth_token': request_tokens['oauth_token'],
        }
        if callback_url:
            auth_url_params['oauth_callback'] = callback_url

        request_tokens['auth_url'] = self.authenticate_url + '?' + urlencode(auth_url_params)

        return request_tokens

    def get_authorized_tokens(self, oauth_verifier):
        """Returns authorized tokens after they go through the auth_url phase.
        """
        response = self.client.get(self.access_token_url,
                                   params={'oauth_verifier': oauth_verifier})

        res = response.content
        if isinstance( response.content, bytes ):
            res = res.decode()

        authorized_tokens = dict(parse_qsl(res))
        if not authorized_tokens:
            raise TumblpyError('Unable to decode authorized tokens.')

        return authorized_tokens

    def request(self, endpoint, method='GET', blog_url=None,
                extra_endpoints=None, params=None):
        params = params or {}
        method = method.lower()

        if not method in ('get', 'post'):
            raise TumblpyError('Method must be of GET or POST')

        url = self.api_url  # http://api.tumblr.com/v2/

        if blog_url is not None:
            # http://api.tumblr.com/v2/blog/blogname.tumblr.com/
            blog_url = blog_url.rstrip('/')
            if blog_url.startswith('http://'):
                blog_url = blog_url[7:]

            url = '%sblog/%s/' % (self.api_url, blog_url)

        url = '%s%s' % (url, endpoint)
        if extra_endpoints is not None:
            # In cases like:
            # http://api.tumblr.com/v2/blog/blogname.tumblr.com/posts/type/
            # 'type' is extra in the url & thought this was the best way
            # Docs: http://www.tumblr.com/docs/en/api/v2#posts

            url = '%s/%s' % (url, '/'.join(extra_endpoints))

        params, files = _split_params_and_files(params)
        params.update(self.default_params)

        func = getattr(self.client, method)
        try:
            if method == 'get':
                response = func(url, params=params, allow_redirects=False)
            else:
                kwargs = {'data': params, 'files': files, 'allow_redirects': False}
                if files:
                    kwargs['params'] = params
                response = func(url, **kwargs)
        except requests.exceptions.RequestException:
            raise TumblpyError('An unknown error occurred.')

        if response.status_code == 401:
            raise TumblpyAuthError('Error: %s, Message: %s' % (response.status_code, response.content))

        content = response.content.decode('utf-8')
        try:
            if endpoint == 'avatar':
                content = {
                    'response': {
                        'url': response.headers.get('location')
                    }
                }
            else:
                content = json.loads(content)
        except ValueError:
            raise TumblpyError('Unable to parse response, invalid JSON.')

        try:
            content = content.get('response', {})
        except AttributeError:
            raise TumblpyError('Unable to parse response, invalid content returned: %s' % content)

        if response.status_code < 200 or response.status_code > 301:
            error_message = ''
            if content and (content.get('errors') or content.get('error')):
                if 'errors' in content:
                    for error in content['errors']:
                        error_message = '%s ' % error
                elif 'error' in content:
                    error_message = content['error']

            error_message = (error_message or
                             'There was an error making your request.')
            raise TumblpyError(error_message, error_code=response.status_code)

        return content

    def get(self, endpoint, blog_url=None, extra_endpoints=None, params=None):
        return self.request(endpoint, blog_url=blog_url,
                            extra_endpoints=extra_endpoints, params=params)

    def post(self, endpoint, blog_url=None, extra_endpoints=None, params=None):
        return self.request(endpoint, method='POST', blog_url=blog_url,
                            extra_endpoints=extra_endpoints, params=params)

    def get_avatar_url(self, blog_url, size=64):
        size = [str(size)] or ['64']
        return self.get('avatar', blog_url=blog_url, extra_endpoints=size)

    def following(self, kwargs=None):
        """
        Gets the blogs that the current user is following.
        :param limit: an int, the number of likes you want returned
        :param offset: an int, the blog you want to start at, for pagination.

            # Start at the 20th blog and get 20 more blogs.
            client.following({'offset': 20, 'limit': 20})

        :returns: A dict created from the JSON response
        """
        return self.get('user/following', params=kwargs)

    def dashboard(self, kwargs=None):
        """
        Gets the dashboard of the current user
        example: dashboard = client.dashboard({'limit': '3'})

        :param limit: an int, the number of posts you want returned
        :param offset: an int, the posts you want to start at, for pagination.
        :param type:   the type of post you want to return
        :param since_id:  return only posts that have appeared after this ID
        :param reblog_info: return reblog information about posts
        :param notes_info:  return notes information about the posts

        :returns: A dict created from the JSON response
        """
        return self.get('user/dashboard', params=kwargs)

    def posts(self, blogname, post_type=None, kwargs=None):
        """
        Gets a list of posts from a particular blog
        :param blogname: a string, the blogname you want to look up posts
                         for. eg: codingjester.tumblr.com
        :param post_type:  the type of posts you want returned, e.g. video. If omitted returns all post types.
        :param limit: an int, the number of likes you want returned
        :param offset: an int, the blog you want to start at, for pagination.
        :returns: A dict created from the JSON response
        """
        if post_type is None:
            url = 'blog/%s/posts' % blogname
        else:
            url = 'blog/%s/posts/%s' % (blogname, post_type)
        return self.get(url, params=kwargs)

    def __repr__(self):
        return u'<TumblrAPI: %s>' % self.app_key
<|endoftext|>


<commit_before>from django.contrib import admin
from django import forms

from filer.admin.videoadmin import VideoAdmin
from filer.models.videomodels import Video

from models import VideoVersion, ArkestraVideo

# class VideoVersionForm(forms.ModelForm):
#     class Meta:
#         model = VideoVersion

class VideoVersionInline(admin.TabularInline):
    # form = VideoVersionForm
    model = VideoVersion
    readonly_fields = ('size', 'codec', 'status')
    extra = 0

class VideoVersionAdmin(admin.ModelAdmin):
    pass

class MyVideoAdmin(VideoAdmin):
    inlines = (VideoVersionInline,)

class ArkestraVideoAdmin(VideoAdmin):
    inlines = (VideoVersionInline,)

admin.site.unregister(Video)
admin.site.register(Video, MyVideoAdmin)
admin.site.register(ArkestraVideo, ArkestraVideoAdmin)
admin.site.register(VideoVersion, VideoVersionAdmin)<commit_msg>video admin<commit_after>from django.contrib import admin
from django import forms

from filer.admin.videoadmin import VideoAdmin
from filer.models.videomodels import Video

from models import VideoVersion, ArkestraVideo

# class VideoVersionForm(forms.ModelForm):
#     class Meta:
#         model = VideoVersion

class VideoVersionInline(admin.TabularInline):
    # form = VideoVersionForm
    model = VideoVersion
    readonly_fields = ('size', 'codec', 'status')
    extra = 0

class VideoVersionAdmin(admin.ModelAdmin):
    pass

class MyVideoAdmin(VideoAdmin):
    inlines = (VideoVersionInline,)

class ArkestraVideoAdmin(admin.ModelAdmin):
    inlines = (VideoVersionInline,)

admin.site.unregister(Video)
admin.site.register(Video, MyVideoAdmin)
admin.site.register(ArkestraVideo, ArkestraVideoAdmin)
admin.site.register(VideoVersion, VideoVersionAdmin)<|endoftext|>