<commit_before>import discord
import os

from .utils.dataIO import fileIO, dataIO
from cogs.utils import checks
from discord.ext import commands
from cogs.utils.chat_formatting import pagify

class Config:
    def __init__(self, bot):
        self.bot = bot
        self.profile = "data/servers/serverlist.json"
        self.riceCog = dataIO.load_json(self.profile)
        self.down_message = "data/servers/down_message.json"
        self.down = dataIO.load_json(self.down_message)

    async def _message_servers(self, ctx, msg):
        for server in self.bot.servers:
            if server.id in self.riceCog:
                try:
                    channel_id  = self.riceCog[server.id]["Channel"]
                    channel = self.bot.get_channel(channel_id)
                    await self.bot.send_message(channel, msg)
                except:
                    pass
            else:
                try:
                    await self.bot.send_message(server, msg)
                except:
                    pass
        await self.bot.say("Message succesfully sent")

    @checks.admin_or_permissions(manage_server=True)
    @commands.command(pass_context=True)
    async def channelset(self, ctx, channel : discord.Channel):
        """Set channel where announcements arrive"""
        server = ctx.message.server
        if server.id in self.riceCog:
            pass
        else:
            self.riceCog[server.id] = {}
        self.riceCog[server.id].update({"Channel" : channel.id})
        dataIO.save_json(self.profile, self.riceCog)
        await self.bot.say("Succesfully changed the channel to get notifications in.")

    @commands.command()
    async def countservers(self):
        """
        Checks how many servers the bot is on"""
        msg = "The bot is in **"
        msg += str(len(self.bot.servers))
        msg += "** servers."
        await self.bot.say(msg)

    @commands.command()
    @checks.is_owner()
    async def shout(self, server_id, channel_id, *, message):
        """
        Shouts a message to the main channel of a certain server"""
        msg = "```asciidoc\n"
        msg += "Announcement :: Information\n"
        msg += message
        msg += "\n```"

        if channel_id == "main":
            channel_id = None
            channel = None
        else:
            channel = discord.utils.get(self.bot.get_all_channels(), id=channel_id)

        for server in self.bot.servers:
            if server.id == server_id:
                if channel in server.channels:
                    try:
                        await self.bot.send_message(channel, msg)
                        await self.bot.say("'{}' has been sent to {} in {}.".format(message, channel.name, channel.server))
                    except discord.errors.Forbidden:
                        await self.bot.say("I'm not allowed to do that.")
                else:
                    try:
                        await self.bot.send_message(server, msg)
                        await self.bot.say("Couldn't find channel with id {}, so I just sent the message to the main channel instead.".format(channel_id))
                    except discord.errors.Forbidden:
                        await self.bot.say("I'm not allowed to do that.")

    @commands.command()
    @checks.is_owner()
    async def listchannels(self, server_id):
        """
        Checks what text channels are in a server"""
        server = self.bot.get_server(server_id)
        msg = "```asciidoc\n"
        # msg += "\n"
        count = 0
        for channel in server.channels:
            if channel.type != "voice":
                channelname = channel.name.replace("_", "-")
                msg += "{} :: {}\n".format(channel.id, channelname)
                count += 1
        await self.bot.say("The server {} has {} text channels:".format(server.name, count))
        await self.bot.say(msg + "```")

    @commands.command()
    @checks.is_owner()
    async def listservers(self):
        """
        Checks what servers the bot is on"""
        servers = self.bot.servers
        await self.bot.say("```asciidoc\nThe bot is in the following {} server(s):\n```".format(str(len(self.bot.servers))))
        msg = "```asciidoc\n"
        msg2 = "```asciidoc\n"
        msg3 = "```asciidoc\n"
        msg4 = "```asciidoc\n"
        # msg += "\n"

        messages = [msg, msg2, msg3, msg4]
        count = 0
        for server in servers:
            if len(server.members) < 10:
                messages[count] += "{:<1} :: 000{} users :: {}".format(server.id, len(server.members), server.name)
            elif len(server.members) < 100:
                messages[count] += "{:<1} :: 00{} users :: {}".format(server.id, len(server.members), server.name)
            elif len(server.members) < 1000:
                messages[count] += "{:<1} :: 0{} users :: {}".format(server.id, len(server.members), server.name)
            else:
                messages[count] += "{:<1} :: {} users :: {}".format(server.id, len(server.members), server.name)
            messages[count] += "\n"
            if len(messages[count]) > 1500:
                count = count + 1

        for message in messages:
            if len(message) > 30:
                await self.bot.say(message + "\n```")

    @commands.command(pass_context=True)
    @checks.is_owner()
    async def discrim(self, ctx, discriminator):
        """
        Find users with this discriminator"""
        msg = "Users with discriminator " + discriminator
        msg += ": \n"
        for r in self.bot.get_all_members():
            if r.discriminator == discriminator:
                if r.name in msg:
                    pass
                else:
                    msg += "```\n"
                    msg += r.name
                    msg += "\n```"
        await self.bot.say(msg)

    @commands.command()
    async def countusers(self):
        """
        Checks how many users the bot is connected to"""
        msg = "The bot is connected to **"
        msg += str(len(set(self.bot.get_all_members())))
        msg += "** users."
        await self.bot.say(msg)

    @commands.command(pass_context=True)
    @checks.is_owner()
    async def notify(self, ctx, *, content):
        """Notifies every server"""
        prefix = ctx.prefix
        if content == "info":
            msg = "```asciidoc\n"
            msg += "Announcement :: Information\n"
            msg += "= -=-=-=-=-=-=-=-=-=-=-=- =\n"
            msg += "Thank you for inviting riceBot!\n"
            msg += "For basic information on the bot, a list of commands, or to contact the owner, use: \n"
            msg += "= {}rice =\n".format(prefix)
            msg += "= {}help =\n".format(prefix)
            msg += "= {}contact =\n".format(prefix)
            msg += "To add the bot to your own server, open this:: https://discordsites.com/ricebot/\n"
            msg += "= -=-=-=-=-=-=-=-=-=-=-=- =\n"
            msg += "riceBot ~ managed by FwiedWice"
            msg += "\n```"
        else:
            msg = "```asciidoc\n"
            msg += "Announcement :: Information\n"
            msg += content
            msg += "\n```"
        await self._message_servers(ctx, msg)

    #def __shutdown(self):
    #   """Credits to Kowlin for this"""
    #    self.bot.loop.create_task(self.task())

    #async def task(self):
    #    msg = ("```asciidoc\n"
    #           "Announcement :: Shutdown\n"
    #           "riceBot shutting down... Will be up again soon!"
    #           "\n```")
    #    for server in self.bot.servers:
    #        try:
    #            await self.bot.send_message(server, msg)
    #        except:
    #            pass
    #    await self.bot.say("asdasd")

    #Credits to Kowlin for formatting!
    @commands.command(pass_context=True)
    @checks.is_owner()
    async def setshutdownmsg(self, ctx, *, msg : str=None):
        """Sets the shutdown message"""
        if msg == None:
            msg = ("```asciidoc\n"
                   "Announcement :: Shutdown\n"
                   "riceBot shutting down... Will be up again soon!"
                   "\n```")
        await self.bot.say("The shutdown message is: ")
        await self.bot.say(msg)
        self.down.update({"Message" : msg})
        dataIO.save_json(self.down_message, self.down)

    @commands.command(pass_context=True)
    @checks.is_owner()
    async def shutdown(self, ctx, silently : bool=False):
        """Shuts down the bot"""
        if "Message" in self.down:
            msg = self.down["Message"]
        else:
            msg = ("```asciidoc\n"
                   "Announcement :: Shutdown\n"
                   "Bot shutting down... Will be up again soon!"
                   "\n```")
        await self._message_servers(ctx, msg)
        try: # We don't want missing perms to stop our shutdown
            if not silently:
                await self.bot.say("Shutting down... ")
        except:
            pass
        await self.bot.shutdown()

    @commands.command()
    @checks.is_owner()
    async def speak(self, *, content):
        """Says something"""
        await self.bot.say(content)

def check_folder():
    if not os.path.exists("data/servers"):
        print("Creating data/servers folder")
        os.makedirs("data/servers")

def check_file():
    data = {}
    f = "data/servers/serverlist.json"
    if not dataIO.is_valid_json(f):
        print("Creating data/servers/serverlist.json")
        dataIO.save_json(f, data)

def check_file1():
    data = {}
    f = "data/servers/down_message.json"
    if not dataIO.is_valid_json(f):
        print("Creating data/servers/down_message.json")
        dataIO.save_json(f, data)

def setup(bot):
    check_folder()
    check_file()
    check_file1()
    bot.remove_command('shutdown')
    bot.add_cog(Config(bot))
<commit_msg>Doesn't say riceBot anywhere anymore.<commit_after>import discord
import os

from .utils.dataIO import fileIO, dataIO
from cogs.utils import checks
from discord.ext import commands
from cogs.utils.chat_formatting import pagify

class Config:
    def __init__(self, bot):
        self.bot = bot
        self.profile = "data/servers/serverlist.json"
        self.riceCog = dataIO.load_json(self.profile)
        self.down_message = "data/servers/down_message.json"
        self.down = dataIO.load_json(self.down_message)

    async def _message_servers(self, ctx, msg):
        for server in self.bot.servers:
            if server.id in self.riceCog:
                try:
                    channel_id  = self.riceCog[server.id]["Channel"]
                    channel = self.bot.get_channel(channel_id)
                    await self.bot.send_message(channel, msg)
                except:
                    pass
            else:
                try:
                    await self.bot.send_message(server, msg)
                except:
                    pass
        await self.bot.say("Message succesfully sent")

    @checks.admin_or_permissions(manage_server=True)
    @commands.command(pass_context=True)
    async def channelset(self, ctx, channel : discord.Channel):
        """Set channel where announcements arrive"""
        server = ctx.message.server
        if server.id in self.riceCog:
            pass
        else:
            self.riceCog[server.id] = {}
        self.riceCog[server.id].update({"Channel" : channel.id})
        dataIO.save_json(self.profile, self.riceCog)
        await self.bot.say("Succesfully changed the channel to get notifications in.")

    @commands.command()
    async def countservers(self):
        """
        Checks how many servers the bot is on"""
        msg = "The bot is in **"
        msg += str(len(self.bot.servers))
        msg += "** servers."
        await self.bot.say(msg)

    @commands.command()
    @checks.is_owner()
    async def shout(self, server_id, channel_id, *, message):
        """
        Shouts a message to the main channel of a certain server"""
        msg = "```asciidoc\n"
        msg += "Announcement :: Information\n"
        msg += message
        msg += "\n```"

        if channel_id == "main":
            channel_id = None
            channel = None
        else:
            channel = discord.utils.get(self.bot.get_all_channels(), id=channel_id)

        for server in self.bot.servers:
            if server.id == server_id:
                if channel in server.channels:
                    try:
                        await self.bot.send_message(channel, msg)
                        await self.bot.say("'{}' has been sent to {} in {}.".format(message, channel.name, channel.server))
                    except discord.errors.Forbidden:
                        await self.bot.say("I'm not allowed to do that.")
                else:
                    try:
                        await self.bot.send_message(server, msg)
                        await self.bot.say("Couldn't find channel with id {}, so I just sent the message to the main channel instead.".format(channel_id))
                    except discord.errors.Forbidden:
                        await self.bot.say("I'm not allowed to do that.")

    @commands.command()
    @checks.is_owner()
    async def listchannels(self, server_id):
        """
        Checks what text channels are in a server"""
        server = self.bot.get_server(server_id)
        msg = "```asciidoc\n"
        # msg += "\n"
        count = 0
        for channel in server.channels:
            if channel.type != "voice":
                channelname = channel.name.replace("_", "-")
                msg += "{} :: {}\n".format(channel.id, channelname)
                count += 1
        await self.bot.say("The server {} has {} text channels:".format(server.name, count))
        await self.bot.say(msg + "```")

    @commands.command()
    @checks.is_owner()
    async def listservers(self):
        """
        Checks what servers the bot is on"""
        servers = self.bot.servers
        await self.bot.say("```asciidoc\nThe bot is in the following {} server(s):\n```".format(str(len(self.bot.servers))))
        msg = "```asciidoc\n"
        msg2 = "```asciidoc\n"
        msg3 = "```asciidoc\n"
        msg4 = "```asciidoc\n"
        # msg += "\n"

        messages = [msg, msg2, msg3, msg4]
        count = 0
        for server in servers:
            if len(server.members) < 10:
                messages[count] += "{:<1} :: 000{} users :: {}".format(server.id, len(server.members), server.name)
            elif len(server.members) < 100:
                messages[count] += "{:<1} :: 00{} users :: {}".format(server.id, len(server.members), server.name)
            elif len(server.members) < 1000:
                messages[count] += "{:<1} :: 0{} users :: {}".format(server.id, len(server.members), server.name)
            else:
                messages[count] += "{:<1} :: {} users :: {}".format(server.id, len(server.members), server.name)
            messages[count] += "\n"
            if len(messages[count]) > 1500:
                count = count + 1

        for message in messages:
            if len(message) > 30:
                await self.bot.say(message + "\n```")

    @commands.command(pass_context=True)
    @checks.is_owner()
    async def discrim(self, ctx, discriminator):
        """
        Find users with this discriminator"""
        msg = "Users with discriminator " + discriminator
        msg += ": \n"
        for r in self.bot.get_all_members():
            if r.discriminator == discriminator:
                if r.name in msg:
                    pass
                else:
                    msg += "```\n"
                    msg += r.name
                    msg += "\n```"
        await self.bot.say(msg)

    @commands.command()
    async def countusers(self):
        """
        Checks how many users the bot is connected to"""
        msg = "The bot is connected to **"
        msg += str(len(set(self.bot.get_all_members())))
        msg += "** users."
        await self.bot.say(msg)

    @commands.command(pass_context=True)
    @checks.is_owner()
    async def notify(self, ctx, *, content):
        """Notifies every server"""
        prefix = ctx.prefix

        msg = "```asciidoc\n"
        msg += "Announcement :: Information\n"
        msg += content
        msg += "\n```"

        await self._message_servers(ctx, msg)

    @commands.command(pass_context=True)
    @checks.is_owner()
    async def setshutdownmsg(self, ctx, *, msg : str=None):
        """Sets the shutdown message"""
        if msg == None:
            msg = ("```asciidoc\n"
                   "Announcement :: Shutdown\n"
                   "Bot shutting down... Will be up again soon!"
                   "\n```")
        await self.bot.say("The shutdown message is: ")
        await self.bot.say(msg)
        self.down.update({"Message" : msg})
        dataIO.save_json(self.down_message, self.down)

    @commands.command(pass_context=True)
    @checks.is_owner()
    async def shutdown(self, ctx, silently : bool=False):
        """Shuts down the bot"""
        if "Message" in self.down:
            msg = self.down["Message"]
        else:
            msg = ("```asciidoc\n"
                   "Announcement :: Shutdown\n"
                   "Bot shutting down... Will be up again soon!"
                   "\n```")
        await self._message_servers(ctx, msg)
        try: # We don't want missing perms to stop our shutdown
            if not silently:
                await self.bot.say("Shutting down... ")
        except:
            pass
        await self.bot.shutdown()

    @commands.command()
    @checks.is_owner()
    async def speak(self, *, content):
        """Says something"""
        await self.bot.say(content)

def check_folder():
    if not os.path.exists("data/servers"):
        print("Creating data/servers folder")
        os.makedirs("data/servers")

def check_file():
    data = {}
    f = "data/servers/serverlist.json"
    if not dataIO.is_valid_json(f):
        print("Creating data/servers/serverlist.json")
        dataIO.save_json(f, data)

def check_file1():
    data = {}
    f = "data/servers/down_message.json"
    if not dataIO.is_valid_json(f):
        print("Creating data/servers/down_message.json")
        dataIO.save_json(f, data)

def setup(bot):
    check_folder()
    check_file()
    check_file1()
    bot.remove_command('shutdown')
    bot.add_cog(Config(bot))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import os
import re
import argparse
import glob
import shlex
import subprocess

try:
    import psycopg2cffi as psycopg2
except ImportError:
    import psycopg2

from filter import get_rsid

BASE_DIR = os.path.dirname(os.path.realpath(__file__))

# TODO: Load from settings.py if exists.
DBNAME = 'dbsnp_b146_GRCh37'
DBUSER = 'dbsnp'

RSID_PATTERN = re.compile('rs(\d+)')

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--source-id', required=True, choices=[3,4], type=int)
    parser.add_argument('--rsids', nargs='+', type=int)
    parser.add_argument('--dbname', default=DBNAME)
    parser.add_argument('--dbuser', default=DBUSER)
    parser.add_argument('--regions', nargs='+', type=str)
    args = parser.parse_args()

    rsids = args.rsids

    if args.regions:
        positions = [tuple(x.split(':')) for x in args.regions]
    else:
        if not args.dbname or not args.dbuser:
            print >>sys.stderr, '[ERROR] If --regions is not set, you need to set --dbname and --dbuser (error code AR1).'
            sys.exit(1)

        # Get current chrom/pos
        conn = psycopg2.connect("dbname={} user={}".format(args.dbname, args.dbuser))
        cur = conn.cursor()
        cur.execute("SELECT chr, pos FROM get_pos_by_rs(ARRAY[%s]);", (rsids,))
        rows = cur.fetchall()
        positions = rows

    # TODO: Get rs id histories
    histories = {}
    for rsid in rsids:
        histories[rsid] = [rsid]

    # TODO: DRY
    sample = {
        3: os.path.join(BASE_DIR, 'sample_ids.1000genomes.phase1.CHB+JPT+CHS.txt'),
        4: os.path.join(BASE_DIR, 'sample_ids.1000genomes.phase3.CHB+JPT+CHS.txt'),
    }[args.source_id]

    vcf = {
        3: os.path.join(BASE_DIR, '..', 'data/1000genomes.phase1/ALL.chr{chrom}.*.vcf.gz'),
        4: os.path.join(BASE_DIR, '..', 'data/1000genomes.phase3/ALL.chr{chrom}.*.vcf.gz'),
    }[args.source_id]

    for i, (chrom, pos) in enumerate(positions):
        vcf_in = glob.glob(vcf.format(chrom=chrom))

        if not vcf_in:
            log(rsids[i], 'vcf record not found (error code NA4). chrom:{}, position:{}'.format(chrom, pos))
            sys.exit(0)

        # Print VCF header lines
        if i == 0:
            cmd = shlex.split('bcftools view --header-only --samples-file {sample} {vcf}'.format(chrpos='{}:{}'.format(chrom, pos),
                                                                                sample=sample,
                                                                                vcf=vcf_in[0]))
            try:
                results = subprocess.check_output(cmd, stderr=subprocess.STDOUT).splitlines()
            except Exception as e:
                log(rsids[i], 'vcf record not found (error code NA3). chrom:{}, position:{}'.format(chrom, pos))
                sys.exit(0)

            for line in results:
                if line.startswith('#'):
                    print line

        # bcftools view         View, subset and filter VCF or BCF files by position and filtering expression. Convert between VCF and BCF. Former bcftools subset.
        #
        # --no-header           suppress the header in VCF output
        # --regions             Comma-separated list of regions, see also -R, --regions-file. Note that -r cannot be used in combination with -R.
        # --samples-file        File of sample names to include or exclude if prefixed with "^". One sample per line.
        cmd = shlex.split('bcftools view --no-header --regions "{chrpos}" --samples-file {sample} {vcf}'.format(chrpos='{}:{}'.format(chrom, pos),
                                                                                sample=sample,
                                                                                vcf=vcf_in[0]))
        try:
            results = subprocess.check_output(cmd, stderr=subprocess.STDOUT).splitlines()
        except Exception as e:
            log(rsids[i], 'vcf record not found (error code NA3). chrom:{}, position:{}'.format(chrom, pos))
            sys.exit(0)

        res_line = ''
        for result in results:
            records = result.split('\t')

            # FIXME: need to handle multiple hits
            if records[0] == chrom and int(records[1]) == int(pos):
                res_line = result
                break
        else:
            log(rsids[i], 'vcf record not found (error code NA1). chrom:{}, position:{}'.format(chrom, pos))
            sys.exit(0)

        if records == ['']:
            log(rsids[i], 'vcf record not found (error code NA2). chrom:{}, position:{}, id:rs{}'.format(chrom, pos))
            sys.exit(0)

        snp_id = get_rsid(records[2])
        if not snp_id in histories[rsids[i]]:
            continue

        # Replace to rs ids in query
        rec = res_line.split('\t')
        print '\t'.join(rec[0:2] + ['rs' + str(rsids[i])] + rec[3:])

def log(rsid, msg):
    print >>sys.stderr, '{}\t{}'.format(rsid, msg)

if __name__ == '__main__':
    main()
<commit_msg>output multiple lines if multiple records hit<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import os
import re
import argparse
import glob
import shlex
import subprocess

try:
    import psycopg2cffi as psycopg2
except ImportError:
    import psycopg2

from filter import get_rsid

BASE_DIR = os.path.dirname(os.path.realpath(__file__))

# TODO: Load from settings.py if exists.
DBNAME = 'dbsnp_b146_GRCh37'
DBUSER = 'dbsnp'

RSID_PATTERN = re.compile('rs(\d+)')

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--source-id', required=True, choices=[3,4], type=int)
    parser.add_argument('--rsids', nargs='+', type=int)
    parser.add_argument('--dbname', default=DBNAME)
    parser.add_argument('--dbuser', default=DBUSER)
    parser.add_argument('--regions', nargs='+', type=str)
    args = parser.parse_args()

    rsids = args.rsids

    if args.regions:
        positions = [tuple(x.split(':')) for x in args.regions]
    else:
        if not args.dbname or not args.dbuser:
            print >>sys.stderr, '[ERROR] If --regions is not set, you need to set --dbname and --dbuser (error code AR1).'
            sys.exit(1)

        # Get current chrom/pos
        conn = psycopg2.connect("dbname={} user={}".format(args.dbname, args.dbuser))
        cur = conn.cursor()
        cur.execute("SELECT chr, pos FROM get_pos_by_rs(ARRAY[%s]);", (rsids,))
        rows = cur.fetchall()
        positions = rows

    # TODO: DRY
    sample = {
        3: os.path.join(BASE_DIR, 'sample_ids.1000genomes.phase1.CHB+JPT+CHS.txt'),
        4: os.path.join(BASE_DIR, 'sample_ids.1000genomes.phase3.CHB+JPT+CHS.txt'),
    }[args.source_id]

    vcf = {
        3: os.path.join(BASE_DIR, '..', 'data/1000genomes.phase1/ALL.chr{chrom}.*.vcf.gz'),
        4: os.path.join(BASE_DIR, '..', 'data/1000genomes.phase3/ALL.chr{chrom}.*.vcf.gz'),
    }[args.source_id]

    for i, (chrom, pos) in enumerate(positions):
        vcf_in = glob.glob(vcf.format(chrom=chrom))

        if not vcf_in:
            log(rsids[i], 'vcf record not found (error code NA4). chrom:{}, position:{}'.format(chrom, pos))
            sys.exit(0)

        # Print VCF header lines
        if i == 0:
            cmd = shlex.split('bcftools view --header-only --samples-file {sample} {vcf}'.format(chrpos='{}:{}'.format(chrom, pos),
                                                                                sample=sample,
                                                                                vcf=vcf_in[0]))
            try:
                results = subprocess.check_output(cmd, stderr=subprocess.STDOUT).splitlines()
            except Exception as e:
                log(rsids[i], 'vcf record not found (error code NA3). chrom:{}, position:{}'.format(chrom, pos))
                sys.exit(0)

            for line in results:
                if line.startswith('#'):
                    print line

        # bcftools view         View, subset and filter VCF or BCF files by position and filtering expression. Convert between VCF and BCF. Former bcftools subset.
        #
        # --no-header           suppress the header in VCF output
        # --regions             Comma-separated list of regions, see also -R, --regions-file. Note that -r cannot be used in combination with -R.
        # --samples-file        File of sample names to include or exclude if prefixed with "^". One sample per line.
        cmd = shlex.split('bcftools view --no-header --regions "{chrpos}" --samples-file {sample} {vcf}'.format(chrpos='{}:{}'.format(chrom, pos),
                                                                                sample=sample,
                                                                                vcf=vcf_in[0]))
        try:
            results = subprocess.check_output(cmd, stderr=subprocess.STDOUT).splitlines()
        except Exception as e:
            log(rsids[i], 'vcf record not found (error code NA3). chrom:{}, position:{}'.format(chrom, pos))
            sys.exit(0)

        founds = []
        for result in results:
            records = result.split('\t')

            # FIXME: need to handle multiple hits
            if records[0] == chrom and int(records[1]) == int(pos):
                founds.append(result)

        if not founds:
            log(rsids[i], 'vcf record not found (error code NA1). chrom:{}, position:{}'.format(chrom, pos))
            sys.exit(0)

        for found in founds:
            # Replace to rs ids in query
            rec = found.split('\t')
            print '\t'.join(rec[0:2] + ['rs' + str(rsids[i])] + rec[3:])

def log(rsid, msg):
    print >>sys.stderr, '{}\t{}'.format(rsid, msg)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>__title__ = 'Controk WebService'
__version__ = '0.0.2'
__author__ = 'Jourdan Rodrigues'
__copyright__ = 'Copyright 2016 Jourdan Rodrigues'

# Version synonym
VERSION = __version__
<commit_msg>Correcting project version<commit_after>__title__ = 'Controk WebService'
__version__ = '0.1.1'
__author__ = 'Jourdan Rodrigues'
__copyright__ = 'Copyright 2016 Jourdan Rodrigues'

# Version synonym
VERSION = __version__
<|endoftext|>


<commit_before>import os
import urllib2
import json

from google.appengine.api import memcache
from google.appengine.ext import ndb
from google.appengine.ext.webapp import template

from base_controller import CacheableHandler

from models.event import Event
from models.team import Team
from models.sitevar import Sitevar
from models.typeahead_entry import TypeaheadEntry
import tba_config

class TypeaheadHandler(CacheableHandler):
    """
    Currently just returns a list of all teams and events
    Needs to be optimized at some point.
    Tried a trie but the datastructure was too big to
    fit into memcache efficiently
    """
    def __init__(self, *args, **kw):
        super(TypeaheadHandler, self).__init__(*args, **kw)
        self._cache_expiration = 60 * 60 * 24
        self._cache_key = "typeahead_entries:{}"
        self._cache_version = 1

    def get(self, search_key):
        search_key = urllib2.unquote(search_key)
        self._cache_key = self._cache_key.format(search_key)
        super(TypeaheadHandler, self).get(search_key)

    def _render(self, search_key):
        self.response.headers['Cache-Control'] = "public, max-age=%d" % self._cache_expiration
        self.response.headers['Pragma'] = 'Public'
        self.response.headers.add_header('content-type', 'application/json', charset='utf-8')

        entry = TypeaheadEntry.get_by_id(search_key)
        if entry is None:
            return '[]'
        else:
            return entry.data_json

class WebcastHandler(CacheableHandler):
    """
    Returns the HTML necessary to generate the webcast embed for a given event
    """
    def __init__(self, *args, **kw):
        super(WebcastHandler, self).__init__(*args, **kw)
        self._cache_expiration = 60 * 60 * 24
        self._cache_key = "webcast_{}_{}"  # (event_key)
        self._cache_version = 1

    def get(self, event_key, webcast_number):
        webcast_number = int(webcast_number) - 1
        self._cache_key = self._cache_key.format(event_key, webcast_number)

        self.response.headers['Cache-Control'] = "public, max-age=%d" % (5 * 60)
        self.response.headers['Pragma'] = 'Public'
        self.response.headers.add_header('content-type', 'application/json', charset='utf-8')

        super(WebcastHandler, self).get(event_key, webcast_number)

    def _render(self, event_key, webcast_number):
        event = Event.get_by_id(event_key)
        output = {}
        if event and event.webcast:
            webcast = event.webcast[webcast_number]
            if 'type' in webcast and 'channel' in webcast:
                output['player'] = self._renderPlayer(webcast)
        else:
            special_webcasts_future = Sitevar.get_by_id_async('gameday.special_webcasts')
            special_webcasts = special_webcasts_future.get_result()
            if special_webcasts:
                special_webcasts = special_webcasts.contents
            else:
                special_webcasts = {}
            if event_key in special_webcasts:
                webcast = special_webcasts[event_key]
                if 'type' in webcast and 'channel' in webcast:
                    output['player'] = self._renderPlayer(webcast)

        return json.dumps(output)

    def _renderPlayer(self, webcast):
        webcast_type = webcast['type']
        template_values = {'webcast': webcast}

        path = os.path.join(os.path.dirname(__file__), '../templates/webcast/' + webcast_type + '.html')
        return template.render(path, template_values)

    def memcacheFlush(self, event_key):
        keys = [self.cache_key.format(event_key, n) for n in range(10)]
        memcache.delete_multi(keys)
        return keys
<commit_msg>set headers within _render<commit_after>import os
import urllib2
import json

from google.appengine.api import memcache
from google.appengine.ext import ndb
from google.appengine.ext.webapp import template

from base_controller import CacheableHandler

from models.event import Event
from models.team import Team
from models.sitevar import Sitevar
from models.typeahead_entry import TypeaheadEntry
import tba_config

class TypeaheadHandler(CacheableHandler):
    """
    Currently just returns a list of all teams and events
    Needs to be optimized at some point.
    Tried a trie but the datastructure was too big to
    fit into memcache efficiently
    """
    def __init__(self, *args, **kw):
        super(TypeaheadHandler, self).__init__(*args, **kw)
        self._cache_expiration = 60 * 60 * 24
        self._cache_key = "typeahead_entries:{}"
        self._cache_version = 1

    def get(self, search_key):
        search_key = urllib2.unquote(search_key)
        self._cache_key = self._cache_key.format(search_key)
        super(TypeaheadHandler, self).get(search_key)

    def _render(self, search_key):
        self.response.headers['Cache-Control'] = "public, max-age=%d" % self._cache_expiration
        self.response.headers['Pragma'] = 'Public'
        self.response.headers.add_header('content-type', 'application/json', charset='utf-8')

        entry = TypeaheadEntry.get_by_id(search_key)
        if entry is None:
            return '[]'
        else:
            return entry.data_json

class WebcastHandler(CacheableHandler):
    """
    Returns the HTML necessary to generate the webcast embed for a given event
    """
    def __init__(self, *args, **kw):
        super(WebcastHandler, self).__init__(*args, **kw)
        self._cache_expiration = 60 * 60 * 24
        self._cache_key = "webcast_{}_{}"  # (event_key)
        self._cache_version = 1

    def get(self, event_key, webcast_number):
        webcast_number = int(webcast_number) - 1
        self._cache_key = self._cache_key.format(event_key, webcast_number)
        super(WebcastHandler, self).get(event_key, webcast_number)

    def _render(self, event_key, webcast_number):
        self.response.headers['Cache-Control'] = "public, max-age=%d" % (5 * 60)
        self.response.headers['Pragma'] = 'Public'
        self.response.headers.add_header('content-type', 'application/json', charset='utf-8')

        event = Event.get_by_id(event_key)
        output = {}
        if event and event.webcast:
            webcast = event.webcast[webcast_number]
            if 'type' in webcast and 'channel' in webcast:
                output['player'] = self._renderPlayer(webcast)
        else:
            special_webcasts_future = Sitevar.get_by_id_async('gameday.special_webcasts')
            special_webcasts = special_webcasts_future.get_result()
            if special_webcasts:
                special_webcasts = special_webcasts.contents
            else:
                special_webcasts = {}
            if event_key in special_webcasts:
                webcast = special_webcasts[event_key]
                if 'type' in webcast and 'channel' in webcast:
                    output['player'] = self._renderPlayer(webcast)

        return json.dumps(output)

    def _renderPlayer(self, webcast):
        webcast_type = webcast['type']
        template_values = {'webcast': webcast}

        path = os.path.join(os.path.dirname(__file__), '../templates/webcast/' + webcast_type + '.html')
        return template.render(path, template_values)

    def memcacheFlush(self, event_key):
        keys = [self.cache_key.format(event_key, n) for n in range(10)]
        memcache.delete_multi(keys)
        return keys
<|endoftext|>


<commit_before>import logging
import webapp2

from time import mktime
from wsgiref.handlers import format_date_time

from google.appengine.api import memcache

import tba_config

from helpers.user_bundle import UserBundle

class CacheableHandler(webapp2.RequestHandler):
    """
    Provides a standard way of caching the output of pages.
    Currently only supports logged-out pages.
    """
    CACHE_KEY_FORMAT = ''
    CACHE_HEADER_LENGTH = 61

    def __init__(self, *args, **kw):
        super(CacheableHandler, self).__init__(*args, **kw)
        self._cache_expiration = 0
        self._last_modified = None  # A datetime object
        if not hasattr(self, '_partial_cache_key'):
            self._partial_cache_key = self.CACHE_KEY_FORMAT
        self.template_values = {}

    @property
    def cache_key(self):
        return self._render_cache_key(self._partial_cache_key)

    @classmethod
    def get_cache_key_from_format(cls, *args):
        return cls._render_cache_key(cls.CACHE_KEY_FORMAT.format(*args))

    @classmethod
    def _render_cache_key(cls, cache_key):
        return "{}:{}:{}".format(
            cache_key,
            cls.CACHE_VERSION,
            tba_config.CONFIG["static_resource_version"])

    def get(self, *args, **kw):
        cached_response = self._read_cache()

        if cached_response is None:
            self._set_cache_header_length(self.CACHE_HEADER_LENGTH)
            self.template_values["cache_key"] = self.cache_key
            rendered = self._render(*args, **kw)
            if self._has_been_modified_since(self._last_modified):
                self.response.out.write(rendered)
                self._write_cache(self.response)
                return
            else:
                return None
        else:
            self.response.headers.update(cached_response.headers)
            if self._has_been_modified_since(self._last_modified):
                self.response.out.write(cached_response.body)
                return
            else:
                return None

    def _has_been_modified_since(self, datetime):
        if datetime is None:
            return True

        last_modified = format_date_time(mktime(datetime.timetuple()))
        if_modified_since = self.request.headers.get('If-Modified-Since')
        self.response.headers['Last-Modified'] = last_modified
        if if_modified_since and if_modified_since == last_modified:
            self.response.set_status(304)
            return False
        else:
            return True

    def memcacheFlush(self):
        memcache.delete(self.cache_key)
        return self.cache_key

    def _read_cache(self):
        result = memcache.get(self.cache_key)
        if result is None:
            return None
        else:
            response, last_modified = result
            self._last_modified = last_modified
            return response

    def _write_cache(self, response):
        if tba_config.CONFIG["memcache"]:
            memcache.set(self.cache_key, (response, self._last_modified), self._cache_expiration)

    @classmethod
    def delete_cache_multi(cls, cache_keys):
        memcache.delete_multi(cache_keys)

    def _render(self):
        raise NotImplementedError("No _render method.")

    def _set_cache_header_length(self, seconds):
        if type(seconds) is not int:
            logging.error("Cache-Control max-age is not integer: {}".format(seconds))
            return

        self.response.headers['Cache-Control'] = "public, max-age=%d" % max(seconds, 61)  # needs to be at least 61 seconds to work
        self.response.headers['Pragma'] = 'Public'

class LoggedInHandler(webapp2.RequestHandler):
    """
    Provides a base set of functionality for pages that need logins.
    Currently does not support caching as easily as CacheableHandler.
    """

    def __init__(self, *args, **kw):
        super(LoggedInHandler, self).__init__(*args, **kw)
        self.user_bundle = UserBundle()
        self.template_values = {
            "user_bundle": self.user_bundle
        }
        self.response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
        self.response.headers['Pragma'] = 'no-cache'
        self.response.headers['Expires'] = '0'

    def _require_admin(self):
        self._require_login()
        if not self.user_bundle.is_current_user_admin:
            return self.redirect(self.user_bundle.login_url, abort=True)

    def _require_login(self, target_url="/"):
        if not self.user_bundle.user:
            return self.redirect(
                self.user_bundle.create_login_url(target_url),
                abort=True
            )

    def _require_permission(self, permission):
        self._require_login()
        logging.info("logged in")
        self._require_registration()
        logging.info("registered")
        if permission not in self.user_bundle.account.permissions:
            return self.redirect(
                "/",
                abort=True
            )

    def _require_registration(self, target_url="/"):
        if not self.user_bundle.account.registered:
            return self.redirect(
                target_url,
                abort=True
            )
<commit_msg>Fix bug with incorrect Content-Length<commit_after>import logging
import webapp2

from time import mktime
from wsgiref.handlers import format_date_time

from google.appengine.api import memcache

import tba_config

from helpers.user_bundle import UserBundle

class CacheableHandler(webapp2.RequestHandler):
    """
    Provides a standard way of caching the output of pages.
    Currently only supports logged-out pages.
    """
    CACHE_KEY_FORMAT = ''
    CACHE_HEADER_LENGTH = 61

    def __init__(self, *args, **kw):
        super(CacheableHandler, self).__init__(*args, **kw)
        self._cache_expiration = 0
        self._last_modified = None  # A datetime object
        if not hasattr(self, '_partial_cache_key'):
            self._partial_cache_key = self.CACHE_KEY_FORMAT
        self.template_values = {}

    @property
    def cache_key(self):
        return self._render_cache_key(self._partial_cache_key)

    @classmethod
    def get_cache_key_from_format(cls, *args):
        return cls._render_cache_key(cls.CACHE_KEY_FORMAT.format(*args))

    @classmethod
    def _render_cache_key(cls, cache_key):
        return "{}:{}:{}".format(
            cache_key,
            cls.CACHE_VERSION,
            tba_config.CONFIG["static_resource_version"])

    def get(self, *args, **kw):
        cached_response = self._read_cache()

        if cached_response is None:
            self._set_cache_header_length(self.CACHE_HEADER_LENGTH)
            self.template_values["cache_key"] = self.cache_key
            rendered = self._render(*args, **kw)
            if self._has_been_modified_since(self._last_modified):
                self.response.out.write(rendered)
                self._write_cache(self.response)
                return
            else:
                return None
        else:
            self.response.headers.update(cached_response.headers)
            del self.response.headers['Content-Length']  # Content-Length gets set automatically
            if self._has_been_modified_since(self._last_modified):
                self.response.out.write(cached_response.body)
                return
            else:
                return None

    def _has_been_modified_since(self, datetime):
        if datetime is None:
            return True

        last_modified = format_date_time(mktime(datetime.timetuple()))
        if_modified_since = self.request.headers.get('If-Modified-Since')
        self.response.headers['Last-Modified'] = last_modified
        if if_modified_since and if_modified_since == last_modified:
            self.response.set_status(304)
            return False
        else:
            return True

    def memcacheFlush(self):
        memcache.delete(self.cache_key)
        return self.cache_key

    def _read_cache(self):
        result = memcache.get(self.cache_key)
        if result is None:
            return None
        else:
            response, last_modified = result
            self._last_modified = last_modified
            return response

    def _write_cache(self, response):
        if tba_config.CONFIG["memcache"]:
            memcache.set(self.cache_key, (response, self._last_modified), self._cache_expiration)

    @classmethod
    def delete_cache_multi(cls, cache_keys):
        memcache.delete_multi(cache_keys)

    def _render(self):
        raise NotImplementedError("No _render method.")

    def _set_cache_header_length(self, seconds):
        if type(seconds) is not int:
            logging.error("Cache-Control max-age is not integer: {}".format(seconds))
            return

        self.response.headers['Cache-Control'] = "public, max-age=%d" % max(seconds, 61)  # needs to be at least 61 seconds to work
        self.response.headers['Pragma'] = 'Public'

class LoggedInHandler(webapp2.RequestHandler):
    """
    Provides a base set of functionality for pages that need logins.
    Currently does not support caching as easily as CacheableHandler.
    """

    def __init__(self, *args, **kw):
        super(LoggedInHandler, self).__init__(*args, **kw)
        self.user_bundle = UserBundle()
        self.template_values = {
            "user_bundle": self.user_bundle
        }
        self.response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
        self.response.headers['Pragma'] = 'no-cache'
        self.response.headers['Expires'] = '0'

    def _require_admin(self):
        self._require_login()
        if not self.user_bundle.is_current_user_admin:
            return self.redirect(self.user_bundle.login_url, abort=True)

    def _require_login(self, target_url="/"):
        if not self.user_bundle.user:
            return self.redirect(
                self.user_bundle.create_login_url(target_url),
                abort=True
            )

    def _require_permission(self, permission):
        self._require_login()
        logging.info("logged in")
        self._require_registration()
        logging.info("registered")
        if permission not in self.user_bundle.account.permissions:
            return self.redirect(
                "/",
                abort=True
            )

    def _require_registration(self, target_url="/"):
        if not self.user_bundle.account.registered:
            return self.redirect(
                target_url,
                abort=True
            )
<|endoftext|>


<commit_before>import datetime
import logging
import os
import json

from google.appengine.api import taskqueue

from google.appengine.ext import ndb

from google.appengine.ext import webapp
from google.appengine.ext.webapp import template
from google.appengine.api import urlfetch

from helpers.event_team_manipulator import EventTeamManipulator
from helpers.event_team_repairer import EventTeamRepairer
from helpers.event_team_updater import EventTeamUpdater

from helpers.insight_manipulator import InsightManipulator
from helpers.team_manipulator import TeamManipulator
from helpers.opr_helper import OprHelper
from helpers.insights_helper import InsightsHelper
from helpers.typeahead_helper import TypeaheadHelper

from models.event import Event
from models.event_team import EventTeam
from models.match import Match
from models.team import Team
from models.sitevar import Sitevar
from models.typeahead_entry import TypeaheadEntry

import tba_config

from models.insight import Insight

class EventTeamRepairDo(webapp.RequestHandler):
    """
    Repair broken EventTeams.
    """
    def get(self):
        event_teams_keys = EventTeam.query(EventTeam.year == None).fetch(keys_only=True)
        event_teams = ndb.get_multi(event_teams_keys)

        event_teams = EventTeamRepairer.repair(event_teams)
        event_teams = EventTeamManipulator.createOrUpdate(event_teams)

        # sigh. -gregmarra
        if type(event_teams) == EventTeam:
            event_teams = [event_teams]

        template_values = {
            'event_teams': event_teams,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/math/eventteam_repair_do.html')
        self.response.out.write(template.render(path, template_values))

class EventTeamUpdate(webapp.RequestHandler):
    """
    Task that adds to the EventTeam index for an Event from Matches.
    Can only update or delete EventTeams for unregistered teams.
    ^^^ Does it actually do this? Eugene -- 2013/07/30
    Removes EventTeams for teams that haven't played any matches.
    """
    def get(self, event_key):
        teams, event_teams, et_keys_to_del = EventTeamUpdater.update(event_key)

        teams = TeamManipulator.createOrUpdate(teams)

        if teams:
            event_teams = EventTeamManipulator.createOrUpdate(event_teams)

        if et_keys_to_del:
            ndb.delete_multi(et_keys_to_del)

        template_values = {
            'event_teams': event_teams,
            'deleted_event_teams_keys': et_keys_to_del
        }

        path = os.path.join(os.path.dirname(__file__),
                            '../templates/math/eventteam_update_do.html')
        self.response.out.write(template.render(path, template_values))

class EventTeamUpdateEnqueue(webapp.RequestHandler):
    """
    Handles enqueing building attendance for Events.
    """
    def get(self, when):
        if when == "all":
            event_keys = Event.query().fetch(10000, keys_only=True)
        else:
            event_keys = Event.query(Event.year == int(when)).fetch(10000, keys_only=True)

        for event_key in event_keys:
            taskqueue.add(
                url='/tasks/math/do/eventteam_update/' + event_key.id(),
                method='GET')

        template_values = {
            'event_keys': event_keys,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/math/eventteam_update_enqueue.html')
        self.response.out.write(template.render(path, template_values))

class EventOprDo(webapp.RequestHandler):
    """
    Calculates the opr for an event
    """
    def get(self, event_key):
        opr = []
        teams = []
        oprs = []
        event = Event.get_by_id(event_key)
        if Match.query(Match.event == event.key).fetch(keys_only=True).count() > 0:
            try:
                opr, teams = OprHelper.opr(event_key)
                oprs.append((opr, teams))
                event.oprs = opr
                event.opr_teams = teams
                event.put()
            except Exception, e:
                logging.error("OPR error on event %s. %s" % (event_key, e))

        logging.info(oprs)

        template_values = {
            'oprs': oprs,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/math/event_opr_do.html')
        self.response.out.write(template.render(path, template_values))

    def post(self):
        self.get()

class EventOprEnqueue(webapp.RequestHandler):
    """
    Enqueues OPR calculation
    """
    def get(self, when):
        if when == "now":
            events = Event.query(Event.end_date <= datetime.datetime.today() + datetime.timedelta(days=4))
            events = Event.query(Event.end_date >= datetime.datetime.today() - datetime.timedelta(days=1))
        else:
            events = Event.query(Event.year == int(when))

        events = events.fetch(500)

        for event in events:
            taskqueue.add(
                url='/tasks/math/do/event_opr/' + event.key_name,
                method='GET')

        template_values = {
            'event_count': len(events),
            'year': when
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/math/event_opr_enqueue.html')
        self.response.out.write(template.render(path, template_values))

class FinalMatchesRepairDo(webapp.RequestHandler):
    """
    Repairs zero-indexed final matches
    """
    def get(self, year):
        year_event_keys = Event.query(Event.year == int(year)).fetch(100, keys_only=True)

        final_match_keys = []
        for event_key in year_event_keys:
            final_match_keys.extend(Match.query(Match.event == event_key, Match.comp_level == 'f').fetch(100, keys_only=True))

        match_keys_to_repair = []
        for match_key in final_match_keys:
            key_name = match_key.id()
            if '_f0' in key_name:
                match_keys_to_repair.append(match_key)

        deleted_keys = []
        matches_to_repair = ndb.get_multi(match_keys_to_repair)
        for match in matches_to_repair:
            deleted_keys.append(match.key)

            event = ndb.get_multi([match.event])[0]
            match.set_number = 1
            match.key = ndb.Key(Match, Match.renderKeyName(
                event,
                match.comp_level,
                match.set_number,
                match.match_number))

        ndb.put_multi(matches_to_repair)
        ndb.delete_multi(deleted_keys)

        template_values = {'deleted_keys': deleted_keys,
                           'new_matches': matches_to_repair}

        path = os.path.join(os.path.dirname(__file__), '../templates/math/final_matches_repair_do.html')
        self.response.out.write(template.render(path, template_values))

class YearInsightsEnqueue(webapp.RequestHandler):
    """
    Enqueues Insights calculation of a given kind for a given year
    """
    def get(self, kind, year):
        taskqueue.add(
            url='/tasks/math/do/insights/{}/{}'.format(kind, year),
            method='GET')

        template_values = {
            'kind': kind,
            'year': year
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/math/year_insights_enqueue.html')
        self.response.out.write(template.render(path, template_values))

class YearInsightsDo(webapp.RequestHandler):
    """
    Calculates insights of a given kind for a given year.
    Calculations of a given kind should reuse items fetched from the datastore.
    """

    def get(self, kind, year):
        year = int(year)

        insights = None
        if kind == 'matches':
            insights = InsightsHelper.doMatchInsights(year)
        elif kind == 'awards':
            insights = InsightsHelper.doAwardInsights(year)

        if insights != None:
            InsightManipulator.createOrUpdate(insights)

        template_values = {
            'insights': insights,
            'year': year,
            'kind': kind,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/math/year_insights_do.html')
        self.response.out.write(template.render(path, template_values))

    def post(self):
        self.get()

class OverallInsightsEnqueue(webapp.RequestHandler):
    """
    Enqueues Overall Insights calculation for a given kind.
    """
    def get(self, kind):
        taskqueue.add(
            url='/tasks/math/do/overallinsights/{}'.format(kind),
            method='GET')

        template_values = {
            'kind': kind,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/math/overall_insights_enqueue.html')
        self.response.out.write(template.render(path, template_values))

class OverallInsightsDo(webapp.RequestHandler):
    """
    Calculates overall insights of a given kind.
    Calculations of a given kind should reuse items fetched from the datastore.
    """

    def get(self, kind):
        insights = None
        if kind == 'matches':
            insights = InsightsHelper.doOverallMatchInsights()
        elif kind == 'awards':
            insights = InsightsHelper.doOverallAwardInsights()

        if insights != None:
            InsightManipulator.createOrUpdate(insights)

        template_values = {
            'insights': insights,
            'kind': kind,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/math/overall_insights_do.html')
        self.response.out.write(template.render(path, template_values))

    def post(self):
        self.get()

class TypeaheadCalcEnqueue(webapp.RequestHandler):
    """
    Enqueues typeahead calculations
    """
    def get(self):
        taskqueue.add(url='/tasks/math/do/typeaheadcalc', method='GET')
        template_values = {}
        path = os.path.join(os.path.dirname(__file__), '../templates/math/typeaheadcalc_enqueue.html')
        self.response.out.write(template.render(path, template_values))

class TypeaheadCalcDo(webapp.RequestHandler):
    """
    Calculates typeahead entries
    """
    def get(self):
        @ndb.tasklet
        def get_events_async():
            event_keys = yield Event.query().order(-Event.year).order(Event.name).fetch_async(keys_only=True)
            events = yield ndb.get_multi_async(event_keys)
            raise ndb.Return(events)

        @ndb.tasklet
        def get_teams_async():
            team_keys = yield Team.query().order(Team.team_number).fetch_async(keys_only=True)
            teams = yield ndb.get_multi_async(team_keys)
            raise ndb.Return(teams)

        @ndb.toplevel
        def get_events_and_teams():
            events, teams = yield get_events_async(), get_teams_async()
            raise ndb.Return((events, teams))

        events, teams = get_events_and_teams()

        results = {}
        for team in teams:
            keys = set()
            keys.add(str(team.team_number)[0])
            if not team.nickname:
                nickname = "Team %s" % team.team_number
            else:
                nickname = team.nickname
                keys = keys.union(TypeaheadHelper.get_search_keys(nickname))
            entry = '%s | %s' % (team.team_number, nickname)
            for key in keys:
                if key in results:
                    results[key].append(entry)
                else:
                    results[key] = [entry]

        for event in events:
            keys = set()
            keys.add(str(event.year)[0])
            keys.add(event.event_short[0])
            keys = keys.union(TypeaheadHelper.get_search_keys(event.name))
            entry = '%s %s [%s]' % (event.year, event.name, event.event_short.upper())
            for key in keys:
                if key in results:
                    results[key].append(entry)
                else:
                    results[key] = [entry]

        entries = []
        for key, data in results.items():
            entries.append(TypeaheadEntry(id=key,
                                          data_json=json.dumps(data)))
        ndb.put_multi(entries)

        template_values = {'results': results}
        path = os.path.join(os.path.dirname(__file__), '../templates/math/typeaheadcalc_do.html')
        self.response.out.write(template.render(path, template_values))
<commit_msg>more rigours check for bad final match<commit_after>import datetime
import logging
import os
import json

from google.appengine.api import taskqueue

from google.appengine.ext import ndb

from google.appengine.ext import webapp
from google.appengine.ext.webapp import template
from google.appengine.api import urlfetch

from helpers.event_team_manipulator import EventTeamManipulator
from helpers.event_team_repairer import EventTeamRepairer
from helpers.event_team_updater import EventTeamUpdater

from helpers.insight_manipulator import InsightManipulator
from helpers.team_manipulator import TeamManipulator
from helpers.opr_helper import OprHelper
from helpers.insights_helper import InsightsHelper
from helpers.typeahead_helper import TypeaheadHelper

from models.event import Event
from models.event_team import EventTeam
from models.match import Match
from models.team import Team
from models.sitevar import Sitevar
from models.typeahead_entry import TypeaheadEntry

import tba_config

from models.insight import Insight

class EventTeamRepairDo(webapp.RequestHandler):
    """
    Repair broken EventTeams.
    """
    def get(self):
        event_teams_keys = EventTeam.query(EventTeam.year == None).fetch(keys_only=True)
        event_teams = ndb.get_multi(event_teams_keys)

        event_teams = EventTeamRepairer.repair(event_teams)
        event_teams = EventTeamManipulator.createOrUpdate(event_teams)

        # sigh. -gregmarra
        if type(event_teams) == EventTeam:
            event_teams = [event_teams]

        template_values = {
            'event_teams': event_teams,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/math/eventteam_repair_do.html')
        self.response.out.write(template.render(path, template_values))

class EventTeamUpdate(webapp.RequestHandler):
    """
    Task that adds to the EventTeam index for an Event from Matches.
    Can only update or delete EventTeams for unregistered teams.
    ^^^ Does it actually do this? Eugene -- 2013/07/30
    Removes EventTeams for teams that haven't played any matches.
    """
    def get(self, event_key):
        teams, event_teams, et_keys_to_del = EventTeamUpdater.update(event_key)

        teams = TeamManipulator.createOrUpdate(teams)

        if teams:
            event_teams = EventTeamManipulator.createOrUpdate(event_teams)

        if et_keys_to_del:
            ndb.delete_multi(et_keys_to_del)

        template_values = {
            'event_teams': event_teams,
            'deleted_event_teams_keys': et_keys_to_del
        }

        path = os.path.join(os.path.dirname(__file__),
                            '../templates/math/eventteam_update_do.html')
        self.response.out.write(template.render(path, template_values))

class EventTeamUpdateEnqueue(webapp.RequestHandler):
    """
    Handles enqueing building attendance for Events.
    """
    def get(self, when):
        if when == "all":
            event_keys = Event.query().fetch(10000, keys_only=True)
        else:
            event_keys = Event.query(Event.year == int(when)).fetch(10000, keys_only=True)

        for event_key in event_keys:
            taskqueue.add(
                url='/tasks/math/do/eventteam_update/' + event_key.id(),
                method='GET')

        template_values = {
            'event_keys': event_keys,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/math/eventteam_update_enqueue.html')
        self.response.out.write(template.render(path, template_values))

class EventOprDo(webapp.RequestHandler):
    """
    Calculates the opr for an event
    """
    def get(self, event_key):
        opr = []
        teams = []
        oprs = []
        event = Event.get_by_id(event_key)
        if Match.query(Match.event == event.key).fetch(keys_only=True).count() > 0:
            try:
                opr, teams = OprHelper.opr(event_key)
                oprs.append((opr, teams))
                event.oprs = opr
                event.opr_teams = teams
                event.put()
            except Exception, e:
                logging.error("OPR error on event %s. %s" % (event_key, e))

        logging.info(oprs)

        template_values = {
            'oprs': oprs,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/math/event_opr_do.html')
        self.response.out.write(template.render(path, template_values))

    def post(self):
        self.get()

class EventOprEnqueue(webapp.RequestHandler):
    """
    Enqueues OPR calculation
    """
    def get(self, when):
        if when == "now":
            events = Event.query(Event.end_date <= datetime.datetime.today() + datetime.timedelta(days=4))
            events = Event.query(Event.end_date >= datetime.datetime.today() - datetime.timedelta(days=1))
        else:
            events = Event.query(Event.year == int(when))

        events = events.fetch(500)

        for event in events:
            taskqueue.add(
                url='/tasks/math/do/event_opr/' + event.key_name,
                method='GET')

        template_values = {
            'event_count': len(events),
            'year': when
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/math/event_opr_enqueue.html')
        self.response.out.write(template.render(path, template_values))

class FinalMatchesRepairDo(webapp.RequestHandler):
    """
    Repairs zero-indexed final matches
    """
    def get(self, year):
        year_event_keys = Event.query(Event.year == int(year)).fetch(100, keys_only=True)

        final_match_keys = []
        for event_key in year_event_keys:
            final_match_keys.extend(Match.query(Match.event == event_key, Match.comp_level == 'f').fetch(100, keys_only=True))

        match_keys_to_repair = []
        for match_key in final_match_keys:
            key_name = match_key.id()
            if '_f0m' in key_name:
                match_keys_to_repair.append(match_key)

        deleted_keys = []
        matches_to_repair = ndb.get_multi(match_keys_to_repair)
        for match in matches_to_repair:
            deleted_keys.append(match.key)

            event = ndb.get_multi([match.event])[0]
            match.set_number = 1
            match.key = ndb.Key(Match, Match.renderKeyName(
                event,
                match.comp_level,
                match.set_number,
                match.match_number))

        ndb.put_multi(matches_to_repair)
        ndb.delete_multi(deleted_keys)

        template_values = {'deleted_keys': deleted_keys,
                           'new_matches': matches_to_repair}

        path = os.path.join(os.path.dirname(__file__), '../templates/math/final_matches_repair_do.html')
        self.response.out.write(template.render(path, template_values))

class YearInsightsEnqueue(webapp.RequestHandler):
    """
    Enqueues Insights calculation of a given kind for a given year
    """
    def get(self, kind, year):
        taskqueue.add(
            url='/tasks/math/do/insights/{}/{}'.format(kind, year),
            method='GET')

        template_values = {
            'kind': kind,
            'year': year
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/math/year_insights_enqueue.html')
        self.response.out.write(template.render(path, template_values))

class YearInsightsDo(webapp.RequestHandler):
    """
    Calculates insights of a given kind for a given year.
    Calculations of a given kind should reuse items fetched from the datastore.
    """

    def get(self, kind, year):
        year = int(year)

        insights = None
        if kind == 'matches':
            insights = InsightsHelper.doMatchInsights(year)
        elif kind == 'awards':
            insights = InsightsHelper.doAwardInsights(year)

        if insights != None:
            InsightManipulator.createOrUpdate(insights)

        template_values = {
            'insights': insights,
            'year': year,
            'kind': kind,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/math/year_insights_do.html')
        self.response.out.write(template.render(path, template_values))

    def post(self):
        self.get()

class OverallInsightsEnqueue(webapp.RequestHandler):
    """
    Enqueues Overall Insights calculation for a given kind.
    """
    def get(self, kind):
        taskqueue.add(
            url='/tasks/math/do/overallinsights/{}'.format(kind),
            method='GET')

        template_values = {
            'kind': kind,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/math/overall_insights_enqueue.html')
        self.response.out.write(template.render(path, template_values))

class OverallInsightsDo(webapp.RequestHandler):
    """
    Calculates overall insights of a given kind.
    Calculations of a given kind should reuse items fetched from the datastore.
    """

    def get(self, kind):
        insights = None
        if kind == 'matches':
            insights = InsightsHelper.doOverallMatchInsights()
        elif kind == 'awards':
            insights = InsightsHelper.doOverallAwardInsights()

        if insights != None:
            InsightManipulator.createOrUpdate(insights)

        template_values = {
            'insights': insights,
            'kind': kind,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/math/overall_insights_do.html')
        self.response.out.write(template.render(path, template_values))

    def post(self):
        self.get()

class TypeaheadCalcEnqueue(webapp.RequestHandler):
    """
    Enqueues typeahead calculations
    """
    def get(self):
        taskqueue.add(url='/tasks/math/do/typeaheadcalc', method='GET')
        template_values = {}
        path = os.path.join(os.path.dirname(__file__), '../templates/math/typeaheadcalc_enqueue.html')
        self.response.out.write(template.render(path, template_values))

class TypeaheadCalcDo(webapp.RequestHandler):
    """
    Calculates typeahead entries
    """
    def get(self):
        @ndb.tasklet
        def get_events_async():
            event_keys = yield Event.query().order(-Event.year).order(Event.name).fetch_async(keys_only=True)
            events = yield ndb.get_multi_async(event_keys)
            raise ndb.Return(events)

        @ndb.tasklet
        def get_teams_async():
            team_keys = yield Team.query().order(Team.team_number).fetch_async(keys_only=True)
            teams = yield ndb.get_multi_async(team_keys)
            raise ndb.Return(teams)

        @ndb.toplevel
        def get_events_and_teams():
            events, teams = yield get_events_async(), get_teams_async()
            raise ndb.Return((events, teams))

        events, teams = get_events_and_teams()

        results = {}
        for team in teams:
            keys = set()
            keys.add(str(team.team_number)[0])
            if not team.nickname:
                nickname = "Team %s" % team.team_number
            else:
                nickname = team.nickname
                keys = keys.union(TypeaheadHelper.get_search_keys(nickname))
            entry = '%s | %s' % (team.team_number, nickname)
            for key in keys:
                if key in results:
                    results[key].append(entry)
                else:
                    results[key] = [entry]

        for event in events:
            keys = set()
            keys.add(str(event.year)[0])
            keys.add(event.event_short[0])
            keys = keys.union(TypeaheadHelper.get_search_keys(event.name))
            entry = '%s %s [%s]' % (event.year, event.name, event.event_short.upper())
            for key in keys:
                if key in results:
                    results[key].append(entry)
                else:
                    results[key] = [entry]

        entries = []
        for key, data in results.items():
            entries.append(TypeaheadEntry(id=key,
                                          data_json=json.dumps(data)))
        ndb.put_multi(entries)

        template_values = {'results': results}
        path = os.path.join(os.path.dirname(__file__), '../templates/math/typeaheadcalc_do.html')
        self.response.out.write(template.render(path, template_values))
<|endoftext|>


<commit_before>from django.conf import settings
from rest_framework.settings import APISettings

USER_SETTINGS = getattr(settings, 'AUTH0', None)

DEFAULTS = {
    'JWT_AUTH_HEADER_PREFIX': 'JWT',
    'AUTHORIZATION_EXTENSION': False,
    'USERNAME_FIELD': 'sub',
    'CLIENT_CODE': 'Client_Code',
    'CLIENTS': {},
    'REPLACE_PIPE_FOR_DOTS_IN_USERNAME': True,
    # Handlers
    'JWT_PAYLOAD_GET_USERNAME_HANDLER':
    'rest_framework_auth0.utils.auth0_get_username_from_payload_handler',
    # Copied from rest_framework_jwt settings, temporal while migrating to pyjwt
    'JWT_SECRET_KEY': settings.SECRET_KEY,
    'JWT_GET_USER_SECRET_KEY': None,
    'JWT_VERIFY_EXPIRATION': True,
    'JWT_PUBLIC_KEY': None,
    'JWT_VERIFY': True,
    'JWT_LEEWAY': 0,
    'JWT_AUDIENCE': None,
    'JWT_ISSUER': None,
    'JWT_ALGORITHM': 'HS256',
    'JWT_AUTH_COOKIE': None,
}

# List of settings that may be in string import notation.
IMPORT_STRINGS = (
    'JWT_PAYLOAD_GET_USERNAME_HANDLER',
)

auth0_api_settings = APISettings(
    USER_SETTINGS,
    DEFAULTS,
    IMPORT_STRINGS
)
<commit_msg>Remove unused default settings #50<commit_after>from django.conf import settings
from rest_framework.settings import APISettings

USER_SETTINGS = getattr(settings, 'AUTH0', None)

DEFAULTS = {
    'JWT_AUTH_HEADER_PREFIX': 'JWT',
    'AUTHORIZATION_EXTENSION': False,
    'USERNAME_FIELD': 'sub',
    'CLIENT_CODE': 'Client_Code',
    'CLIENTS': {},
    'REPLACE_PIPE_FOR_DOTS_IN_USERNAME': True,
    # Handlers
    'JWT_PAYLOAD_GET_USERNAME_HANDLER':
    'rest_framework_auth0.utils.auth0_get_username_from_payload_handler',
    # Copied from rest_framework_jwt settings, temporal while migrating to pyjwt
    'JWT_AUTH_COOKIE': None,
}

# List of settings that may be in string import notation.
IMPORT_STRINGS = (
    'JWT_PAYLOAD_GET_USERNAME_HANDLER',
)

auth0_api_settings = APISettings(
    USER_SETTINGS,
    DEFAULTS,
    IMPORT_STRINGS
)
<|endoftext|>


<commit_before>"""
Retrieve anime info from AnimePlanet.
"""
from collections import deque
from difflib import SequenceMatcher
from typing import List, Optional
from urllib.parse import quote

from pyquery import PyQuery

from roboragi.session_manager import SessionManager

def sanitize_search_text(text: str) -> str:
    """
    Sanitize text for Anime-Planet use.
    :param text: the text to be escaped.
    :return: the escaped text.
    """
    return text.replace('(TV)', 'TV')

async def get_anime_url(
        session_manager: SessionManager,
        query: str,
        names: list) -> Optional[str]:
    """
    Get anime url by search query.
    :param session_manager: the `SessionManager` instance.
    :param query: a search query.
    :param names: a list of synonyms
    :return: the anime url if it's found.
    """
    query = sanitize_search_text(query)
    params = {
        'name': quote(query)
    }
    async with await session_manager.get(
            "http://www.anime-planet.com/anime/all?",
            params=params) as resp:
        html = await resp.text()
    ap = PyQuery(html)

    if ap.find('.cardDeck.pure-g.cd-narrow[data-type="anime"]'):
        anime_list = []
        for entry in ap.find('.card.pure-1-6'):
            anime = {
                'title': PyQuery(entry).find('h4').text(),
                'url': f'''http://www.anime-planet.com{PyQuery(entry).find("a").attr("href")}'''
            }
            anime_list.append(anime)
        return __get_closest(query, anime_list, names).get('url')
    else:
        return ap.find("meta[property='og:url']").attr('content')

async def get_manga_url(
        session_manager: SessionManager,
        query: str,
        names: list,
        author_name: str=None) -> Optional[str]:
    """
    Get manga url by search query.
    :param session_manager: the `SessionManager` instance.
    :param query: a search query.
    :return: the anime url if it's found.
    """
    params = {
            'name': quote(query)
        }
    if author_name:
        params['author'] = quote(author_name)
        async with await session_manager.get(
                "http://www.anime-planet.com/manga/all?",
                params=params) as resp:
            html = await resp.text()
        if "No results found" in html:
            rearranged_author_names = deque(
                author_name.split(' '))
            rearranged_author_names.rotate(-1)
            rearranged_name = ' '.join(rearranged_author_names)
            params['author'] = quote(rearranged_name)
            async with await session_manager.get(
                    "http://www.anime-planet.com/manga/all?",
                    params=params) as resp:
                html = await resp.text()
    else:
        async with await session_manager.get(
                "http://www.anime-planet.com/manga/all?",
                params=params) as resp:
            html = await resp.text()
    ap = PyQuery(html)

    if ap.find('.cardDeck.pure-g.cd-narrow[data-type="manga"]'):
        manga_list = []
        for entry in ap.find('.card.pure-1-6'):
            anime = {
                'title': PyQuery(entry).find('h4').text(),
                'url': f'''http://www.anime-planet.com{PyQuery(entry).find("a").attr("href")}'''
            }
            manga_list.append(anime)

            if author_name:
                author_name = author_name.lower().split(' ')
                for manga in manga_list:
                    manga['title'] = manga['title'].lower()
                    for name in author_name:
                        manga['title'] = manga['title'].replace(name, '')
                    manga['title'] = manga['title'].replace('(', '')
                    manga['title'] = manga['title'].replace(')', '').strip()

        return __get_closest(query, manga_list).get('url')
    else:
        return ap.find("meta[property='og:url']").attr('content')
    pass

def get_anime_url_by_id(anime_id: str) -> str:
    """
    Returns anime url by id.
    :param anime_id: an anime id.
    :return: the anime url.
    """
    return 'http://www.anime-planet.com/anime/' + str(anime_id)

def get_manga_url_by_id(manga_id: str) -> str:
    """
    Returns manga url by id.
    :param manga_id: a manga id.
    :return: the manga url.
    """
    return 'http://www.anime-planet.com/manga/' + str(manga_id)

def __get_closest(query: str, anime_list: List[dict], names) -> dict:
    """
    Get the closest matching anime by search query.
    :param query: the search term.
    :param anime_list: a list of anime.

    :return:
        Closest matching anime by search query if found else an empty dict.
    """
    synonyms_list = list(names[0])
    synonyms_list.insert(0, query)
    match = None
    for name in synonyms_list:
        max_ratio = 0
        matcher = SequenceMatcher(b=name.lower())
        for anime in anime_list:
            matcher.set_seq1(anime['title'].lower())
            ratio = matcher.ratio()
            if ratio > max_ratio and ratio >= 0.85:
                max_ratio = ratio
                match = anime
        if match:
            break
    return match or {}
<commit_msg>fixing internal documentation<commit_after>"""
Retrieve anime info from AnimePlanet.
"""
from collections import deque
from difflib import SequenceMatcher
from typing import List, Optional
from urllib.parse import quote

from pyquery import PyQuery

from roboragi.session_manager import SessionManager

def sanitize_search_text(text: str) -> str:
    """
    Sanitize text for Anime-Planet use.
    :param text: the text to be escaped.
    :return: the escaped text.
    """
    return text.replace('(TV)', 'TV')

async def get_anime_url(
        session_manager: SessionManager,
        query: str,
        names: list) -> Optional[str]:
    """
    Get anime url by search query.

    :param session_manager: the `SessionManager` instance.

    :param query: a search query.

    :param names: a list of synonyms

    :return: the anime url if it's found.
    """
    query = sanitize_search_text(query)
    params = {
        'name': quote(query)
    }
    async with await session_manager.get(
            "http://www.anime-planet.com/anime/all?",
            params=params) as resp:
        html = await resp.text()
    ap = PyQuery(html)

    if ap.find('.cardDeck.pure-g.cd-narrow[data-type="anime"]'):
        anime_list = []
        for entry in ap.find('.card.pure-1-6'):
            anime = {
                'title': PyQuery(entry).find('h4').text(),
                'url': f'''http://www.anime-planet.com{PyQuery(entry).find("a").attr("href")}'''
            }
            anime_list.append(anime)
        return __get_closest(query, anime_list, names).get('url')
    else:
        return ap.find("meta[property='og:url']").attr('content')

async def get_manga_url(
        session_manager: SessionManager,
        query: str,
        names: list,
        author_name: str=None) -> Optional[str]:
    """
    Get manga url by search query.

    :param session_manager: the `SessionManager` instance.

    :param query: a search query.

    :param names: a list of known names

    :author_name: name to search for manga

    :return: the anime url if it's found.
    """
    params = {
            'name': quote(query)
        }
    if author_name:
        params['author'] = quote(author_name)
        async with await session_manager.get(
                "http://www.anime-planet.com/manga/all?",
                params=params) as resp:
            html = await resp.text()
        if "No results found" in html:
            rearranged_author_names = deque(
                author_name.split(' '))
            rearranged_author_names.rotate(-1)
            rearranged_name = ' '.join(rearranged_author_names)
            params['author'] = quote(rearranged_name)
            async with await session_manager.get(
                    "http://www.anime-planet.com/manga/all?",
                    params=params) as resp:
                html = await resp.text()
    else:
        async with await session_manager.get(
                "http://www.anime-planet.com/manga/all?",
                params=params) as resp:
            html = await resp.text()
    ap = PyQuery(html)

    if ap.find('.cardDeck.pure-g.cd-narrow[data-type="manga"]'):
        manga_list = []
        for entry in ap.find('.card.pure-1-6'):
            anime = {
                'title': PyQuery(entry).find('h4').text(),
                'url': f'''http://www.anime-planet.com{PyQuery(entry).find("a").attr("href")}'''
            }
            manga_list.append(anime)

            if author_name:
                author_name = author_name.lower().split(' ')
                for manga in manga_list:
                    manga['title'] = manga['title'].lower()
                    for name in author_name:
                        manga['title'] = manga['title'].replace(name, '')
                    manga['title'] = manga['title'].replace('(', '')
                    manga['title'] = manga['title'].replace(')', '').strip()

        return __get_closest(query, manga_list).get('url')
    else:
        return ap.find("meta[property='og:url']").attr('content')
    pass

def get_anime_url_by_id(anime_id: str) -> str:
    """
    Returns anime url by id.

    :param anime_id: an anime id.

    :return: the anime url.
    """
    return 'http://www.anime-planet.com/anime/' + str(anime_id)

def get_manga_url_by_id(manga_id: str) -> str:
    """
    Returns manga url by id.

    :param manga_id: a manga id.

    :return: the manga url.
    """
    return 'http://www.anime-planet.com/manga/' + str(manga_id)

def __get_closest(query: str, anime_list: List[dict], names) -> dict:
    """
    Get the closest matching anime by search query.

    :param query: the search term.

    :param anime_list: a list of anime.

    :param names: a list of known names

    :return:
        Closest matching anime by search query if found else an empty dict.
    """
    synonyms_list = list(names[0])
    synonyms_list.insert(0, query)
    match = None
    for name in synonyms_list:
        max_ratio = 0
        matcher = SequenceMatcher(b=name.lower())
        for anime in anime_list:
            matcher.set_seq1(anime['title'].lower())
            ratio = matcher.ratio()
            if ratio > max_ratio and ratio >= 0.85:
                max_ratio = ratio
                match = anime
        if match:
            break
    return match or {}
<|endoftext|>


<commit_before><commit_msg>ref(py3): Correct encoding verification in cloudflare integration (#20328)<commit_after><|endoftext|>


<commit_before><commit_msg>Revert to using cv2 but keep the Wrapper changes<commit_after><|endoftext|>


<commit_before><commit_msg>Tests for sparse_vector_simulator<commit_after><|endoftext|>


<commit_before><commit_msg>unnecessary pass<commit_after><|endoftext|>


<commit_before><commit_msg>fix Arriva Midlands duplicates (?)<commit_after><|endoftext|>


<commit_before>import datetime as dt
import logging
from math import radians, tan
from urllib.request import urlretrieve

import click
import pandas as pd
import toml
from tqdm import tqdm

import planetpy.pdstools.data

try:
    from importlib_resources import path
except ModuleNotFoundError:
    from importlib.resources import path

logger = logging.getLogger(__name__)

try:
    from osgeo import gdal
except ImportError:
    GDAL_INSTALLED = False
    logger.warning("No GDAL found.Some util funcs not working, but okay.")
else:
    GDAL_INSTALLED = True

class ProgressBar(tqdm):
    """Provides `update_to(n)` which uses `tqdm.update(delta_n)`."""

    def update_to(self, b=1, bsize=1, tsize=None):
        """
        b  : int, optional
            Number of blocks transferred so far [default: 1].
        bsize  : int, optional
            Size of each block (in tqdm units) [default: 1].
        tsize  : int, optional
            Total size (in tqdm units). If [default: None] remains unchanged.
        """
        if tsize is not None:
            self.total = tsize
        self.update(b * bsize - self.n)  # will also set self.n = b * bsize

nasa_date_format = '%Y-%j'
nasa_dt_format = nasa_date_format + 'T%H:%M:%S'
nasa_dt_format_with_ms = nasa_dt_format + '.%f'
standard_date_format = '%Y-%m-%d'
standard_dt_format = standard_date_format + 'T%H:%M:%S'

def nasa_date_to_iso(datestr):
    """Convert the day-number based NASA format to ISO.

    Parameters
    ----------
    datestr : str
        Date string in the form Y-j

    Returns
    -------
    Datestring in ISO standard yyyy-mm-ddTHH:MM:SS.MMMMMM
    """
    date = dt.datetime.strptime(datestr, nasa_date_format)
    return date.isoformat()

@click.command('nasa_date_to_iso')
@click.argument('datestr')
def nasa_date_to_iso_command(datestr):
    click.echo(nasa_date_to_iso(datestr))

def iso_to_nasa_date(datestr):
    date = dt.datetime.strptime(datestr, standard_date_format)
    return date.strftime(nasa_date_format)

def nasa_datetime_to_iso(dtimestr):
    if dtimestr.split('.')[1]:
        tformat = nasa_dt_format_with_ms
    else:
        tformat = nasa_dt_format
    time = dt.datetime.strptime(dtimestr, tformat)
    return time.isoformat()

def iso_to_nasa_datetime(dtimestr):
    date = dt.datetime.strptime(dtimestr, standard_dt_format)
    return date.strftime(nasa_dt_format)

def replace_all_nasa_times(df):
    for col in [col for col in df.columns if 'TIME' in col]:
        if 'T' in df[col].iloc[0]:
            df[col] = pd.to_datetime(df[col].map(nasa_datetime_to_iso))

def get_gdal_center_coords(imgpath):
    if not GDAL_INSTALLED:
        logger.error("GDAL not installed. Returning")
        return
    ds = gdal.Open(str(imgpath))
    xmean = ds.RasterXSize // 2
    ymean = ds.RasterYSize // 2
    return xmean, ymean

def download(url, localpath='.', **kwargs):
    """Simple wrapper of urlretrieve

    Adding a default path to urlretrieve

    Parameters:
    ----------
    url : str
        HTTP(S) URL to download
    localpath : str,pathlib.Path
        Local path where to store the download.
    **kwargs : {dict}
        Keyword args to be handed to urlretrieve.
    Returns
    -------
    Tuple
        Tuple returned by urlretrieve
    """
    return urlretrieve(url, localpath, **kwargs)

def height_from_shadow(shadow_in_pixels, sun_elev):
    """Calculate height of an object from its shadow length.

    Note, that your image might have been binned.
    You need to correct `shadow_in_pixels` for that.

    Parameters
    ----------
    shadow_in_pixels : float
        Measured length of shadow in pixels
    sun_elev : angle(float)
        Angle of sun over horizon

    Returns
    -------
    height [meter]
    """
    return tan(radians(sun_elev)) * shadow_in_pixels
<commit_msg>fix download (and black-formatting)<commit_after>import datetime as dt
import logging
from math import radians, tan
from pathlib import Path
from urllib.request import urlretrieve

import click
import pandas as pd
import toml
from tqdm import tqdm

import planetpy.pdstools.data

try:
    from importlib_resources import path
except ModuleNotFoundError:
    from importlib.resources import path

logger = logging.getLogger(__name__)

try:
    from osgeo import gdal
except ImportError:
    GDAL_INSTALLED = False
    logger.warning("No GDAL found.Some util funcs not working, but okay.")
else:
    GDAL_INSTALLED = True

class ProgressBar(tqdm):
    """Provides `update_to(n)` which uses `tqdm.update(delta_n)`."""

    def update_to(self, b=1, bsize=1, tsize=None):
        """
        b  : int, optional
            Number of blocks transferred so far [default: 1].
        bsize  : int, optional
            Size of each block (in tqdm units) [default: 1].
        tsize  : int, optional
            Total size (in tqdm units). If [default: None] remains unchanged.
        """
        if tsize is not None:
            self.total = tsize
        self.update(b * bsize - self.n)  # will also set self.n = b * bsize

nasa_date_format = "%Y-%j"
nasa_dt_format = nasa_date_format + "T%H:%M:%S"
nasa_dt_format_with_ms = nasa_dt_format + ".%f"
standard_date_format = "%Y-%m-%d"
standard_dt_format = standard_date_format + "T%H:%M:%S"

def nasa_date_to_iso(datestr):
    """Convert the day-number based NASA format to ISO.

    Parameters
    ----------
    datestr : str
        Date string in the form Y-j

    Returns
    -------
    Datestring in ISO standard yyyy-mm-ddTHH:MM:SS.MMMMMM
    """
    date = dt.datetime.strptime(datestr, nasa_date_format)
    return date.isoformat()

@click.command("nasa_date_to_iso")
@click.argument("datestr")
def nasa_date_to_iso_command(datestr):
    click.echo(nasa_date_to_iso(datestr))

def iso_to_nasa_date(datestr):
    date = dt.datetime.strptime(datestr, standard_date_format)
    return date.strftime(nasa_date_format)

def nasa_datetime_to_iso(dtimestr):
    if dtimestr.split(".")[1]:
        tformat = nasa_dt_format_with_ms
    else:
        tformat = nasa_dt_format
    time = dt.datetime.strptime(dtimestr, tformat)
    return time.isoformat()

def iso_to_nasa_datetime(dtimestr):
    date = dt.datetime.strptime(dtimestr, standard_dt_format)
    return date.strftime(nasa_dt_format)

def replace_all_nasa_times(df):
    for col in [col for col in df.columns if "TIME" in col]:
        if "T" in df[col].iloc[0]:
            df[col] = pd.to_datetime(df[col].map(nasa_datetime_to_iso))

def get_gdal_center_coords(imgpath):
    if not GDAL_INSTALLED:
        logger.error("GDAL not installed. Returning")
        return
    ds = gdal.Open(str(imgpath))
    xmean = ds.RasterXSize // 2
    ymean = ds.RasterYSize // 2
    return xmean, ymean

def download(url, localpath=".", **kwargs):
    """Simple wrapper of urlretrieve

    Adding a default path to urlretrieve

    Parameters:
    ----------
    url : str
        HTTP(S) URL to download
    localpath : str,pathlib.Path
        Local path where to store the download.
    **kwargs : {dict}
        Keyword args to be handed to urlretrieve.
    Returns
    -------
    Tuple
        Tuple returned by urlretrieve
    """
    urlpath = Path(url)
    local = Path(localpath)
    savepath = local / urlpath.name if local.is_dir() else local
    return urlretrieve(url, savepath, **kwargs)

def height_from_shadow(shadow_in_pixels, sun_elev):
    """Calculate height of an object from its shadow length.

    Note, that your image might have been binned.
    You need to correct `shadow_in_pixels` for that.

    Parameters
    ----------
    shadow_in_pixels : float
        Measured length of shadow in pixels
    sun_elev : angle(float)
        Angle of sun over horizon

    Returns
    -------
    height [meter]
    """
    return tan(radians(sun_elev)) * shadow_in_pixels
<|endoftext|>