<commit_before>import discord
import aiohttp
import json
from __main__ import send_cmd_help
from .utils import chat_formatting as chat
from discord.ext import commands

numbs = {
    "next": "➡",
    "back": "⬅",
    "exit": "❌"
}
BASEURL = 'http://dnd5eapi.co/api/'

class DND:
    '''D&D Lookup Stuff'''

    def __init__(self, bot):
        self.bot = bot

    @commands.group(pass_context=True)
    async def dnd(self, ctx):
        if ctx.invoked_subcommand is None:
            await send_cmd_help(ctx)
            return

    @dnd.command(name='spells', pass_context=True)
    async def lookup_spells(self, ctx, *, spell=None):
        '''Lookup Spells'''
        CHANNEL = ctx.message.channel
        CATEGORY = 'Spells'
        if spell is None:
            url = '{}{}'.format(BASEURL, CATEGORY)
            menu_pages = await _present_list(self, url, CATEGORY)
            ## // process menu_pages
            await self.cogs_menu(ctx, menu_pages)

        elif spell is not None:
            await self.bot.say('spell: {}'.format(spell))

    #
    # @dnd.command(name='classes', pass_context=True)
    # async def lookup_classes(self, klass=None):
    #     '''Lookup Classes'''
    #     baseurl = BASEURL+'classes'
    #     #Your code will go here
    #     await self.bot.say("Lookup Classes initiated.")
    #     await self.bot.say("<{}>".format(baseurl))
    #
    # @dnd.command(name='monsters', pass_context=True)
    # async def lookup_monsters(self, monster=None):
    #     '''Lookup Monsters'''
    #     baseurl = BASEURL+'monsters'
    #     #Your code will go here
    #     await self.bot.say("Lookup Monsters initiated.")
    #     await self.bot.say("<{}>".format(baseurl))
    #
    # @dnd.command(name='equipment', pass_context=True)
    # async def lookup_equipment(self, equiped=None):
    #     '''Lookup Equpiment'''
    #     baseurl = BASEURL+'equipment'
    #     #Your code will go here
    #     await self.bot.say("Lookup Spells initiated.")
    #     await self.bot.say("<{}>".format(baseurl))

    async def cogs_menu(self, ctx, cog_list: list, message: discord.Message=None, page=0, timeout: int=30):
        """menu control logic for this taken from
           https://github.com/Lunar-Dust/Dusty-Cogs/blob/master/menu/menu.py"""
        cog = cog_list[page]
        if not message:
            message =\
                await self.bot.send_message(ctx.message.channel, embed=cog)
            await self.bot.add_reaction(message, "⬅")
            await self.bot.add_reaction(message, "❌")
            await self.bot.add_reaction(message, "➡")
        else:
            message = await self.bot.edit_message(message, embed=cog)
        react = await self.bot.wait_for_reaction(
            message=message, user=ctx.message.author, timeout=timeout,
            emoji=["➡", "⬅", "❌"]
        )
        if react is None:
            try:
                try:
                    await self.bot.clear_reactions(message)
                except:
                    await self.bot.remove_reaction(message, "⬅", self.bot.user)
                    await self.bot.remove_reaction(message, "❌", self.bot.user)
                    await self.bot.remove_reaction(message, "➡", self.bot.user)
            except:
                pass
            return None
        reacts = {v: k for k, v in numbs.items()}
        react = reacts[react.reaction.emoji]
        if react == "next":
            next_page = 0
            if page == len(cog_list) - 1:
                next_page = 0  # Loop around to the first item
            else:
                next_page = page + 1
            return await self.cogs_menu(ctx, cog_list, message=message,
                                        page=next_page, timeout=timeout)
        elif react == "back":
            next_page = 0
            if page == 0:
                next_page = len(cog_list) - 1  # Loop around to the last item
            else:
                next_page = page - 1
            return await self.cogs_menu(ctx, cog_list, message=message,
                                        page=next_page, timeout=timeout)
        else:
            try:
                return await\
                    self.bot.delete_message(message)
            except:
                pass

async def _get_file(url):
    async with aiohttp.ClientSession() as session:
        async with session.get(url) as response:
            print('_get_file('+url+')')
            json_file = await response.json()
            if json_file is not None:
                return json_file

async def _present_list(self, url, category):
    json_file = await _get_file(url)
    urllength = len(url)
    print(urllength)
    await self.bot.say('{}'.format(urllength))
    if json_file is not None:
        count = json_file['count']
        results = json_file['results']
        package = []
        for i in range(0,int(count)):
            c = i+1
            package.append('{} {}'.format(c, results[i]['name']))

        pages = chat.pagify('\n'.join(package), delims=['\n'], escape=True, shorten_by=8, page_length=750)
        menu_pages = []
        for page in pages:
            em=discord.Embed(color=discord.Color.red(), title=category, description=page)
            em.set_footer(text='From dnd5eapi.co',icon_url='http://www.dnd5eapi.co/public/favicon.ico')
            menu_pages.append(em)
            return menu_pages

def setup(bot):
    bot.add_cog(DND(bot))
<commit_msg><commit_after>import discord
import aiohttp
import json
from __main__ import send_cmd_help
from .utils import chat_formatting as chat
from discord.ext import commands

numbs = {
    "next": "➡",
    "back": "⬅",
    "exit": "❌"
}
BASEURL = 'http://dnd5eapi.co/api/'

class DND:
    '''D&D Lookup Stuff'''

    def __init__(self, bot):
        self.bot = bot

    @commands.group(pass_context=True)
    async def dnd(self, ctx):
        if ctx.invoked_subcommand is None:
            await send_cmd_help(ctx)
            return

    @dnd.command(name='spells', pass_context=True)
    async def lookup_spells(self, ctx, *, spell=None):
        '''Lookup Spells'''
        CHANNEL = ctx.message.channel
        CATEGORY = 'Spells'
        if spell is None:
            url = '{}{}'.format(BASEURL, CATEGORY)
            menu_pages = await _present_list(self, url, CATEGORY)
            ## // process menu_pages
            await self.bot.say('{} pages'.format(len(menu_pages)))
            await self.cogs_menu(ctx, menu_pages, message=None, page=0, timeout=30)

        elif spell is not None:
            await self.bot.say('spell: {}'.format(spell))

    #
    # @dnd.command(name='classes', pass_context=True)
    # async def lookup_classes(self, klass=None):
    #     '''Lookup Classes'''
    #     baseurl = BASEURL+'classes'
    #     #Your code will go here
    #     await self.bot.say("Lookup Classes initiated.")
    #     await self.bot.say("<{}>".format(baseurl))
    #
    # @dnd.command(name='monsters', pass_context=True)
    # async def lookup_monsters(self, monster=None):
    #     '''Lookup Monsters'''
    #     baseurl = BASEURL+'monsters'
    #     #Your code will go here
    #     await self.bot.say("Lookup Monsters initiated.")
    #     await self.bot.say("<{}>".format(baseurl))
    #
    # @dnd.command(name='equipment', pass_context=True)
    # async def lookup_equipment(self, equiped=None):
    #     '''Lookup Equpiment'''
    #     baseurl = BASEURL+'equipment'
    #     #Your code will go here
    #     await self.bot.say("Lookup Spells initiated.")
    #     await self.bot.say("<{}>".format(baseurl))

    async def cogs_menu(self, ctx, cog_list: list, message: discord.Message=None, page=0, timeout: int=30):
        """menu control logic for this taken from
           https://github.com/Lunar-Dust/Dusty-Cogs/blob/master/menu/menu.py"""
        cog = cog_list[page]
        if not message:
            message =\
                await self.bot.send_message(ctx.message.channel, embed=cog)
            await self.bot.add_reaction(message, "⬅")
            await self.bot.add_reaction(message, "❌")
            await self.bot.add_reaction(message, "➡")
        else:
            message = await self.bot.edit_message(message, embed=cog)
        react = await self.bot.wait_for_reaction(
            message=message, user=ctx.message.author, timeout=timeout,
            emoji=["➡", "⬅", "❌"]
        )
        if react is None:
            try:
                try:
                    await self.bot.clear_reactions(message)
                except:
                    await self.bot.remove_reaction(message, "⬅", self.bot.user)
                    await self.bot.remove_reaction(message, "❌", self.bot.user)
                    await self.bot.remove_reaction(message, "➡", self.bot.user)
            except:
                pass
            return None
        reacts = {v: k for k, v in numbs.items()}
        react = reacts[react.reaction.emoji]
        if react == "next":
            next_page = 0
            if page == len(cog_list) - 1:
                next_page = 0  # Loop around to the first item
            else:
                next_page = page + 1
            return await self.cogs_menu(ctx, cog_list, message=message,
                                        page=next_page, timeout=timeout)
        elif react == "back":
            next_page = 0
            if page == 0:
                next_page = len(cog_list) - 1  # Loop around to the last item
            else:
                next_page = page - 1
            return await self.cogs_menu(ctx, cog_list, message=message,
                                        page=next_page, timeout=timeout)
        else:
            try:
                return await\
                    self.bot.delete_message(message)
            except:
                pass

async def _get_file(url):
    async with aiohttp.ClientSession() as session:
        async with session.get(url) as response:
            print('_get_file('+url+')')
            json_file = await response.json()
            if json_file is not None:
                return json_file

async def _present_list(self, url, category):
    json_file = await _get_file(url)
    if json_file is not None:
        results = json_file['results']
        package = []
        for i in range(0,int(json_file['count'])):
            c = i+1
            package.append('{} {}'.format(c, json_file['results'][i]['name']))

        pages = chat.pagify('\n'.join(package), delims=['\n'], escape=True, shorten_by=8, page_length=750)
        menu_pages = []
        for page in pages:
            em=discord.Embed(color=discord.Color.red(), title=category, description=page)
            em.set_footer(text='From dnd5eapi.co',icon_url='http://www.dnd5eapi.co/public/favicon.ico')
            menu_pages.append(em)
            return menu_pages

def setup(bot):
    bot.add_cog(DND(bot))
<|endoftext|>


<commit_before>import play

class MightyMight(play.Play):
    def __init__(self):
        super().__init__(continuous=True)
        <commit_msg>Removes MightyMight cause we're not going to implement it.<commit_after><|endoftext|>


<commit_before>import simplejson
import math
import DQXMathUtils
import os
import re
import DQXEncoder
import random

#############################################################################################

def FilterValuesToFloat(list):
    rs=[]
    for vl in list:
        if vl!=None:
            if vl!='\N':
                rs.append(float(vl))
    return rs

class ValueList:
    def __init__(self,ilist):
        self.list=ilist
        self.floatlist=None
        self.sortedfloatlist=None
    def GetOrigList(self):
        return self.list
    def GetFloatList(self):
        if not(self.floatlist):
            self.floatlist=FilterValuesToFloat(self.list)
        return self.floatlist
    def GetSortedFloatList(self):
        if not(self.sortedfloatlist):
            self.sortedfloatlist=sorted(self.GetFloatList())
        return self.sortedfloatlist

class Summariser:
    def __init__(self,info):
        self.propID=info['PropID']
        self.IDExt=info['IDExt']
        self.ID=self.propID+'_'+self.IDExt
        self.encoder=DQXEncoder.GetEncoder(info['Encoder'])
    def calcSummary(self,list):
        raise Exception('Summary function not implemented')
    def Perform(self,list):#list is of class ValueList
        summvalue=self.calcSummary(list)
        rs=self.encoder.perform(summvalue)
        if len(rs)!=self.encoder.getlength():
            raise Exception('Encoder returned wrong size')
        return rs

class SummariserPickRandom(Summariser):
    def __init__(self,info):
        Summariser.__init__(self,info)
    def calcSummary(self,list):
        lst=list.GetOrigList()
        if len(lst)==0:
            return None
        else:
            idx=int(random.randint(0,len(lst)-1))
            return lst[idx]
    def getInfo(self):
        return {'ID':'PickRandom' }

class SummariserMostFrequent(Summariser):
    def __init__(self,info):
        Summariser.__init__(self,info)
    def calcSummary(self,list):
        lst=list.GetOrigList()
        if len(lst)==0:
            return None
        else:
            freqdict={}
            for item in lst:
                if not(item in freqdict):
                    freqdict[item]=0
                freqdict[item]+=1
            maxfreq=0
            mostfrequentitemlist=[]
            for item in freqdict:
                if freqdict[item]>maxfreq:
                    maxfreq=freqdict[item]
                    mostfrequentitemlist=[]
                if freqdict[item]==maxfreq:
                    mostfrequentitemlist.append(item)
            mostfrequentitem=mostfrequentitemlist[random.randint(0,len(mostfrequentitemlist)-1)]
#            if len(lst)>2:
#                print('----------------')
#                print(str(freqdict))
#                print(str(mostfrequentitemlist))
#                print(str(mostfrequentitem))
            return mostfrequentitem
    def getInfo(self):
        return {'ID':'MostFrequent' }

class SummariserAverage(Summariser):
    def __init__(self,info):
        Summariser.__init__(self,info)
    def calcSummary(self,list):
        floatlist=list.GetFloatList()
        if len(floatlist)==0:
            return None
        return sum(floatlist)/float(len(floatlist))
    def getInfo(self):
        return {'ID':'Average' }

class SummariserMax(Summariser):
    def __init__(self,info):
        Summariser.__init__(self,info)
    def calcSummary(self,list):
        floatlist=list.GetFloatList()
        if len(floatlist)==0:
            return None
        return max(floatlist)
    def getInfo(self):
        return {'ID':'Max' }

class SummariserMin(Summariser):
    def __init__(self,info):
        Summariser.__init__(self,info)
    def calcSummary(self,list):
        floatlist=list.GetFloatList()
        if len(floatlist)==0:
            return None
        return min(floatlist)
    def getInfo(self):
        return {'ID':'Min' }

class SummariserQuantile(Summariser):
    def __init__(self,info):
        Summariser.__init__(self,info)
        self.frac=info['Fraction']
    def calcSummary(self,list):
        sortfloatlist=list.GetSortedFloatList()
        if len(sortfloatlist)==0:
            return None
        return DQXMathUtils.quantile(sortfloatlist,self.frac,7,True)
    def getInfo(self):
        return {'ID':'Quantile', 'Frac':self.frac }

class SummariserHistogram(Summariser):
    def __init__(self,info):
        Summariser.__init__(self,info)
        self.count=info['Count']
        self.minval=info['MinVal']
        self.stepsize=info['StepSize']
    def calcSummary(self,list):
        histo=[0]*self.count
        floatlist=list.GetFloatList()
        for val in floatlist:
            idx=int(0.01+math.floor((val-self.minval)*1.0/self.stepsize))
            if idx<0:idx=0
            if idx>=self.count: idx=self.count-1
            histo[idx]+=1
        if len(floatlist)>0:
            normhisto=[val*1.0/len(floatlist) for val in histo]
        else:
            normhisto=histo
        return normhisto
    def getInfo(self):
        return {'ID':'Histogram', 'Count':self.count, 'MinVal':self.minval, 'StepSize':self.stepsize }

#############################################################################################

class DataProperty:
    def __init__(self,info):
        self.ID=info['ID']

def ReadJsonFile(filename):
    f = open(filename, 'r')
    body=''
    for line in f.readlines():
        line=line.lstrip()
        if (len(line)>0) and (line[0]!='#'):
            body+=line
    f.close()
    data=simplejson.loads(body)
    return data

#############################################################################################

class DataProvider_TabbedFile:
    def __init__(self,ifilename):
        self.inputfilename=ifilename
    def GetRowIterator(self):
        inputfile=open(self.inputfilename,'r')
        separator='\t'
        header=inputfile.readline().rstrip('\n')
        if header.find('|')>0:
            separator='|'
        inputcolumns=header.split(separator)[1:]
        while True:
            line=inputfile.readline().rstrip('\n')
            if not(line):
                break
            linecomps=line.split(separator)
            rs={}
            rs['position']=int(linecomps[0])
            for colnr in range(len(inputcolumns)):
                rs[inputcolumns[colnr]]=[x for x in linecomps[colnr+1].split(',')]
            yield rs
        inputfile.close()

def CreateDataProvider(filename,filetype):
    if filetype=="TabDelimitedFile":
        return DataProvider_TabbedFile(filename)
    raise Exception('Unknown data type {0}'.format(filetype))

#############################################################################################

class Creator:

    def __init__(self,ibasedir,ifolder,iconfig):
        self.basedir=ibasedir
        self.folder=ifolder
        self.config=iconfig
        self.datadir=self.basedir+'/'+self.folder
        print('Initialising Creator, directory="{0}", config="{1}"'.format(self.datadir,self.config))
        configdata=ReadJsonFile(self.datadir+'/'+self.config+".cnf")

        if 'SourceFilePattern' in configdata:
            self.SourceFilePattern=configdata['SourceFilePattern']
        if 'SourceFileType' in configdata:
            self.SourceFileType=configdata['SourceFileType']
        self.BlockSizeStart=int(configdata['BlockSizeStart'])
        self.BlockSizeIncrFactor=int(configdata['BlockSizeIncrFactor'])
        self.BlockSizeMax=int(configdata['BlockSizeMax'])

        self.properties=[DataProperty(prop) for prop in configdata['Properties']]

        self.summarisers=[]
        self.summarisersIdx={}
        for summinfo in configdata['Summarisers']:
            summ=self.CreateSummariser(summinfo)
            self.summarisersIdx[summ.ID]=len(self.summarisers)
            self.summarisers.append(summ)

        self.encodedRowSize=sum([summ.encoder.getlength() for summ in self.summarisers])

    def CreateSummariser(self,summinfo):
        mysumm=None
        if summinfo['Method']=='Average':
            mysumm=SummariserAverage(summinfo)
        if summinfo['Method']=='Max':
            mysumm=SummariserMax(summinfo)
        if summinfo['Method']=='Min':
            mysumm=SummariserMin(summinfo)
        if summinfo['Method']=='Quantile':
            mysumm=SummariserQuantile(summinfo)
        if summinfo['Method']=='Histogram':
            mysumm=SummariserHistogram(summinfo)
        if summinfo['Method']=='PickRandom':
            mysumm=SummariserPickRandom(summinfo)
        if summinfo['Method']=='MostFrequent':
            mysumm=SummariserMostFrequent(summinfo)
        if mysumm is None:
            raise Exception('Unknown summariser '+summinfo['Method'])
        return mysumm

    def getSummariserNr(self,summid):
        return self.summarisersIdx[summid]

    def GetBlocks(self,blockSize):
        lst=[]
        currentBlockStart=1
        for row in self.dataprovider.GetRowIterator():
            while row['position']>=currentBlockStart+blockSize:
                yield lst
                lst=[]
                currentBlockStart+=blockSize
            lst.append(row)
        yield lst

    def Summarise(self):

        #create output directory if necessary
        outputdir=os.path.join(self.datadir,'Summaries')
        if not os.path.exists(outputdir):
            os.makedirs(outputdir)

        #remove all summary files that correspond to this configuration
        for filename in os.listdir(outputdir):
            if filename.startswith(self.config):
                os.remove(os.path.join(outputdir,filename))

        expr=re.compile(self.SourceFilePattern)
        for sourcefilename in os.listdir(self.datadir):
            fullsourcefilename=os.path.join(self.datadir,sourcefilename)
            if not os.path.isdir(fullsourcefilename):
                if not expr.match(sourcefilename):
                    print('[Skipping file "{0}" (not matched)]'.format(sourcefilename))
                else:
                    print('Processing file "{0}"'.format(sourcefilename))
                    dataid=sourcefilename.split('.')[0]

                    self.dataprovider=CreateDataProvider(fullsourcefilename,self.SourceFileType)
                    outputbasefilename=os.path.join(self.datadir,'Summaries',dataid)
                    blockSize=self.BlockSizeStart
                    while blockSize<=self.BlockSizeMax:
                        print('EXECUTING {0}, BLOCK SIZE {1}'.format(dataid,blockSize))
                        outputfilename=os.path.join(self.datadir,'Summaries','{0}_{1}_{2}'.format(self.config,dataid,str(blockSize)))
                        outputfile=open(outputfilename,'w')

                        blocknr=0
                        for block in self.GetBlocks(blockSize):
                            #for each property, get a list of data in this block
                            proplists={prop.ID: ValueList([val for row in block for val in row[prop.ID]]) for prop in self.properties}
                            #run all the summarisers, merge & write to the output file as a row record
                            summarystr=''.join([summ.Perform(proplists[summ.propID]) for summ in self.summarisers])
                            outputfile.write(summarystr)
                            blocknr+=1
                            if blocknr%5000==0: print('  {0} blocks processed'.format(blocknr))

                        outputfile.close()
                        blockSize *=self.BlockSizeIncrFactor

    def GetData(self,dataid,blockSize,start,length,summarylist):
        outputbasefilename=self.datadir+'/Summaries/'+self.config+'_'+dataid
        filename=outputbasefilename+'_'+str(blockSize)
        #print('Fetching from '+filename)
        f=open(filename,'r')
        f.seek(start*self.encodedRowSize)
        linelength=self.encodedRowSize
        strblock=f.read(length*linelength)
        f.close()
        result={}
        for summid in summarylist:
            nr=self.getSummariserNr(summid)
            strrs=''
            offset=sum([self.summarisers[i].encoder.getlength() for i in range(nr)])
            complength=self.summarisers[nr].encoder.getlength()
            for i in range(length):
                strrs+=strblock[offset:offset+complength]
                offset+=linelength

            result[self.folder+'_'+self.config+'_'+summid]={'data':strrs, 'summariser':self.summarisers[nr].getInfo(), 'encoder':self.summarisers[nr].encoder.getInfo() }
        return result
<commit_msg>Improved handling of absent summary data<commit_after>import simplejson
import math
import DQXMathUtils
import os
import re
import DQXEncoder
import random

#############################################################################################

def FilterValuesToFloat(list):
    rs=[]
    for vl in list:
        if vl!=None:
            if vl!='\N':
                rs.append(float(vl))
    return rs

class ValueList:
    def __init__(self,ilist):
        self.list=ilist
        self.floatlist=None
        self.sortedfloatlist=None
    def GetOrigList(self):
        return self.list
    def GetFloatList(self):
        if not(self.floatlist):
            self.floatlist=FilterValuesToFloat(self.list)
        return self.floatlist
    def GetSortedFloatList(self):
        if not(self.sortedfloatlist):
            self.sortedfloatlist=sorted(self.GetFloatList())
        return self.sortedfloatlist

class Summariser:
    def __init__(self,info):
        self.propID=info['PropID']
        self.IDExt=info['IDExt']
        self.ID=self.propID+'_'+self.IDExt
        self.encoder=DQXEncoder.GetEncoder(info['Encoder'])
    def calcSummary(self,list):
        raise Exception('Summary function not implemented')
    def Perform(self,list):#list is of class ValueList
        summvalue=self.calcSummary(list)
        rs=self.encoder.perform(summvalue)
        if len(rs)!=self.encoder.getlength():
            raise Exception('Encoder returned wrong size')
        return rs

class SummariserPickRandom(Summariser):
    def __init__(self,info):
        Summariser.__init__(self,info)
    def calcSummary(self,list):
        lst=list.GetOrigList()
        if len(lst)==0:
            return None
        else:
            idx=int(random.randint(0,len(lst)-1))
            return lst[idx]
    def getInfo(self):
        return {'ID':'PickRandom' }

class SummariserMostFrequent(Summariser):
    def __init__(self,info):
        Summariser.__init__(self,info)
    def calcSummary(self,list):
        lst=list.GetOrigList()
        if len(lst)==0:
            return None
        else:
            freqdict={}
            for item in lst:
                if not(item in freqdict):
                    freqdict[item]=0
                freqdict[item]+=1
            maxfreq=0
            mostfrequentitemlist=[]
            for item in freqdict:
                if freqdict[item]>maxfreq:
                    maxfreq=freqdict[item]
                    mostfrequentitemlist=[]
                if freqdict[item]==maxfreq:
                    mostfrequentitemlist.append(item)
            mostfrequentitem=mostfrequentitemlist[random.randint(0,len(mostfrequentitemlist)-1)]
#            if len(lst)>2:
#                print('----------------')
#                print(str(freqdict))
#                print(str(mostfrequentitemlist))
#                print(str(mostfrequentitem))
            return mostfrequentitem
    def getInfo(self):
        return {'ID':'MostFrequent' }

class SummariserAverage(Summariser):
    def __init__(self,info):
        Summariser.__init__(self,info)
    def calcSummary(self,list):
        floatlist=list.GetFloatList()
        if len(floatlist)==0:
            return None
        return sum(floatlist)/float(len(floatlist))
    def getInfo(self):
        return {'ID':'Average' }

class SummariserMax(Summariser):
    def __init__(self,info):
        Summariser.__init__(self,info)
    def calcSummary(self,list):
        floatlist=list.GetFloatList()
        if len(floatlist)==0:
            return None
        return max(floatlist)
    def getInfo(self):
        return {'ID':'Max' }

class SummariserMin(Summariser):
    def __init__(self,info):
        Summariser.__init__(self,info)
    def calcSummary(self,list):
        floatlist=list.GetFloatList()
        if len(floatlist)==0:
            return None
        return min(floatlist)
    def getInfo(self):
        return {'ID':'Min' }

class SummariserQuantile(Summariser):
    def __init__(self,info):
        Summariser.__init__(self,info)
        self.frac=info['Fraction']
    def calcSummary(self,list):
        sortfloatlist=list.GetSortedFloatList()
        if len(sortfloatlist)==0:
            return None
        return DQXMathUtils.quantile(sortfloatlist,self.frac,7,True)
    def getInfo(self):
        return {'ID':'Quantile', 'Frac':self.frac }

class SummariserHistogram(Summariser):
    def __init__(self,info):
        Summariser.__init__(self,info)
        self.count=info['Count']
        self.minval=info['MinVal']
        self.stepsize=info['StepSize']
    def calcSummary(self,list):
        histo=[0]*self.count
        floatlist=list.GetFloatList()
        for val in floatlist:
            idx=int(0.01+math.floor((val-self.minval)*1.0/self.stepsize))
            if idx<0:idx=0
            if idx>=self.count: idx=self.count-1
            histo[idx]+=1
        if len(floatlist)>0:
            normhisto=[val*1.0/len(floatlist) for val in histo]
        else:
            normhisto=histo
        return normhisto
    def getInfo(self):
        return {'ID':'Histogram', 'Count':self.count, 'MinVal':self.minval, 'StepSize':self.stepsize }

#############################################################################################

class DataProperty:
    def __init__(self,info):
        self.ID=info['ID']

def ReadJsonFile(filename):
    if not os.path.isfile(filename):
        print('ERROR: MISSING FILE '+filename)
        return None
    f = open(filename, 'r')
    body=''
    for line in f.readlines():
        line=line.lstrip()
        if (len(line)>0) and (line[0]!='#'):
            body+=line
    f.close()
    data=simplejson.loads(body)
    return data

#############################################################################################

class DataProvider_TabbedFile:
    def __init__(self,ifilename):
        self.inputfilename=ifilename
    def GetRowIterator(self):
        inputfile=open(self.inputfilename,'r')
        separator='\t'
        header=inputfile.readline().rstrip('\n')
        if header.find('|')>0:
            separator='|'
        inputcolumns=header.split(separator)[1:]
        while True:
            line=inputfile.readline().rstrip('\n')
            if not(line):
                break
            linecomps=line.split(separator)
            rs={}
            rs['position']=int(linecomps[0])
            for colnr in range(len(inputcolumns)):
                rs[inputcolumns[colnr]]=[x for x in linecomps[colnr+1].split(',')]
            yield rs
        inputfile.close()

def CreateDataProvider(filename,filetype):
    if filetype=="TabDelimitedFile":
        return DataProvider_TabbedFile(filename)
    raise Exception('Unknown data type {0}'.format(filetype))

#############################################################################################

class Creator:

    def __init__(self,ibasedir,ifolder,iconfig):
        self.basedir=ibasedir
        self.folder=ifolder
        self.config=iconfig
        self.datadir=self.basedir+'/'+self.folder
        print('Initialising Creator, directory="{0}", config="{1}"'.format(self.datadir,self.config))
        configdata=ReadJsonFile(self.datadir+'/'+self.config+".cnf")
        if configdata is None:
            self.present = False
            return
        else:
            self.present = True

        if 'SourceFilePattern' in configdata:
            self.SourceFilePattern=configdata['SourceFilePattern']
        if 'SourceFileType' in configdata:
            self.SourceFileType=configdata['SourceFileType']
        self.BlockSizeStart=int(configdata['BlockSizeStart'])
        self.BlockSizeIncrFactor=int(configdata['BlockSizeIncrFactor'])
        self.BlockSizeMax=int(configdata['BlockSizeMax'])

        self.properties=[DataProperty(prop) for prop in configdata['Properties']]

        self.summarisers=[]
        self.summarisersIdx={}
        for summinfo in configdata['Summarisers']:
            summ=self.CreateSummariser(summinfo)
            self.summarisersIdx[summ.ID]=len(self.summarisers)
            self.summarisers.append(summ)

        self.encodedRowSize=sum([summ.encoder.getlength() for summ in self.summarisers])

    def CreateSummariser(self,summinfo):
        mysumm=None
        if summinfo['Method']=='Average':
            mysumm=SummariserAverage(summinfo)
        if summinfo['Method']=='Max':
            mysumm=SummariserMax(summinfo)
        if summinfo['Method']=='Min':
            mysumm=SummariserMin(summinfo)
        if summinfo['Method']=='Quantile':
            mysumm=SummariserQuantile(summinfo)
        if summinfo['Method']=='Histogram':
            mysumm=SummariserHistogram(summinfo)
        if summinfo['Method']=='PickRandom':
            mysumm=SummariserPickRandom(summinfo)
        if summinfo['Method']=='MostFrequent':
            mysumm=SummariserMostFrequent(summinfo)
        if mysumm is None:
            raise Exception('Unknown summariser '+summinfo['Method'])
        return mysumm

    def getSummariserNr(self,summid):
        return self.summarisersIdx[summid]

    def GetBlocks(self,blockSize):
        lst=[]
        currentBlockStart=1
        for row in self.dataprovider.GetRowIterator():
            while row['position']>=currentBlockStart+blockSize:
                yield lst
                lst=[]
                currentBlockStart+=blockSize
            lst.append(row)
        yield lst

    def Summarise(self):

        #create output directory if necessary
        outputdir=os.path.join(self.datadir,'Summaries')
        if not os.path.exists(outputdir):
            os.makedirs(outputdir)

        #remove all summary files that correspond to this configuration
        for filename in os.listdir(outputdir):
            if filename.startswith(self.config):
                os.remove(os.path.join(outputdir,filename))

        expr=re.compile(self.SourceFilePattern)
        for sourcefilename in os.listdir(self.datadir):
            fullsourcefilename=os.path.join(self.datadir,sourcefilename)
            if not os.path.isdir(fullsourcefilename):
                if not expr.match(sourcefilename):
                    print('[Skipping file "{0}" (not matched)]'.format(sourcefilename))
                else:
                    print('Processing file "{0}"'.format(sourcefilename))
                    dataid=sourcefilename.split('.')[0]

                    self.dataprovider=CreateDataProvider(fullsourcefilename,self.SourceFileType)
                    outputbasefilename=os.path.join(self.datadir,'Summaries',dataid)
                    blockSize=self.BlockSizeStart
                    while blockSize<=self.BlockSizeMax:
                        print('EXECUTING {0}, BLOCK SIZE {1}'.format(dataid,blockSize))
                        outputfilename=os.path.join(self.datadir,'Summaries','{0}_{1}_{2}'.format(self.config,dataid,str(blockSize)))
                        outputfile=open(outputfilename,'w')

                        blocknr=0
                        for block in self.GetBlocks(blockSize):
                            #for each property, get a list of data in this block
                            proplists={prop.ID: ValueList([val for row in block for val in row[prop.ID]]) for prop in self.properties}
                            #run all the summarisers, merge & write to the output file as a row record
                            summarystr=''.join([summ.Perform(proplists[summ.propID]) for summ in self.summarisers])
                            outputfile.write(summarystr)
                            blocknr+=1
                            if blocknr%5000==0: print('  {0} blocks processed'.format(blocknr))

                        outputfile.close()
                        blockSize *=self.BlockSizeIncrFactor

    def GetData(self,dataid,blockSize,start,length,summarylist):
        result = {}
        for summid in summarylist:
            result[self.folder+'_'+self.config+'_'+summid] = None
        if not self.present:
            return result
        outputbasefilename=self.datadir+'/Summaries/'+self.config+'_'+dataid
        filename=outputbasefilename+'_'+str(blockSize)
        #print('Fetching from '+filename)
        if not os.path.isfile(filename):
            print('ERROR: MISSING FILE '+filename)
            return result
        f=open(filename,'r')
        f.seek(start*self.encodedRowSize)
        linelength=self.encodedRowSize
        strblock=f.read(length*linelength)
        f.close()
        for summid in summarylist:
            nr=self.getSummariserNr(summid)
            strrs=''
            offset=sum([self.summarisers[i].encoder.getlength() for i in range(nr)])
            complength=self.summarisers[nr].encoder.getlength()
            for i in range(length):
                strrs+=strblock[offset:offset+complength]
                offset+=linelength

            result[self.folder+'_'+self.config+'_'+summid]={'data':strrs, 'summariser':self.summarisers[nr].getInfo(), 'encoder':self.summarisers[nr].encoder.getInfo() }
        return result
<|endoftext|>


<commit_before>#!/usr/bin/env python

from __future__ import print_function

import argparse
import json
import os
import re
import sys
import zipfile

from PIL import Image
import numpy as np

def extractZip(path, dest, flatten=True):
    """
    Extract a zip file, optionally flattening it into a single directory.
    """
    try:
        os.makedirs(dest)
    except OSError:
        if not os.path.exists(dest):
            raise

    with zipfile.ZipFile(path) as zf:
        if flatten:
            for name in zf.namelist():
                out = os.path.join(dest, os.path.basename(name))
                with open(out, 'wb') as ofh:
                    with zf.open(name) as ifh:
                        while True:
                            buf = ifh.read(65536)
                            if buf:
                                ofh.write(buf)
                            else:
                                break
        else:
            zf.extractall(dest)

def matchInputFile(truthFile, testDir):
    # truthFile ~= 'ISIC_0000003_Segmentation.png'
    for testFile in os.listdir(testDir):
        truthFileId = truthFile.split('_')[1]
        if truthFileId in testFile:
            testPath = os.path.join(testDir, testFile)
            return testPath

    raise Exception('No matching submission image for: %s' % truthFile)

def loadImage(imagePath, rsize=-1):
    try:
        image = Image.open(imagePath)
    except:
        raise Exception('Could not decode image: %s' % imagePath)

    if image.mode != 'L':
        raise Exception('Image %s is not single-channel (grayscale).' %
                        os.path.basename(imagePath))

    return np.array(image)

def runScoringP1(truthPath, testPath):
    truthImage = loadImage(truthPath)
    testImage = loadImage(testPath)

    if testImage.shape[0:2] != truthImage.shape[0:2]:
        raise Exception('Image %s has dimensions %s; expected %s.' %
                        (os.path.basename(testPath), testImage.shape[0:2],
                         truthImage.shape[0:2]))

    truthBinaryImage = (truthImage > 128)
    testBinaryImage = (testImage > 128)  # TODO: make dynamic

    truthBinaryNegativeImage = 1 - truthBinaryImage
    testBinaryNegativeImage = 1 - testBinaryImage

    truthPixelSum = float(np.sum(truthBinaryImage))
    testPixelSum = float(np.sum(testBinaryImage))

    truePositive = float(np.sum(np.logical_and(truthBinaryImage,
                                               testBinaryImage)))
    trueNegative = float(np.sum(np.logical_and(truthBinaryNegativeImage,
                                               testBinaryNegativeImage)))
    falsePositive = float(np.sum(np.logical_and(truthBinaryNegativeImage,
                                                testBinaryImage)))
    falseNegative = float(np.sum(np.logical_and(truthBinaryImage,
                                                testBinaryNegativeImage)))

    metrics = [
        {
            'name': 'accuracy',
            'value': (truePositive + trueNegative) /
                     (truePositive + trueNegative + falsePositive + falseNegative)
        },
        {
            'name': 'jaccard',
            'value': (truePositive) / (truePositive + falseNegative + falsePositive)
        },
        {
            'name': 'dice',
            'value': (2 * truePositive) / (truthPixelSum + testPixelSum)
        },
        {
            'name': 'sensitivity',
            'value': (truePositive) / (truePositive + falseNegative)
        },
        {
            'name': 'specificity',
            'value': (trueNegative) / (trueNegative + falsePositive)
        }
    ]
    return metrics

def scoreAll(args):
    truthDir = '/covalic/Data/ISBI2016_ISIC_Part1_Test_GroundTruth'

    # Unzip the input files into appropriate folders
    testZipPath = args.submission
    testBaseDir = os.path.dirname(testZipPath)
    testDir = os.path.join(testBaseDir, 'submission')
    extractZip(testZipPath, testDir)

    # Unzip any zip files that were contained in the submission zip file
    zipFiles = [f for f in os.listdir(testDir) if f.lower().endswith('.zip')]
    for zipFile in zipFiles:
        zipFile = os.path.join(testDir, zipFile)
        extractZip(zipFile, testDir)

    # Iterate over each file and call scoring executable on the pair
    scores = []
    for truthFile in os.listdir(truthDir):
        # truthFile ~= 'ISIC_0000003_Segmentation.png'
        try:
            testPath = matchInputFile(truthFile, testDir)
            truthPath = os.path.join(truthDir, truthFile)

            datasetName = truthFile.rsplit('_', 1)[0]
            metrics = runScoringP1(truthPath, testPath)
        except Exception as e:
            print(str(e), file=sys.stderr)
            # TODO: Don't fail completely
            raise

        scores.append({
            'dataset': datasetName,
            'metrics': metrics
        })

    print(json.dumps(scores))

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Submission scoring helper script')
    # parser.add_argument('-g', '--groundtruth', required=True,
    #                     help='path to the ground truth zip file')
    parser.add_argument('-s', '--submission', required=True,
                        help='path to the submission zip file')
    args = parser.parse_args()

    scoreAll(args)
<commit_msg>Skip directories when flattening zip file<commit_after>#!/usr/bin/env python

from __future__ import print_function

import argparse
import json
import os
import re
import sys
import zipfile

from PIL import Image
import numpy as np

def extractZip(path, dest, flatten=True):
    """
    Extract a zip file, optionally flattening it into a single directory.
    """
    try:
        os.makedirs(dest)
    except OSError:
        if not os.path.exists(dest):
            raise

    with zipfile.ZipFile(path) as zf:
        if flatten:
            for name in zf.namelist():
                outName = os.path.basename(name)
                # Skip directories
                if not outName:
                    continue
                out = os.path.join(dest, outName)
                with open(out, 'wb') as ofh:
                    with zf.open(name) as ifh:
                        while True:
                            buf = ifh.read(65536)
                            if buf:
                                ofh.write(buf)
                            else:
                                break
        else:
            zf.extractall(dest)

def matchInputFile(truthFile, testDir):
    # truthFile ~= 'ISIC_0000003_Segmentation.png'
    for testFile in os.listdir(testDir):
        truthFileId = truthFile.split('_')[1]
        if truthFileId in testFile:
            testPath = os.path.join(testDir, testFile)
            return testPath

    raise Exception('No matching submission image for: %s' % truthFile)

def loadImage(imagePath, rsize=-1):
    try:
        image = Image.open(imagePath)
    except:
        raise Exception('Could not decode image: %s' % imagePath)

    if image.mode != 'L':
        raise Exception('Image %s is not single-channel (grayscale).' %
                        os.path.basename(imagePath))

    return np.array(image)

def runScoringP1(truthPath, testPath):
    truthImage = loadImage(truthPath)
    testImage = loadImage(testPath)

    if testImage.shape[0:2] != truthImage.shape[0:2]:
        raise Exception('Image %s has dimensions %s; expected %s.' %
                        (os.path.basename(testPath), testImage.shape[0:2],
                         truthImage.shape[0:2]))

    truthBinaryImage = (truthImage > 128)
    testBinaryImage = (testImage > 128)  # TODO: make dynamic

    truthBinaryNegativeImage = 1 - truthBinaryImage
    testBinaryNegativeImage = 1 - testBinaryImage

    truthPixelSum = float(np.sum(truthBinaryImage))
    testPixelSum = float(np.sum(testBinaryImage))

    truePositive = float(np.sum(np.logical_and(truthBinaryImage,
                                               testBinaryImage)))
    trueNegative = float(np.sum(np.logical_and(truthBinaryNegativeImage,
                                               testBinaryNegativeImage)))
    falsePositive = float(np.sum(np.logical_and(truthBinaryNegativeImage,
                                                testBinaryImage)))
    falseNegative = float(np.sum(np.logical_and(truthBinaryImage,
                                                testBinaryNegativeImage)))

    metrics = [
        {
            'name': 'accuracy',
            'value': (truePositive + trueNegative) /
                     (truePositive + trueNegative + falsePositive + falseNegative)
        },
        {
            'name': 'jaccard',
            'value': (truePositive) / (truePositive + falseNegative + falsePositive)
        },
        {
            'name': 'dice',
            'value': (2 * truePositive) / (truthPixelSum + testPixelSum)
        },
        {
            'name': 'sensitivity',
            'value': (truePositive) / (truePositive + falseNegative)
        },
        {
            'name': 'specificity',
            'value': (trueNegative) / (trueNegative + falsePositive)
        }
    ]
    return metrics

def scoreAll(args):
    truthDir = '/covalic/Data/ISBI2016_ISIC_Part1_Test_GroundTruth'

    # Unzip the input files into appropriate folders
    testZipPath = args.submission
    testBaseDir = os.path.dirname(testZipPath)
    testDir = os.path.join(testBaseDir, 'submission')
    extractZip(testZipPath, testDir)

    # Unzip any zip files that were contained in the submission zip file
    zipFiles = [f for f in os.listdir(testDir) if f.lower().endswith('.zip')]
    for zipFile in zipFiles:
        zipFile = os.path.join(testDir, zipFile)
        extractZip(zipFile, testDir)

    # Iterate over each file and call scoring executable on the pair
    scores = []
    for truthFile in os.listdir(truthDir):
        # truthFile ~= 'ISIC_0000003_Segmentation.png'
        try:
            testPath = matchInputFile(truthFile, testDir)
            truthPath = os.path.join(truthDir, truthFile)

            datasetName = truthFile.rsplit('_', 1)[0]
            metrics = runScoringP1(truthPath, testPath)
        except Exception as e:
            print(str(e), file=sys.stderr)
            # TODO: Don't fail completely
            raise

        scores.append({
            'dataset': datasetName,
            'metrics': metrics
        })

    print(json.dumps(scores))

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Submission scoring helper script')
    # parser.add_argument('-g', '--groundtruth', required=True,
    #                     help='path to the ground truth zip file')
    parser.add_argument('-s', '--submission', required=True,
                        help='path to the submission zip file')
    args = parser.parse_args()

    scoreAll(args)
<|endoftext|>


<commit_before>"""Test suite for pylab_import_all magic
Modified from the irunner module but using regex.
"""

# Global to make tests extra verbose and help debugging
VERBOSE = True

# stdlib imports
import StringIO
import sys
import unittest
import re

# IPython imports
from IPython.lib import irunner
from IPython.testing import decorators

# Testing code begins
class RunnerTestCase(unittest.TestCase):

    def setUp(self):
        self.out = StringIO.StringIO()
        #self.out = sys.stdout

    def _test_runner(self,runner,source,output):
        """Test that a given runner's input/output match."""

        runner.run_source(source)
        out = self.out.getvalue()
        #out = ''
        # this output contains nasty \r\n lineends, and the initial ipython
        # banner.  clean it up for comparison, removing lines of whitespace
        output_l = [l for l in output.splitlines() if l and not l.isspace()]
        out_l = [l for l in out.splitlines() if l and not l.isspace()]
        mismatch  = 0
        if len(output_l) != len(out_l):
            message = ("Mismatch in number of lines\n\n"
                       "Expected:\n"
                       "~~~~~~~~~\n"
                       "%s\n\n"
                       "Got:\n"
                       "~~~~~~~~~\n"
                       "%s"
                       ) % ("\n".join(output_l), "\n".join(out_l))
            self.fail(message)
        for n in range(len(output_l)):
            # Do a line-by-line comparison
            ol1 = output_l[n].strip()
            ol2 = out_l[n].strip()
            if not re.match(ol1,ol2):
                mismatch += 1
                if VERBOSE:
                    print '<<< line %s does not match:' % n
                    print repr(ol1)
                    print repr(ol2)
                    print '>>>'
        self.assert_(mismatch==0,'Number of mismatched lines: %s' %
                     mismatch)

    @decorators.skipif_not_matplotlib
    def test_pylab_import_all_enabled(self):
        "Verify that plot is available when pylab_import_all = True"
        source = """
from IPython.config.application import Application
app = Application.instance()
app.pylab_import_all = True
pylab
ip=get_ipython()
'plot' in ip.user_ns
        """
        output = """
In \[1\]: from IPython\.config\.application import Application
In \[2\]: app = Application\.instance\(\)
In \[3\]: app\.pylab_import_all = True
In \[4\]: pylab
^Welcome to pylab, a matplotlib-based Python environment
For more information, type 'help\(pylab\)'\.
In \[5\]: ip=get_ipython\(\)
In \[6\]: \'plot\' in ip\.user_ns
Out\[6\]: True
"""
        runner = irunner.IPythonRunner(out=self.out)
        self._test_runner(runner,source,output)

    @decorators.skipif_not_matplotlib
    def test_pylab_import_all_disabled(self):
        "Verify that plot is not available when pylab_import_all = False"
        source = """
from IPython.config.application import Application
app = Application.instance()
app.pylab_import_all = False
pylab
ip=get_ipython()
'plot' in ip.user_ns
        """
        output = """
In \[1\]: from IPython\.config\.application import Application
In \[2\]: app = Application\.instance\(\)
In \[3\]: app\.pylab_import_all = False
In \[4\]: pylab
^Welcome to pylab, a matplotlib-based Python environment
For more information, type 'help\(pylab\)'\.
In \[5\]: ip=get_ipython\(\)
In \[6\]: \'plot\' in ip\.user_ns
Out\[6\]: False
"""
        runner = irunner.IPythonRunner(out=self.out)
        self._test_runner(runner,source,output)
<commit_msg>Skip tests that require X, when importing pylab results in RuntimeError.<commit_after>"""Test suite for pylab_import_all magic
Modified from the irunner module but using regex.
"""

# Global to make tests extra verbose and help debugging
VERBOSE = True

# stdlib imports
import StringIO
import sys
import unittest
import re

# IPython imports
from IPython.lib import irunner
from IPython.testing import decorators

def pylab_not_importable():
    """Test if importing pylab fails with RuntimeError (true when having no display)"""
    try:
        import pylab
        return False
    except RuntimeError:
        return True

# Testing code begins
class RunnerTestCase(unittest.TestCase):

    def setUp(self):
        self.out = StringIO.StringIO()
        #self.out = sys.stdout

    def _test_runner(self,runner,source,output):
        """Test that a given runner's input/output match."""

        runner.run_source(source)
        out = self.out.getvalue()
        #out = ''
        # this output contains nasty \r\n lineends, and the initial ipython
        # banner.  clean it up for comparison, removing lines of whitespace
        output_l = [l for l in output.splitlines() if l and not l.isspace()]
        out_l = [l for l in out.splitlines() if l and not l.isspace()]
        mismatch  = 0
        if len(output_l) != len(out_l):
            message = ("Mismatch in number of lines\n\n"
                       "Expected:\n"
                       "~~~~~~~~~\n"
                       "%s\n\n"
                       "Got:\n"
                       "~~~~~~~~~\n"
                       "%s"
                       ) % ("\n".join(output_l), "\n".join(out_l))
            self.fail(message)
        for n in range(len(output_l)):
            # Do a line-by-line comparison
            ol1 = output_l[n].strip()
            ol2 = out_l[n].strip()
            if not re.match(ol1,ol2):
                mismatch += 1
                if VERBOSE:
                    print '<<< line %s does not match:' % n
                    print repr(ol1)
                    print repr(ol2)
                    print '>>>'
        self.assert_(mismatch==0,'Number of mismatched lines: %s' %
                     mismatch)

    @decorators.skipif(pylab_not_importable, "Likely a run without X.")
    @decorators.skipif_not_matplotlib
    def test_pylab_import_all_enabled(self):
        "Verify that plot is available when pylab_import_all = True"
        source = """
from IPython.config.application import Application
app = Application.instance()
app.pylab_import_all = True
pylab
ip=get_ipython()
'plot' in ip.user_ns
        """
        output = """
In \[1\]: from IPython\.config\.application import Application
In \[2\]: app = Application\.instance\(\)
In \[3\]: app\.pylab_import_all = True
In \[4\]: pylab
^Welcome to pylab, a matplotlib-based Python environment
For more information, type 'help\(pylab\)'\.
In \[5\]: ip=get_ipython\(\)
In \[6\]: \'plot\' in ip\.user_ns
Out\[6\]: True
"""
        runner = irunner.IPythonRunner(out=self.out)
        self._test_runner(runner,source,output)

    @decorators.skipif(pylab_not_importable, "Likely a run without X.")
    @decorators.skipif_not_matplotlib
    def test_pylab_import_all_disabled(self):
        "Verify that plot is not available when pylab_import_all = False"
        source = """
from IPython.config.application import Application
app = Application.instance()
app.pylab_import_all = False
pylab
ip=get_ipython()
'plot' in ip.user_ns
        """
        output = """
In \[1\]: from IPython\.config\.application import Application
In \[2\]: app = Application\.instance\(\)
In \[3\]: app\.pylab_import_all = False
In \[4\]: pylab
^Welcome to pylab, a matplotlib-based Python environment
For more information, type 'help\(pylab\)'\.
In \[5\]: ip=get_ipython\(\)
In \[6\]: \'plot\' in ip\.user_ns
Out\[6\]: False
"""
        runner = irunner.IPythonRunner(out=self.out)
        self._test_runner(runner,source,output)
<|endoftext|>


<commit_before>
from requests.packages.urllib3.exceptions import InsecureRequestWarning
import requests
import json
import time
import simplejson
from time import gmtime, strftime
from datetime import datetime, timedelta
from cassandra.cluster import Cluster

requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

base_url = "https://earthquake.usgs.gov/fdsnws/event/1/query?format=geojson"

def writeFile(fileName, data):
    """Write the information to a file"""
    print("###Writing file")
    with open("./"+fileName, 'w') as outfile:
        json.dump(data, outfile, indent=0, sort_keys=True)

def printLog(msg):
    print(strftime("%d-%m-%Y %H:%M:%S", gmtime()), "->", msg)

def getX(longitude):
    x = 0

    if 0 <= longitude < 30:
        x = 1
    elif 30 <= longitude < 60:
        x = 2
    elif 60 <= longitude < 90:
        x = 3
    elif 90 <= longitude < 120:
        x = 4
    elif 120 <= longitude < 150:
        x = 5
    elif 150 <= longitude <= 180:
        x = 6
    elif -180 <= longitude < -150:
        x = 7
    elif -150 <= longitude < -120:
        x = 8
    elif -120 <= longitude < -90:
        x = 9
    elif -90 <= longitude < -60:
        x = 10
    elif -60 <= longitude < -30:
        x = 11
    elif -30 <= longitude < 0:
        x = 12
    return x

def getY(latitude):
    y = 0

    if 60 <= latitude < 80:
        y = 1
    elif 40 <= latitude < 60:
        y = 2
    elif 20 <= latitude < 40:
        y = 3
    elif 0 <= latitude < 20:
        y = 4
    elif -20 <= latitude < 0:
        y = 5
    elif -40 <= latitude < -20:
        y = 6
    elif -60 <= latitude < -40:
        y = 7
    elif -80 <= latitude < -60:
        y = 8
    elif latitude < -80:
        y = 8
    elif 80 <= latitude:
        y = 1

    return y

def insertIntoDB(data):
    eventId = data.get("id")
    properties = data.get("properties")
    location = data.get("geometry")
    latitude = location.get("coordinates")[1]
    longitude = location.get("coordinates")[0]
    depth = location.get("coordinates")[2]
    magnitude = properties["mag"]
    time = properties["time"]
    place = properties["place"]
    fecha = datetime.fromtimestamp(float(properties["time"])/1000).strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]
    geoJson = str(data)
    quadrantX = getX(float(longitude))
    quadrantY = getY(float(latitude))
    quadrantXY = str(quadrantX)+","+str(quadrantY)

    magnitude = float(str(magnitude).replace(",", ".")) if not (magnitude is None) else None
    depth = float(str(depth).replace(",", ".")) if not (depth is None) else None

    if str(place) and str(time):
        session.execute("INSERT INTO Earthquake (\"eventId\",\"place\",\"time\",\"fecha\",\"magnitude\",\"depth\",\"longitude\",\"latitude\",\"geojson\",\"quadrant\",\"quadrantX\", \"quadrantY\") \
        VALUES (%s, %s, %s, %s, %s, %s, %s, %s,%s,%s,%s,%s)",
        [str(eventId), str(place), str(time), fecha, magnitude, depth, longitude, latitude, geoJson,quadrantXY,int(quadrantX),int(quadrantY)])

def getEarthQuakesWorld(yearFrom):

    now = datetime.now()
    while yearFrom <= now.year:
        startDateStr = str(yearFrom)+"-01-01"
        endDateStr = str(yearFrom)+"-12-31"

        try:
            getData(startDateStr,endDateStr)
        except simplejson.scanner.JSONDecodeError:
            printLog("####ERROR!!!"+str(yearFrom))
            getDataMonthly(yearFrom)
        finally:
            yearFrom = yearFrom+1

def getDataMonthly(yearFrom):
    for i in range(1, 12):
        startDateStr = str(yearFrom)+"-"+str(i)+"-01"
        endDateStr = str(yearFrom)+"-"+str((i+1)%13)+"-01"
        getData(startDateStr, endDateStr)
        time.sleep(10)

def getData(fromDate, toDate):
    url = "&starttime="+fromDate+"&endtime="+toDate
    printLog("Calculating from "+str(fromDate)+" to " + str(toDate))

    response = requests.get(base_url+url)
    jsonData = response.json()
    for feature in jsonData.get("features"):
        properties = feature.get("properties")
        dateTime =  datetime.fromtimestamp(float(properties["time"])/1000).strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]
        try:
            printLog("Processing :" + str(properties["place"])+" for: " + str(dateTime))
            insertIntoDB(feature)
        except UnicodeEncodeError:
            printLog("Bad encoding")

# Main Function #
if __name__ == "__main__":
    cluster = Cluster(['192.168.246.236'])
    session = cluster.connect("dev")
    getEarthQuakesWorld(2017)
    printLog("END")
<commit_msg>Auto year detection<commit_after>
from requests.packages.urllib3.exceptions import InsecureRequestWarning
import requests
import json
import time
import simplejson
from time import gmtime, strftime
from datetime import datetime, timedelta
from cassandra.cluster import Cluster

requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

base_url = "https://earthquake.usgs.gov/fdsnws/event/1/query?format=geojson"

def writeFile(fileName, data):
    """Write the information to a file"""
    print("###Writing file")
    with open("./"+fileName, 'w') as outfile:
        json.dump(data, outfile, indent=0, sort_keys=True)

def printLog(msg):
    print(strftime("%d-%m-%Y %H:%M:%S", gmtime()), "->", msg)

def getX(longitude):
    x = 0

    if 0 <= longitude < 30:
        x = 1
    elif 30 <= longitude < 60:
        x = 2
    elif 60 <= longitude < 90:
        x = 3
    elif 90 <= longitude < 120:
        x = 4
    elif 120 <= longitude < 150:
        x = 5
    elif 150 <= longitude <= 180:
        x = 6
    elif -180 <= longitude < -150:
        x = 7
    elif -150 <= longitude < -120:
        x = 8
    elif -120 <= longitude < -90:
        x = 9
    elif -90 <= longitude < -60:
        x = 10
    elif -60 <= longitude < -30:
        x = 11
    elif -30 <= longitude < 0:
        x = 12
    return x

def getY(latitude):
    y = 0

    if 60 <= latitude < 80:
        y = 1
    elif 40 <= latitude < 60:
        y = 2
    elif 20 <= latitude < 40:
        y = 3
    elif 0 <= latitude < 20:
        y = 4
    elif -20 <= latitude < 0:
        y = 5
    elif -40 <= latitude < -20:
        y = 6
    elif -60 <= latitude < -40:
        y = 7
    elif -80 <= latitude < -60:
        y = 8
    elif latitude < -80:
        y = 8
    elif 80 <= latitude:
        y = 1

    return y

def insertIntoDB(data):
    eventId = data.get("id")
    properties = data.get("properties")
    location = data.get("geometry")
    latitude = location.get("coordinates")[1]
    longitude = location.get("coordinates")[0]
    depth = location.get("coordinates")[2]
    magnitude = properties["mag"]
    time = properties["time"]
    place = properties["place"]
    fecha = datetime.fromtimestamp(float(properties["time"])/1000).strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]
    geoJson = str(data)
    quadrantX = getX(float(longitude))
    quadrantY = getY(float(latitude))
    quadrantXY = str(quadrantX)+","+str(quadrantY)

    magnitude = float(str(magnitude).replace(",", ".")) if not (magnitude is None) else None
    depth = float(str(depth).replace(",", ".")) if not (depth is None) else None

    if str(place) and str(time):
        session.execute("INSERT INTO Earthquake (\"eventId\",\"place\",\"time\",\"fecha\",\"magnitude\",\"depth\",\"longitude\",\"latitude\",\"geojson\",\"quadrant\",\"quadrantX\", \"quadrantY\") \
        VALUES (%s, %s, %s, %s, %s, %s, %s, %s,%s,%s,%s,%s)",
        [str(eventId), str(place), str(time), fecha, magnitude, depth, longitude, latitude, geoJson,quadrantXY,int(quadrantX),int(quadrantY)])

def getEarthQuakesWorld(yearFrom):

    now = datetime.now()
    while yearFrom <= now.year:
        startDateStr = str(yearFrom)+"-01-01"
        endDateStr = str(yearFrom)+"-12-31"

        try:
            getData(startDateStr,endDateStr)
        except simplejson.scanner.JSONDecodeError:
            printLog("####ERROR!!!"+str(yearFrom))
            getDataMonthly(yearFrom)
        finally:
            yearFrom = yearFrom+1

def getDataMonthly(yearFrom):
    for i in range(1, 12):
        startDateStr = str(yearFrom)+"-"+str(i)+"-01"
        endDateStr = str(yearFrom)+"-"+str((i+1)%13)+"-01"
        getData(startDateStr, endDateStr)
        time.sleep(10)

def getData(fromDate, toDate):
    url = "&starttime="+fromDate+"&endtime="+toDate
    printLog("Calculating from "+str(fromDate)+" to " + str(toDate))

    response = requests.get(base_url+url)
    jsonData = response.json()
    for feature in jsonData.get("features"):
        properties = feature.get("properties")
        dateTime =  datetime.fromtimestamp(float(properties["time"])/1000).strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]
        try:
            printLog("Processing :" + str(properties["place"])+" for: " + str(dateTime))
            insertIntoDB(feature)
        except UnicodeEncodeError:
            printLog("Bad encoding")

# Main Function #
if __name__ == "__main__":
    cluster = Cluster(['192.168.246.236'])
    session = cluster.connect("dev")
    now = datetime.now()
    getEarthQuakesWorld(now.year)
    printLog("END")
<|endoftext|>


<commit_before>import discord
import aiohttp
from discord.ext import commands

class DND:
    """D&D Lookup Stuff"""
    baseurl = "http://dnd5eapi.co/api/"

    def __init__(self, bot):
        self.bot = bot

    @commands.group(pass_context=True, invoke_without_command=False)
    async def dnd(self, ctx, *, hargs=''):
        if ctx.invoked_subcommand is None:
            await self.bot.say('DEBUG: hargs = '+ hargs)
            return

    @dnd.command()
    async def spells(self, hargs):
        """Lookup Spells"""
        baseurl = self.baseurl+'spells'
        #Your code will go here
        await self.bot.say("Lookup Spells initiated.")
        await self.bot.say("<{}>".format(baseurl))

    @dnd.command()
    async def classes(self, hargs):
        """Lookup Classes"""
        baseurl = self.baseurl+'classes'
        #Your code will go here
        await self.bot.say("Lookup Classes initiated.")
        await self.bot.say("<{}>".format(baseurl))

    @dnd.command()
    async def monsters(self, hargs):
        """Lookup Monsters"""
        baseurl = self.baseurl+'monsters'
        #Your code will go here
        await self.bot.say("Lookup Monsters initiated.")
        await self.bot.say("<{}>".format(baseurl))

    @dnd.command()
    async def equipment(self, hargs):
        """Lookup Equpiment"""
        baseurl = self.baseurl+'equipment'
        #Your code will go here
        await self.bot.say("Lookup Spells initiated.")
        await self.bot.say("<{}>".format(baseurl))

    async def _get_file(url):
        async with aiohttp.ClientSession() as session:
            async with session.get(url) as response:
                file_txt = await response.text()
                if file_txt is not None:
                    await self.bot.say('DEBUG: Got the file.')

def setup(bot):
    bot.add_cog(DND(bot))
<commit_msg><commit_after>import discord
import aiohttp
from discord.ext import commands

class DND:
    """D&D Lookup Stuff"""
    baseurl = "http://dnd5eapi.co/api/"

    def __init__(self, bot):
        self.bot = bot

    @commands.group(pass_context=True)
    async def dnd(self, ctx, *, hargs=''):
        if ctx.invoked_subcommand is None:
            await self.bot.say('DEBUG: hargs = '+ hargs)
            return

    @dnd.command()
    async def spells(self, hargs):
        """Lookup Spells"""
        baseurl = self.baseurl+'spells'
        #Your code will go here
        await self.bot.say("Lookup Spells initiated.")
        await self.bot.say("<{}>".format(baseurl))

    @dnd.command()
    async def classes(self, hargs):
        """Lookup Classes"""
        baseurl = self.baseurl+'classes'
        #Your code will go here
        await self.bot.say("Lookup Classes initiated.")
        await self.bot.say("<{}>".format(baseurl))

    @dnd.command()
    async def monsters(self, hargs):
        """Lookup Monsters"""
        baseurl = self.baseurl+'monsters'
        #Your code will go here
        await self.bot.say("Lookup Monsters initiated.")
        await self.bot.say("<{}>".format(baseurl))

    @dnd.command()
    async def equipment(self, hargs):
        """Lookup Equpiment"""
        baseurl = self.baseurl+'equipment'
        #Your code will go here
        await self.bot.say("Lookup Spells initiated.")
        await self.bot.say("<{}>".format(baseurl))

    async def _get_file(url):
        async with aiohttp.ClientSession() as session:
            async with session.get(url) as response:
                file_txt = await response.text()
                if file_txt is not None:
                    await self.bot.say('DEBUG: Got the file.')

def setup(bot):
    bot.add_cog(DND(bot))
<|endoftext|>


<commit_before>#!/usr/bin/env python
#-*- coding: utf-8 -*-

import sys
import json
import random
import urllib2
import datetime

try:
    import irclib
    import ircbot
except ImportError:
    print >>sys.stderr, "This bot needs python-irclib."
    print >>sys.stderr, "http://python-irclib.sourceforge.net"

try:
    import twitter
except ImportError:
    print >>sys.stderr, "This bot needs python-twitter."
    print >>sys.stderr, "http://code.google.com/p/python-twitter/"

NICK = "ec09bot"
SERVERS = [("irc.freenode.net", 6667)]
CHANNEL = "#ec09"

class EC09Bot(ircbot.SingleServerIRCBot):
    def __init__(self, *args):
        ircbot.SingleServerIRCBot.__init__(self, SERVERS, NICK, NICK)

        self.connection.add_global_handler("pubmsg", self._on_pubmsg)

        self.twitter = twitter.Api()
        self.batima_cache = []
        self.batima_cache_age_max = 30*60 # seconds
        self.batima_cache_time = datetime.datetime.now()

    def start(self):
        self._connect()
        self.connection.join(CHANNEL)
        irclib.SimpleIRCClient.start(self)

    def _on_pubmsg(self, connection, event):
        sendernick, _ = event.source().split('!', 1)
        message = event.arguments()[0].strip()

        if not message.startswith("!"):
            return

        command = message[1:]
        handler = getattr(self, "command_" + command, None)

        if callable(handler):
            self.connection.privmsg(CHANNEL, "%s: %s" % (sendernick, handler()))

    def command_bandeco(self):
        bandeco_url = \
            "http://www.students.ic.unicamp.br/~ra091187/bandeco_svg/json.php"

        try:
            u = urllib2.urlopen(bandeco_url, timeout = 5)
            st = u.read()
        except urllib2.URLError:
            return "Indisponível :("

        st = st[len("bandeco("):-2]

        d = json.loads(st)
        d.update((k, v.lower()) for k, v in d.items())

        bandeco_str = \
            "%(prato)s, com suco de %(suco)s, " \
            "salada %(salada)s e sobremesa %(sobremesa)s." % d

        bandeco_str = bandeco_str[0].upper() + bandeco_str[1:]
        return bandeco_str.encode("utf-8")

    def command_batima(self):
        age = (datetime.datetime.now() - self.batima_cache_time).seconds
        if not self.batima_cache or age > self.batima_cache_age_max:
            self._rebuild_batima_cache()

        if not self.batima_cache:
            return "FODEU! SERÁ QUE O TWITTER CAIU?"

        return random.choice(self.batima_cache)

    def command_leave(self):
        self.connection.quit("How do I deal with mortality? "
                            "Thanks to denial, I'm immortal!")
        sys.exit(0)

    def _rebuild_batima_cache(self):
        from itertools import dropwhile

        status = self.twitter.GetUserTimeline("falasdobatima", count = 50)
        tweets = [s.GetText() for s in status]

        self.batima_cache = []

        for t in tweets:
            # Strip @usernames from replies, hope they are all in the beginning
            words = t.split()
            batima_line = " ".join(dropwhile(lambda w: w[0] == "@", words))

            self.batima_cache.append(batima_line.encode("utf-8"))

        self.batima_cache_time = datetime.datetime.now()

if __name__ == "__main__":
    bot = EC09Bot()
    bot.start()
<commit_msg>Beautification.<commit_after>#!/usr/bin/env python
#-*- coding: utf-8 -*-

import sys
import json
import random
import urllib2
import datetime

try:
    import irclib
    import ircbot
except ImportError:
    print >>sys.stderr, "This bot needs python-irclib."
    print >>sys.stderr, "http://python-irclib.sourceforge.net"

try:
    import twitter
except ImportError:
    print >>sys.stderr, "This bot needs python-twitter."
    print >>sys.stderr, "http://code.google.com/p/python-twitter/"

class EC09Bot(ircbot.SingleServerIRCBot):
    NICK = "ec09bot"
    SERVERS = [("irc.freenode.net", 6667)]
    CHANNEL = "#ec09"

    def __init__(self):
        ircbot.SingleServerIRCBot.__init__(self, self.SERVERS,
                                           self.NICK, self.NICK)

        self.connection.add_global_handler("pubmsg", self._on_pubmsg)

        self.twitter = twitter.Api()
        self.batima_cache = []
        self.batima_cache_age_max = 30*60 # seconds
        self.batima_cache_time = datetime.datetime.now()

    def start(self):
        self._connect()
        self.connection.join(self.CHANNEL)
        irclib.SimpleIRCClient.start(self)

    def _on_pubmsg(self, connection, event):
        sendernick, _ = event.source().split('!', 1)
        message = event.arguments()[0].strip()

        if not message.startswith("!"):
            return

        command = message[1:]
        handler = getattr(self, "command_" + command, None)

        if callable(handler):
            reply = "%s: %s" % (sendernick, handler())
            self.connection.privmsg(self.CHANNEL, reply)

    def command_bandeco(self):
        bandeco_url = \
            "http://www.students.ic.unicamp.br/~ra091187/bandeco_svg/json.php"

        try:
            u = urllib2.urlopen(bandeco_url, timeout = 5)
            st = u.read()
        except urllib2.URLError:
            return "Indisponível :("

        st = st[len("bandeco("):-2]

        d = json.loads(st)
        d.update((k, v.lower()) for k, v in d.items())

        bandeco_str = \
            "%(prato)s, com suco de %(suco)s, " \
            "salada %(salada)s e sobremesa %(sobremesa)s." % d

        bandeco_str = bandeco_str[0].upper() + bandeco_str[1:]
        return bandeco_str.encode("utf-8")

    def command_batima(self):
        age = (datetime.datetime.now() - self.batima_cache_time).seconds
        if not self.batima_cache or age > self.batima_cache_age_max:
            self._rebuild_batima_cache()

        if not self.batima_cache:
            return "FODEU! SERÁ QUE O TWITTER CAIU?"

        return random.choice(self.batima_cache)

    def _rebuild_batima_cache(self):
        from itertools import dropwhile

        status = self.twitter.GetUserTimeline("falasdobatima", count = 50)
        tweets = [s.GetText() for s in status]

        self.batima_cache = []

        for t in tweets:
            # Strip @usernames from replies, hope they are in the beginning
            words = t.split()
            batima_line = " ".join(dropwhile(lambda w: w[0] == "@", words))

            self.batima_cache.append(batima_line.encode("utf-8"))

        self.batima_cache_time = datetime.datetime.now()

    def command_leave(self):
        self.connection.quit("How do I deal with mortality? "
                             "Thanks to denial, I'm immortal!")
        sys.exit(0)

if __name__ == "__main__":
    bot = EC09Bot()
    bot.start()
<|endoftext|>


<commit_before><commit_msg>fix typo<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python3
import subprocess as sb
import html
import re
import time
from datetime import datetime as dt
import datetime
import sys
try:
    from tinydb import TinyDB, Query
    def execute_query(cardapio, code):
        days = ["Segunda", "Terça", "Quarta", "Quinta", "Sexta", "Sábado", "Domingo"]
        rests = ["fisica", "quimica", "prefeitura", "central"]
        ts = time.time()
        ts = dt.fromtimestamp(ts).strftime("%d-%m-%Y")
        dict_res = {key : value for (key, value) in zip([8, 9, 7, 6], rests)}
        refeicao = cardapio[days[dt.today().weekday()]]
        weekday = days[dt.today().weekday()].lower()
        db = TinyDB("/root/cardapio.json")
        ingr_table = db.table("ingrediente")
        rest_table = db.table(dict_res[code])
        almoco = generate_refeicao(refeicao[0].strip().split("\n"), ingr_table)
        jantar = generate_refeicao(refeicao[1].strip().split("\n"), ingr_table)
        almoco = almoco if almoco else False
        jantar = jantar if jantar else False
        if almoco or jantar:
            rest_table.insert({"data" : ts, "diasemana" : weekday,
                               "almoco" : almoco, "jantar" : jantar})

    def generate_refeicao(list_i, table):
        list_i = list(filter(None, list_i))
        if(len(list_i) < 2): return None
        dict_keys = ["comum", "principal", "opcao", "acomp", "adicional", "sobremesa"]
        dict_refeicao = dict()
        for (key, value) in zip(dict_keys, list_i):
            dict_refeicao[key] = value.replace("Opção:","").strip().lower()
        queryS = Query()
        for ingr in list_i:
            ingr = ingr.strip().lower()
            query_result = table.search(queryS.tipo == ingr)
            if(len(query_result) == 0):
                table.insert({"tipo" : ingr , "qtd" : 1})
            else:
                preCount = query_result[0]["qtd"]
                table.update({"qtd" : preCount + 1}, queryS.tipo == ingr)
        return dict_refeicao
except ImportError:
    pass

# Global variables
WHITE = "\033[38;5;7m"
GRAY = "\033[38;5;252m"
bCodes = {"Física" :8, "Química" :9, "Prefeitura":7, "Central" :6}
colors  = {8 :"\033[38;5;9m", 9 :"\033[38;5;120m", 7 :"\033[38;5;220m", 6 :"\033[38;5;190m"}
godlike = True
'''
List with featured FOODS! For each item in the list, the first
item is the featured one, and all the others followed by a ","
are a string to NOT match.
For example, if you don't like "pudim de abacate" you put "pudim, abacate",
so then it won't match if the word contains "pudim", but not "abacate"!
'''
featured = ["mel", "queijo", "doce, batata", "pudim", "flan", "sugo, berinjela",
            "batata, doce", "abacaxi", "nhoque", "mousse", "chocolate", "estrogonofe",
	    "brigadeiro", "milanesa", "pizzaiolo", "fantasia", "mandioca","sorvete", "fricassé", "almôndegas",
	    "madeira", "escondidinho", "cocada", "bolonhesa", "lasanha"]
def get_command(rID):
    cmd = ["curl", "-sw", "-H", "\"Host:uspdigital.usp.br\nConnection:keep-alive\nContent-Length:280\nOrigin:https://uspdigital.usp.br\nUser-Agent:Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Mobile Safari/537.36\nContent-Type:text/plain\nAccept:*/*\nReferer:https://uspdigital.usp.br/rucard/Jsp/cardapioSAS.jsp\nAccept-Encoding:gzip, deflate, br\nAccept-Language:pt-BR,pt;q=0.8,en-US;q=0.6,en;q=0.4,fr;q=0.2\nRequest Payload:\"", "-X", "POST", "-d","callCount=1\nwindowName=1\nnextReverseAjaxIndex=0\nc0-scriptName=CardapioControleDWR\nc0-methodName=obterCardapioRestUSP\nc0-id=0\nc0-param0=string:{}\nbatchId=1\ninstanceId=0\npage=%2Frucard%2FJsp%2FcardapioSAS.jsp%3Fcodrtn%3D0\nscriptSessionId=".format(rID), "https://uspdigital.usp.br/rucard/dwr/call/plaincall/CardapioControleDWR.obterCardapioRestUSP.dwr"]
    return cmd

def parse_cardapio(script_res):
    r = str(script_res).split("\\n")
    r = list(filter(lambda s: "dwr.engine.remote." in s, r))
    r = html.unescape(r[0]).split("[")[1].encode().decode('unicode_escape')
    r = list(filter(None, clean_str(r.replace("\/",", ")).split("{")))
    for i in range(len(r)):
        k = list(filter(None, r[i].split("cdpdia:")))
        k = list(filter(None, "".join(k).split("\"")))
        r[i] = k[0].encode().decode('unicode_escape') if k[0] != " " else "Fechado"
    r = list(zip(r[::2], r[1::2]))
    return r

def clean_str(r):
    d = {"
":"\n", "}":""}
    regex = re.compile('|'.join(d.keys()))
    return regex.sub(lambda x: d[x.group()], r)

def create_cardapio(r):
    days = ["Segunda", "Terça", "Quarta", "Quinta", "Sexta", "Sábado", "Domingo"]
    cardapio = {dia: refeicao for dia, refeicao in zip(days, r)}
    return cardapio

def app_h(k):
    for f in featured:
        f = [x.strip().lower() for x in f.split(",")]
        if f[0] in k.lower().split(" "):
            if(len(list((x for x in f[1:] if x in k.lower()))) == 0):
                return "\033[4m\033[38;5;165m*"+k.strip()+"*\033[0m"
    return k

def print_refeicao(rf, color):
    uLine = lambda x: "\033[38;5;14m"+x+"\033[0m"+color
    tmp = list(filter(None, rf.split("\n")))
    if(len(tmp) > 2):
        for k, idx in zip(tmp, range(len(tmp))):
            print(WHITE+"➤ "+GRAY, end="")
            if idx == 1 or idx == 5 or idx == 3:
                temp = app_h(k) if godlike else k
                print(uLine(temp))
            else: print(k)
    else:
        print(rf)

def print_day(cardapio, tag, color, code=None, day=""):
    days = ["Segunda", "Terça", "Quarta", "Quinta", "Sexta", "Sábado", "Domingo"]
    d_Index = datetime.datetime.today().weekday() if not day else day - 1
    refeicao = cardapio[days[d_Index]]
    if tag != "":
        tag = 0 if tag == " (Almoço)" else 1
        print_refeicao(refeicao[tag], GRAY)
    else:
        for i, title in zip(refeicao,["Almoço", "Jantar"]):
            print(colors[code]+format_str(title,"",char="-", sz=20)+color)
            print_refeicao(i, GRAY)

def get_TimeTag():
        now = datetime.datetime.now()
        aInf = now.replace(hour=6, minute=0, second=0, microsecond=0)
        aSup = now.replace(hour=14, minute=0, second=0, microsecond=0)
        jInf = now.replace(hour=14, minute=1, second=0, microsecond=0)
        jSup = now.replace(hour=20, minute=1, second=0, microsecond=0)
        if now > aInf and now < aSup:
            return " (Almoço)"
        elif now > jInf and now < jSup:
            return " (Jantar)"
        return ""

def format_str(name, tag, char="=", sz=50, addIcons=True):
    bdSt = list(name+tag)
    l = True
    while(len(bdSt) < sz):
        if l:   bdSt.append(char)
        else:   bdSt = [char] + bdSt
        l = not l
    if(addIcons):
        bdSt[0] = "#"
        bdSt[-1] = "#"
    # Centralize
    if(len(bdSt) < 50):
        bdSt = [" " for x in range((50 - len(bdSt))//2)] + bdSt
    return "".join(bdSt)

def print_AllBdex(tag, dump=False, day="", logo=True):
    days = ["Segunda", "Terça", "Quarta", "Quinta", "Sexta", "Sábado", "Domingo"]
    if logo: print_logo()
    if day:
        print("\n\n"+format_str(days[day-1],"",char="~", sz=30))
    for name, code in bCodes.items():
        print("\n"+colors[code]+format_str(name, tag)+GRAY)
        cdp = get_command(str(code))
        cdp = parse_cardapio(sb.check_output(cdp))
        cdp = create_cardapio(cdp)
        if(not dump):
            print_day(cdp, tag, GRAY, code=code, day=day)
        else:
            execute_query(cdp, code)

def print_logo():
    fadeColors = ["\033[38;5;{}m".format(i) for i in range(118, 124)]

    logo = [" ____                     _\n",
              "| __ )   __ _  _ __    __| |  ___ __  __\n",
              "|  _ \  / _` || '_ \  / _` | / _ \\\ \/ /\n",
              "| |_) || (_| || | | || (_| ||  __/ >  <\n",
              "|____/  \__,_||_| |_| \__,_| \___|/_/\_\\\n"]
    for color, line  in zip(fadeColors, logo):
        print(color+line, end="", sep="")

def print_usage(wrong_arg=None):
    err = lambda x: "\033[38;5;160m "+x+"\033[38;5;7m"
    runF = lambda x: "\033[38;5;4m"+x+"\033[38;5;7m"
    flagF = lambda x: "\033[38;5;46m"+x+"\033[38;5;7m"
    if(wrong_arg != None):
        print("😞😞😞 Poxa "+sb.check_output(["whoami"]).decode("utf-8").replace("\n","")+
              "," +err("argumento \""+wrong_arg+"\" é inválido! ")+"😞😞😞")
    print("Uso:\n $ "+runF("bandex")+"\n\t Imprime o cardápio do dia de todos os restau"
          "rantes, com as refeições de acordo com o horário de execução do script;\n"+
          flagF("FLAGS:")+"\n\t"+flagF("-a")+" : Imprime apenas os almoços;"+"\n\t"+
          flagF("-j")+" : Imprime apenas os jantares;"+"\n\t"+flagF("-all")+" : Imprime"
          " todas as refeições do dia;"+"\n\t"+flagF("-d número")+" : Imprime"
          " todas as refeições do dia escolhido (segunda = 1, terça = 2, etc);"+
          "\n\t"+flagF("-E ou --EVERYTHING")+" : Imprime todas as refeições da semana! D: ;"+"\n\t"
          +flagF("-h ou --help")+" : Imprime esta página de informação.\n")
    exit()

def main():
    #print(sb.check_output(["whoami"]))

    if(len(sys.argv) == 1):
        tag = get_TimeTag()
        print_AllBdex(tag)
    elif(len(sys.argv) == 2):
        if(sys.argv[1] == "-a"):
            tag = " (Almoço)"
        elif(sys.argv[1] == "-j"):
            tag = " (Jantar)"
        elif(sys.argv[1] == "-all"):
            tag = ""
        elif(sys.argv[1] == "--EVERYTHING" or sys.argv[1] == "-E"):
            for i in range(1, 8):
                print_AllBdex("", day=i, logo = False)
            exit()
        elif(sys.argv[1] == "-h" or sys.argv[1] == "--help"):
            print_usage()
        else:
            try:
                k = sb.check_output(["cat", "/root/naoZOA"])
                k = k.decode("utf-8")
                k = k.replace("\n","")
                print("K=|",k,"|",sep="")
                if(sys.argv[1] == k):
                     print_AllBdex("", dump=True)
                exit()
            except Exception as e:
                print(e)
            print_usage(sys.argv[1])
        print_AllBdex(tag)
    elif(len(sys.argv) == 3 and sys.argv[1] == "-d"):
        try:
            day_choosen = int(sys.argv[2])
            if day_choosen in range(1,8):
                print_AllBdex("", day=day_choosen)
        except (TypeError, ValueError):
            print_usage()
    else:
        print_usage()

if __name__ == '__main__':
    main()

<commit_msg>update bandex<commit_after>#!/usr/bin/env python3
import subprocess as sb
import html
import re
import time
from datetime import datetime as dt
import datetime
import sys
try:
    from tinydb import TinyDB, Query
    def execute_query(cardapio, code):
        days = ["Segunda", "Terça", "Quarta", "Quinta", "Sexta", "Sábado", "Domingo"]
        rests = ["fisica", "quimica", "prefeitura", "central"]
        ts = time.time()
        ts = dt.fromtimestamp(ts).strftime("%d-%m-%Y")
        dict_res = {key : value for (key, value) in zip([8, 9, 7, 6], rests)}
        refeicao = cardapio[days[dt.today().weekday()]]
        weekday = days[dt.today().weekday()].lower()
        db = TinyDB("/root/cardapio.json")
        ingr_table = db.table("ingrediente")
        rest_table = db.table(dict_res[code])
        almoco = generate_refeicao(refeicao[0].strip().split("\n"), ingr_table)
        jantar = generate_refeicao(refeicao[1].strip().split("\n"), ingr_table)
        almoco = almoco if almoco else False
        jantar = jantar if jantar else False
        if almoco or jantar:
            rest_table.insert({"data" : ts, "diasemana" : weekday,
                               "almoco" : almoco, "jantar" : jantar})

    def generate_refeicao(list_i, table):
        list_i = list(filter(None, list_i))
        if(len(list_i) < 2): return None
        dict_keys = ["comum", "principal", "opcao", "acomp", "adicional", "sobremesa"]
        dict_refeicao = dict()
        for (key, value) in zip(dict_keys, list_i):
            dict_refeicao[key] = value.replace("Opção:","").strip().lower()
        queryS = Query()
        for ingr in list_i:
            ingr = ingr.strip().lower()
            query_result = table.search(queryS.tipo == ingr)
            if(len(query_result) == 0):
                table.insert({"tipo" : ingr , "qtd" : 1})
            else:
                preCount = query_result[0]["qtd"]
                table.update({"qtd" : preCount + 1}, queryS.tipo == ingr)
        return dict_refeicao
except ImportError:
    pass

# Global variables
WHITE = "\033[38;5;7m"
GRAY = "\033[38;5;252m"
bCodes = {"Física" :8, "Química" :9, "Prefeitura":7, "Central" :6}
colors  = {8 :"\033[38;5;9m", 9 :"\033[38;5;120m", 7 :"\033[38;5;220m", 6 :"\033[38;5;190m"}
godlike = True
'''
List with featured FOODS! For each item in the list, the first
item is the featured one, and all the others followed by a ","
are a string to NOT match.
For example, if you don't like "pudim de abacate" you put "pudim, abacate",
so then it won't match if the word contains "pudim", but not "abacate"!
'''
featured = ["mel", "queijo", "doce, batata", "pudim", "flan", "sugo, berinjela",
            "batata, doce", "abacaxi", "nhoque", "mousse", "chocolate", "estrogonofe",
	    "brigadeiro", "milanesa", "pizzaiolo", "fantasia", "mandioca","sorvete", "fricassé", "almôndegas",
	    "madeira", "escondidinho", "cocada", "bolonhesa", "lasanha"]
def get_command(rID):
    cmd = ["curl", "-sw", "-H", "\"Host:uspdigital.usp.br\nConnection:keep-alive\nContent-Length:280\nOrigin:https://uspdigital.usp.br\nUser-Agent:Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Mobile Safari/537.36\nContent-Type:text/plain\nAccept:*/*\nReferer:https://uspdigital.usp.br/rucard/Jsp/cardapioSAS.jsp\nAccept-Encoding:gzip, deflate, br\nAccept-Language:pt-BR,pt;q=0.8,en-US;q=0.6,en;q=0.4,fr;q=0.2\nRequest Payload:\"", "-X", "POST", "-d","callCount=1\nwindowName=1\nnextReverseAjaxIndex=0\nc0-scriptName=CardapioControleDWR\nc0-methodName=obterCardapioRestUSP\nc0-id=0\nc0-param0=string:{}\nbatchId=1\ninstanceId=0\npage=%2Frucard%2FJsp%2FcardapioSAS.jsp%3Fcodrtn%3D0\nscriptSessionId=".format(rID), "https://uspdigital.usp.br/rucard/dwr/call/plaincall/CardapioControleDWR.obterCardapioRestUSP.dwr"]
    return cmd

def parse_cardapio(script_res):
    r = str(script_res).split("\\n")
    r = list(filter(lambda s: "dwr.engine.remote." in s, r))
    r = html.unescape(r[0]).split("[")[1].encode().decode('unicode_escape')
    r = list(filter(None, clean_str(r.replace("\/",", ")).split("{")))
    for i in range(len(r)):
        k = list(filter(None, r[i].split("cdpdia:")))
        k = list(filter(None, "".join(k).split("\"")))
        r[i] = k[0].encode().decode('unicode_escape') if k[0] != " " else "Fechado"
    r = list(zip(r[::2], r[1::2]))
    return r

def clean_str(r):
    d = {"
":"\n", "}":""}
    regex = re.compile('|'.join(d.keys()))
    return regex.sub(lambda x: d[x.group()], r)

def create_cardapio(r):
    days = ["Segunda", "Terça", "Quarta", "Quinta", "Sexta", "Sábado", "Domingo"]
    cardapio = {dia: refeicao for dia, refeicao in zip(days, r)}
    return cardapio

def app_h(k, isMarkd):
    for f in featured:
        f = [x.strip().lower() for x in f.split(",")]
        if f[0] in k.lower().split(" "):
            if(len(list((x for x in f[1:] if x in k.lower()))) == 0):
                if not isMarkd:
                    return "\033[4m\033[38;5;165m*"+k.strip()+"*\033[0m"
                else:
                    return "`"+k.strip()+"`"
    return k

def print_refeicao(rf, color, isMarkd=False):
    uLine = lambda x: "\033[38;5;14m"+x+"\033[0m"+color
    tmp = list(filter(None, rf.split("\n")))
    if(len(tmp) > 2):
        for k, idx in zip(tmp, range(len(tmp))):
            if not isMarkd:
                print(WHITE+"➤ "+GRAY, end="")
            else:
                print("➤ ", end="")
            if idx == 1 or idx == 3 or idx == 5:
                temp = app_h(k, isMarkd) if godlike else k
                if not isMarkd:
                    print(uLine(temp))
                else:
                    print((lambda x : "*"+x+"*" if x[0] != "`" else x)(temp))
            else: print(k)
    else:
        print(rf)

def print_day(cardapio, tag, color, code=None, day="", isMarkd=False):
    days = ["Segunda", "Terça", "Quarta", "Quinta", "Sexta", "Sábado", "Domingo"]
    d_Index = datetime.datetime.today().weekday() if not day else day - 1
    refeicao = cardapio[days[d_Index]]
    if tag != "":
        tag = 0 if tag == " (Almoço)" else 1
        print_refeicao(refeicao[tag], GRAY, isMarkd=isMarkd)
    else:
        for i, title in zip(refeicao,["Almoço", "Jantar"]):
            print(colors[code]+format_str(title,"",char="-", sz=20)+color)
            print_refeicao(i, GRAY)

def get_TimeTag():
        now = datetime.datetime.now()
        aInf = now.replace(hour=6, minute=0, second=0, microsecond=0)
        aSup = now.replace(hour=14, minute=0, second=0, microsecond=0)
        jInf = now.replace(hour=14, minute=1, second=0, microsecond=0)
        jSup = now.replace(hour=20, minute=1, second=0, microsecond=0)
        if now > aInf and now < aSup:
            return " (Almoço)"
        elif now > jInf and now < jSup:
            return " (Jantar)"
        return ""

def format_str(name, tag, char="=", sz=50, addIcons=True):
    bdSt = list(name+tag)
    l = True
    while(len(bdSt) < sz):
        if l:   bdSt.append(char)
        else:   bdSt = [char] + bdSt
        l = not l
    if(addIcons):
        bdSt[0] = "#"
        bdSt[-1] = "#"
    # Centralize
    if(len(bdSt) < 50):
        bdSt = [" " for x in range((50 - len(bdSt))//2)] + bdSt
    return "".join(bdSt)

def markdown_formatter(tag, day):
    for name, code in bCodes.items():
        print("\n*"+format_str(name, tag)+"*")
        cdp = get_command(str(code))
        cdp = parse_cardapio(sb.check_output(cdp))
        cdp = create_cardapio(cdp)
        print_day(cdp, tag, GRAY, code=code, day=day, isMarkd=True)

def print_AllBdex(tag, dump=False, day="", logo=True, mkdump=False):
    days = ["Segunda", "Terça", "Quarta", "Quinta", "Sexta", "Sábado", "Domingo"]
    if logo: print_logo()
    if day:
        print("\n\n"+format_str(days[day-1],"",char="~", sz=30))
    if mkdump:
        markdown_formatter(tag, day)
        return
    for name, code in bCodes.items():
        print("\n"+colors[code]+format_str(name, tag)+GRAY)
        cdp = get_command(str(code))
        cdp = parse_cardapio(sb.check_output(cdp))
        cdp = create_cardapio(cdp)
        if not dump:
            print_day(cdp, tag, GRAY, code=code, day=day)
        else:
            execute_query(cdp, code)

def print_logo():
    fadeColors = ["\033[38;5;{}m".format(i) for i in range(118, 124)]

    logo = [" ____                     _\n",
              "| __ )   __ _  _ __    __| |  ___ __  __\n",
              "|  _ \  / _` || '_ \  / _` | / _ \\\ \/ /\n",
              "| |_) || (_| || | | || (_| ||  __/ >  <\n",
              "|____/  \__,_||_| |_| \__,_| \___|/_/\_\\\n"]
    for color, line  in zip(fadeColors, logo):
        print(color+line, end="", sep="")

def print_usage(wrong_arg=None):
    err = lambda x: "\033[38;5;160m "+x+"\033[38;5;7m"
    runF = lambda x: "\033[38;5;4m"+x+"\033[38;5;7m"
    flagF = lambda x: "\033[38;5;46m"+x+"\033[38;5;7m"
    if(wrong_arg != None):
        print("😞😞😞 Poxa "+sb.check_output(["whoami"]).decode("utf-8").replace("\n","")+
              "," +err("argumento \""+wrong_arg+"\" é inválido! ")+"😞😞😞")
    print("Uso:\n $ "+runF("bandex")+"\n\t Imprime o cardápio do dia de todos os restau"
          "rantes, com as refeições de acordo com o horário de execução do script;\n"+
          flagF("FLAGS:")+"\n\t"+flagF("-a")+" : Imprime apenas os almoços;"+"\n\t"+
          flagF("-j")+" : Imprime apenas os jantares;"+"\n\t"+flagF("-all")+" : Imprime"
          " todas as refeições do dia;"+"\n\t"+flagF("-d número")+" : Imprime"
          " todas as refeições do dia escolhido (segunda = 1, terça = 2, etc);"+
          "\n\t"+flagF("-E ou -EVERYTHING")+" : Imprime todas as refeições da semana! D: ;"+"\n\t"
          +flagF("-h ou --help")+" : Imprime esta página de informação.\n")
    exit()

def main():
    if(len(sys.argv) == 1):
        tag = get_TimeTag()
        print_AllBdex(tag)
    elif(len(sys.argv) == 2):
        if(sys.argv[1] == "-a"):
            tag = " (Almoço)"
        elif(sys.argv[1] == "-j"):
            tag = " (Jantar)"
        elif(sys.argv[1] == "-all"):
            tag = ""
        elif(sys.argv[1] == "-EVERYTHING" or sys.argv[1] == "-E"):
            for i in range(1, 8):
                print_AllBdex("", day=i, logo = False)
            exit()
        elif(sys.argv[1] == "-markdown"):
            tag = get_TimeTag()
            print_AllBdex(tag, logo=False, mkdump=True)
            exit()
        elif(sys.argv[1] == "-h" or sys.argv[1] == "--help"):
            print_usage()
        else:
            try:
                k = sb.check_output(["cat", "/root/naoZOA"])
                k = k.decode("utf-8")
                k = k.replace("\n","")
                print("K=|",k,"|",sep="")
                if(sys.argv[1] == k):
                     print_AllBdex("", dump=True)
                exit()
            except Exception as e:
                print(e)
            print_usage(sys.argv[1])
        print_AllBdex(tag)
    elif(len(sys.argv) == 3 and sys.argv[1] == "-d"):
        try:
            day_choosen = int(sys.argv[2])
            if day_choosen in range(1,8):
                print_AllBdex("", day=day_choosen)
        except (TypeError, ValueError):
            print_usage()
    else:
        print_usage()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
Module with tests for rst.py
"""

#-----------------------------------------------------------------------------
# Copyright (c) 2013, the IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

from .base import ExportersTestsBase
from ..rst import RstExporter

#-----------------------------------------------------------------------------
# Class
#-----------------------------------------------------------------------------

class Test_RstExporter(ExportersTestsBase):
    """Contains test functions for rst.py"""

    def test_constructor(self):
        """
        Can a RstExporter be constructed?
        """
        RstExporter()

    def test_export(self):
        """
        Can a RstExporter export something?
        """
        (output, resources) = RstExporter().from_filename(self._get_notebook())
        assert len(output) > 0<commit_msg>RST capitalization fix<commit_after>"""
Module with tests for rst.py
"""

#-----------------------------------------------------------------------------
# Copyright (c) 2013, the IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

from .base import ExportersTestsBase
from ..rst import RSTExporter

#-----------------------------------------------------------------------------
# Class
#-----------------------------------------------------------------------------

class Test_RstExporter(ExportersTestsBase):
    """Contains test functions for rst.py"""

    def test_constructor(self):
        """
        Can a RSTExporter be constructed?
        """
        RSTExporter()

    def test_export(self):
        """
        Can a RSTExporter export something?
        """
        (output, resources) = RSTExporter().from_filename(self._get_notebook())
        assert len(output) > 0<|endoftext|>