<commit_before>import sys
import wx
import time
import threading
sys.path.insert(0, "../settings/")
import settings
sys.path.insert(0, "../control/")
import motorControl

class MyFrame(wx.Frame):
    def __init__(self, parent, id, title):
        wx.Frame.__init__(self, parent, id, title, wx.DefaultPosition, (600, 600))
        #panel = wx.Panel(self, -1, style=wx.SIMPLE_BORDER)

        vbox = wx.BoxSizer(wx.VERTICAL)
        hboxcontrols = wx.BoxSizer(wx.HORIZONTAL)
        hboxlist = wx.BoxSizer(wx.HORIZONTAL)

        vbox.Add(hboxcontrols)
        vbox.Add(hboxlist, 1, wx.EXPAND)

        motorSettings = settings.MotorSettings().get()
        motorControler = motorControl.MotorControl(motorSettings)
        motorControler.setAllSpeed(10)
        #parse motor setting to get min and max angle, store result in hash motors
        motors = {}
        motorsConfig=motorSettings["motorConfig"]
        for motor in motorsConfig:
            motors[motor[3]] = [motor[1], motor[2]]

        #thread for sequence of movement
        self.play = PlayMove(motorControler)

        #visual element

        #bowl button
        pnlBowl = wx.Panel(self, -1, style=wx.SIMPLE_BORDER)
        hboxcontrols.Add(pnlBowl, 1, wx.ALL, 1)
        self.sldBowl = wx.Slider(pnlBowl, -1, motors["bowl"][0], motors["bowl"][0], motors["bowl"][1], wx.DefaultPosition, (-1, -1), wx.SL_VERTICAL | wx.SL_LABELS)
        textBowl = wx.StaticText(pnlBowl, -1, 'Bowl')
        vboxBowl = wx.BoxSizer(wx.VERTICAL)
        vboxBowl.Add(self.sldBowl, 1, wx.CENTER, 0)
        vboxBowl.Add(textBowl, 1, wx.CENTER)
        pnlBowl.SetSizer(vboxBowl)

        #bottom button
        pnlBottom = wx.Panel(self, -1, style=wx.SIMPLE_BORDER)
        hboxcontrols.Add(pnlBottom, 1, wx.ALL, 1)
        self.sldBottom = wx.Slider(pnlBottom, -1, motors["bottom"][0], motors["bottom"][0], motors["bottom"][1], wx.DefaultPosition, (-1, -1), wx.SL_VERTICAL | wx.SL_LABELS)
        textBottom = wx.StaticText(pnlBottom, -1, 'Bottom')
        vboxBottom = wx.BoxSizer(wx.VERTICAL)
        vboxBottom.Add(self.sldBottom, 1, wx.CENTER)
        vboxBottom.Add(textBottom, 1, wx.CENTER)
        pnlBottom.SetSizer(vboxBottom)

        #middle button
        pnlMiddle = wx.Panel(self, -1, style=wx.SIMPLE_BORDER)
        hboxcontrols.Add(pnlMiddle, 1, wx.ALL, 1)
        self.sldMiddle = wx.Slider(pnlMiddle, -1, motors["middle"][0], motors["middle"][0], motors["middle"][1], wx.DefaultPosition, (-1, -1), wx.SL_VERTICAL | wx.SL_LABELS)
        textMiddle = wx.StaticText(pnlMiddle, -1, 'Middle')
        vboxMiddle= wx.BoxSizer(wx.VERTICAL)
        vboxMiddle.Add(self.sldMiddle, 1, wx.CENTER)
        vboxMiddle.Add(textMiddle, 1, wx.CENTER)
        pnlMiddle.SetSizer(vboxMiddle)

        #top button
        pnlTop = wx.Panel(self, -1, style=wx.SIMPLE_BORDER)
        hboxcontrols.Add(pnlTop, 1, wx.ALL, 1)
        self.sldTop = wx.Slider(pnlTop, -1, motors["top"][0], motors["top"][0], motors["top"][1], wx.DefaultPosition, (-1, -1), wx.SL_VERTICAL | wx.SL_LABELS)
        textTop = wx.StaticText(pnlTop, -1, 'Top')
        vboxTop = wx.BoxSizer(wx.VERTICAL)
        vboxTop.Add(self.sldTop, 1, wx.CENTER)
        vboxTop.Add(textTop, 1, wx.CENTER)
        pnlTop.SetSizer(vboxTop)

        #head button
        pnlHead = wx.Panel(self, -1, style=wx.SIMPLE_BORDER)
        hboxcontrols.Add(pnlHead, 1, wx.ALL, 1)
        self.sldHead = wx.Slider(pnlHead, -1, motors["head"][0], motors["head"][0], motors["head"][1], wx.DefaultPosition, (-1, -1), wx.SL_VERTICAL | wx.SL_LABELS)
        textHead = wx.StaticText(pnlHead, -1, 'Head')
        vboxHead = wx.BoxSizer(wx.VERTICAL)
        vboxHead.Add(self.sldHead, 1, wx.CENTER)
        vboxHead.Add(textHead, 1, wx.CENTER)
        pnlHead.SetSizer(vboxHead)

        #buttons
        pnlButtons = wx.Panel(self, -1, style=wx.SIMPLE_BORDER)
        hboxcontrols.Add(pnlButtons, 1, wx.EXPAND | wx.ALL, 1)
        vboxButton = wx.BoxSizer(wx.VERTICAL)

        gotoButton = wx.Button(pnlButtons, 7, 'Go To')
        vboxButton.Add(gotoButton, 1, wx.ALIGN_CENTER | wx.TOP, 15)
        wx.EVT_BUTTON(self, 7, self.OnGoto)

        savePositionButton = wx.Button(pnlButtons, 8, 'Save position to list')
        vboxButton.Add(savePositionButton, 1, wx.ALIGN_CENTER | wx.TOP, 15)
        wx.EVT_BUTTON(self, 8, self.OnSavePosition)

        deletePositionButton = wx.Button(pnlButtons, 9, 'delete position from list')
        vboxButton.Add(deletePositionButton, 1, wx.ALIGN_CENTER | wx.TOP, 15)
        wx.EVT_BUTTON(self, 9, self.OnDeletePosition)

        playButton = wx.Button(pnlButtons, 10, 'Play')
        vboxButton.Add(playButton, 1, wx.ALIGN_CENTER | wx.TOP, 15)
        wx.EVT_BUTTON(self, 10, self.OnPlay)

        stopButton = wx.Button(pnlButtons, 11, 'Stop')
        vboxButton.Add(stopButton, 1, wx.ALIGN_CENTER | wx.TOP | wx.BOTTOM, 15)
        wx.EVT_BUTTON(self, 11, self.OnStop)

        pnlButtons.SetSizer(vboxButton)

        #list of position
        self.positionList = wx.ListCtrl(self, -1, style=wx.LC_REPORT)
        self.positionList.InsertColumn(0, 'bowl')
        self.positionList.InsertColumn(1, 'bottom')
        self.positionList.InsertColumn(3, 'middle')
        self.positionList.InsertColumn(4, 'top')
        self.positionList.InsertColumn(5, 'head')
        hboxlist.Add(self.positionList, 1, wx.EXPAND)

        #panel.SetSizer(vbox)
        self.SetSizer(vbox)

    def OnPlay(self, event):
        count = self.positionList.GetItemCount()
        list=[]
        for row in range(count):
            value = self.positionList.GetItem(row, 0)
            list.append(["bowl", value])
            value = self.positionList.GetItem(row, 1)
            list.append(["bottom", value])
            value = self.positionList.GetItem(row, 2)
            list.append(["middle", value])
            value = self.positionList.GetItem(row, 3)
            list.append(["top", value])
            value = self.positionList.GetItem(row, 4)
            list.append(["head", value])

        self.play.listmove(list)
        self.play.start(motorControl)

    def OnStop(self, event):
        #print "caramel"
        self.play.stop()

    def OnGoto(self, event):
        list=[]
        list.append(["bowl", self.sldBowl.GetValue()])
        list.append(["bottom", self.sldBottom.GetValue()])
        list.append(["middle", self.sldMiddle.GetValue()])
        list.append(["top", self.sldTop.GetValue()])
        list.append(["head", self.sldHead.GetValue()])

        motorControl.setMotorByName(list)

    def OnSavePosition(self, event):
        print "biscuit"
        num_positions = self.positionList.GetItemCount()
        self.positionList.InsertStringItem(num_positions, str(self.sldBowl.GetValue()))
        self.positionList.SetStringItem(num_positions, 1, str(self.sldBottom.GetValue()))
        self.positionList.SetStringItem(num_positions, 2, str(self.sldMiddle.GetValue()))
        self.positionList.SetStringItem(num_positions, 3, str(self.sldTop.GetValue()))
        self.positionList.SetStringItem(num_positions, 4, str(self.sldHead.GetValue()))

    def OnDeletePosition(self, event):
        print "framboise"
        while self.positionList.GetSelectedItemCount() != 0 :
            index = self.positionList.GetFirstSelected()
            self.positionList.DeleteItem(index)

class MyApp(wx.App):
    def OnInit(self):
        #wx.App.__init__(self)
        #motorsConfig = [[0, 0, 100, "bowl"], [0, 9, 100, "bottom"], [1, 30, 100, "middle"], [2, 45, 1000, "top"], [3, 10, 400, "head"]]
        frame = MyFrame(None, -1, 'Pinokio alpha 0.1')
        frame.Show(True)
        frame.Centre()
        return True

class PlayMove(threading.Thread):
    def __init__(self, motorControler):
        threading.Thread.__init__(self)
        self.play = False
        self.stoplock = threading.Lock()
        self.motorControler = motorControler

    def run(self):
        self.play = True
        while self.play:
            print "chocolat"
            time.sleep(2)

    def stop(self):
        with self.stoplock:
            self.play = False
            print "caramel2"

    def listMove(self, list):
        with self.stoplock:
            self.moves = list

motorsConfig = [[0, 0, 100, "bowl"], [0, 9, 100, "bottom"], [1, 30, 100, "middle"], [2, 45, 1000, "top"], [3, 10, 400, "head"]]
app = MyApp(0)
app.MainLoop()<commit_msg>modification on gui<commit_after>import sys
import wx
import time
import threading
sys.path.insert(0, "../settings/")
import settings
sys.path.insert(0, "../control/")
import motorControl

class MyFrame(wx.Frame):
    def __init__(self, parent, id, title):
        wx.Frame.__init__(self, parent, id, title, wx.DefaultPosition, (600, 600))
        #panel = wx.Panel(self, -1, style=wx.SIMPLE_BORDER)

        vbox = wx.BoxSizer(wx.VERTICAL)
        hboxcontrols = wx.BoxSizer(wx.HORIZONTAL)
        hboxlist = wx.BoxSizer(wx.HORIZONTAL)

        vbox.Add(hboxcontrols)
        vbox.Add(hboxlist, 1, wx.EXPAND)

        motorSettings = settings.MotorSettings().get()
        self.motorControler = motorControl.MotorControl(motorSettings)
        self.motorControler.setAllSpeed(100)
        #parse motor setting to get min and max angle, store result in hash motors
        motors = {}
        motorsConfig=motorSettings["motorConfig"]
        for motor in motorsConfig:
            motors[motor[3]] = [motor[1], motor[2]]

        #thread for sequence of movement
        self.play = PlayMove(self.motorControler)

        #visual element

        #bowl button
        pnlBowl = wx.Panel(self, -1, style=wx.SIMPLE_BORDER)
        hboxcontrols.Add(pnlBowl, 1, wx.ALL |wx.EXPAND, 1)
        self.sldBowl = wx.Slider(pnlBowl, -1, motors["bowl"][0], motors["bowl"][0], motors["bowl"][1], wx.DefaultPosition, (-1, -1), wx.SL_VERTICAL | wx.SL_LABELS)
        textBowl = wx.StaticText(pnlBowl, -1, 'Bowl')
        vboxBowl = wx.BoxSizer(wx.VERTICAL)
        vboxBowl.Add(self.sldBowl, 1, wx.CENTER, 0)
        vboxBowl.Add(textBowl, 1, wx.CENTER)
        pnlBowl.SetSizer(vboxBowl)

        #bottom button
        pnlBottom = wx.Panel(self, -1, style=wx.SIMPLE_BORDER)
        hboxcontrols.Add(pnlBottom, 1, wx.ALL | wx.EXPAND, 1)
        self.sldBottom = wx.Slider(pnlBottom, -1, motors["bottom"][0], motors["bottom"][0], motors["bottom"][1], wx.DefaultPosition, (-1, -1), wx.SL_VERTICAL | wx.SL_LABELS)
        textBottom = wx.StaticText(pnlBottom, -1, 'Bottom')
        vboxBottom = wx.BoxSizer(wx.VERTICAL)
        vboxBottom.Add(self.sldBottom, 1, wx.CENTER)
        vboxBottom.Add(textBottom, 1, wx.CENTER)
        pnlBottom.SetSizer(vboxBottom)

        #middle button
        pnlMiddle = wx.Panel(self, -1, style=wx.SIMPLE_BORDER)
        hboxcontrols.Add(pnlMiddle, 1, wx.ALL | wx.EXPAND, 1)
        self.sldMiddle = wx.Slider(pnlMiddle, -1, motors["mid"][0], motors["mid"][0], motors["mid"][1], wx.DefaultPosition, (-1, -1), wx.SL_VERTICAL | wx.SL_LABELS)
        textMiddle = wx.StaticText(pnlMiddle, -1, 'Middle')
        vboxMiddle= wx.BoxSizer(wx.VERTICAL)
        vboxMiddle.Add(self.sldMiddle, 1, wx.CENTER)
        vboxMiddle.Add(textMiddle, 1, wx.CENTER)
        pnlMiddle.SetSizer(vboxMiddle)

        #top button
        pnlTop = wx.Panel(self, -1, style=wx.SIMPLE_BORDER)
        hboxcontrols.Add(pnlTop, 1, wx.ALL | wx.EXPAND, 1)
        self.sldTop = wx.Slider(pnlTop, -1, motors["top"][0], motors["top"][0], motors["top"][1], wx.DefaultPosition, (-1, -1), wx.SL_VERTICAL | wx.SL_LABELS)
        textTop = wx.StaticText(pnlTop, -1, 'Top')
        vboxTop = wx.BoxSizer(wx.VERTICAL)
        vboxTop.Add(self.sldTop, 1, wx.CENTER)
        vboxTop.Add(textTop, 1, wx.CENTER)
        pnlTop.SetSizer(vboxTop)

        #head button
        pnlHead = wx.Panel(self, -1, style=wx.SIMPLE_BORDER)
        hboxcontrols.Add(pnlHead, 1, wx.ALL | wx.EXPAND, 1)
        self.sldHead = wx.Slider(pnlHead, -1, motors["head"][0], motors["head"][0], motors["head"][1], wx.DefaultPosition, (-1, -1), wx.SL_VERTICAL | wx.SL_LABELS)
        textHead = wx.StaticText(pnlHead, -1, 'Head')
        vboxHead = wx.BoxSizer(wx.VERTICAL)
        vboxHead.Add(self.sldHead, 1, wx.CENTER)
        vboxHead.Add(textHead, 1, wx.CENTER)
        pnlHead.SetSizer(vboxHead)

        #buttons
        pnlButtons = wx.Panel(self, -1, style=wx.SIMPLE_BORDER)
        hboxcontrols.Add(pnlButtons, 1, wx.EXPAND | wx.ALL, 1)
        vboxButton = wx.BoxSizer(wx.VERTICAL)

        gotoButton = wx.Button(pnlButtons, 7, 'Go To')
        vboxButton.Add(gotoButton, 1, wx.ALIGN_CENTER | wx.TOP, 15)
        wx.EVT_BUTTON(self, 7, self.OnGoto)

        savePositionButton = wx.Button(pnlButtons, 8, 'Save position to list')
        vboxButton.Add(savePositionButton, 1, wx.ALIGN_CENTER | wx.TOP, 15)
        wx.EVT_BUTTON(self, 8, self.OnSavePosition)

        deletePositionButton = wx.Button(pnlButtons, 9, 'delete position from list')
        vboxButton.Add(deletePositionButton, 1, wx.ALIGN_CENTER | wx.TOP, 15)
        wx.EVT_BUTTON(self, 9, self.OnDeletePosition)

        playButton = wx.Button(pnlButtons, 10, 'Play')
        vboxButton.Add(playButton, 1, wx.ALIGN_CENTER | wx.TOP, 15)
        wx.EVT_BUTTON(self, 10, self.OnPlay)

        stopButton = wx.Button(pnlButtons, 11, 'Stop')
        vboxButton.Add(stopButton, 1, wx.ALIGN_CENTER | wx.TOP | wx.BOTTOM, 15)
        wx.EVT_BUTTON(self, 11, self.OnStop)

        pnlButtons.SetSizer(vboxButton)

        #list of position
        self.positionList = wx.ListCtrl(self, -1, style=wx.LC_REPORT)
        self.positionList.InsertColumn(0, 'bowl')
        self.positionList.InsertColumn(1, 'bottom')
        self.positionList.InsertColumn(3, 'middle')
        self.positionList.InsertColumn(4, 'top')
        self.positionList.InsertColumn(5, 'head')
        hboxlist.Add(self.positionList, 1, wx.EXPAND)

        #panel.SetSizer(vbox)
        self.SetSizer(vbox)

    def OnPlay(self, event):
        count = self.positionList.GetItemCount()
        list=[]
        for row in range(count):
            value = self.positionList.GetItem(row, 0)
            list.append(["bowl", value])
            value = self.positionList.GetItem(row, 1)
            list.append(["bottom", value])
            value = self.positionList.GetItem(row, 2)
            list.append(["mid", value])
            value = self.positionList.GetItem(row, 3)
            list.append(["top", value])
            value = self.positionList.GetItem(row, 4)
            list.append(["head", value])

        self.play.listmove(list)
        self.play.start(motorControl)

    def OnStop(self, event):
        #print "caramel"
        self.play.stop()

    def OnGoto(self, event):
        list=[]
        list.append(["bowl", self.sldBowl.GetValue()])
        list.append(["bottom", self.sldBottom.GetValue()])
        list.append(["mid", self.sldMiddle.GetValue()])
        list.append(["top", self.sldTop.GetValue()])
        list.append(["head", self.sldHead.GetValue()])

        self.motorControler.setMotorsByName(list)

    def OnSavePosition(self, event):
        print "biscuit"
        num_positions = self.positionList.GetItemCount()
        self.positionList.InsertStringItem(num_positions, str(self.sldBowl.GetValue()))
        self.positionList.SetStringItem(num_positions, 1, str(self.sldBottom.GetValue()))
        self.positionList.SetStringItem(num_positions, 2, str(self.sldMiddle.GetValue()))
        self.positionList.SetStringItem(num_positions, 3, str(self.sldTop.GetValue()))
        self.positionList.SetStringItem(num_positions, 4, str(self.sldHead.GetValue()))

    def OnDeletePosition(self, event):
        print "framboise"
        while self.positionList.GetSelectedItemCount() != 0 :
            index = self.positionList.GetFirstSelected()
            self.positionList.DeleteItem(index)

class MyApp(wx.App):
    def OnInit(self):
        #wx.App.__init__(self)
        #motorsConfig = [[0, 0, 100, "bowl"], [0, 9, 100, "bottom"], [1, 30, 100, "middle"], [2, 45, 1000, "top"], [3, 10, 400, "head"]]
        frame = MyFrame(None, -1, 'Pinokio alpha 0.1')
        frame.Show(True)
        frame.Centre()
        return True

class PlayMove(threading.Thread):
    def __init__(self, motorControler):
        threading.Thread.__init__(self)
        self.play = False
        self.stoplock = threading.Lock()
        self.motorControler = motorControler

    def run(self):
        self.play = True
        while self.play:
            print "chocolat"
            time.sleep(2)

    def stop(self):
        with self.stoplock:
            self.play = False
            print "caramel2"

    def listMove(self, list):
        with self.stoplock:
            self.moves = list

motorsConfig = [[0, 0, 100, "bowl"], [0, 9, 100, "bottom"], [1, 30, 100, "mid"], [2, 45, 1000, "top"], [3, 10, 400, "head"]]
app = MyApp(0)
app.MainLoop()
<|endoftext|>


<commit_before>from sklearn import preprocessing

def transform(dataframe):
    df = preprocessing.scale(dataframe[:-1])
    return df

<commit_msg>typo removed<commit_after>from sklearn import preprocessing

def transform(dataframe):
    df = preprocessing.scale(dataframe[:,:-1])
    return df

<|endoftext|>


<commit_before>
from tests.fixtures.constants import *
from sqlalchemy import Integer, Column, Numeric, String
from sqlalchemy import create_engine, MetaData, Table
from sqlalchemy.orm import mapper, sessionmaker
from sqlalchemy.ext.declarative import declarative_base
from collections import Counter

class Filters(object):
    """Contains filters used to filter the contents from textfile"""
    def __init__(self):
        self.order = ['lowercase','split','badchars','whitespace']

    def filter(self,chunk):
        """Filters chunk in specified order of filter layers"""
        for name in self.order:
            func = getattr(self,name)
            chunk = func(chunk)
        return chunk

    def split(self,chunk):
        """Parses chunk into list"""
        return chunk.split(' ')

    def badchars(self,chunk):
        """Removes non-alphabetical characters"""
        return [''.join(filter(str.isalpha,word)) for word in chunk]

    def lowercase(self,chunk):
        """Lowercase all characters"""
        return chunk.lower()

    def whitespace(self,chunk):
        """Removes whitespace and blanks"""
        space,nothing = ' ',''
        while space in chunk:
            chunk.remove(space)
        while nothing in chunk:
            chunk.remove(nothing)
        return chunk

class FileObj(object):
    """Read in file"""
    def __init__(self,file_path,chunk_size=1):
        self.file_path = file_path
        self.chunk_size = chunk_size
        self.pipeline = self.chunk(self.openfile())
        self.filters = Filters()

    def openfile(self,*kwargs):
        """Opens file and generates lines
        Args:
            file_path = Path to text file
        Notes:
            Accepts plain text files
        """
        with open(self.file_path, 'r') as infile:
            for line in infile:
                line_generator = line.rstrip('\n')
                yield line_generator

    def chunk(self,line_generator):
        """Package lines in file into specific chunk size, return the chunk
        Args:
            line_gen = String Generator of lines of file
            chunk_size = Number of lines from file wanted in each chunk
        """
        lines = []
        try:
            while True:
                for i in range(0,self.chunk_size):
                    lines.append(next(line_generator))
                dirty_chunk = ' '.join(lines)
                yield dirty_chunk
                lines = []
        except StopIteration as e:
            if lines:
                dirty_chunk = ' '.join(lines)
                yield dirty_chunk

    def filter(self):
        for chunk in self.pipeline:
            yield self.filters.filter(chunk)

class Data(object):
    def __init__(self,content=None):
        #Object with word and word values
        self.content = content
        self.counted_words = Counter()
        self.dbvalues = {}

    def countwords(self):
        for chunk in self.content:
            self.counted_words.update(Counter(chunk))

    def updatedbvalues(self,dbvalues):
        for value in dbvalues:
            self.dbvalues.update(value)

    def rejectwords(self):
        self.rejectedwords = {key for key,value in self.dbvalues.items() if not value}

    def cleanvalues(self):
        for word in self.rejectedwords:
            self.dbvalues.pop(word,None)
            self.counted_words.pop(word,None)

        self.dbvalues = Counter(self.dbvalues)
        return self.dbvalues

    def sentimentvalue(self):
        totalvalue = 0
        frequency = sum(self.counted_words.values())
        for key,value in self.counted_words.items():
            totalvalue += value * self.dbvalues[key]

        sv = totalvalue/ frequency
        return round(sv,4)

Base = declarative_base()
class WordBank(Base):
    __tablename__ = 'Warriner-English'
    word = Column(String, primary_key=True)
    value = Column(Integer)

class Database(object):
    def __init__(self,db_path=None):
        engine = create_engine('sqlite:///%s' % db_path, echo=False)
        metadata = MetaData(engine)
        Warriner_English = Table('Warriner-English', metadata, autoload=True)
        Session = sessionmaker(bind=engine)
        self.session = Session()

    def query(self,word):
        row = self.session.query(WordBank).filter_by(word=word).first()
        return row

    def queryall(self,dictobj):
        for word in dictobj:
            row = self.session.query(WordBank).filter_by(word=word).first()
            if row:
                yield {row.word:row.value}
            elif not row:
                yield {word:None}

class TextSentiment(object):
    def __init__(self,file_path=None,chunk_size=3):
        #Create content genenerator
        self.content = FileObj(file_path=file_path,
                               chunk_size=chunk_size).filter()
        #Create database connection
        self.db = Database(db_path=DB_PATH)
        #Create Data Object that holds working information
        self.data = Data(content=self.content)

    def countwords(self):
        try:
            self.data.countwords()
        except Exception as e:
            print("Error: ",e)
        finally:
            return self.data.counted_words

    def query(self,word):
        return self.db.query(word)

    def queryall(self,dictobj):
        return self.db.queryall(dictobj)

    def updatedbvalues(self,values):
        return self.data.updatedbvalues(values)

    def rejectwords(self):
        return self.data.rejectwords()

    def cleanvalues(self):
        return self.data.cleanvalues()

    def sentimentvalue(self):
        return self.data.sentimentvalue()

def main():
    ts = TextSentiment(file_path=TEST_DOC,chunk_size=10)
    data = ts.countwords()
    dbvalues = ts.queryall(data)
    ts.updatedbvalues(dbvalues)
    ts.rejectwords()
    ts.cleanvalues()
    sv = ts.sentimentvalue()
    print("Sentiment Value: ",sv)

if __name__ == '__main__':
    main()
<commit_msg>Refactor Data & TextSentiment Class<commit_after>
from tests.fixtures.constants import *
from sqlalchemy import Integer, Column, Numeric, String
from sqlalchemy import create_engine, MetaData, Table
from sqlalchemy.orm import mapper, sessionmaker
from sqlalchemy.ext.declarative import declarative_base
from collections import Counter

class Filters(object):
    """Contain filters used to filter the contents from textfile"""
    def __init__(self):
        self.order = ['lowercase','split','badchars','whitespace']

    def filter(self,chunk):
        """Filter chunk in specified order of filter layers"""
        for name in self.order:
            func = getattr(self,name)
            chunk = func(chunk)
        return chunk

    def split(self,chunk):
        """Parse chunk into list"""
        return chunk.split(' ')

    def badchars(self,chunk):
        """Remove non-alphabetical characters"""
        return [''.join(filter(str.isalpha,word)) for word in chunk]

    def lowercase(self,chunk):
        """Lowercase all characters"""
        return chunk.lower()

    def whitespace(self,chunk):
        """Remove whitespace and blanks"""
        space,nothing = ' ',''
        while space in chunk:
            chunk.remove(space)
        while nothing in chunk:
            chunk.remove(nothing)
        return chunk

class FileObj(object):
    """Read in textfile and create content generator"""
    def __init__(self,file_path,chunk_size=1):
        self.file_path = file_path
        self.chunk_size = chunk_size
        self.pipeline = self.chunk(self.openfile())
        self.filters = Filters()

    def openfile(self,*kwargs):
        """Opens file and generates lines
        Args:
            file_path = Path to text file
        Notes:
            Accepts plain text files
        """
        with open(self.file_path, 'r') as infile:
            for line in infile:
                line_generator = line.rstrip('\n')
                yield line_generator

    def chunk(self,line_generator):
        """Package lines in file into specific chunk size, return the chunk
        Args:
            line_gen = String Generator of lines of file
            chunk_size = Number of lines from file wanted in each chunk
        """
        lines = []
        try:
            while True:
                for i in range(0,self.chunk_size):
                    lines.append(next(line_generator))
                dirty_chunk = ' '.join(lines)
                yield dirty_chunk
                lines = []
        except StopIteration as e:
            if lines:
                dirty_chunk = ' '.join(lines)
                yield dirty_chunk

    def filter(self):
        """Filter content in textfile"""
        for chunk in self.pipeline:
            yield self.filters.filter(chunk)

Base = declarative_base()
class WordBank(Base):
    __tablename__ = 'Warriner-English'
    word = Column(String, primary_key=True)
    value = Column(Integer)

class Database(object):
    def __init__(self,db_path=None):
        engine = create_engine('sqlite:///%s' % db_path, echo=False)
        metadata = MetaData(engine)
        Warriner_English = Table('Warriner-English', metadata, autoload=True)
        Session = sessionmaker(bind=engine)
        self.session = Session()

    def query(self,content):
        found = {}
        notfound = {}
        for word in content:
            row = self.session.query(WordBank).filter_by(word=word).first()
            try:
                found.update({row.word:row.value})
            except AttributeError:
                notfound.update({word:None})
            """
            if row:
                found.update({row.word:row.value})
            elif not row:
                notfound.update({word:None})
            """

        return found, notfound

class Data(object):
    def __init__(self):
        self.content = None
        self.wordcount = Counter()
        self.wordsfound = Counter()
        self.wordsnotfound = {}

    def __call__(self):
        return self.__dict__

class TextSentiment(object):
    def __init__(self,file_path=None,chunk_size=3):
        self.content = FileObj(file_path=file_path,
                               chunk_size=chunk_size).filter()
        self.db = Database(db_path=DB_PATH)
        self.data = Data()
        self.data.content = self.content

    def wordcountcalc(self):
        for chunk in self.content:
            self.data.wordcount.update(Counter(chunk))

    def query(self):
        keys = self.data.wordcount.keys()
        self.data.wordsfound, self.data.wordsnotfound = self.db.query(keys)

    def sentimentvalue(self):
        totalvalue, totalfrequency = 0, 0
        for key, value in self.data.wordsfound.items():
            wordfrequency = self.data.wordcount[key]
            totalfrequency += wordfrequency
            totalvalue += wordfrequency * value
        sv = totalvalue / totalfrequency
        return round(sv,4)

    def __getitem__(self,key):
        return self.data.__dict__[key]

    def __call__(self):
        return self.data.__dict__

def main():
    ts = TextSentiment(file_path=TEST_DOC,chunk_size=10)
    ts.wordcountcalc() #Count each word
    ts.query() #Split words into wordsfound and wordsnotfound
    sv = ts.sentimentvalue() #Calculate the average sentiment value
    print("Sentiment Value: ",sv)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from vasppy import atom, cell, rdf
import numpy as np

class Configuration:

    def __init__( self, cell, atoms ):
        self.cell  = cell
        self.atoms = atoms

    def dr( self, atom1, atom2 ):
        return( self.cell.dr( atom1.r, atom2.r ) )

    def minimum_image_dr( self, atom1, atom2, cutoff = None ):
        return( self.cell.minimum_image_dr( atom1.r, atom2.r, cutoff = cutoff ) )

    def interatomic_distances( self, minimum_image_convention = True ):
        return( np.array( [ [ self.minimum_image_dr( atom_i, atom_j ) for atom_j in self.atoms ] for atom_i in self.atoms ] ) )

    def interatomic_distances_for_atom( self, atom1, minimum_image_convention = True ):
        return( np.array( [ self.minimum_image_dr( atom1, atom2 ) for atom2 in self.atoms ] ) )

    def atoms_with_label( self, label ):
        return filter( lambda atom: atom.label == label, self.atoms )

    def partial_rdf( self, spec_i, spec_j, max_r, number_of_bins ):
        this_rdf = rdf.Rdf( max_r, number_of_bins )
        atoms_i = self.atoms_with_label( spec_i )
        atoms_j = self.atoms_with_label( spec_j )
        for atom_i in atoms_i:
            for atom_j in atoms_j:
                if atom_i is atom_j:
                    continue
                dr = self.minimum_image_dr( atom_i, atom_j )
                try:
                    this_rdf.add_dr( dr )
                except IndexError:
                    pass
                except:
                    raise
        return this_rdf
<commit_msg>added per-atom rdf method. fixed bug in rdf calculation method.<commit_after>from vasppy import atom, cell, rdf
import numpy as np

class Configuration:

    def __init__( self, cell, atoms ):
        self.cell  = cell
        self.atoms = atoms

    def dr( self, atom1, atom2 ):
        return( self.cell.dr( atom1.r, atom2.r ) )

    def minimum_image_dr( self, atom1, atom2, cutoff = None ):
        return( self.cell.minimum_image_dr( atom1.r, atom2.r, cutoff = cutoff ) )

    def interatomic_distances( self, minimum_image_convention = True ):
        return( np.array( [ [ self.minimum_image_dr( atom_i, atom_j ) for atom_j in self.atoms ] for atom_i in self.atoms ] ) )

    def interatomic_distances_for_atom( self, atom1, minimum_image_convention = True ):
        return( np.array( [ self.minimum_image_dr( atom1, atom2 ) for atom2 in self.atoms ] ) )

    def atoms_with_label( self, label ):
        return filter( lambda atom: atom.label == label, self.atoms )

    def partial_rdf( self, spec_i, spec_j, max_r, number_of_bins ):
        this_rdf = rdf.Rdf( max_r, number_of_bins )
        atoms_i = list( self.atoms_with_label( spec_i ) )
        atoms_j = list( self.atoms_with_label( spec_j ) )
        for atom_i in atoms_i:
            for atom_j in atoms_j:
                if atom_i is atom_j:
                    continue
                dr = self.minimum_image_dr( atom_i, atom_j )
                try:
                    this_rdf.add_dr( dr )
                except IndexError:
                    pass
                except:
                    raise
        return this_rdf

    def per_atom_rdf( self, spec_i, spec_j, max_r, number_of_bins ):
        rdfs = []
        atoms_i = list( self.atoms_with_label( spec_i ) )
        atoms_j = list( self.atoms_with_label( spec_j ) )
        for atom_i in atoms_i:
            this_rdf = rdf.Rdf( max_r, number_of_bins )
            for atom_j in atoms_j:
                if atom_i is atom_j:
                    continue
                dr = self.minimum_image_dr( atom_i, atom_j )
                try:
                    this_rdf.add_dr( dr )
                except IndexError:
                    pass
                except:
                    raise
            rdfs.append( this_rdf )
        return( rdfs )

<|endoftext|>


<commit_before># Copyright (c) 2012-2013 SHIFT.com
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
# the Software, and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

from datetime import datetime
from decimal import Decimal as D

from thunderdome.tests.base import BaseCassEngTestCase

from thunderdome.columns import Column
from thunderdome.columns import Text
from thunderdome.columns import Integer
from thunderdome.columns import DateTime
from thunderdome.columns import Dictionary
from thunderdome.columns import UUID
from thunderdome.columns import Boolean
from thunderdome.columns import Float
from thunderdome.columns import List
from thunderdome.columns import Decimal

from thunderdome.models import Vertex

from thunderdome.exceptions import ValidationError

class DatetimeTest(Vertex):
    test_id = Integer(primary_key=True)
    created_at = DateTime(required=False)

class DatetimeCoercionTest(Vertex):
    test_id = Integer(primary_key=True)
    created_at = DateTime(required=False, strict=False)

class TestDatetime(BaseCassEngTestCase):

    def test_datetime_io(self):
        now = datetime.now()
        dt = DatetimeTest.create(test_id=0, created_at=now)
        dt2 = DatetimeTest.get(dt.vid)
        assert dt2.created_at.timetuple()[:6] == now.timetuple()[:6]

    def test_none_handling(self):
        """
        Tests the handling of NoneType
        :return:
        """
        dt = DatetimeTest.create(test_id=0, created_at=None)

    def test_coercion_of_floats(self):
        with self.assertRaises(ValidationError):
            dt = DatetimeTest.create(test_id=1)
            dt.created_at = 12309834.234
            dt.save()

        dt2 = DatetimeCoercionTest.create(test_id=2)
        dt2.created_at = 123098123.123
        dt2.save()
        dt2.reload()
        assert isinstance(dt2.created_at, datetime)

class DecimalTest(Vertex):
    test_id = Integer(primary_key=True)
    dec_val = Decimal()

class TestDecimal(BaseCassEngTestCase):

    def test_datetime_io(self):
        dt = DecimalTest.create(test_id=0, dec_val=D('0.00'))
        dt2 = DecimalTest.get(dt.vid)
        assert dt2.dec_val == dt.dec_val

        dt = DecimalTest.create(test_id=0, dec_val=5)
        dt2 = DecimalTest.get(dt.vid)
        assert dt2.dec_val == D('5')

class TestText(BaseCassEngTestCase):

    def test_max_length_validation(self):
        """
        Tests that the max_length kwarg works
        """

class TestInteger(BaseCassEngTestCase):

    def test_non_integral_validation(self):
        """
        Tests that attempting to save non integral values raises a ValidationError
        """

class TestFloat(BaseCassEngTestCase):

    def test_non_numberic_validation(self):
        """
        Tests that attempting to save a non numeric value raises a ValidationError
        """

class DictionaryTestVertex(Vertex):
    test_id = Integer(primary_key=True)
    map_val = Dictionary()

class TestDictionary(BaseCassEngTestCase):

    def test_dictionary_io(self):
        """ Tests that dictionary objects are saved and loaded successfully """
        dict_val = {'blake':31, 'something_else':'that'}
        v1 = DictionaryTestVertex.create(test_id=5, map_val=dict_val)
        v2 = DictionaryTestVertex.get(v1.vid)

        assert v2.map_val == dict_val

    def test_validation(self):
        """ Tests that the Dictionary column validates values properly """

        with self.assertRaises(ValidationError):
            Dictionary().validate([1,2,3])

        with self.assertRaises(ValidationError):
            Dictionary().validate('stringy')

        with self.assertRaises(ValidationError):
            Dictionary().validate(1)

class ListTestVertex(Vertex):
    test_id = Integer(primary_key=True)
    list_val = List()

class TestList(BaseCassEngTestCase):

    def test_dictionary_io(self):
        """ Tests that dictionary objects are saved and loaded successfully """
        list_val = ['blake', 31, 'something_else', 'that']
        v1 = ListTestVertex.create(test_id=5, list_val=list_val)
        v2 = ListTestVertex.get(v1.vid)

        assert v2.list_val == list_val

    def test_validation(self):
        """ Tests that the Dictionary column validates values properly """

        with self.assertRaises(ValidationError):
            List().validate({'blake':31, 'something_else':'that'})

        with self.assertRaises(ValidationError):
            List().validate('stringy')

        with self.assertRaises(ValidationError):
            List().validate(1)

<commit_msg>Add Additional Validation Tests<commit_after># Copyright (c) 2012-2013 SHIFT.com
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
# the Software, and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

from datetime import datetime
from decimal import Decimal as D

from thunderdome.tests.base import BaseCassEngTestCase

from thunderdome.columns import Column
from thunderdome.columns import Text
from thunderdome.columns import Integer
from thunderdome.columns import DateTime
from thunderdome.columns import Dictionary
from thunderdome.columns import UUID
from thunderdome.columns import Boolean
from thunderdome.columns import Float
from thunderdome.columns import List
from thunderdome.columns import Decimal

from thunderdome.models import Vertex

from thunderdome.exceptions import ValidationError

class DatetimeTest(Vertex):
    test_id = Integer(primary_key=True)
    created_at = DateTime(required=False)

class DatetimeCoercionTest(Vertex):
    test_id = Integer(primary_key=True)
    created_at = DateTime(required=False, strict=False)

class TestDatetime(BaseCassEngTestCase):

    def test_datetime_io(self):
        now = datetime.now()
        dt = DatetimeTest.create(test_id=0, created_at=now)
        dt2 = DatetimeTest.get(dt.vid)
        assert dt2.created_at.timetuple()[:6] == now.timetuple()[:6]

    def test_none_handling(self):
        """
        Tests the handling of NoneType
        :return:
        """
        dt = DatetimeTest.create(test_id=0, created_at=None)

    def test_coercion_of_floats(self):
        with self.assertRaises(ValidationError):
            dt = DatetimeTest.create(test_id=1)
            dt.created_at = 12309834.234
            dt.save()

        dt2 = DatetimeCoercionTest.create(test_id=2)
        dt2.created_at = 1362470400
        dt2.save()
        dt2.created_at = 1098234098.2098
        dt2.save()
        dt2.created_at = '120398231'
        dt2.save()
        dt2.created_at = '12039823.198'
        dt2.save()
        dt2.reload()
        assert isinstance(dt2.created_at, datetime)

class DecimalTest(Vertex):
    test_id = Integer(primary_key=True)
    dec_val = Decimal()

class TestDecimal(BaseCassEngTestCase):

    def test_datetime_io(self):
        dt = DecimalTest.create(test_id=0, dec_val=D('0.00'))
        dt2 = DecimalTest.get(dt.vid)
        assert dt2.dec_val == dt.dec_val

        dt = DecimalTest.create(test_id=0, dec_val=5)
        dt2 = DecimalTest.get(dt.vid)
        assert dt2.dec_val == D('5')

class TestText(BaseCassEngTestCase):

    def test_max_length_validation(self):
        """
        Tests that the max_length kwarg works
        """

class TestInteger(BaseCassEngTestCase):

    def test_non_integral_validation(self):
        """
        Tests that attempting to save non integral values raises a ValidationError
        """

class TestFloat(BaseCassEngTestCase):

    def test_non_numberic_validation(self):
        """
        Tests that attempting to save a non numeric value raises a ValidationError
        """

class DictionaryTestVertex(Vertex):
    test_id = Integer(primary_key=True)
    map_val = Dictionary()

class TestDictionary(BaseCassEngTestCase):

    def test_dictionary_io(self):
        """ Tests that dictionary objects are saved and loaded successfully """
        dict_val = {'blake':31, 'something_else':'that'}
        v1 = DictionaryTestVertex.create(test_id=5, map_val=dict_val)
        v2 = DictionaryTestVertex.get(v1.vid)

        assert v2.map_val == dict_val

    def test_validation(self):
        """ Tests that the Dictionary column validates values properly """

        with self.assertRaises(ValidationError):
            Dictionary().validate([1,2,3])

        with self.assertRaises(ValidationError):
            Dictionary().validate('stringy')

        with self.assertRaises(ValidationError):
            Dictionary().validate(1)

class ListTestVertex(Vertex):
    test_id = Integer(primary_key=True)
    list_val = List()

class TestList(BaseCassEngTestCase):

    def test_dictionary_io(self):
        """ Tests that dictionary objects are saved and loaded successfully """
        list_val = ['blake', 31, 'something_else', 'that']
        v1 = ListTestVertex.create(test_id=5, list_val=list_val)
        v2 = ListTestVertex.get(v1.vid)

        assert v2.list_val == list_val

    def test_validation(self):
        """ Tests that the Dictionary column validates values properly """

        with self.assertRaises(ValidationError):
            List().validate({'blake':31, 'something_else':'that'})

        with self.assertRaises(ValidationError):
            List().validate('stringy')

        with self.assertRaises(ValidationError):
            List().validate(1)

<|endoftext|>


<commit_before># -----------------------------
# IMPORTS
# -----------------------------

import argparse
import numpy as np
import tensorflow as tf
from math import ceil

# -----------------------------
# DATASETS
# -----------------------------

class DataSet(object):

  def __init__(self, images, labels, fake_data=False, preprocessed=False):
    if fake_data:
      self._num_examples = 10000
    elif preprocessed:
        self._num_examples = images.shape[0]
    else:
      assert images.shape[0] == labels.shape[0], (
          "images.shape: %s labels.shape: %s" % (images.shape,
                                                 labels.shape))
      self._num_examples = images.shape[0]

      # Convert shape from [num examples, rows, columns, depth]
      # to [num examples, rows*columns] (assuming depth == 1)
      # assert images.shape[3] == 1
      images = images.reshape(images.shape[0],
                              images.shape[1] * images.shape[2])
      # Convert from [0, 255] -> [0.0, 1.0].
      images = images.astype(np.float32)
      images = np.multiply(images, 1.0 / 255.0)

    self._images = images
    self._labels = labels
    self._epochs_completed = 0
    self._index_in_epoch = 0

  @property
  def images(self):
    return self._images

  @property
  def labels(self):
    return self._labels

  @property
  def num_examples(self):
    return self._num_examples

  @property
  def epochs_completed(self):
    return self._epochs_completed

  def next_batch(self, batch_size, fake_data=False):
    """Return the next `batch_size` examples from this data set."""
    if fake_data:
      fake_image = [1.0 for _ in range(784)]
      fake_label = 0
      return [fake_image for _ in range(batch_size)], [
          fake_label for _ in range(batch_size)]
    start = self._index_in_epoch
    self._index_in_epoch += batch_size
    if self._index_in_epoch > self._num_examples:
      # Finished epoch
      self._epochs_completed += 1
      # Shuffle the data
      perm = np.arange(self._num_examples)
      np.random.shuffle(perm)
      self._images = self._images[perm]
      self._labels = self._labels[perm]
      # Start next epoch
      start = 0
      self._index_in_epoch = batch_size
      assert batch_size <= self._num_examples
    end = self._index_in_epoch
    return self._images[start:end], self._labels[start:end]

class SemiDataSet(object):
    def __init__(self, images, labels, n_labeled, disjoint=False,
                 preprocessed=False):
        self._n_labeled = n_labeled
        self.disjoint = disjoint
        l_images, l_labels, u_images, u_labels = self.sample_balanced_labeled(images, labels, num_labeled=n_labeled)

        # Unlabeled DataSet
        if disjoint:
            self._unlabeled_ds = DataSet(u_images, u_labels, preprocessed=preprocessed)
        else:
            self._unlabeled_ds = DataSet(images, labels, preprocessed=preprocessed)

        # Labeled DataSet
        self._labeled_ds = DataSet(l_images, l_labels, preprocessed=preprocessed)

    def next_batch(self, batch_size, ul_batch_size=None):
        if ul_batch_size is None:
            ul_batch_size = batch_size
        unlabeled_images, _ = self.unlabeled_ds.next_batch(ul_batch_size)
        if batch_size > self.n_labeled:
            labeled_images, labels = self.labeled_ds.next_batch(self.n_labeled)
        else:
            labeled_images, labels = self.labeled_ds.next_batch(batch_size)
        images = np.vstack([labeled_images, unlabeled_images])
        return images, labels

    def sample_balanced_labeled(self, images, labels, num_labeled):
        n_total, n_classes = labels.shape

        # First create a fully balanced set larger than desired
        nl_per_class = int(ceil(num_labeled / n_classes))
        # rng = self.rng
        idx = []

        for c in range(n_classes):
            c_idx = np.where(labels[:,c]==1)[0]
            idx.append(np.random.choice(c_idx, nl_per_class))
        l_idx = np.concatenate(idx)

        # Now sample uniformly without replacement from the larger set to get desired
        l_idx = np.random.choice(l_idx, size=num_labeled, replace=False)

        u_idx = np.setdiff1d(np.arange(n_total), l_idx, assume_unique=True)
        self.l_idx = l_idx
        self.u_idx = u_idx

        return images[l_idx], labels[l_idx], images[u_idx], labels[u_idx]

    @property
    def n_labeled(self):
        return self._n_labeled

    @property
    def num_examples(self):
        if self.disjoint:
            return self.num_labeled + self.num_unlabeled
        else:
            return self.num_unlabeled

    @property
    def labeled_ds(self):
        return self._labeled_ds

    @property
    def unlabeled_ds(self):
        return self._unlabeled_ds

    @property
    def num_labeled(self):
        return self._labeled_ds.num_examples

    @property
    def num_unlabeled(self):
        return self._unlabeled_ds.num_examples

def dense_to_one_hot(labels_dense, num_classes=10):
    """Convert class labels from scalars to one-hot vectors."""
    num_labels = labels_dense.shape[0]
    index_offset = np.arange(num_labels) * num_classes
    labels_one_hot = np.zeros((num_labels, num_classes))
    labels_one_hot.flat[(index_offset + labels_dense.ravel()).astype(int)] = 1
    return labels_one_hot

# -----------------------------
# PARAMETER PARSING
# -----------------------------

def parse_argstring(argstring, dtype=float, sep='-'):
    return list(map(dtype, argstring.split(sep)))

def get_cli_params():
    parser = argparse.ArgumentParser()
    add = parser.add_argument

    # -------------------------
    # LOGGING
    # -------------------------
    add('--id', default='ladder')
    add('--logdir', default='results/logs/')
    add('--write_to', default=None)
    # description to print
    add('--description', default=None)

    # option to not save the model at all
    add('--do_not_save', action='store_true')
    add('--verbose', action='store_true')

    # -------------------------
    # DATA
    # -------------------------
    add('--dataset', default='mnist', choices=['mnist', 'svhn'])
    add('--input_size', default=784, type=int)
    # -------------------------
    # EVALUATE
    # -------------------------
    add('--test_frequency_in_epochs', default=5, type=float)
    # validation
    add('--validation', default=0, nargs='?', const=1000, type=int)

    add('--tb', default=False, nargs='?', const='tb/')
    # -------------------------
    # TRAINING
    # -------------------------

    add('--which_gpu', default=0, type=int)
    add('--seed', default=1, type=int)

    add('--end_epoch', default=150, type=int)
    add('--num_labeled', default=100, type=int)
    add('--batch_size', default=100, type=int)
    add('--ul_batch_size', default=100, type=int)

    add('--initial_learning_rate', default=0.002, type=float)
    add('--decay_start', default=0.67, type=float)
    add('--lr_decay_frequency', default=5, type=int)

    add('--beta1', default=0.9, type=float) # first momentum coefficient
    add('--beta1_during_decay', default=0.9, type=float)

    # -------------------------
    # LADDER STRUCTURE
    # -------------------------
    # Specify encoder layers
    add('--encoder_layers',
                        default='1000-500-250-250-250-10')

    # Standard deviation of the Gaussian noise to inject at each level
    add('--corrupt_sd', default=0.3, type=float)

    # Default RC cost corresponds to the gamma network
    add('--rc_weights', default='2000-20-0.2-0.2-0.2-0.2-0.2')

    # Batch norm decay weight mode
    add('--static_bn', default=0.99, type=float)

    # Use lrelu
    add('--lrelu_a', default=0.0, type=float)

    # Batch norm the top logits
    add('--top_bn', action='store_true')

    # -------------------------
    # VAT SETTINGS
    # -------------------------
    # vat params
    add('--epsilon', default='5.0')  # vary this instead of vat_weight
    add('--num_power_iters', default=1, type=int)
    add('--xi', default=1e-6, type=float, help="small constant for finite difference")
    add('--vadv_sd', default=0.0, type=float,
        help="noise to add at each layer of forward pass for stability")

    # -------------------------

    # VAL SETTINGS
    # -------------------------
    add('--model', default="c", choices=["n", "nlw", "c", "clw", "ladder",
                                         "vat", "gamma"])
    add('--measure_smoothness', action='store_true')
    add('--measure_vat', action='store_true', help='compute vat_cost but do '
                                                   'not use for optimisation')

    # -------------------------
    # CNN LADDER
    # -------------------------
    add('--cnn', action='store_true')

    # arguments for the cnn encoder/decoder
    add('--cnn_layer_types', default='c-c-c-max-c-c-c-max-c-c-c-avg-fc')
    add('--cnn_fan', default='3-96-96-96-96-192-192-192-192-192-192-192-192-10')
    add('--cnn_ksizes', default='3-3-3-3-3-3-3-3-3-1-1-0-0')
    add('--cnn_strides', default='1-1-1-2-1-1-1-2-1-1-1-0-0')
    add('--cnn_dims', default='32-32-32-32-16-16-16-16-8-8-8-8-1')

    # -------------------------
    # HYPEROPT
    # -------------------------
    parser.add_argument('--x0', default=None)
    parser.add_argument('--y0', default=None, type=float)
    parser.add_argument('--npi', default='1-2-3-4')

    # -------------------------
    # ADVERSARIAL ATTACKS
    # -------------------------
    add('--ord', default=2, choices=['inf', '1', '2'])

    params = parser.parse_args()

    return params

def enum_dict(list_):
    return dict(zip(range(len(list_)), list_))

def process_cli_params(params):
    # Specify base structure

    params.decay_start_epoch = int(params.decay_start * params.end_epoch)
    params.eval_batch_size = params.batch_size  # this should be redundant
    params.input_size = 768 if params.dataset == 'svhn' else 784

    if params.cnn:
        params.cnn_layer_types = parse_argstring(params.cnn_layer_types,
                                                 dtype=str)
        params.cnn_fan = parse_argstring(params.cnn_fan, dtype=int)
        params.cnn_ksizes = parse_argstring(params.cnn_ksizes, dtype=int)
        params.cnn_strides = parse_argstring(params.cnn_strides, dtype=int)
        params.cnn_dims = parse_argstring(params.cnn_dims, dtype=int)
        params.encoder_layers = params.cnn_fan
        # params.rc_weights = enum_dict(([0] * (len(params.cnn_fan)-1)) + [float(
        #     params.rc_weights)])

        if params.dataset == "mnist":
            params.cnn_init_size = 28
            params.cnn_fan[0] = 1
        elif params.dataset == "cifar":
            params.cnn_init_size = 32
            params.cnn_fan[0] = 3
        else:
            params.cnn_init_size = 28

    else:
        params.encoder_layers = parse_argstring(params.encoder_layers,
                                                dtype=int)

    params.encoder_layers = [params.input_size] + params.encoder_layers
    params.rc_weights = enum_dict(
        parse_argstring(params.rc_weights, dtype=float))

    params.num_layers = len(params.encoder_layers) - 1
    params.epsilon = enum_dict(parse_argstring(params.epsilon, dtype=float))
    # if params.model == 'vat':
    #     params.epsilon = params.epsilon[0]

    return params

def count_trainable_params():
    trainables = tf.trainable_variables()
    return np.sum([np.prod(var.get_shape()) for var in trainables])

def order_param_settings(params):
    param_dict = vars(params)
    param_list = []
    for k in sorted(param_dict.keys()):
        param_list.append(str(k) + ": " + str(param_dict[k]))

    return param_list

def preprocess(placeholder, params):
    return tf.reshape(placeholder, shape=[
        -1, params.cnn_init_size, params.cnn_init_size, params.cnn_fan[0]
    ]) if params.cnn else placeholder

def get_batch_ops(batch_size):
    join = lambda l, u: tf.concat([l, u], 0)
    split_lu = lambda x: (labeled(x), unlabeled(x))
    labeled = lambda x: x[:batch_size] if x is not None else x
    unlabeled = lambda x: x[batch_size:] if x is not None else x
    return join, split_lu, labeled, unlabeled
<commit_msg>Allow unrestricted choice of model cli<commit_after># -----------------------------
# IMPORTS
# -----------------------------

import argparse
import numpy as np
import tensorflow as tf
from math import ceil

# -----------------------------
# DATASETS
# -----------------------------

class DataSet(object):

  def __init__(self, images, labels, fake_data=False, preprocessed=False):
    if fake_data:
      self._num_examples = 10000
    elif preprocessed:
        self._num_examples = images.shape[0]
    else:
      assert images.shape[0] == labels.shape[0], (
          "images.shape: %s labels.shape: %s" % (images.shape,
                                                 labels.shape))
      self._num_examples = images.shape[0]

      # Convert shape from [num examples, rows, columns, depth]
      # to [num examples, rows*columns] (assuming depth == 1)
      # assert images.shape[3] == 1
      images = images.reshape(images.shape[0],
                              images.shape[1] * images.shape[2])
      # Convert from [0, 255] -> [0.0, 1.0].
      images = images.astype(np.float32)
      images = np.multiply(images, 1.0 / 255.0)

    self._images = images
    self._labels = labels
    self._epochs_completed = 0
    self._index_in_epoch = 0

  @property
  def images(self):
    return self._images

  @property
  def labels(self):
    return self._labels

  @property
  def num_examples(self):
    return self._num_examples

  @property
  def epochs_completed(self):
    return self._epochs_completed

  def next_batch(self, batch_size, fake_data=False):
    """Return the next `batch_size` examples from this data set."""
    if fake_data:
      fake_image = [1.0 for _ in range(784)]
      fake_label = 0
      return [fake_image for _ in range(batch_size)], [
          fake_label for _ in range(batch_size)]
    start = self._index_in_epoch
    self._index_in_epoch += batch_size
    if self._index_in_epoch > self._num_examples:
      # Finished epoch
      self._epochs_completed += 1
      # Shuffle the data
      perm = np.arange(self._num_examples)
      np.random.shuffle(perm)
      self._images = self._images[perm]
      self._labels = self._labels[perm]
      # Start next epoch
      start = 0
      self._index_in_epoch = batch_size
      assert batch_size <= self._num_examples
    end = self._index_in_epoch
    return self._images[start:end], self._labels[start:end]

class SemiDataSet(object):
    def __init__(self, images, labels, n_labeled, disjoint=False,
                 preprocessed=False):
        self._n_labeled = n_labeled
        self.disjoint = disjoint
        l_images, l_labels, u_images, u_labels = self.sample_balanced_labeled(images, labels, num_labeled=n_labeled)

        # Unlabeled DataSet
        if disjoint:
            self._unlabeled_ds = DataSet(u_images, u_labels, preprocessed=preprocessed)
        else:
            self._unlabeled_ds = DataSet(images, labels, preprocessed=preprocessed)

        # Labeled DataSet
        self._labeled_ds = DataSet(l_images, l_labels, preprocessed=preprocessed)

    def next_batch(self, batch_size, ul_batch_size=None):
        if ul_batch_size is None:
            ul_batch_size = batch_size
        unlabeled_images, _ = self.unlabeled_ds.next_batch(ul_batch_size)
        if batch_size > self.n_labeled:
            labeled_images, labels = self.labeled_ds.next_batch(self.n_labeled)
        else:
            labeled_images, labels = self.labeled_ds.next_batch(batch_size)
        images = np.vstack([labeled_images, unlabeled_images])
        return images, labels

    def sample_balanced_labeled(self, images, labels, num_labeled):
        n_total, n_classes = labels.shape

        # First create a fully balanced set larger than desired
        nl_per_class = int(ceil(num_labeled / n_classes))
        # rng = self.rng
        idx = []

        for c in range(n_classes):
            c_idx = np.where(labels[:,c]==1)[0]
            idx.append(np.random.choice(c_idx, nl_per_class))
        l_idx = np.concatenate(idx)

        # Now sample uniformly without replacement from the larger set to get desired
        l_idx = np.random.choice(l_idx, size=num_labeled, replace=False)

        u_idx = np.setdiff1d(np.arange(n_total), l_idx, assume_unique=True)
        self.l_idx = l_idx
        self.u_idx = u_idx

        return images[l_idx], labels[l_idx], images[u_idx], labels[u_idx]

    @property
    def n_labeled(self):
        return self._n_labeled

    @property
    def num_examples(self):
        if self.disjoint:
            return self.num_labeled + self.num_unlabeled
        else:
            return self.num_unlabeled

    @property
    def labeled_ds(self):
        return self._labeled_ds

    @property
    def unlabeled_ds(self):
        return self._unlabeled_ds

    @property
    def num_labeled(self):
        return self._labeled_ds.num_examples

    @property
    def num_unlabeled(self):
        return self._unlabeled_ds.num_examples

def dense_to_one_hot(labels_dense, num_classes=10):
    """Convert class labels from scalars to one-hot vectors."""
    num_labels = labels_dense.shape[0]
    index_offset = np.arange(num_labels) * num_classes
    labels_one_hot = np.zeros((num_labels, num_classes))
    labels_one_hot.flat[(index_offset + labels_dense.ravel()).astype(int)] = 1
    return labels_one_hot

# -----------------------------
# PARAMETER PARSING
# -----------------------------

def parse_argstring(argstring, dtype=float, sep='-'):
    return list(map(dtype, argstring.split(sep)))

def get_cli_params():
    parser = argparse.ArgumentParser()
    add = parser.add_argument

    # -------------------------
    # LOGGING
    # -------------------------
    add('--id', default='ladder')
    add('--logdir', default='results/logs/')
    add('--write_to', default=None)
    # description to print
    add('--description', default=None)

    # option to not save the model at all
    add('--do_not_save', action='store_true')
    add('--verbose', action='store_true')

    # -------------------------
    # DATA
    # -------------------------
    add('--dataset', default='mnist', choices=['mnist', 'svhn'])
    add('--input_size', default=784, type=int)
    # -------------------------
    # EVALUATE
    # -------------------------
    add('--test_frequency_in_epochs', default=5, type=float)
    # validation
    add('--validation', default=0, nargs='?', const=1000, type=int)

    add('--tb', default=False, nargs='?', const='tb/')
    # -------------------------
    # TRAINING
    # -------------------------

    add('--which_gpu', default=0, type=int)
    add('--seed', default=1, type=int)

    add('--end_epoch', default=150, type=int)
    add('--num_labeled', default=100, type=int)
    add('--batch_size', default=100, type=int)
    add('--ul_batch_size', default=100, type=int)

    add('--initial_learning_rate', default=0.002, type=float)
    add('--decay_start', default=0.67, type=float)
    add('--lr_decay_frequency', default=5, type=int)

    add('--beta1', default=0.9, type=float) # first momentum coefficient
    add('--beta1_during_decay', default=0.9, type=float)

    # -------------------------
    # LADDER STRUCTURE
    # -------------------------
    # Specify encoder layers
    add('--encoder_layers',
                        default='1000-500-250-250-250-10')

    # Standard deviation of the Gaussian noise to inject at each level
    add('--corrupt_sd', default=0.3, type=float)

    # Default RC cost corresponds to the gamma network
    add('--rc_weights', default='2000-20-0.2-0.2-0.2-0.2-0.2')

    # Batch norm decay weight mode
    add('--static_bn', default=0.99, type=float)

    # Use lrelu
    add('--lrelu_a', default=0.0, type=float)

    # Batch norm the top logits
    add('--top_bn', action='store_true')

    # -------------------------
    # VAT SETTINGS
    # -------------------------
    # vat params
    add('--epsilon', default='5.0')  # vary this instead of vat_weight
    add('--num_power_iters', default=1, type=int)
    add('--xi', default=1e-6, type=float, help="small constant for finite difference")
    add('--vadv_sd', default=0.0, type=float,
        help="noise to add at each layer of forward pass for stability")

    # -------------------------

    # VAL SETTINGS
    # -------------------------
    add('--model', default="c")
    add('--measure_smoothness', action='store_true')
    add('--measure_vat', action='store_true', help='compute vat_cost but do '
                                                   'not use for optimisation')

    # -------------------------
    # CNN LADDER
    # -------------------------
    add('--cnn', action='store_true')

    # arguments for the cnn encoder/decoder
    add('--cnn_layer_types', default='c-c-c-max-c-c-c-max-c-c-c-avg-fc')
    add('--cnn_fan', default='3-96-96-96-96-192-192-192-192-192-192-192-192-10')
    add('--cnn_ksizes', default='3-3-3-3-3-3-3-3-3-1-1-0-0')
    add('--cnn_strides', default='1-1-1-2-1-1-1-2-1-1-1-0-0')
    add('--cnn_dims', default='32-32-32-32-16-16-16-16-8-8-8-8-1')

    # -------------------------
    # HYPEROPT
    # -------------------------
    parser.add_argument('--x0', default=None)
    parser.add_argument('--y0', default=None, type=float)
    parser.add_argument('--npi', default='1-2-3-4')

    # -------------------------
    # ADVERSARIAL ATTACKS
    # -------------------------
    add('--ord', default=2, choices=['inf', '1', '2'])

    params = parser.parse_args()

    return params

def enum_dict(list_):
    return dict(zip(range(len(list_)), list_))

def process_cli_params(params):
    # Specify base structure

    params.decay_start_epoch = int(params.decay_start * params.end_epoch)
    params.eval_batch_size = params.batch_size  # this should be redundant
    params.input_size = 768 if params.dataset == 'svhn' else 784

    if params.cnn:
        params.cnn_layer_types = parse_argstring(params.cnn_layer_types,
                                                 dtype=str)
        params.cnn_fan = parse_argstring(params.cnn_fan, dtype=int)
        params.cnn_ksizes = parse_argstring(params.cnn_ksizes, dtype=int)
        params.cnn_strides = parse_argstring(params.cnn_strides, dtype=int)
        params.cnn_dims = parse_argstring(params.cnn_dims, dtype=int)
        params.encoder_layers = params.cnn_fan
        # params.rc_weights = enum_dict(([0] * (len(params.cnn_fan)-1)) + [float(
        #     params.rc_weights)])

        if params.dataset == "mnist":
            params.cnn_init_size = 28
            params.cnn_fan[0] = 1
        elif params.dataset == "cifar":
            params.cnn_init_size = 32
            params.cnn_fan[0] = 3
        else:
            params.cnn_init_size = 28

    else:
        params.encoder_layers = parse_argstring(params.encoder_layers,
                                                dtype=int)

    params.encoder_layers = [params.input_size] + params.encoder_layers
    params.rc_weights = enum_dict(
        parse_argstring(params.rc_weights, dtype=float))

    params.num_layers = len(params.encoder_layers) - 1
    params.epsilon = enum_dict(parse_argstring(params.epsilon, dtype=float))
    # if params.model == 'vat':
    #     params.epsilon = params.epsilon[0]

    return params

def count_trainable_params():
    trainables = tf.trainable_variables()
    return np.sum([np.prod(var.get_shape()) for var in trainables])

def order_param_settings(params):
    param_dict = vars(params)
    param_list = []
    for k in sorted(param_dict.keys()):
        param_list.append(str(k) + ": " + str(param_dict[k]))

    return param_list

def preprocess(placeholder, params):
    return tf.reshape(placeholder, shape=[
        -1, params.cnn_init_size, params.cnn_init_size, params.cnn_fan[0]
    ]) if params.cnn else placeholder

def get_batch_ops(batch_size):
    join = lambda l, u: tf.concat([l, u], 0)
    split_lu = lambda x: (labeled(x), unlabeled(x))
    labeled = lambda x: x[:batch_size] if x is not None else x
    unlabeled = lambda x: x[batch_size:] if x is not None else x
    return join, split_lu, labeled, unlabeled
<|endoftext|>


<commit_before>from . import config
from elasticsearch import Elasticsearch, client, helpers
from .config import logger
from .relationship import relationship
from .es_types import *
from .base import VWBase
import json
import inspect

class MapperError(Exception):
    pass

# tools for creating or reindexing elasticsearch mapping
class Mapper(object):
    def __init__(self,connect_args={}):
        self._es = Elasticsearch(config.dsn,**connect_args)
        self._esc = client.IndicesClient(self._es)

    # Retrieves the mapping as defined by the server
    def get_server_mapping(self,**kwargs):
        indexes = []
        if isinstance(kwargs.get('index'),list):
            indexes = kwargs.get('index')
        elif kwargs.get('index'):
            indexes.append(kwargs.get('index'))

        # if the model arguent is a VWBase object
        if isinstance(kwargs.get('index'), VWBase):
            try:
                indexes.append(kwargs.get('index').__index__)
            except AttributeError:
                pass

        if not indexes:
            indexes.append(config.default_index)

        return self._esc.get_mapping(index=indexes)

    # Retrieves what the map should be according to the defined models
    def get_index_map(self, **kwargs ):
        # recursively find all the subclasses of base

        # options
        """
        * index = "string"
            only map the index defined by "string"

        * index = ['index1','index2' ...]
            map the indexes defined by entries in list

        """

        subclasses = []
        self.get_subclasses(VWBase,subclasses)

        indexes = {}

        index_list = []
        if kwargs.get('index'):
            if isinstance(kwargs.get('index'), str):
                index_list.append( kwargs.get('index'))

            elif isinstance(kwargs.get('index'), list):
                index_list.extend( kwargs.get('index') )
            else:
                raise TypeError('"index" argument must be a string or list')

        for sc in subclasses:

            try:
                idx = sc.__index__
            except AttributeError:
                idx = config.default_index

            if len(index_list) > 0 and idx not in index_list:
                continue

            if idx not in indexes:
                indexes[idx] = {"mappings": {} }

            try:
                # create the basic body
                sc_body = { sc.__type__: { "properties": {} } }
            except AttributeError:
                # fails when no __type__ is found. Likely a subclass
                # to add other features. We will skip mapping
                continue

            for k,v in inspect.getmembers(sc):
                try:
                    if v.__metaclass__ == ESType:
                        sc_body[sc.__type__]['properties'][k] = v.prop_dict()
                except AttributeError:
                    pass

            indexes[idx]['mappings'].update(sc_body)

        return indexes

    def create_indices(self, **kwargs):

        suffix = kwargs.get('suffix')
        indexes = self.get_index_map(**kwargs)

        for k,v in indexes.iteritems():
            if suffix:
                idx = k + suffix
            else:
                idx = k

            self._esc.create( index=idx, body=v )

            if suffix:
                self._esc.put_alias(index=idx, name=k)

    def get_index_for_alias(self, alias):
        aliasd = self._esc.get_aliases(index=alias)
        index = ''
        for k,v in aliasd.iteritems():
            index = k
            break

        if index == alias:
            return None

        return index

    def reindex(self, idx, newindex, alias_name=None, remap_alias=None, **kwargs):
        # are we an alias or an actual index?
        index = idx;
        alias = None
        alias_exists = False

        if self._esc.exists_alias(idx):
            alias = idx
            index = self.get_index_for_alias(idx)
            alias_exists = True

        if alias_name:
            if self._esc.exists_alias(alias_name):
                if remap_alias:
                    alias_exists = True
                else:
                    raise MapperError('%s already exists as an alias. If you wish to delete the old alias pass remap_alias=True' % alias_name)

            alias = alias_name

        # does the new index exist?
        if not self._esc.exists( newindex ):
            # if new doesn't exist then create the mapping
            # as a copy of the old one. The idea being that the mapping
            # was changed
            index_mapping = self.get_index_map(index=idx) # using "idx" intentionally because models will be defined as alias
            self._esc.create( index=newindex, body=index_mapping.get(idx)) # have to use the index name as the key to the dict even though only one is returned.  .create() only takes the mapping

        # map our documents
        helpers.reindex(self._es, index, newindex, **kwargs)

        if alias and (remap_alias or alias_name):
            if alias_exists:
                self._esc.delete_alias(name=alias, index=index)

            self._esc.put_alias(name=alias, index=newindex)

    def get_subclasses(self,cls,subs):
        this_subs = cls.__subclasses__()
        if len(this_subs) == 0:
            subs.append(cls)
        else:
            for sc in this_subs:
                self.get_subclasses(sc,subs)

    def describe(self,cls):
        body = {}
        for k,v in cls.__dict__.iteritems():
            try:
                if v.__metaclass__ == ESType:
                    body[k] = v.prop_dict()
            except AttributeError:
                pass

            if not body.get(k):
                body[k] = { "type": type(v).__name__ }

        return body
<commit_msg>updated the connection in the mapper<commit_after>from . import config
from elasticsearch import Elasticsearch, client, helpers
from .config import logger
from .relationship import relationship
from .es_types import *
from .base import VWBase
import json
import inspect

class MapperError(Exception):
    pass

# tools for creating or reindexing elasticsearch mapping
class Mapper(object):
    def __init__(self,connect_args={}):
        self._es = Elasticsearch(config.dsn,**config.connection_params)
        self._esc = client.IndicesClient(self._es)

    # Retrieves the mapping as defined by the server
    def get_server_mapping(self,**kwargs):
        indexes = []
        if isinstance(kwargs.get('index'),list):
            indexes = kwargs.get('index')
        elif kwargs.get('index'):
            indexes.append(kwargs.get('index'))

        # if the model arguent is a VWBase object
        if isinstance(kwargs.get('index'), VWBase):
            try:
                indexes.append(kwargs.get('index').__index__)
            except AttributeError:
                pass

        if not indexes:
            indexes.append(config.default_index)

        return self._esc.get_mapping(index=indexes)

    # Retrieves what the map should be according to the defined models
    def get_index_map(self, **kwargs ):
        # recursively find all the subclasses of base

        # options
        """
        * index = "string"
            only map the index defined by "string"

        * index = ['index1','index2' ...]
            map the indexes defined by entries in list

        """

        subclasses = []
        self.get_subclasses(VWBase,subclasses)

        indexes = {}

        index_list = []
        if kwargs.get('index'):
            if isinstance(kwargs.get('index'), str):
                index_list.append( kwargs.get('index'))

            elif isinstance(kwargs.get('index'), list):
                index_list.extend( kwargs.get('index') )
            else:
                raise TypeError('"index" argument must be a string or list')

        for sc in subclasses:

            try:
                idx = sc.__index__
            except AttributeError:
                idx = config.default_index

            if len(index_list) > 0 and idx not in index_list:
                continue

            if idx not in indexes:
                indexes[idx] = {"mappings": {} }

            try:
                # create the basic body
                sc_body = { sc.__type__: { "properties": {} } }
            except AttributeError:
                # fails when no __type__ is found. Likely a subclass
                # to add other features. We will skip mapping
                continue

            for k,v in inspect.getmembers(sc):
                try:
                    if v.__metaclass__ == ESType:
                        sc_body[sc.__type__]['properties'][k] = v.prop_dict()
                except AttributeError:
                    pass

            indexes[idx]['mappings'].update(sc_body)

        return indexes

    def create_indices(self, **kwargs):

        suffix = kwargs.get('suffix')
        indexes = self.get_index_map(**kwargs)

        for k,v in indexes.iteritems():
            if suffix:
                idx = k + suffix
            else:
                idx = k

            self._esc.create( index=idx, body=v )

            if suffix:
                self._esc.put_alias(index=idx, name=k)

    def get_index_for_alias(self, alias):
        aliasd = self._esc.get_aliases(index=alias)
        index = ''
        for k,v in aliasd.iteritems():
            index = k
            break

        if index == alias:
            return None

        return index

    def reindex(self, idx, newindex, alias_name=None, remap_alias=None, **kwargs):
        # are we an alias or an actual index?
        index = idx;
        alias = None
        alias_exists = False

        if self._esc.exists_alias(idx):
            alias = idx
            index = self.get_index_for_alias(idx)
            alias_exists = True

        if alias_name:
            if self._esc.exists_alias(alias_name):
                if remap_alias:
                    alias_exists = True
                else:
                    raise MapperError('%s already exists as an alias. If you wish to delete the old alias pass remap_alias=True' % alias_name)

            alias = alias_name

        # does the new index exist?
        if not self._esc.exists( newindex ):
            # if new doesn't exist then create the mapping
            # as a copy of the old one. The idea being that the mapping
            # was changed
            index_mapping = self.get_index_map(index=idx) # using "idx" intentionally because models will be defined as alias
            self._esc.create( index=newindex, body=index_mapping.get(idx)) # have to use the index name as the key to the dict even though only one is returned.  .create() only takes the mapping

        # map our documents
        helpers.reindex(self._es, index, newindex, **kwargs)

        if alias and (remap_alias or alias_name):
            if alias_exists:
                self._esc.delete_alias(name=alias, index=index)

            self._esc.put_alias(name=alias, index=newindex)

    def get_subclasses(self,cls,subs):
        this_subs = cls.__subclasses__()
        if len(this_subs) == 0:
            subs.append(cls)
        else:
            for sc in this_subs:
                self.get_subclasses(sc,subs)

    def describe(self,cls):
        body = {}
        for k,v in cls.__dict__.iteritems():
            try:
                if v.__metaclass__ == ESType:
                    body[k] = v.prop_dict()
            except AttributeError:
                pass

            if not body.get(k):
                body[k] = { "type": type(v).__name__ }

        return body
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import lxml.html

import re
import time

class PluginParseError(Exception): pass

class Oharas(object):
    def __init__(self):
        self.url = "https://www.gastropub.net/oharas/keikkakalenteri/"
        self.name = "O'Hara's"
        self.city = "Tampere"
        self.country = "Finland"

        # Parsing patterns
        self.datepat = re.compile("^[0-9]{,2}.[0-9]{,2}. ")

    def getVenueName(self):
        return self.name

    def getCity(self):
        return self.city

    def getCountry(self):
        return self.country

    def eventSQLentity(self):
        """
        This method is used to ensure venue exists in venue SQL table.
        """
        return { u"name" : self.name, \
                 u"city" : self.city, \
                 u"country" : self.country }

    def parsePrice(self, line):
        return u"0" if line == "-" else u"%s€" % line

    def parseDate(self, tag):
        month_now = time.strftime("%m")
        year = int(time.strftime("%Y"))

        if tag == None:
            return u""

        day, month = tag.group().rstrip(" .").split(".")
        # Are we on the new year already?
        if int(month) < int(month_now):
            year += 1

        return "%.4d-%.2d-%.2d" % (int(year), int(month), int(day))

    def parseEvent(self, tag):
        """
        Example format:
        04.12. LifeLine + Vaara
        """
        date = u""
        title = u""

        datedata = re.search(self.datepat, tag.text_content())
        date = self.parseDate(datedata)

        title = tag.text_content().split(" ", 1)
        if len(title) > 1:
            title = title[1]
        else:
            title = u""

        price = u"0"

        return { u"venue" : self.getVenueName(),  \
                 u"date" : date,                  \
                 u"name" : "%s" % (title),        \
                 u"price" : price }

    def parseEvents(self, data):
        doc = lxml.html.fromstring(data)
        eventtags = doc.xpath('//section[@id="main-content"]/p')

        for et in eventtags:
            if et.text_content() == u'\xa0': # We've reached the last event
                return
            yield self.parseEvent(et)

if __name__ == '__main__':
    import requests

    k = Oharas()
    r = requests.get(k.url)

    #with open("venues/oharas.html") as f:
    #    r = f.read()

    #for e in k.parseEvents(r):
    for e in k.parseEvents(r.content):
        for k, v in e.iteritems():
            print "%-10s: %s" % (k, v)
        print

<commit_msg>Fix plugin for O'Haras's<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import lxml.html

import re
import time

class PluginParseError(Exception): pass

class Oharas(object):
    def __init__(self):
        self.url = "https://www.gastropub.net/oharas/keikkakalenteri/"
        self.name = "O'Hara's"
        self.city = "Tampere"
        self.country = "Finland"

        # Parsing patterns
        self.datepat = re.compile("^[0-9]{,2}.[0-9]{,2}. ")

    def getVenueName(self):
        return self.name

    def getCity(self):
        return self.city

    def getCountry(self):
        return self.country

    def eventSQLentity(self):
        """
        This method is used to ensure venue exists in venue SQL table.
        """
        return { u"name" : self.name, \
                 u"city" : self.city, \
                 u"country" : self.country }

    def parsePrice(self, line):
        return u"0" if line == "-" else u"%s€" % line

    def parseDate(self, tag):
        month_now = time.strftime("%m")
        year = int(time.strftime("%Y"))

        if tag == None:
            return u""

        day, month = tag.group().rstrip(" .").split(".")
        # Are we on the new year already?
        if int(month) < int(month_now):
            year += 1

        return "%.4d-%.2d-%.2d" % (int(year), int(month), int(day))

    def parseEvent(self, tag):
        """
        Example format:
        04.12. LifeLine + Vaara
        """
        date = u""
        title = u""

        datedata = re.search(self.datepat, tag.text_content())
        date = self.parseDate(datedata)

        title = tag.text_content().split(" ", 1)
        if len(title) > 1:
            title = title[1]
        else:
            title = u""

        price = u"0"

        return { u"venue" : self.getVenueName(),  \
                 u"date" : date,                  \
                 u"name" : "%s" % (title),        \
                 u"price" : price }

    def parseEvents(self, data):
        doc = lxml.html.fromstring(data)
        eventtags = doc.xpath('//section[@id="main-content"]/p')

        for et in eventtags:
            parsed = self.parseEvent(et)
            if len(parsed["date"]) == 0:
                continue
            yield parsed

if __name__ == '__main__':
    import requests

    k = Oharas()
    r = requests.get(k.url)

    #with open("venues/oharas.html") as f:
    #    r = f.read()

    #for e in k.parseEvents(r):
    for e in k.parseEvents(r.content):
        for k, v in e.iteritems():
            print "%-10s: %s" % (k, v)
        print

<|endoftext|>


<commit_before>"""
CheckiOReferee is a base referee for checking you code.
    arguments:
        tests -- the dict contains tests in the specific structure.
            You can find an example in tests.py.
        cover_code -- is a wrapper for the user function and additional operations before give data
            in the user function. You can use some predefined codes from checkio.referee.cover_codes
        checker -- is replacement for the default checking of an user function result. If given, then
            instead simple "==" will be using the checker function which return tuple with result
            (false or true) and some additional info (some message).
            You can use some predefined codes from checkio.referee.checkers
        add_allowed_modules -- additional module which will be allowed for your task.
        add_close_builtins -- some closed builtin words, as example, if you want, you can close "eval"
        remove_allowed_modules -- close standard library modules, as example "math"

checkio.referee.checkers
    checkers.float_comparison -- Checking function fabric for check result with float numbers.
        Syntax: checkers.float_comparison(digits) -- where "digits" is a quantity of significant
            digits after coma.

checkio.referee.cover_codes
    cover_codes.unwrap_args -- Your "input" from test can be given as a list. if you want unwrap this
        before user function calling, then using this function. For example: if your test's input
        is [2, 2] and you use this cover_code, then user function will be called as checkio(2, 2)
    cover_codes.unwrap_kwargs -- the same as unwrap_kwargs, but unwrap dict.

"""

from checkio.signals import ON_CONNECT
from checkio import api
from checkio.referees.io import CheckiOReferee
from checkio.referees import cover_codes
from checkio.referees import checkers

from tests import TESTS

TYPE_ERROR = False, {"error_code": 1, "message": "You should return a callable function."}
ALL_OK = True, {"error_code": 100,
                "message": "All ok."}

def check_data(answer_data, user_data):
    if not hasattr(user_data,'__call__'):
        return TYPE_ERROR
    return ALL_OK

api.add_listener(
    ON_CONNECT,
    CheckiOReferee(
        tests=TESTS,
        #cover_code={
        #    'python-27': cover,
        #    'python-3': cover
        #},
        # checker=None,  # checkers.float.comparison(2)
        checker=check_data
        # add_allowed_modules=[],
        # add_close_builtins=[],
        # remove_allowed_modules=[]
    ).on_ready)
<commit_msg>Update referee.py<commit_after>"""
CheckiOReferee is a base referee for checking you code.
    arguments:
        tests -- the dict contains tests in the specific structure.
            You can find an example in tests.py.
        cover_code -- is a wrapper for the user function and additional operations before give data
            in the user function. You can use some predefined codes from checkio.referee.cover_codes
        checker -- is replacement for the default checking of an user function result. If given, then
            instead simple "==" will be using the checker function which return tuple with result
            (false or true) and some additional info (some message).
            You can use some predefined codes from checkio.referee.checkers
        add_allowed_modules -- additional module which will be allowed for your task.
        add_close_builtins -- some closed builtin words, as example, if you want, you can close "eval"
        remove_allowed_modules -- close standard library modules, as example "math"

checkio.referee.checkers
    checkers.float_comparison -- Checking function fabric for check result with float numbers.
        Syntax: checkers.float_comparison(digits) -- where "digits" is a quantity of significant
            digits after coma.

checkio.referee.cover_codes
    cover_codes.unwrap_args -- Your "input" from test can be given as a list. if you want unwrap this
        before user function calling, then using this function. For example: if your test's input
        is [2, 2] and you use this cover_code, then user function will be called as checkio(2, 2)
    cover_codes.unwrap_kwargs -- the same as unwrap_kwargs, but unwrap dict.

"""

from checkio.signals import ON_CONNECT
from checkio import api
from checkio.referees.io import CheckiOReferee
from checkio.referees import cover_codes
from checkio.referees import checkers

from tests import TESTS

TYPE_ERROR = False, {"error_code": 1, "message": "You should return a callable function."}
ALL_OK = True, {"error_code": 100,
                "message": "All ok."}

def check_data(answer_data, user_data):
    if not hasattr(user_data,'__call__'):
        return TYPE_ERROR
    return ALL_OK

api.add_listener(
    ON_CONNECT,
    CheckiOReferee(
        tests=TESTS,
        cover_code={
            'python-27': cover,
            'python-3': cover
        },
        # checker=None,  # checkers.float.comparison(2)
        checker=check_data
        # add_allowed_modules=[],
        # add_close_builtins=[],
        # remove_allowed_modules=[]
    ).on_ready)
<|endoftext|>


<commit_before>import unittest2

from tempfile import NamedTemporaryFile
from xml.etree.ElementTree import ElementTree as ET
from random import randint

from penchy.maven import BootstrapPOM, POM, MavenDependency, IntegrityError

class PomTest(unittest2.TestCase):
    """
    A POM generated by :class:`POM` should look something like::

        <project>
          <dependencies>
            <dependency>
              <version>version</version>
              <groupId>groupId</groupId>
              <artifactId>artifactId</artifactId>
            </dependency>
          </dependencies>
          <repositories>
            <repository>
              <url>repo</url>
              <id>repo</id>
            </repository>
          </repositories>
          <modelVersion>4.0.0</modelVersion>
        </project>

    """
    def test_pom_attribs(self):
        for attrib in ('groupId', 'artifactId', 'version', 'repo'):
            with NamedTemporaryFile() as tf:
                rand = str(randint(0, 1000))
                p = POM(**{attrib: rand})
                p.write(tf.name)
                tree = ET()
                tree.parse(tf.name)
                self.assertEqual(tree.getroot().find(attrib).text, rand)

    def test_pom_dependency(self):
        dep = MavenDependency('groupId', 'artifactId', 'version', 'repo')

        with NamedTemporaryFile() as tf:
            p = POM()
            p.add_dependency(dep)
            p.write(tf.name)
            tree = ET()
            tree.parse(tf.name)
            root = tree.getroot()
            xdep = root.find('dependencies/dependency')
            for attrib in ('groupId', 'artifactId', 'version'):
                self.assertEqual(xdep.find(attrib).text, attrib)

            self.assertEqual(root.find('repositories/repository/url').text, 'repo')

    def test_bootstrap_pom(self):
        with NamedTemporaryFile() as tf:
            p = BootstrapPOM()
            p.write(tf.name)
            tree = ET()
            tree.parse(tf.name)
            root = tree.getroot()
            from penchy import __version__ as penchy_version

            self.assertEqual(root.find('artifactId').text, 'penchy-bootstrap')
            self.assertEqual(root.find('version').text, penchy_version)

    def test_mavendep_equal(self):
        d1 = MavenDependency(
            'org.scalabench.benchmarks',
            'scala-benchmark-suite',
            '0.1.0-20110908.085753-2',
            'http://repo.scalabench.org/snapshots/')

        d2 = MavenDependency(
            'org.scalabench.benchmarks',
            'scala-benchmark-suite',
            '0.1.0-20110908.085753-2',
            'http://repo.scalabench.org/snapshots/')

        self.assertEqual(d1, d2)
<commit_msg>more maven tests<commit_after>import unittest2

from tempfile import NamedTemporaryFile
from xml.etree.ElementTree import ElementTree as ET
from random import randint

from penchy.maven import BootstrapPOM, POM, MavenDependency, IntegrityError

class PomTest(unittest2.TestCase):
    """
    A POM generated by :class:`POM` should look something like::

        <project>
          <dependencies>
            <dependency>
              <version>version</version>
              <groupId>groupId</groupId>
              <artifactId>artifactId</artifactId>
            </dependency>
          </dependencies>
          <repositories>
            <repository>
              <url>repo</url>
              <id>repo</id>
            </repository>
          </repositories>
          <modelVersion>4.0.0</modelVersion>
        </project>

    """
    def test_pom_attribs(self):
        for attrib in ('groupId', 'artifactId', 'version', 'repo'):
            with NamedTemporaryFile() as tf:
                rand = str(randint(0, 1000))
                p = POM(**{attrib: rand})
                p.write(tf.name)
                tree = ET()
                tree.parse(tf.name)
                self.assertEqual(tree.getroot().find(attrib).text, rand)

    def test_pom_dependency(self):
        dep = MavenDependency('groupId', 'artifactId', 'version', 'repo')

        with NamedTemporaryFile() as tf:
            p = POM()
            p.add_dependency(dep)
            p.write(tf.name)
            tree = ET()
            tree.parse(tf.name)
            root = tree.getroot()
            xdep = root.find('dependencies/dependency')
            for attrib in ('groupId', 'artifactId', 'version'):
                self.assertEqual(xdep.find(attrib).text, attrib)

            self.assertEqual(root.find('repositories/repository/url').text, 'repo')

    def test_bootstrap_pom(self):
        with NamedTemporaryFile() as tf:
            p = BootstrapPOM()
            p.write(tf.name)
            tree = ET()
            tree.parse(tf.name)
            root = tree.getroot()
            from penchy import __version__ as penchy_version

            self.assertEqual(root.find('artifactId').text, 'penchy-bootstrap')
            self.assertEqual(root.find('version').text, penchy_version)

class MavenTest(unittest2.TestCase):
    def setUp(self):
        self.d1 = MavenDependency(
                'org.scalabench.benchmarks',
                'scala-benchmark-suite',
                '0.1.0-20110908.085753-2',
                'http://repo.scalabench.org/snapshots/')

        self.d2 = MavenDependency(
                'org.scalabench.benchmarks',
                'scala-benchmark-suite',
                '0.1.0-20110908.085753-2',
                'http://repo.scalabench.org/snapshots/')

    def test_mavendep_equal(self):
        self.assertEquals(self.d1, self.d2)

    def test_mavendep_duplicates(self):
        p = POM()
        p.add_dependency(self.d1)
        p.add_dependency(self.d2)
        self.assertEquals(p.dependency_list, set((self.d1,)))
<|endoftext|>


<commit_before>import os
import validators
import jwt
import datetime
import urllib
from flask import request, jsonify, redirect, url_for, flash
from flask_login import current_user, login_required
from pepper.utils import get_default_dashboard_for_role

# sends an access token to the callback provided by the url
@login_required
def auth():
    app_id = request.args.get('app_id')
    callback = request.args.get('callback')
    if app_id != os.getenv('INNOVATION_PORTAL_KEY'):
        return jsonify({'error': 'Invalid app_id provided'}), 422
    elif callback is None or not validators.url(callback):
        return jsonify({'error': 'Invalid callback provided'}), 422
    elif current_user.status == 'NEW':
        print("YOOOOOOOO")
        flash('You must finish registering before starting the puzzle challenge', 'warning')
        return redirect(url_for(get_default_dashboard_for_role()))
    else:
        access_token = {'t': jwt.encode({'exp': datetime.datetime.utcnow() + datetime.timedelta(seconds=30),
            'id': current_user.id, 'fname': current_user.fname, 'lname': current_user.lname}, 'secRetkEy', algorithm='HS256')}
        return redirect(callback + "?" + urllib.urlencode(access_token))

# returns the current user's full name and database id
def get_user_info():
    app_id = request.args.get('app_id')
    access_token = request.args.get('t') #token
    if app_id != os.getenv('INNOVATION_PORTAL_KEY'):
        return jsonify({'error': 'Invalid app_id provided'}), 422
    try:
        decoded = jwt.decode(access_token, 'secRetkEy', algorithms=['HS256'])
        return jsonify({'fname': decoded['fname'], 'lname': decoded['lname'], 'id': decoded['id']})
    except jwt.ExpiredSignatureError:
        return jsonify({'error': 'Token expired'}), 422
    except jwt.InvalidTokenError:
        return jsonify({'error': 'Token validation failed'}), 422
    except jwt.DecodeError:
        return jsonify({'error': 'Token cannot be decoded'}), 422
    except jwt.InvalidSignatureError:
        return jsonify({'error': 'Token has invalid signature'}), 422
<commit_msg>Update views.py<commit_after>import os
import validators
import jwt
import datetime
import urllib
from flask import request, jsonify, redirect, url_for, flash
from flask_login import current_user, login_required
from pepper.utils import get_default_dashboard_for_role

# sends an access token to the callback provided by the url
@login_required
def auth():
    app_id = request.args.get('app_id')
    callback = request.args.get('callback')
    if app_id != os.getenv('INNOVATION_PORTAL_KEY'):
        return jsonify({'error': 'Invalid app_id provided'}), 422
    elif callback is None or not validators.url(callback):
        return jsonify({'error': 'Invalid callback provided'}), 422
    elif current_user.status == 'NEW':
        flash('You must finish registering before starting the puzzle challenge', 'warning')
        return redirect(url_for(get_default_dashboard_for_role()))
    else:
        access_token = {'t': jwt.encode({'exp': datetime.datetime.utcnow() + datetime.timedelta(seconds=30),
            'id': current_user.id, 'fname': current_user.fname, 'lname': current_user.lname}, 'secRetkEy', algorithm='HS256')}
        return redirect(callback + "?" + urllib.urlencode(access_token))

# returns the current user's full name and database id
def get_user_info():
    app_id = request.args.get('app_id')
    access_token = request.args.get('t') #token
    if app_id != os.getenv('INNOVATION_PORTAL_KEY'):
        return jsonify({'error': 'Invalid app_id provided'}), 422
    try:
        decoded = jwt.decode(access_token, 'secRetkEy', algorithms=['HS256'])
        return jsonify({'fname': decoded['fname'], 'lname': decoded['lname'], 'id': decoded['id']})
    except jwt.ExpiredSignatureError:
        return jsonify({'error': 'Token expired'}), 422
    except jwt.InvalidTokenError:
        return jsonify({'error': 'Token validation failed'}), 422
    except jwt.DecodeError:
        return jsonify({'error': 'Token cannot be decoded'}), 422
    except jwt.InvalidSignatureError:
        return jsonify({'error': 'Token has invalid signature'}), 422
<|endoftext|>