<commit_before>import glob
import sys
import os
import shutil

def log(x):
    print x

def create_offsets(baserxdir, outdir, num_offsets=4, period=5):
    orig_keys = glob.glob(os.path.join(baserxdir, '*_original.key'))
    if len(orig_keys) != 1:
        log("!!!!! Need one and only one *_original.key file per directory")
        sys.exit(1)

    for inkey in orig_keys:
        keybase = os.path.splitext(os.path.basename(inkey))[0].replace("_original", "")
        # GO = grow only = a special case
        # if keybase == "GO":
        #     continue
        log("Processing %s keys" % keybase)

        outkey = os.path.join(outdir, keybase + "_00.key")
        shutil.copy(inkey, outkey)
        log("\tbase key %s_00.key" % keybase)

        for offset in range(1, num_offsets + 1):
            offset_years = (offset) * period
            outkey = os.path.join(outdir, "%s_%02d.key" % (keybase, offset))
            log("\toffset key %s_%02d.key" % (keybase, offset))
            ofh = open(outkey, 'w')
            for line in open(inkey, 'r'):
                newline = line
                if line.startswith("Offset ="):
                    newline = "Offset = %d\r\n" % offset_years
                ofh.write(newline)
            ofh.close()

        # Create final key; grow only
        # gokey = os.path.join(baserxdir, "GO_01.key")
        # outkey = os.path.join(outdir, "%s_%02d.key" % (keybase, num_offsets))
        # shutil.copy(gokey, outkey)
        # log("  grow-only key GO_%s.key" % num_offsets)

if __name__ == "__main__":
    if len(sys.argv) == 3:
        create_offsets(sys.argv[1], sys.argv[2])
    elif len(sys.argv) == 5:
        create_offsets(sys.argv[1], sys.argv[2], num_offsets=int(sys.argv[3]), period=int(sys.argv[4]))
    else:
        raise Exception
<commit_msg>take a stab at grow only offset strategy; not sure if this is the correct approach<commit_after>import glob
import sys
import os
import shutil

def log(x):
    print x

def create_offsets(baserxdir, outdir, num_offsets=4, period=5):
    orig_keys = glob.glob(os.path.join(baserxdir, '*_original.key'))
    if len(orig_keys) != 1:
        log("!!!!! Need one and only one *_original.key file per directory")
        sys.exit(1)

    for inkey in orig_keys:
        keybase = os.path.splitext(os.path.basename(inkey))[0].replace("_original", "")
        log("Processing %s keys" % keybase)

        outkey = os.path.join(outdir, keybase + "_00.key")
        shutil.copy(inkey, outkey)
        log("\tbase key %s_00.key" % keybase)

        for offset in range(1, num_offsets + 1):
            offset_years = (offset) * period
            outkey = os.path.join(outdir, "%s_%02d.key" % (keybase, offset))
            log("\toffset key %s_%02d.key" % (keybase, offset))
            ofh = open(outkey, 'w')
            for line in open(inkey, 'r'):
                newline = line
                if line.startswith("Offset ="):
                    newline = "Offset = %d\r\n" % offset_years
                ofh.write(newline)
            ofh.close()

        # create the grow-only
        offset_years = 100
        outkey = os.path.join(outdir, "%s_99.key" % (keybase,))
        log("\toffset key %s_99.key" % (keybase,))
        ofh = open(outkey, 'w')
        for line in open(inkey, 'r'):
            newline = line
            if line.startswith("Offset ="):
                newline = "Offset = %d\r\n" % offset_years
            ofh.write(newline)
        ofh.close()

if __name__ == "__main__":
    if len(sys.argv) == 3:
        create_offsets(sys.argv[1], sys.argv[2])
    elif len(sys.argv) == 5:
        create_offsets(sys.argv[1], sys.argv[2], num_offsets=int(sys.argv[3]), period=int(sys.argv[4]))
    else:
        raise Exception
<|endoftext|>


<commit_before>#
# Copyright (c) 2005-2006 rPath, Inc.
#
# All rights reserved
#

import os
from raa.modules.raasrvplugin import rAASrvPlugin
from conary.repository.netrepos.netserver import ServerConfig
from conary.lib.cfgtypes import CfgEnvironmentError

class ConaryServer(rAASrvPlugin):
    cnrPath = '/srv/conary/repository.cnr'
    conaryrcPath = '/srv/www/html/conaryrc'
    apacheRestart = '/usr/bin/killall -USR1 httpd'

    def doTask(self, schedId, execId):
        '''
            Updates serverName in repository.cnr if no commits have occured in the
            repository.
        '''

        data = self.server.getData()
        if not len(data):
            data = ('localhost',)
        try:
            cfg = ServerConfig()
            cfg.read(self.cnrPath)
            cfg.serverName = data
        except CfgEnvironmentError:
            pass

        try:
            f = open(self.cnrPath, "w")
            try:
                cfg.display(f)
            finally:
                f.close()
        except IOError, e:
            pass

        pipeCmd = os.popen('hostname --fqdn')
        try:
            srvName = pipeCmd.read().strip()
        finally:
            pipeCmd.close()

        cfgData = '\n'.join(['repositoryMap %s http://%s/conary/' % (x, srvName) for x in data if x != 'localhost'])
        cfgData = cfgData and (cfgData + '\n') or cfgData
        f = open(self.conaryrcPath, 'w')
        try:
            f.write(cfgData)
        finally:
            f.close()

        try:
            os.system(self.apacheRestart)
        except:
            pass
<commit_msg>modify conaryserver plugin to use generated config file<commit_after>#
# Copyright (c) 2005-2006 rPath, Inc.
#
# All rights reserved
#

import os
from raa.modules.raasrvplugin import rAASrvPlugin
from conary.repository.netrepos.netserver import ServerConfig
from conary.lib.cfgtypes import CfgEnvironmentError

class ConaryServer(rAASrvPlugin):
    cnrPath = '/srv/conary/repository.cnr'
    generatedFile = '/srv/conary/repository-generated.cnr'
    conaryrcPath = '/srv/www/html/conaryrc'
    apacheRestart = '/usr/bin/killall -USR1 httpd'

    def doTask(self, schedId, execId):
        '''
            Updates serverName in repository.cnr if no commits have occured in the
            repository.
        '''

        data = self.server.getData()
        if not len(data):
            data = ('localhost',)
        try:
            cfg = ServerConfig()
            cfg.serverName = data
        except CfgEnvironmentError:
            pass

        try:
            genFile = open(self.generatedFile, 'w')
            try:
                cfg.displayKey('serverName', out=genFile)
            finally:
                genFile.close()
        except IOError, e:
            pass

        pipeCmd = os.popen('hostname --fqdn')
        try:
            srvName = pipeCmd.read().strip()
        finally:
            pipeCmd.close()

        cfgData = '\n'.join(['repositoryMap %s http://%s/conary/' % (x, srvName) for x in data if x != 'localhost'])
        cfgData = cfgData and (cfgData + '\n') or cfgData
        f = open(self.conaryrcPath, 'w')
        try:
            f.write(cfgData)
        finally:
            f.close()

        try:
            os.system(self.apacheRestart)
        except:
            pass
<|endoftext|>


<commit_before>#!/usr/bin/env python3

class Unit(object):
    name = ''
    symbol = ''
    id = None
    moveSteps = ()
    moveMaxLength = 1
    classByName = {}
    classBySymbol = {}
    classList = []

    @classmethod
    def registerClass(myCls, cls):
        myCls.classByName[cls.name] = cls
        myCls.classBySymbol[cls.symbol] = cls

        cls.id = len(myCls.classList)
        myCls.classList.append(cls)

    def __init__(self, rowNum, colNum):
        """
        rowNum: row number, starting from 0
        colNum: column number, starting from 0
        """
        self.setPos(rowNum, colNum)

    def setPos(self, rowNum, colNum):
        """set position of unit

        rowNum: row number, starting from 0
        colNum: column number, starting from 0
        """
        self.rowNum = rowNum
        self.colNum = colNum

    def getPos(self):
        """return the current position as tuple (col, row)"""
        return (self.rowNum, self.colNum)

    def attacksPos(self, rowNum, colNum):
        raise NotImplementedError

    def attacksUnit(self, other):
        return self.attacksPos(other.rowNum, other.colNum)

    def attacksBoard(self, board):
        """check if this unit can attck (threatens) any unit on board
            return True if it can, False otherwise

        board: a dict { (rowNum, colNum) => unitSymbol }
        """
        for rowNum, colNum in board.keys():
            if self.attacksPos(rowNum, colNum):
                return True

        return False

    @classmethod
    def posAttckedByBoard(cls, rowNum, colNum, board):
        for (bRowNum, bColNum), symbol in board.items():
            other = cls.classBySymbol[symbol](bRowNum, bColNum)
            if other.attacksPos(rowNum, colNum):
                return True

        return False

    def canPutOnBoard(self, board):
        """check if this unit can be added to the board without threatening
            or being threatened by any unit on board
            return True if it can, False otherwise

        board: a dict { (rowNum, colNum) => unitSymbol }
            we use dict instead of 2-dimentional array bcoz the number of units
            on board is probably small comparing to the whole table (N*M)
            should we use numpy matrix? FIXME
        """
        for (rowNum, colNum), symbol in board.items():
            if self.attacksPos(rowNum, colNum):
                return False

            other = self.classBySymbol[symbol](rowNum, colNum)
            if other.attacksUnit(self):
                return False

        return True

@Unit.registerClass
class King(Unit):
    name = 'king'
    symbol = 'K'
    moveSteps = (
        (-1, -1),
        (-1, 0),
        (-1, 1),
        (0, -1),
        (0, 1),
        (1, -1),
        (1, 0),
        (1, 1),
    )
    moveMaxLength = 1

    def attacksPos(self, rowNum, colNum):
        return 1 == max(
            abs(rowNum - self.rowNum),
            abs(colNum - self.colNum),
        )

@Unit.registerClass
class Queen(Unit):
    name = 'queen'
    symbol = 'Q'
    moveSteps = (
        (-1, -1),
        (-1, 0),
        (-1, 1),
        (0, -1),
        (0, 1),
        (1, -1),
        (1, 0),
        (1, 1),
    )
    moveMaxLength = -1

    def attacksPos(self, rowNum, colNum):
        return rowNum == self.rowNum or colNum == self.colNum or \
            abs(rowNum - self.rowNum) == abs(colNum - self.colNum)

@Unit.registerClass
class Bishop(Unit):
    name = 'bishop'
    symbol = 'B'
    moveSteps = (
        (-1, -1),
        (-1, 1),
        (1, -1),
        (1, 1),
    )
    moveMaxLength = -1

    def attacksPos(self, rowNum, colNum):
        return abs(rowNum - self.rowNum) == abs(colNum - self.colNum)

@Unit.registerClass
class Rook(Unit):
    name = 'rook'
    symbol = 'R'
    moveSteps = (
        (-1, 0),
        (0, -1),
        (0, 1),
        (1, 0),
    )
    moveMaxLength = -1

    def attacksPos(self, rowNum, colNum):
        return rowNum == self.rowNum or colNum == self.colNum

@Unit.registerClass
class Knight(Unit):
    name = 'knight'
    symbol = 'N'
    moveSteps = (
        (-2, -1),
        (-2, 1),
        (-1, -2),
        (-1, 2),
        (1, -2),
        (1, 2),
        (2, -1),
        (2, 1),
    )
    moveMaxLength = 1

    def attacksPos(self, rowNum, colNum):
        return {1, 2} == {
            abs(rowNum - self.rowNum),
            abs(colNum - self.colNum),
        }

if __name__ == '__main__':
    from pprint import pprint, pformat
    print('classByName = %s' % pformat(Unit.classByName))
    print('classBySymbol = %s' % pformat(Unit.classBySymbol))
<commit_msg>pep-8 fixes in units.py<commit_after>#!/usr/bin/env python3

class Unit(object):
    name = ''
    symbol = ''
    id = None
    moveSteps = ()
    moveMaxLength = 1
    classByName = {}
    classBySymbol = {}
    classList = []

    @classmethod
    def registerClass(myCls, cls):
        myCls.classByName[cls.name] = cls
        myCls.classBySymbol[cls.symbol] = cls

        cls.id = len(myCls.classList)
        myCls.classList.append(cls)

    def __init__(self, rowNum, colNum):
        """
        rowNum: row number, starting from 0
        colNum: column number, starting from 0
        """
        self.setPos(rowNum, colNum)

    def setPos(self, rowNum, colNum):
        """set position of unit

        rowNum: row number, starting from 0
        colNum: column number, starting from 0
        """
        self.rowNum = rowNum
        self.colNum = colNum

    def getPos(self):
        """return the current position as tuple (col, row)"""
        return (self.rowNum, self.colNum)

    def attacksPos(self, rowNum, colNum):
        raise NotImplementedError

    def attacksUnit(self, other):
        return self.attacksPos(other.rowNum, other.colNum)

    def attacksBoard(self, board):
        """check if this unit can attck (threatens) any unit on board
            return True if it can, False otherwise

        board: a dict { (rowNum, colNum) => unitSymbol }
        """
        for rowNum, colNum in board.keys():
            if self.attacksPos(rowNum, colNum):
                return True

        return False

    @classmethod
    def posAttckedByBoard(cls, rowNum, colNum, board):
        for (bRowNum, bColNum), symbol in board.items():
            other = cls.classBySymbol[symbol](bRowNum, bColNum)
            if other.attacksPos(rowNum, colNum):
                return True

        return False

    def canPutOnBoard(self, board):
        """check if this unit can be added to the board without threatening
            or being threatened by any unit on board
            return True if it can, False otherwise

        board: a dict { (rowNum, colNum) => unitSymbol }
            we use dict instead of 2-dimentional array bcoz the number of units
            on board is probably small comparing to the whole table (N*M)
            should we use numpy matrix? FIXME
        """
        for (rowNum, colNum), symbol in board.items():
            if self.attacksPos(rowNum, colNum):
                return False

            other = self.classBySymbol[symbol](rowNum, colNum)
            if other.attacksUnit(self):
                return False

        return True

@Unit.registerClass
class King(Unit):
    name = 'king'
    symbol = 'K'
    moveSteps = (
        (-1, -1),
        (-1, 0),
        (-1, 1),
        (0, -1),
        (0, 1),
        (1, -1),
        (1, 0),
        (1, 1),
    )
    moveMaxLength = 1

    def attacksPos(self, rowNum, colNum):
        return 1 == max(
            abs(rowNum - self.rowNum),
            abs(colNum - self.colNum),
        )

@Unit.registerClass
class Queen(Unit):
    name = 'queen'
    symbol = 'Q'
    moveSteps = (
        (-1, -1),
        (-1, 0),
        (-1, 1),
        (0, -1),
        (0, 1),
        (1, -1),
        (1, 0),
        (1, 1),
    )
    moveMaxLength = -1

    def attacksPos(self, rowNum, colNum):
        return rowNum == self.rowNum or colNum == self.colNum or \
            abs(rowNum - self.rowNum) == abs(colNum - self.colNum)

@Unit.registerClass
class Bishop(Unit):
    name = 'bishop'
    symbol = 'B'
    moveSteps = (
        (-1, -1),
        (-1, 1),
        (1, -1),
        (1, 1),
    )
    moveMaxLength = -1

    def attacksPos(self, rowNum, colNum):
        return abs(rowNum - self.rowNum) == abs(colNum - self.colNum)

@Unit.registerClass
class Rook(Unit):
    name = 'rook'
    symbol = 'R'
    moveSteps = (
        (-1, 0),
        (0, -1),
        (0, 1),
        (1, 0),
    )
    moveMaxLength = -1

    def attacksPos(self, rowNum, colNum):
        return rowNum == self.rowNum or colNum == self.colNum

@Unit.registerClass
class Knight(Unit):
    name = 'knight'
    symbol = 'N'
    moveSteps = (
        (-2, -1),
        (-2, 1),
        (-1, -2),
        (-1, 2),
        (1, -2),
        (1, 2),
        (2, -1),
        (2, 1),
    )
    moveMaxLength = 1

    def attacksPos(self, rowNum, colNum):
        return {1, 2} == {
            abs(rowNum - self.rowNum),
            abs(colNum - self.colNum),
        }

if __name__ == '__main__':
    from pprint import pprint, pformat
    print('classByName = %s' % pformat(Unit.classByName))
    print('classBySymbol = %s' % pformat(Unit.classBySymbol))
<|endoftext|>


<commit_before>import subprocess

def main():
    st = subprocess.getoutput("mpc")
    lin = st.split("\n")
    if len(lin) > 1:
        sn_status = lin[1]
        duration = lin[1].split()
        if "paused" in sn_status:
            print(lin[0].split("-")[-1] + " [paused]")
        elif "playing" in sn_status:
            print(lin[0].split("-")[-1] + "   " + duration[2])
        else:
            print("stopped")
    else:
        print("stopped")

if __name__ == "__main__":
    main()
<commit_msg>[track] Use `spotify-now` to get current song info in i3blocks<commit_after>import subprocess

def text_split(text):
    new_text = text.split()
    new_text_len = len(new_text)
    if new_text_len < 2:
        return new_text[0]
    elif new_text_len == 2:
        return text
    else:
        return " ".join(new_text[0:2]) + "..."

def main():
    st = subprocess.getoutput("mpc")
    lin = st.split("\n")
    if len(lin) > 1:
        sn_status = lin[1]
        duration = lin[1].split()
        if "paused" in sn_status:
            print(lin[0].split("-")[-1] + " [paused]")
        elif "playing" in sn_status:
            print(lin[0].split("-")[-1] + "   " + duration[2])
        else:
            print("stopped")
    else:
        title = subprocess.getoutput("spotify-now -i %title -p 'paused' -e stopped")
        if title == "paused" or title == "stopped":
            print(title)
        elif title == "":
            print("empty")
        else:
            artist = subprocess.getoutput("spotify-now -i '%artist'")
            new_title = text_split(title)
            new_artist = text_split(artist)
            print(new_title + ' - ' + new_artist)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python3
# coding: utf-8
import os
import os.path
import time
import sys
import time
import subprocess
contents = 'I:\\迅雷下载'
print(contents)
from rarfile import RarFile
pass_lists = os.listdir(contents)
pass_lists2 =[]
for i in pass_lists:
    j  = i.replace('星号','*')#.replace('&','^&')
    pass_lists2.append(j)
print(pass_lists2)
s =[]
error_files=[]
for root, dirs, files in os.walk(contents):
    for name in files:

        if ('rar' in name[-4:] ) :
            print('    --------')
            END_ = 0
            #print(f'    name ={name}')
            a_ = root[len(contents):]
            #print(f'a_ ={a_}')
            a2_ = a_.split('\\')
            #print(a2_)
            PASSWORD1_ = a2_[1]
            if "星号" in PASSWORD1_:
                PASSWORD1_ = PASSWORD1_.replace('星号','*')

            #print(f'    password ={PASSWORD1_}')
            PATH2_ = a2_[1:]
            PATH2str = '\\'.join(PATH2_)
            #print(f'path2 = {PATH2str}')
            #print(a2_)
            ex_dir = root  +"\\"
            ex_dir = '\"' + ex_dir  + '\"'
            #ex_dir = ex_dir.replace('&','`&')
            #print(f'    exdir={ex_dir}')
            #print(ex_dir.split(':'))
            FILENAME1_ = root +'\\'+ name
            FILENAME2_ = '\"' + FILENAME1_  + '\"'
            print(f'    filename1 ={FILENAME1_}')
            PASSWORD1_ = PASSWORD1_.replace('&','^&')

            #解压流程

            SSTR = ' '.join(["unrar.exe","x",FILENAME2_,f"-p{PASSWORD1_}","-o+",ex_dir])
            print(f"    SSTR = {SSTR}")
            obj = subprocess.Popen(SSTR,bufsize=1 ,stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
            cmd_out,cmd_error = obj.communicate()

            if ("All OK" in cmd_out):
                print(f'')
                print(f'')
                print(f'')
                print(f'解压成功,删除文件{FILENAME1_}')
                print(f'')
                os.remove(FILENAME1_)
                #print(f'-----------')

            else:
                #print(cmd_out)
                print("             ",cmd_error)
                print('')
                print('')
                print('')
                print(f'                解压失败 ')
                print('')
                if ("Incorrect password" in cmd_error) or ('error' in cmd_error):
                    for i in pass_lists2:
                        SSTR = ' '.join(["unrar.exe","x",FILENAME2_,f"-p{i}","-o+",ex_dir])
                        #print(f"SSTR = {SSTR}")
                        #print('')
                        print(f"                                   尝试密码  {i}                ",end='\r')
                        obj = subprocess.Popen(SSTR,bufsize=1 , stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
                        cmd_out,cmd_error = obj.communicate()

                        #print(cmd_error)
                        if ("All OK" in cmd_out):
                            print('')
                            print('')
                            print('')
                            print(f'解压成功！！！！！！！！密码=    {i}     ！！！！！！！！')
                            print(f'删除文件{FILENAME1_}')
                            with open(FILENAME1_+f"成功密码{i}.txt",mode="w",encoding="utf-8") as f:  #写文件,当文件不存在时,就直接创建此文件
                                pass
                            END_ = 1
                            os.remove(FILENAME1_)
                            #print(f'-----------')
                            time.sleep(1)
                            break

                    if END_ == 0:
                        #print('     尝试无密码解压')
                        SSTR = ' '.join(["unrar.exe","x",FILENAME2_,f"-p{i}","-o+",ex_dir])
                        #print(f"SSTR = {SSTR}")

                        print("")
                        #print(f"        FILE=={FILENAME1_[10:30]}，  无密码              ",end='\r')
                        obj = subprocess.Popen(SSTR,bufsize=1 , stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
                        cmd_out,cmd_error = obj.communicate()

                        #print(cmd_error)
                        if ("All OK" in cmd_out):
                            print("")
                            print("")
                            print("")
                            print(f'解压成功！！！！！！！！无密码解压！！！！！！！！')
                            print(f'删除文件{FILENAME1_}')
                            with open(FILENAME1_+f"无密码解压.txt",mode="w",encoding="utf-8") as f:  #写文件,当文件不存在时,就直接创建此文件

                                pass
                            os.remove(FILENAME1_)
                            print(f'-----------')
                            #time.sleep(2)
                            break

                        print("未尝试出密码")
                        error_files.append(FILENAME1_)
                break

print('=============================================')
print('end \n\r 未解压文件如下')
for i in error_files:
    print(i)
<commit_msg>Update unrar.py<commit_after>#!/usr/bin/python3
# coding: utf-8
import os
import os.path
import time
import sys
import time
import subprocess
contents = 'I:\\迅雷下载'
print(contents)
from rarfile import RarFile
pass_lists = os.listdir(contents)
pass_lists2 =[]
for i in pass_lists:
    j  = i.replace('星号','*')#.replace('&','^&')
    pass_lists2.append(j)
print(pass_lists2)
s =[]
error_files=[]
for root, dirs, files in os.walk(contents):
    for name in files:

        if ('rar' in name[-4:] ) :
            print('    --------')
            END_ = 0
            #print(f'    name ={name}')
            a_ = root[len(contents):]
            #print(f'a_ ={a_}')
            a2_ = a_.split('\\')
            #print(a2_)
            PASSWORD1_ = a2_[1]
            if "星号" in PASSWORD1_:
                PASSWORD1_ = PASSWORD1_.replace('星号','*')

            #print(f'    password ={PASSWORD1_}')
            PATH2_ = a2_[1:]
            PATH2str = '\\'.join(PATH2_)
            #print(f'path2 = {PATH2str}')
            #print(a2_)
            ex_dir = root  +"\\"
            ex_dir = '\"' + ex_dir  + '\"'
            #ex_dir = ex_dir.replace('&','`&')
            #print(f'    exdir={ex_dir}')
            #print(ex_dir.split(':'))
            FILENAME1_ = root +'\\'+ name
            FILENAME2_ = '\"' + FILENAME1_  + '\"'
            print(f'    filename1 ={FILENAME1_}')
            PASSWORD1_ = PASSWORD1_.replace('&','^&')

            #解压流程

            SSTR = ' '.join(["unrar.exe","x",FILENAME2_,f"-p{PASSWORD1_}","-o+",ex_dir])
            print(f"    SSTR = {SSTR}")
            obj = subprocess.Popen(SSTR,bufsize=1 ,stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
            cmd_out,cmd_error = obj.communicate()

            if ("All OK" in cmd_out):
                print(f'')
                print(f'')
                print(f'')
                print(f'解压成功,删除文件{FILENAME1_}')
                print(f'')
                os.remove(FILENAME1_)
                #print(f'-----------')

            else:
                #print(cmd_out)
                print("             ",cmd_error)
                print('')
                print('')
                print('')
                print(f'                解压失败 ')
                print('')
                if ("Incorrect password" in cmd_error) or ('error' in cmd_error):
                    for i in pass_lists2:
                        SSTR = ' '.join(["unrar.exe","x",FILENAME2_,f"-p{i}","-o+",ex_dir])
                        #print(f"SSTR = {SSTR}")
                        #print('')
                        print(f"                                   尝试密码  {i}                ",end='\r')
                        obj = subprocess.Popen(SSTR,bufsize=1 , stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
                        cmd_out,cmd_error = obj.communicate()

                        #print(cmd_error)
                        if ("All OK" in cmd_out):
                            print('')
                            print('')
                            print('')
                            print(f'解压成功！！！！！！！！密码=    {i}     ！！！！！！！！')
                            print(f'删除文件{FILENAME1_}')
                            with open(FILENAME1_+f"成功密码{i}.txt",mode="w",encoding="utf-8") as f:  #写文件,当文件不存在时,就直接创建此文件
                                pass
                            END_ = 1
                            os.remove(FILENAME1_)
                            #print(f'-----------')
                            time.sleep(1)
                            break

                    if END_ == 0:
                        #print('     尝试无密码解压')
                        SSTR = ' '.join(["unrar.exe","x",FILENAME2_,f"-p{i}","-o+",ex_dir])
                        #print(f"SSTR = {SSTR}")

                        print("")
                        #print(f"        FILE=={FILENAME1_[10:30]}，  无密码              ",end='\r')
                        obj = subprocess.Popen(SSTR,bufsize=1 , stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
                        cmd_out,cmd_error = obj.communicate()

                        #print(cmd_error)
                        if ("All OK" in cmd_out):
                            print("")
                            print("")
                            print("")
                            print(f'解压成功！！！！！！！！无密码解压！！！！！！！！')
                            print(f'删除文件{FILENAME1_}')
                            with open(FILENAME1_+f"无密码解压.txt",mode="w",encoding="utf-8") as f:  #写文件,当文件不存在时,就直接创建此文件

                                pass
                            os.remove(FILENAME1_)
                            print(f'-----------')
                            #time.sleep(2)
                            break

                        print("未尝试出密码")
                        error_files.append(FILENAME1_)
                        break

print('=============================================')
print('end \n\r 未解压文件如下')
for i in error_files:
    print(i)
<|endoftext|>


<commit_before>import os
import logging
import re
import typing
from typing import Any, Dict, Optional, Text

from rasa.nlu import utils
from rasa.nlu.components import Component
from rasa.nlu.training_data import Message

logger = logging.getLogger(__name__)

if typing.TYPE_CHECKING:
    from rasa.nlu.config import RasaNLUModelConfig
    from rasa.nlu.training_data import TrainingData
    from rasa.nlu.model import Metadata
    from rasa.nlu.training_data import Message

class KeywordIntentClassifier(Component):
    """Intent classifier using simple keyword matching.

    The classifier takes a list of keywords and associated intents as an input.
    A input sentence is checked for the keywords and the intent is returned.

    """

    provides = ["intent"]

    defaults = {"case_sensitive": False}

    def __init__(self,
                 component_config: Dict[Text, Any] = None,
                 intent_keyword_map: Optional[Dict] = None):

        super(KeywordIntentClassifier, self).__init__(component_config)

        if intent_keyword_map is None:
            self.intent_keyword_map = {}
        else:
            self.intent_keyword_map = intent_keyword_map

    def train(
        self,
        training_data: "TrainingData",
        cfg: Optional["RasaNLUModelConfig"] = None,
        **kwargs: Any
    ) -> None:

        for intent in training_data.intents:
            self.intent_keyword_map[intent] = [
                ex.text
                for ex in training_data.training_examples
                if ex.get("intent") == intent
            ]

    def process(self, message: Message, **kwargs: Any) -> None:
        intent_name = self._map_keyword_to_intent(message.text)
        if intent_name is not None:
            intent = {"name": intent_name, "confidence": 1.0}
            message.set("intent", intent, add_to_output=True)

    def _map_keyword_to_intent(self, text: Text) -> Optional[Text]:
        re_flags = 0 if self.component_config["case_sensitive"] else re.IGNORECASE
        for intent, keywords in self.intent_keyword_map.items():
            for string in keywords:
                if re.search(r"\b" + string + r"\b", text, flags=re_flags):
                    return intent

        return None

    def persist(self, file_name: Text, model_dir: Text) -> Dict[Text, Any]:
        """Persist this model into the passed directory.

        Return the metadata necessary to load the model again.
        """

        file_name = file_name + ".json"
        keyword_file = os.path.join(model_dir, file_name)
        utils.write_json_to_file(keyword_file, self.intent_keyword_map)

        return {"file": file_name}

    @classmethod
    def load(
        cls,
        meta: Dict[Text, Any],
        model_dir: Optional[Text] = None,
        model_metadata: "Metadata" = None,
        cached_component: Optional["KeywordIntentClassifier"] = None,
        **kwargs: Any
    ) -> "KeywordIntentClassifier":

        if model_dir and meta.get("file"):
            file_name = meta.get("file")
            keyword_file = os.path.join(model_dir, file_name)
            if os.path.exists(keyword_file):
                intent_keyword_map = utils.read_json_file(keyword_file)
            else:
                logger.warning(
                    "Failed to load IntentKeywordClassifier, maybe "
                    "{} does not exist.".format(keyword_file)
                )
        return cls(intent_keyword_map)
<commit_msg>case sensitive by default, fixed loading<commit_after>import os
import logging
import re
import typing
from typing import Any, Dict, Optional, Text

from rasa.nlu import utils
from rasa.nlu.components import Component
from rasa.nlu.training_data import Message

logger = logging.getLogger(__name__)

if typing.TYPE_CHECKING:
    from rasa.nlu.config import RasaNLUModelConfig
    from rasa.nlu.training_data import TrainingData
    from rasa.nlu.model import Metadata
    from rasa.nlu.training_data import Message

class KeywordIntentClassifier(Component):
    """Intent classifier using simple keyword matching.

    The classifier takes a list of keywords and associated intents as an input.
    A input sentence is checked for the keywords and the intent is returned.

    """

    provides = ["intent"]

    defaults = {"case_sensitive": True}

    def __init__(self,
                 component_config: Dict[Text, Any] = None,
                 intent_keyword_map: Optional[Dict] = None):

        super(KeywordIntentClassifier, self).__init__(component_config)

        if intent_keyword_map is None:
            self.intent_keyword_map = {}
        else:
            self.intent_keyword_map = intent_keyword_map

    def train(
        self,
        training_data: "TrainingData",
        cfg: Optional["RasaNLUModelConfig"] = None,
        **kwargs: Any
    ) -> None:

        for intent in training_data.intents:
            self.intent_keyword_map[intent] = [
                ex.text
                for ex in training_data.training_examples
                if ex.get("intent") == intent
            ]

    def process(self, message: Message, **kwargs: Any) -> None:
        intent_name = self._map_keyword_to_intent(message.text)
        if intent_name is not None:
            intent = {"name": intent_name, "confidence": 1.0}
            message.set("intent", intent, add_to_output=True)

    def _map_keyword_to_intent(self, text: Text) -> Optional[Text]:
        re_flags = 0 if self.component_config["case_sensitive"] else re.IGNORECASE
        for intent, keywords in self.intent_keyword_map.items():
            for string in keywords:
                if re.search(r"\b" + string + r"\b", text, flags=re_flags):
                    return intent

        return None

    def persist(self, file_name: Text, model_dir: Text) -> Dict[Text, Any]:
        """Persist this model into the passed directory.

        Return the metadata necessary to load the model again.
        """

        file_name = file_name + ".json"
        keyword_file = os.path.join(model_dir, file_name)
        utils.write_json_to_file(keyword_file, self.intent_keyword_map)

        return {"file": file_name}

    @classmethod
    def load(
        cls,
        meta: Dict[Text, Any],
        model_dir: Optional[Text] = None,
        model_metadata: "Metadata" = None,
        cached_component: Optional["KeywordIntentClassifier"] = None,
        **kwargs: Any
    ) -> "KeywordIntentClassifier":

        if model_dir and meta.get("file"):
            file_name = meta.get("file")
            keyword_file = os.path.join(model_dir, file_name)
            if os.path.exists(keyword_file):
                intent_keyword_map = utils.read_json_file(keyword_file)
            else:
                logger.warning(
                    "Failed to load IntentKeywordClassifier, maybe "
                    "{} does not exist.".format(keyword_file)
                )
        return cls(meta, intent_keyword_map)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# run_slavelastic.py: Runs a test based off of a slavelastic manifest file.

import glob
import hashlib
import json
import optparse
import os
import socket
import sys
import time
import urllib
import urllib2
import zipfile

DESCRIPTION = """This script takes a slavelastic manifest file, packages it,
and sends a swarm manifest file to the swarm server.  This is expected to be
called as a build step with the cwd as the parent of the src/ directory.
"""

CLEANUP_SCRIPT_NAME = 'swarm_cleanup.py'
CLEANUP_SCRIPT_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                   CLEANUP_SCRIPT_NAME)

WINDOWS_SCRIPT_NAME = 'kill_processes.py'
WINDOWS_SCRIPT_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                   WINDOWS_SCRIPT_NAME)

HANDLE_EXE = 'handle.exe'
HANDLE_EXE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                               '..', '..', 'third_party', 'psutils',
                               HANDLE_EXE)

class Manifest(object):
  RUN_TEST_PATH = os.path.join(
      'src', 'tools', 'isolate', 'run_test_from_archive.py')

  def __init__(self, manifest_hash, test_name, shards, switches):
    """Populates a manifest object.
      Args:
        manifest_hash - The manifest's sha-1 that the slave is going to fetch.
        shards - The number of swarm shards to request.
        test_name - The name to give the test request.
        switches - An object with properties to apply to the test request.
    """
    platform_mapping =  {
      'darwin': 'Mac',
      'cygwin': 'Windows',
      'linux2': 'Linux',
      'win32': 'Windows'
      }

    self.manifest_hash = manifest_hash
    self.shards = shards

    self.zipfile_fullpath = os.path.join(switches.data_dir, test_name + '.zip')
    self.tasks = []
    self.target_platform = platform_mapping[switches.os_image]
    self.working_dir = switches.working_dir
    self.test_name = test_name
    self.data_dir = switches.data_dir

  def add_task(self, task_name, actions, time_out=600):
    """Appends a new task to the swarm manifest file."""
    self.tasks.append({
          'test_name': task_name,
          'action': actions,
          'time_out': time_out
    })

  def zip(self):
    """Zip up all the files necessary to run a shard."""
    start_time = time.time()

    try:
      zip_file = zipfile.ZipFile(self.zipfile_fullpath, 'w')
    except IOError as e:
      print 'Error creating zip files %s' % str(e)
      return False

    zip_file.write(self.RUN_TEST_PATH)
    zip_file.write(CLEANUP_SCRIPT_PATH, CLEANUP_SCRIPT_NAME)

    if self.target_platform == 'Windows':
      zip_file.write(WINDOWS_SCRIPT_PATH, WINDOWS_SCRIPT_NAME)
      zip_file.write(HANDLE_EXE_PATH, HANDLE_EXE)

    zip_file.close()

    print 'Zipping completed, time elapsed: %f' % (time.time() - start_time)
    return True

  def to_json(self):
    """Export the current configuration into a swarm-readable manifest file"""
    self.add_task(
        'Run Test',
        ['python', self.RUN_TEST_PATH, '--hash', self.manifest_hash,
         '--remote', self.data_dir, '-v'])

    # Clean up
    self.add_task('Clean Up', ['python', CLEANUP_SCRIPT_NAME])

    # Call kill_processes.py if on windows
    if self.target_platform == 'Windows':
      self.add_task('Kill Processes',
          ['python', WINDOWS_SCRIPT_NAME,
           '--handle_exe', HANDLE_EXE])

    # This separation of vlans isn't required anymore, but it is
    # still a useful separation to keep.
    hostname = socket.gethostname().lower().split('.', 1)[0]
    vlan = ''
    if hostname.endswith('-m1'):
      vlan = 'm1'
    elif hostname.endswith('m4'):
      vlan = 'm4'

    data_scheme = 'file://'
    if self.target_platform == 'Windows':
      data_scheme += '/'

    # Construct test case
    test_case = {
      'test_case_name': self.test_name,
      'data': [
        urllib.quote(data_scheme + self.zipfile_fullpath.replace(os.sep, '/'),
                     ':/'),
      ],
      'tests': self.tasks,
      'env_vars': {
        'GTEST_TOTAL_SHARDS': '%(num_instances)s',
        'GTEST_SHARD_INDEX': '%(instance_index)s',
      },
      'configurations': [
        {
          'min_instances': self.shards,
          'config_name': self.target_platform,
          'dimensions': {
            'os': self.target_platform,
            'vlan': vlan
          },
        },
      ],
      'working_dir': self.working_dir,
      'cleanup': 'data',
    }

    return json.dumps(test_case)

def ProcessManifest(filepath, shards, options):
  """Process the manifest file and send off the swarm test request."""
  test_name = os.path.splitext(os.path.basename(filepath))[0]
  test_full_name = options.test_name_prefix + test_name

  if not os.path.exists(filepath):
    print ("Manifest file, %s, not found. Unable to send swarm request "
           "for %s" % (filepath, test_full_name))
    return 1

  # Parses manifest file
  print "Parsing file %s..." % filepath
  file_sha1 = hashlib.sha1(open(filepath, 'rb').read()).hexdigest()
  manifest = Manifest(file_sha1, test_full_name, shards, options)

  # Zip up relevent files
  print "Zipping up files..."
  if not manifest.zip():
    return 1

  # Send test requests off to swarm.
  print 'Sending test requests to swarm'
  test_url = options.swarm_url + '/test'
  manifest_text = manifest.to_json()
  try:
    result = urllib2.urlopen(test_url, manifest_text).read()

    # Check that we can read the output as a JSON string
    json.loads(result)
  except (ValueError, TypeError, urllib2.URLError) as e:
    print 'Failed to send test for ' + test_name
    print e
    return 1

  return 0

def main():
  """Packages up a Slavelastic test and send it to swarm.  Receive output from
  all shards and print it to stdout.

  Args
    slavelastic manifest file
    number of shards
    ...
  """
  # Parses arguments
  parser = optparse.OptionParser(
      usage='%prog [options]',
      description=DESCRIPTION)
  parser.add_option('-w', '--working_dir', default='swarm_tests',
                    help='Desired working direction on the swarm slave side. '
                    'Defaults to %default.')
  parser.add_option('-o', '--os_image',
                    help='Swarm OS image to request.')
  parser.add_option('-u', '--swarm-url', default='http://localhost:8080',
                    help='Specify the url of the Swarm server. '
                    'Defaults to %default')
  parser.add_option('-d', '--data-dir',
                    help='The directory where all the test data is stored.'
                    'This should path must be valid for all the swarm bots')
  parser.add_option('-t', '--test-name-prefix', default='',
                    help='Specify the prefix to give the swarm test request. '
                    'Defaults to %default')
  parser.add_option('-n', '--manifest_name', action='append',
                    help='The name of a manifest to send to swarm. This may '
                    'be given multiple times to send multiple manifests.')
  parser.add_option('-s', '--shards', type='int', action='append',
                    help='The number of shards to request for a manifest. '
                    'This must be listed once for each -n.')
  parser.add_option('-v', '--verbose', action='store_true',
                    help='Print verbose logging')
  # TODO(csharp): Remove after the next try server restart.
  parser.add_option('--hashtable-dir', help='Unused.')
  parser.add_option('--data-dest-dir', help='Unused.')
  (options, args) = parser.parse_args()

  # TODO(csharp): Remove after next try server restart
  # The data_dir use to be called the data_dest_dir, so remap it if it exists
  if options.data_dest_dir:
    options.data_dir = options.data_dest_dir

  if args:
    parser.error('Unknown args, ' + args)

  if not options.os_image:
    parser.error('Must specify an os image')
  if not options.data_dir:
    parser.error('Must specify the data directory')

  if len(options.manifest_name) != len(options.shards):
    parser.error('Number of min shards given doesn\'t match the number '
                 'of manifests')

  # Remove the old data from this builder if there is any.
  if os.path.isdir(options.data_dest_dir):
    print 'Removing old swarm files...'

    # We want to extract and use the name of the builder from the test name
    # prefix because the test name prefix contains the build number, which is
    # different for older zip files (so they would fail to match the
    # expression).
    builder_name = options.test_name_prefix.split('-')[0]
    for filename in glob.glob(os.path.join(options.data_dest_dir,
                                           builder_name + '*.zip')):
      os.remove(filename)

  # Send off the swarm test requests.
  highest_exit_code = 0
  for i in range(len(options.manifest_name)):
    highest_exit_code = max(highest_exit_code,
                            ProcessManifest(options.manifest_name[i],
                                            options.shards[i],
                                            options))

  return highest_exit_code

if __name__ == '__main__':
  sys.exit(main())
<commit_msg>Remove old run_slavelastic parameters<commit_after>#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# run_slavelastic.py: Runs a test based off of a slavelastic manifest file.

import glob
import hashlib
import json
import optparse
import os
import socket
import sys
import time
import urllib
import urllib2
import zipfile

DESCRIPTION = """This script takes a slavelastic manifest file, packages it,
and sends a swarm manifest file to the swarm server.  This is expected to be
called as a build step with the cwd as the parent of the src/ directory.
"""

CLEANUP_SCRIPT_NAME = 'swarm_cleanup.py'
CLEANUP_SCRIPT_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                   CLEANUP_SCRIPT_NAME)

WINDOWS_SCRIPT_NAME = 'kill_processes.py'
WINDOWS_SCRIPT_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                   WINDOWS_SCRIPT_NAME)

HANDLE_EXE = 'handle.exe'
HANDLE_EXE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                               '..', '..', 'third_party', 'psutils',
                               HANDLE_EXE)

class Manifest(object):
  RUN_TEST_PATH = os.path.join(
      'src', 'tools', 'isolate', 'run_test_from_archive.py')

  def __init__(self, manifest_hash, test_name, shards, switches):
    """Populates a manifest object.
      Args:
        manifest_hash - The manifest's sha-1 that the slave is going to fetch.
        shards - The number of swarm shards to request.
        test_name - The name to give the test request.
        switches - An object with properties to apply to the test request.
    """
    platform_mapping =  {
      'darwin': 'Mac',
      'cygwin': 'Windows',
      'linux2': 'Linux',
      'win32': 'Windows'
      }

    self.manifest_hash = manifest_hash
    self.shards = shards

    self.zipfile_fullpath = os.path.join(switches.data_dir, test_name + '.zip')
    self.tasks = []
    self.target_platform = platform_mapping[switches.os_image]
    self.working_dir = switches.working_dir
    self.test_name = test_name
    self.data_dir = switches.data_dir

  def add_task(self, task_name, actions, time_out=600):
    """Appends a new task to the swarm manifest file."""
    self.tasks.append({
          'test_name': task_name,
          'action': actions,
          'time_out': time_out
    })

  def zip(self):
    """Zip up all the files necessary to run a shard."""
    start_time = time.time()

    try:
      zip_file = zipfile.ZipFile(self.zipfile_fullpath, 'w')
    except IOError as e:
      print 'Error creating zip files %s' % str(e)
      return False

    zip_file.write(self.RUN_TEST_PATH)
    zip_file.write(CLEANUP_SCRIPT_PATH, CLEANUP_SCRIPT_NAME)

    if self.target_platform == 'Windows':
      zip_file.write(WINDOWS_SCRIPT_PATH, WINDOWS_SCRIPT_NAME)
      zip_file.write(HANDLE_EXE_PATH, HANDLE_EXE)

    zip_file.close()

    print 'Zipping completed, time elapsed: %f' % (time.time() - start_time)
    return True

  def to_json(self):
    """Export the current configuration into a swarm-readable manifest file"""
    self.add_task(
        'Run Test',
        ['python', self.RUN_TEST_PATH, '--hash', self.manifest_hash,
         '--remote', self.data_dir, '-v'])

    # Clean up
    self.add_task('Clean Up', ['python', CLEANUP_SCRIPT_NAME])

    # Call kill_processes.py if on windows
    if self.target_platform == 'Windows':
      self.add_task('Kill Processes',
          ['python', WINDOWS_SCRIPT_NAME,
           '--handle_exe', HANDLE_EXE])

    # This separation of vlans isn't required anymore, but it is
    # still a useful separation to keep.
    hostname = socket.gethostname().lower().split('.', 1)[0]
    vlan = ''
    if hostname.endswith('-m1'):
      vlan = 'm1'
    elif hostname.endswith('m4'):
      vlan = 'm4'

    data_scheme = 'file://'
    if self.target_platform == 'Windows':
      data_scheme += '/'

    # Construct test case
    test_case = {
      'test_case_name': self.test_name,
      'data': [
        urllib.quote(data_scheme + self.zipfile_fullpath.replace(os.sep, '/'),
                     ':/'),
      ],
      'tests': self.tasks,
      'env_vars': {
        'GTEST_TOTAL_SHARDS': '%(num_instances)s',
        'GTEST_SHARD_INDEX': '%(instance_index)s',
      },
      'configurations': [
        {
          'min_instances': self.shards,
          'config_name': self.target_platform,
          'dimensions': {
            'os': self.target_platform,
            'vlan': vlan
          },
        },
      ],
      'working_dir': self.working_dir,
      'cleanup': 'data',
    }

    return json.dumps(test_case)

def ProcessManifest(filepath, shards, options):
  """Process the manifest file and send off the swarm test request."""
  test_name = os.path.splitext(os.path.basename(filepath))[0]
  test_full_name = options.test_name_prefix + test_name

  if not os.path.exists(filepath):
    print ("Manifest file, %s, not found. Unable to send swarm request "
           "for %s" % (filepath, test_full_name))
    return 1

  # Parses manifest file
  print "Parsing file %s..." % filepath
  file_sha1 = hashlib.sha1(open(filepath, 'rb').read()).hexdigest()
  manifest = Manifest(file_sha1, test_full_name, shards, options)

  # Zip up relevent files
  print "Zipping up files..."
  if not manifest.zip():
    return 1

  # Send test requests off to swarm.
  print 'Sending test requests to swarm'
  test_url = options.swarm_url + '/test'
  manifest_text = manifest.to_json()
  try:
    result = urllib2.urlopen(test_url, manifest_text).read()

    # Check that we can read the output as a JSON string
    json.loads(result)
  except (ValueError, TypeError, urllib2.URLError) as e:
    print 'Failed to send test for ' + test_name
    print e
    return 1

  return 0

def main():
  """Packages up a Slavelastic test and send it to swarm.  Receive output from
  all shards and print it to stdout.

  Args
    slavelastic manifest file
    number of shards
    ...
  """
  # Parses arguments
  parser = optparse.OptionParser(
      usage='%prog [options]',
      description=DESCRIPTION)
  parser.add_option('-w', '--working_dir', default='swarm_tests',
                    help='Desired working direction on the swarm slave side. '
                    'Defaults to %default.')
  parser.add_option('-o', '--os_image',
                    help='Swarm OS image to request.')
  parser.add_option('-u', '--swarm-url', default='http://localhost:8080',
                    help='Specify the url of the Swarm server. '
                    'Defaults to %default')
  parser.add_option('-d', '--data-dir',
                    help='The directory where all the test data is stored.'
                    'This should path must be valid for all the swarm bots')
  parser.add_option('-t', '--test-name-prefix', default='',
                    help='Specify the prefix to give the swarm test request. '
                    'Defaults to %default')
  parser.add_option('-n', '--manifest_name', action='append',
                    help='The name of a manifest to send to swarm. This may '
                    'be given multiple times to send multiple manifests.')
  parser.add_option('-s', '--shards', type='int', action='append',
                    help='The number of shards to request for a manifest. '
                    'This must be listed once for each -n.')
  parser.add_option('-v', '--verbose', action='store_true',
                    help='Print verbose logging')
  (options, args) = parser.parse_args()

  if args:
    parser.error('Unknown args, ' + args)

  if not options.os_image:
    parser.error('Must specify an os image')
  if not options.data_dir:
    parser.error('Must specify the data directory')

  if len(options.manifest_name) != len(options.shards):
    parser.error('Number of min shards given doesn\'t match the number '
                 'of manifests')

  # Remove the old data from this builder if there is any.
  if os.path.isdir(options.data_dir):
    print 'Removing old swarm files...'

    # We want to extract and use the name of the builder from the test name
    # prefix because the test name prefix contains the build number, which is
    # different for older zip files (so they would fail to match the
    # expression).
    builder_name = options.test_name_prefix.split('-')[0]
    for filename in glob.glob(os.path.join(options.data_dir,
                                           builder_name + '*.zip')):
      os.remove(filename)

  # Send off the swarm test requests.
  highest_exit_code = 0
  for i in range(len(options.manifest_name)):
    highest_exit_code = max(highest_exit_code,
                            ProcessManifest(options.manifest_name[i],
                                            options.shards[i],
                                            options))

  return highest_exit_code

if __name__ == '__main__':
  sys.exit(main())
<|endoftext|>


<commit_before>__all__ = ['LoggerMiddleware']

import logging
from fastapi import __version__

from rollbar.contrib.asgi.integration import integrate
from rollbar.contrib.starlette import LoggerMiddleware as StarletteLoggerMiddleware

log = logging.getLogger(__name__)

@integrate(framework_name=f'fastapi {__version__}')
class LoggerMiddleware(StarletteLoggerMiddleware):
    ...
<commit_msg>Remove unused logger<commit_after>__all__ = ['LoggerMiddleware']

from fastapi import __version__

from rollbar.contrib.asgi.integration import integrate
from rollbar.contrib.starlette import LoggerMiddleware as StarletteLoggerMiddleware

@integrate(framework_name=f'fastapi {__version__}')
class LoggerMiddleware(StarletteLoggerMiddleware):
    ...
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import neovim
from .api import TogglAPI

@neovim.plugin
class Toggl(object):

    def __init__(self, nvim):
        self.nvim = nvim
        self.api_token = nvim.eval("g:toggl_api_token")
        self.api = TogglAPI(self.api_token)
        self.wid = self.api.workspaces()[0]["id"]

    def update(self):
        self.projects = self.get_projects([])

    def echo(self, msg):
        self.nvim.command("echo '{}'".format(msg))

    @neovim.function("TogglAPIToken", sync=True)
    def api_token(self, args):
        return self.api_token

    @neovim.function("TogglGetCurrent", sync=True)
    def get_current(self, args):
        return self.api.time_entries.current()

    @neovim.function("TogglGetProjects", sync=True)
    def get_projects(self, args):
        return self.api.workspaces.projects(self.wid)

    @neovim.function("TogglGetTags", sync=True)
    def get_tags(self, args):
        return self.api.workspaces.tags(self.wid)

    @neovim.command("TogglStart", range='', nargs="*")
    def start(self, args, range):
        projects = [arg[1:] for arg in args if arg[0] == "+"]
        if len(projects) > 1:
            raise RuntimeError("Multiple projects are specified.")
        tags = [arg[1:] for arg in args if arg[0] == "@"]
        desc = " ".join([arg for arg in args
                         if not arg.startswith(("+", "@"))])
        self.api.time_entries.start({
            "time_entry": {
                "description": desc,
                "pid": 0,
                "tags": tags,
                "created_with": "toggl.nvim",
            }
        })
        self.echo("Task Start: {}".format(desc))

    @neovim.command("TogglStop")
    def stop(self):
        current = self.api.time_entries.current()
        if current is None:
            self.echo("No task is running.")
            return
        self.api.time_entries.stop(current["id"])
        self.echo("Task Stop: {}".format(current["description"]))
<commit_msg>Get project id from cache<commit_after># -*- coding: utf-8 -*-

import neovim
from .api import TogglAPI

@neovim.plugin
class Toggl(object):

    def __init__(self, nvim):
        self.nvim = nvim
        self.api_token = nvim.eval("g:toggl_api_token")
        self.api = TogglAPI(self.api_token)
        self.update()

    @neovim.function("TogglUpdate", sync=True)
    def update(self):
        self.wid = self.api.workspaces()[0]["id"]
        self.projects = self.get_projects([])

    def echo(self, msg):
        self.nvim.command("echo '{}'".format(msg))

    @neovim.function("TogglAPIToken", sync=True)
    def api_token(self, args):
        return self.api_token

    @neovim.function("TogglGetCurrent", sync=True)
    def get_current(self, args):
        return self.api.time_entries.current()

    @neovim.function("TogglGetProjects", sync=True)
    def get_projects(self, args):
        return self.api.workspaces.projects(self.wid)

    @neovim.function("TogglGetTags", sync=True)
    def get_tags(self, args):
        return self.api.workspaces.tags(self.wid)

    @neovim.command("TogglStart", range='', nargs="*")
    def start(self, args, range):
        projects = [arg[1:] for arg in args if arg[0] == "+"]
        if len(projects) > 1:
            raise RuntimeError("Multiple projects are specified.")
        if len(projects) == 1:
            name = projects[0]
        else:
            name = ""
        for p in self.projects:
            if p["name"] == name:
                pid = p["id"]
                break
        else:
            pid = 0

        tags = [arg[1:] for arg in args if arg[0] == "@"]
        desc = " ".join([arg for arg in args
                         if not arg.startswith(("+", "@"))])

        self.api.time_entries.start({
            "time_entry": {
                "description": desc,
                "pid": pid,
                "tags": tags,
                "created_with": "toggl.nvim",
            }
        })
        self.echo("Task Start: {}".format(desc))

    @neovim.command("TogglStop")
    def stop(self):
        current = self.api.time_entries.current()
        if current is None:
            self.echo("No task is running.")
            return
        self.api.time_entries.stop(current["id"])
        self.echo("Task Stop: {}".format(current["description"]))
<|endoftext|>


<commit_before># utile.py - Collection of useful functions
# Copyright (C) 2013 Marwan Alsabbagh
# license: BSD, see LICENSE for more details.

__version__ = '0.3.dev'

import time
import re
import os
import os.path
import sys
import itertools
from functools import wraps
from shutil import rmtree
from hashlib import sha256
from inspect import getargspec
from subprocess import check_output, check_call, Popen, PIPE
from tempfile import mkdtemp
from contextlib import contextmanager
from fcntl import flock, LOCK_EX, LOCK_NB
from datetime import timedelta, datetime
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, RawDescriptionHelpFormatter

def safe_import(name, default=None):
    try:
        package, name = name.rsplit('.', 1)
        return getattr(__import__(package, fromlist=[name]), name)
    except ImportError:
        return default

etree = safe_import('lxml.etree')
AES = safe_import('Crypto.Cipher.AES')
bunch_or_dict = safe_import('bunch.Bunch', dict)

# an alias for easier importing
now = datetime.now

def _root_to_dict(item, return_tuple=True):
    if return_tuple:
        return item.tag, dict(map(_root_to_dict, item)) or item.text
    else:
        return dict(map(_root_to_dict, item))

def xml_to_dict(xml, *args, **kwargs):
    enforce(etree, 'lxml is not installed.')
    root = etree.fromstring(xml, etree.XMLParser(*args, **kwargs))
    return _root_to_dict(root, False)

def git_version(version):
    git_details = ''
    if which('git'):
        git_details = Popen(['git', 'describe'], stdout=PIPE, stderr=PIPE).communicate()[0]
    if 'dev' not in version:
        return version
    elif git_details:
        return version + git_details.split('-')[1]
    elif os.path.exists('PKG-INFO'):
        info = open('PKG-INFO').read()
        return re.findall(r'^Version: (.*)$', info, re.MULTILINE)[0]
    else:
        return version

def save_args(f):
    @wraps(f)
    def wrapper(self, *args, **kwargs):
        names = getargspec(f).args[1:]
        defaults = zip(reversed(names), reversed(getargspec(f).defaults))
        items = zip(names, args) + kwargs.items() + defaults
        for k, v in items:
            if not hasattr(self, k):
                setattr(self, k, v)
        return f(self, *args, **kwargs)
    return wrapper

def flatten(data):
    return list(itertools.chain.from_iterable(data))

def mac_address(interface='eth0'):
    output = check_output(['ifconfig', interface])
    return re.search(r'HWaddr ([\w:]+)', output).group(1)

@contextmanager
def TemporaryDirectory(suffix='', prefix='tmp', dir=None):
    path = mkdtemp(suffix, prefix, dir)
    try:
        yield path
    finally:
        rmtree(path)

@contextmanager
def file_lock(path):
    try:
        f = open(path, 'w')
        flock(f, LOCK_EX | LOCK_NB)
    except IOError:
        raise IOError("Could not lock '%s'" % path)
    try:
        yield
    finally:
        f.close()
        os.remove(path)

def shell_quote(s):
    return "'" + s.replace("'", "'\"'\"'") + "'"

def _cipher(key):
    enforce(AES, 'pycrypto is not installed.')
    return AES.new(sha256(key).digest(), AES.MODE_CFB, '\x00' * AES.block_size)

def encrypt(key, data):
    return _cipher(key).encrypt(data)

def decrypt(key, data):
    return _cipher(key).decrypt(data)

class EnforcementError(Exception):
    pass

def enforce(rule, msg):
    if not rule:
        raise EnforcementError(msg)

def enforce_clean_exit(func):
    @wraps(func)
    def wrapper(*args, **kwds):
        try:
            return func(*args, **kwds)
        except EnforcementError as err:
            sys.stderr.write("ERROR: {0}\n".format(err))
            sys.exit(1)
    return wrapper

def which(cmd):
    paths = [os.path.join(i, cmd) for i in os.environ['PATH'].split(os.pathsep)]
    return [i for i in paths if os.path.exists(i)]

def commands_required(commands):
    missing = [cmd for cmd in commands.split() if not which(cmd)]
    enforce(not missing, "'%s' command(s) not found" % ' '.join(missing))

def shell(cmd=None, msg=None, caller=check_call, strict=False, **kwargs):
    msg = msg if msg else cmd
    if strict:
        if not shell or not isinstance(cmd, basestring):
            raise ValueError('strict can only be used when shell=True and cmd is a string')
        cmd = 'set -e;' + cmd
    print ' {} '.format(msg).center(60, '-')
    start = time.time()
    kwargs.setdefault('shell', True)
    if kwargs.get('shell'):
        kwargs.setdefault('executable', '/bin/bash')
    returncode = caller(cmd, **kwargs)
    print 'duration: %s' % timedelta(seconds=time.time() - start)
    return returncode

class ArgDefaultRawDescrHelpFormatter(ArgumentDefaultsHelpFormatter, RawDescriptionHelpFormatter):
    """Help message formatter which adds default values to argument help and
    which retains any formatting in descriptions.
    """

class Arg(object):
    def __init__(self, *args, **kwargs):
        self.args = args
        self.kwargs = kwargs

def parse_args(description, *args, **kwargs):
    kwargs['description'] = description
    kwargs.setdefault('formatter_class', ArgDefaultRawDescrHelpFormatter)
    parser = ArgumentParser(**kwargs)
    for i in args:
        parser.add_argument(*i.args, **i.kwargs)
    return bunch_or_dict(parser.parse_args().__dict__)
<commit_msg>changed xml_to_dict to return a bunch_or_dict, renamed _root_to_dict to element_to_dict<commit_after># utile.py - Collection of useful functions
# Copyright (C) 2013 Marwan Alsabbagh
# license: BSD, see LICENSE for more details.

__version__ = '0.3.dev'

import time
import re
import os
import os.path
import sys
import itertools
from functools import wraps
from shutil import rmtree
from hashlib import sha256
from inspect import getargspec
from subprocess import check_output, check_call, Popen, PIPE
from tempfile import mkdtemp
from contextlib import contextmanager
from fcntl import flock, LOCK_EX, LOCK_NB
from datetime import timedelta, datetime
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, RawDescriptionHelpFormatter

def safe_import(name, default=None):
    try:
        package, name = name.rsplit('.', 1)
        return getattr(__import__(package, fromlist=[name]), name)
    except ImportError:
        return default

etree = safe_import('lxml.etree')
AES = safe_import('Crypto.Cipher.AES')
bunch_or_dict = safe_import('bunch.Bunch', dict)

# an alias for easier importing
now = datetime.now

def element_to_dict(elem, return_tuple=False):
    children = bunch_or_dict(element_to_dict(i, True) for i in elem)
    if return_tuple:
        return elem.tag, children or elem.text
    else:
        return children

def xml_to_dict(xml, *args, **kwargs):
    enforce(etree, 'lxml is not installed.')
    root = etree.fromstring(xml, etree.XMLParser(*args, **kwargs))
    return element_to_dict(root)

def git_version(version):
    git_details = ''
    if which('git'):
        git_details = Popen(['git', 'describe'], stdout=PIPE, stderr=PIPE).communicate()[0]
    if 'dev' not in version:
        return version
    elif git_details:
        return version + git_details.split('-')[1]
    elif os.path.exists('PKG-INFO'):
        info = open('PKG-INFO').read()
        return re.findall(r'^Version: (.*)$', info, re.MULTILINE)[0]
    else:
        return version

def save_args(f):
    @wraps(f)
    def wrapper(self, *args, **kwargs):
        names = getargspec(f).args[1:]
        defaults = zip(reversed(names), reversed(getargspec(f).defaults))
        items = zip(names, args) + kwargs.items() + defaults
        for k, v in items:
            if not hasattr(self, k):
                setattr(self, k, v)
        return f(self, *args, **kwargs)
    return wrapper

def flatten(data):
    return list(itertools.chain.from_iterable(data))

def mac_address(interface='eth0'):
    output = check_output(['ifconfig', interface])
    return re.search(r'HWaddr ([\w:]+)', output).group(1)

@contextmanager
def TemporaryDirectory(suffix='', prefix='tmp', dir=None):
    path = mkdtemp(suffix, prefix, dir)
    try:
        yield path
    finally:
        rmtree(path)

@contextmanager
def file_lock(path):
    try:
        f = open(path, 'w')
        flock(f, LOCK_EX | LOCK_NB)
    except IOError:
        raise IOError("Could not lock '%s'" % path)
    try:
        yield
    finally:
        f.close()
        os.remove(path)

def shell_quote(s):
    return "'" + s.replace("'", "'\"'\"'") + "'"

def _cipher(key):
    enforce(AES, 'pycrypto is not installed.')
    return AES.new(sha256(key).digest(), AES.MODE_CFB, '\x00' * AES.block_size)

def encrypt(key, data):
    return _cipher(key).encrypt(data)

def decrypt(key, data):
    return _cipher(key).decrypt(data)

class EnforcementError(Exception):
    pass

def enforce(rule, msg):
    if not rule:
        raise EnforcementError(msg)

def enforce_clean_exit(func):
    @wraps(func)
    def wrapper(*args, **kwds):
        try:
            return func(*args, **kwds)
        except EnforcementError as err:
            sys.stderr.write("ERROR: {0}\n".format(err))
            sys.exit(1)
    return wrapper

def which(cmd):
    paths = [os.path.join(i, cmd) for i in os.environ['PATH'].split(os.pathsep)]
    return [i for i in paths if os.path.exists(i)]

def commands_required(commands):
    missing = [cmd for cmd in commands.split() if not which(cmd)]
    enforce(not missing, "'%s' command(s) not found" % ' '.join(missing))

def shell(cmd=None, msg=None, caller=check_call, strict=False, **kwargs):
    msg = msg if msg else cmd
    if strict:
        if not shell or not isinstance(cmd, basestring):
            raise ValueError('strict can only be used when shell=True and cmd is a string')
        cmd = 'set -e;' + cmd
    print ' {} '.format(msg).center(60, '-')
    start = time.time()
    kwargs.setdefault('shell', True)
    if kwargs.get('shell'):
        kwargs.setdefault('executable', '/bin/bash')
    returncode = caller(cmd, **kwargs)
    print 'duration: %s' % timedelta(seconds=time.time() - start)
    return returncode

class ArgDefaultRawDescrHelpFormatter(ArgumentDefaultsHelpFormatter, RawDescriptionHelpFormatter):
    """Help message formatter which adds default values to argument help and
    which retains any formatting in descriptions.
    """

class Arg(object):
    def __init__(self, *args, **kwargs):
        self.args = args
        self.kwargs = kwargs

def parse_args(description, *args, **kwargs):
    kwargs['description'] = description
    kwargs.setdefault('formatter_class', ArgDefaultRawDescrHelpFormatter)
    parser = ArgumentParser(**kwargs)
    for i in args:
        parser.add_argument(*i.args, **i.kwargs)
    return bunch_or_dict(parser.parse_args().__dict__)
<|endoftext|>


<commit_before>from ..core import db
from sqlalchemy.sql.functions import GenericFunction
from sqlalchemy.types import Float
from sqlalchemy.ext.compiler import compiles

class hit_decay(GenericFunction):
    """
    An exponential decay function

    It is used for assigning older hits a lower weight
    :param hit_time: The time the hit occurred
    :param half_life: The number of days, the weight reaches the half of its
    initial weight
    :param life_span: The number of days, a hit is valid. Hits older than the
    life_span have a weight of 0
    """
    type = Float

@compiles(hit_decay, 'postgresql')
def pg_compile_hit_decay(element, compiler, **kw):
    if len(element.clauses) != 3:
        raise TypeError("hit_decay requires 3 parameters")
    hit_time, half_life, life_span = list(element.clauses)
    return (
        "CASE WHEN age(TIMEZONE('utc', CURRENT_TIMESTAMP), %s) "
        " > %s * interval '1 day' THEN 0.0 ELSE "
        " exp(-ln(2)/%s * floor("
        "  extract(epoch from age(TIMEZONE('utc', CURRENT_TIMESTAMP), %s))"
        "  / (3600 * 24)))"
        "END"
        )\
        % (
            compiler.process(hit_time),
            compiler.process(life_span),
            compiler.process(half_life),
            compiler.process(hit_time)
        )

@compiles(hit_decay, 'sqlite')
def sq_compile_hit_decay(element, compiler, **kw):
    if len(element.clauses) != 3:
        raise TypeError("hit_decay requires 3 parameters")
    # As sqlite is used for testing, definition of the decay is omitted
    return "1.0"

class SearchQuery(db.Model):
    __tablename__ = 'search_query'

    query_string = db.Column(
        db.Text(), index=True, primary_key=True, nullable=False)

    def __repr__(self):
        return '{0}: {1}'.format(self.__class__.__name__, self.query_string)

class SearchSession(db.Model):
    __tablename__ = 'search_session'

    session_id = db.Column(db.String(256), primary_key=True)
    time_created = db.Column(
        db.DateTime(), index=True, nullable=False)

class Record(db.Model):
    __tablename__ = 'record'

    record_id = db.Column(
        db.String(512), primary_key=True, index=True, nullable=False)
    active = db.Column(
        db.Boolean(), default=True, nullable=False)
    is_internal = db.Column(
        db.Boolean(), default=True, nullable=False)

class Action(db.Model):
    __tablename__ = 'action'

    record_id = db.Column(
        db.String(512), db.ForeignKey('record.record_id'),
        index=True, nullable=False, primary_key=True)

    session_id = db.Column(
        db.String(256), db.ForeignKey('search_session.session_id'),
        index=True, nullable=False, primary_key=True)

    action_type = db.Column(
        db.Enum("view", "copy", name="action_types"),
        index=True, nullable=False, primary_key=True)

    query_string = db.Column(
        db.Text(), db.ForeignKey('search_query.query_string'),
        index=True, nullable=True)

    time_created = db.Column(
        db.DateTime(), index=True, nullable=False)

class ImportedRecordSimilarity(db.Model):
    __tablename__ = 'imported_record_similarity'

    from_record_id = db.Column(
        db.String(512), db.ForeignKey('record.record_id'),
        index=True, nullable=False, primary_key=True)

    to_record_id = db.Column(
        db.String(512), db.ForeignKey('record.record_id'),
        index=True, nullable=False, primary_key=True)

    similarity_value = db.Column(
        db.Float, nullable=False)
<commit_msg>No use of enum type<commit_after>from ..core import db
from sqlalchemy.sql.functions import GenericFunction
from sqlalchemy.types import Float
from sqlalchemy.ext.compiler import compiles

class hit_decay(GenericFunction):
    """
    An exponential decay function

    It is used for assigning older hits a lower weight
    :param hit_time: The time the hit occurred
    :param half_life: The number of days, the weight reaches the half of its
    initial weight
    :param life_span: The number of days, a hit is valid. Hits older than the
    life_span have a weight of 0
    """
    type = Float

@compiles(hit_decay, 'postgresql')
def pg_compile_hit_decay(element, compiler, **kw):
    if len(element.clauses) != 3:
        raise TypeError("hit_decay requires 3 parameters")
    hit_time, half_life, life_span = list(element.clauses)
    return (
        "CASE WHEN age(TIMEZONE('utc', CURRENT_TIMESTAMP), %s) "
        " > %s * interval '1 day' THEN 0.0 ELSE "
        " exp(-ln(2)/%s * floor("
        "  extract(epoch from age(TIMEZONE('utc', CURRENT_TIMESTAMP), %s))"
        "  / (3600 * 24)))"
        "END"
        )\
        % (
            compiler.process(hit_time),
            compiler.process(life_span),
            compiler.process(half_life),
            compiler.process(hit_time)
        )

@compiles(hit_decay, 'sqlite')
def sq_compile_hit_decay(element, compiler, **kw):
    if len(element.clauses) != 3:
        raise TypeError("hit_decay requires 3 parameters")
    # As sqlite is used for testing, definition of the decay is omitted
    return "1.0"

class SearchQuery(db.Model):
    __tablename__ = 'search_query'

    query_string = db.Column(
        db.Text(), index=True, primary_key=True, nullable=False)

    def __repr__(self):
        return '{0}: {1}'.format(self.__class__.__name__, self.query_string)

class SearchSession(db.Model):
    __tablename__ = 'search_session'

    session_id = db.Column(db.String(256), primary_key=True)
    time_created = db.Column(
        db.DateTime(), index=True, nullable=False)

class Record(db.Model):
    __tablename__ = 'record'

    record_id = db.Column(
        db.String(512), primary_key=True, index=True, nullable=False)
    active = db.Column(
        db.Boolean(), default=True, nullable=False)
    is_internal = db.Column(
        db.Boolean(), default=True, nullable=False)

class ActionType(object):
    view = 'view'
    copy = 'copy'

class Action(db.Model):
    __tablename__ = 'action'

    record_id = db.Column(
        db.String(512), db.ForeignKey('record.record_id'),
        index=True, nullable=False, primary_key=True)

    session_id = db.Column(
        db.String(256), db.ForeignKey('search_session.session_id'),
        index=True, nullable=False, primary_key=True)

    action_type = db.Column(
        db.Enum(ActionType.view, ActionType.copy, name="action_types"),
        index=True, nullable=False, primary_key=True)

    query_string = db.Column(
        db.Text(), db.ForeignKey('search_query.query_string'),
        index=True, nullable=True)

    time_created = db.Column(
        db.DateTime(), index=True, nullable=False)

class ImportedRecordSimilarity(db.Model):
    __tablename__ = 'imported_record_similarity'

    from_record_id = db.Column(
        db.String(512), db.ForeignKey('record.record_id'),
        index=True, nullable=False, primary_key=True)

    to_record_id = db.Column(
        db.String(512), db.ForeignKey('record.record_id'),
        index=True, nullable=False, primary_key=True)

    similarity_value = db.Column(
        db.Float, nullable=False)
<|endoftext|>


<commit_before>## Lexer Implementation

import re
import sre_parse

from .utils import Str, classify
from .common import is_terminal, PatternStr, PatternRE, TokenDef

class LexError(Exception):
    pass

class UnexpectedInput(LexError):
    def __init__(self, seq, lex_pos, line, column):
        context = seq[lex_pos:lex_pos+5]
        message = "No token defined for: '%s' in %r at line %d" % (seq[lex_pos], context, line)

        super(UnexpectedInput, self).__init__(message)

        self.line = line
        self.column = column
        self.context = context

class Token(Str):
    def __new__(cls, type_, value, pos_in_stream=None, line=None, column=None):
        inst = Str.__new__(cls, value)
        inst.type = type_
        inst.pos_in_stream = pos_in_stream
        inst.value = value
        inst.line = line
        inst.column = column
        return inst

    @classmethod
    def new_borrow_pos(cls, type_, value, borrow_t):
        return cls(type_, value, borrow_t.pos_in_stream, line=borrow_t.line, column=borrow_t.column)

    def __repr__(self):
        return 'Token(%s, %r)' % (self.type, self.value)

    def __deepcopy__(self, memo):
        return Token(self.type, self.value, self.pos_in_stream, self.line, self.column)

class Regex:
    def __init__(self, pattern, flags=()):
        self.pattern = pattern
        self.flags = flags

def _regexp_has_newline(r):
    return '\n' in r or '\\n' in r or ('(?s)' in r and '.' in r)

def _create_unless_callback(strs):
    mres = build_mres(strs, match_whole=True)
    def unless_callback(t):
        # if t in strs:
        #     t.type = strs[t]
        for mre, type_from_index in mres:
            m = mre.match(t.value)
            if m:
                value = m.group(0)
                t.type = type_from_index[m.lastindex]
                break
        return t
    return unless_callback

def _create_unless(tokens):
    tokens_by_type = classify(tokens, lambda t: type(t.pattern))
    assert len(tokens_by_type) <= 2, tokens_by_type.keys()
    embedded_strs = set()
    delayed_strs = []
    callback = {}
    for retok in tokens_by_type.get(PatternRE, []):
        unless = [] # {}
        for strtok in tokens_by_type.get(PatternStr, []):
            s = strtok.pattern.value
            m = re.match(retok.pattern.to_regexp(), s)
            if m and m.group(0) == s:
                if strtok.pattern.flags:
                    delayed_strs.append(strtok)
                embedded_strs.add(strtok.name)
                unless.append(strtok)
        if unless:
            callback[retok.name] = _create_unless_callback(unless)

    tokens = [t for t in tokens if t.name not in embedded_strs] + delayed_strs
    return tokens, callback

def _build_mres(tokens, max_size, match_whole):
    # Python sets an unreasonable group limit (currently 100) in its re module
    # Worse, the only way to know we reached it is by catching an AssertionError!
    # This function recursively tries less and less groups until it's successful.
    postfix = '$' if match_whole else ''
    mres = []
    while tokens:
        try:
            mre = re.compile(u'|'.join(u'(?P<%s>%s)'%(t.name, t.pattern.to_regexp()+postfix) for t in tokens[:max_size]))
        except AssertionError:  # Yes, this is what Python provides us.. :/
            return _build_mres(tokens, max_size//2, match_whole)

        mres.append((mre, {i:n for n,i in mre.groupindex.items()} ))
        tokens = tokens[max_size:]
    return mres

def build_mres(tokens, match_whole=False):
    return _build_mres(tokens, len(tokens), match_whole)

class Lexer(object):
    def __init__(self, tokens, ignore=()):
        assert all(isinstance(t, TokenDef) for t in tokens), tokens

        self.ignore = ignore
        self.newline_char = '\n'
        tokens = list(tokens)

        # Sanitization
        for t in tokens:
            try:
                re.compile(t.pattern.to_regexp())
            except:
                raise LexError("Cannot compile token %s: %s" % (t.name, t.pattern))

            width = sre_parse.parse(t.pattern.to_regexp()).getwidth()
            if width[0] == 0:
                raise LexError("Lexer does not allow zero-width tokens. (%s: %s)" % (t.name, t.pattern))

        token_names = {t.name for t in tokens}
        for t in ignore:
            if t not in token_names:
                raise LexError("Token '%s' was marked to ignore but it is not defined!" % t)

        # Init
        self.newline_types = [t.name for t in tokens if _regexp_has_newline(t.pattern.to_regexp())]
        self.ignore_types = [t for t in ignore]

        tokens.sort(key=lambda x:(x.pattern.priority, len(x.pattern.value)), reverse=True)

        tokens, self.callback = _create_unless(tokens)
        assert all(self.callback.values())

        self.tokens = tokens

        self.mres = build_mres(tokens)

    def lex(self, stream):
        lex_pos = 0
        line = 1
        col_start_pos = 0
        newline_types = list(self.newline_types)
        ignore_types = list(self.ignore_types)
        while True:
            for mre, type_from_index in self.mres:
                m = mre.match(stream, lex_pos)
                if m:
                    value = m.group(0)
                    type_ = type_from_index[m.lastindex]
                    to_yield = type_ not in ignore_types

                    if to_yield:
                        t = Token(type_, value, lex_pos, line, lex_pos - col_start_pos)
                        if t.type in self.callback:
                            t = self.callback[t.type](t)

                    end_col = t.column + len(value)
                    if type_ in newline_types:
                        newlines = value.count(self.newline_char)
                        if newlines:
                            line += newlines
                            last_newline_index = value.rindex(self.newline_char) + 1
                            col_start_pos = lex_pos + last_newline_index
                            end_col = len(value) - last_newline_index

                    if to_yield:
                        t.end_line = line
                        t.end_col = end_col
                        yield t

                    lex_pos += len(value)
                    break
            else:
                if lex_pos < len(stream):
                    raise UnexpectedInput(stream, lex_pos, line, lex_pos - col_start_pos)
                break

class ContextualLexer:
    def __init__(self, tokens, states, ignore=(), always_accept=()):
        tokens_by_name = {}
        for t in tokens:
            assert t.name not in tokens_by_name, t
            tokens_by_name[t.name] = t

        lexer_by_tokens = {}
        self.lexers = {}
        for state, accepts in states.items():
            key = frozenset(accepts)
            try:
                lexer = lexer_by_tokens[key]
            except KeyError:
                accepts = set(accepts) | set(ignore) | set(always_accept)
                state_tokens = [tokens_by_name[n] for n in accepts if is_terminal(n) and n!='$end']
                lexer = Lexer(state_tokens, ignore=ignore)
                lexer_by_tokens[key] = lexer

            self.lexers[state] = lexer

        self.root_lexer = Lexer(tokens, ignore=ignore)

        self.set_parser_state(None) # Needs to be set on the outside

    def set_parser_state(self, state):
        self.parser_state = state

    def lex(self, stream):
        lex_pos = 0
        line = 1
        col_start_pos = 0
        newline_types = list(self.root_lexer.newline_types)
        ignore_types = list(self.root_lexer.ignore_types)
        while True:
            lexer = self.lexers[self.parser_state]
            for mre, type_from_index in lexer.mres:
                m = mre.match(stream, lex_pos)
                if m:
                    value = m.group(0)
                    type_ = type_from_index[m.lastindex]
                    if type_ not in ignore_types:
                        t = Token(type_, value, lex_pos, line, lex_pos - col_start_pos)
                        if t.type in lexer.callback:
                            t = lexer.callback[t.type](t)
                        yield t

                    if type_ in newline_types:
                        newlines = value.count(lexer.newline_char)
                        if newlines:
                            line += newlines
                            col_start_pos = lex_pos + value.rindex(lexer.newline_char)
                    lex_pos += len(value)
                    break
            else:
                if lex_pos < len(stream):
                    print("Allowed tokens:", lexer.tokens)
                    raise UnexpectedInput(stream, lex_pos, line, lex_pos - col_start_pos)
                break

<commit_msg>Tiny fix in lexer<commit_after>## Lexer Implementation

import re
import sre_parse

from .utils import Str, classify
from .common import is_terminal, PatternStr, PatternRE, TokenDef

class LexError(Exception):
    pass

class UnexpectedInput(LexError):
    def __init__(self, seq, lex_pos, line, column):
        context = seq[lex_pos:lex_pos+5]
        message = "No token defined for: '%s' in %r at line %d" % (seq[lex_pos], context, line)

        super(UnexpectedInput, self).__init__(message)

        self.line = line
        self.column = column
        self.context = context

class Token(Str):
    def __new__(cls, type_, value, pos_in_stream=None, line=None, column=None):
        inst = Str.__new__(cls, value)
        inst.type = type_
        inst.pos_in_stream = pos_in_stream
        inst.value = value
        inst.line = line
        inst.column = column
        return inst

    @classmethod
    def new_borrow_pos(cls, type_, value, borrow_t):
        return cls(type_, value, borrow_t.pos_in_stream, line=borrow_t.line, column=borrow_t.column)

    def __repr__(self):
        return 'Token(%s, %r)' % (self.type, self.value)

    def __deepcopy__(self, memo):
        return Token(self.type, self.value, self.pos_in_stream, self.line, self.column)

class Regex:
    def __init__(self, pattern, flags=()):
        self.pattern = pattern
        self.flags = flags

def _regexp_has_newline(r):
    return '\n' in r or '\\n' in r or ('(?s)' in r and '.' in r)

def _create_unless_callback(strs):
    mres = build_mres(strs, match_whole=True)
    def unless_callback(t):
        # if t in strs:
        #     t.type = strs[t]
        for mre, type_from_index in mres:
            m = mre.match(t.value)
            if m:
                value = m.group(0)
                t.type = type_from_index[m.lastindex]
                break
        return t
    return unless_callback

def _create_unless(tokens):
    tokens_by_type = classify(tokens, lambda t: type(t.pattern))
    assert len(tokens_by_type) <= 2, tokens_by_type.keys()
    embedded_strs = set()
    delayed_strs = []
    callback = {}
    for retok in tokens_by_type.get(PatternRE, []):
        unless = [] # {}
        for strtok in tokens_by_type.get(PatternStr, []):
            s = strtok.pattern.value
            m = re.match(retok.pattern.to_regexp(), s)
            if m and m.group(0) == s:
                if strtok.pattern.flags:
                    delayed_strs.append(strtok)
                embedded_strs.add(strtok.name)
                unless.append(strtok)
        if unless:
            callback[retok.name] = _create_unless_callback(unless)

    tokens = [t for t in tokens if t.name not in embedded_strs] + delayed_strs
    return tokens, callback

def _build_mres(tokens, max_size, match_whole):
    # Python sets an unreasonable group limit (currently 100) in its re module
    # Worse, the only way to know we reached it is by catching an AssertionError!
    # This function recursively tries less and less groups until it's successful.
    postfix = '$' if match_whole else ''
    mres = []
    while tokens:
        try:
            mre = re.compile(u'|'.join(u'(?P<%s>%s)'%(t.name, t.pattern.to_regexp()+postfix) for t in tokens[:max_size]))
        except AssertionError:  # Yes, this is what Python provides us.. :/
            return _build_mres(tokens, max_size//2, match_whole)

        mres.append((mre, {i:n for n,i in mre.groupindex.items()} ))
        tokens = tokens[max_size:]
    return mres

def build_mres(tokens, match_whole=False):
    return _build_mres(tokens, len(tokens), match_whole)

class Lexer(object):
    def __init__(self, tokens, ignore=()):
        assert all(isinstance(t, TokenDef) for t in tokens), tokens

        self.ignore = ignore
        self.newline_char = '\n'
        tokens = list(tokens)

        # Sanitization
        for t in tokens:
            try:
                re.compile(t.pattern.to_regexp())
            except:
                raise LexError("Cannot compile token %s: %s" % (t.name, t.pattern))

            width = sre_parse.parse(t.pattern.to_regexp()).getwidth()
            if width[0] == 0:
                raise LexError("Lexer does not allow zero-width tokens. (%s: %s)" % (t.name, t.pattern))

        token_names = {t.name for t in tokens}
        for t in ignore:
            if t not in token_names:
                raise LexError("Token '%s' was marked to ignore but it is not defined!" % t)

        # Init
        self.newline_types = [t.name for t in tokens if _regexp_has_newline(t.pattern.to_regexp())]
        self.ignore_types = [t for t in ignore]

        tokens.sort(key=lambda x:(x.pattern.priority, len(x.pattern.value)), reverse=True)

        tokens, self.callback = _create_unless(tokens)
        assert all(self.callback.values())

        self.tokens = tokens

        self.mres = build_mres(tokens)

    def lex(self, stream):
        lex_pos = 0
        line = 1
        col_start_pos = 0
        newline_types = list(self.newline_types)
        ignore_types = list(self.ignore_types)
        while True:
            for mre, type_from_index in self.mres:
                m = mre.match(stream, lex_pos)
                if m:
                    value = m.group(0)
                    type_ = type_from_index[m.lastindex]
                    to_yield = type_ not in ignore_types

                    if to_yield:
                        t = Token(type_, value, lex_pos, line, lex_pos - col_start_pos)
                        end_col = t.column + len(value)
                        if t.type in self.callback:
                            t = self.callback[t.type](t)

                    if type_ in newline_types:
                        newlines = value.count(self.newline_char)
                        if newlines:
                            line += newlines
                            last_newline_index = value.rindex(self.newline_char) + 1
                            col_start_pos = lex_pos + last_newline_index
                            end_col = len(value) - last_newline_index

                    if to_yield:
                        t.end_line = line
                        t.end_col = end_col
                        yield t

                    lex_pos += len(value)
                    break
            else:
                if lex_pos < len(stream):
                    raise UnexpectedInput(stream, lex_pos, line, lex_pos - col_start_pos)
                break

class ContextualLexer:
    def __init__(self, tokens, states, ignore=(), always_accept=()):
        tokens_by_name = {}
        for t in tokens:
            assert t.name not in tokens_by_name, t
            tokens_by_name[t.name] = t

        lexer_by_tokens = {}
        self.lexers = {}
        for state, accepts in states.items():
            key = frozenset(accepts)
            try:
                lexer = lexer_by_tokens[key]
            except KeyError:
                accepts = set(accepts) | set(ignore) | set(always_accept)
                state_tokens = [tokens_by_name[n] for n in accepts if is_terminal(n) and n!='$end']
                lexer = Lexer(state_tokens, ignore=ignore)
                lexer_by_tokens[key] = lexer

            self.lexers[state] = lexer

        self.root_lexer = Lexer(tokens, ignore=ignore)

        self.set_parser_state(None) # Needs to be set on the outside

    def set_parser_state(self, state):
        self.parser_state = state

    def lex(self, stream):
        lex_pos = 0
        line = 1
        col_start_pos = 0
        newline_types = list(self.root_lexer.newline_types)
        ignore_types = list(self.root_lexer.ignore_types)
        while True:
            lexer = self.lexers[self.parser_state]
            for mre, type_from_index in lexer.mres:
                m = mre.match(stream, lex_pos)
                if m:
                    value = m.group(0)
                    type_ = type_from_index[m.lastindex]
                    if type_ not in ignore_types:
                        t = Token(type_, value, lex_pos, line, lex_pos - col_start_pos)
                        if t.type in lexer.callback:
                            t = lexer.callback[t.type](t)
                        yield t

                    if type_ in newline_types:
                        newlines = value.count(lexer.newline_char)
                        if newlines:
                            line += newlines
                            col_start_pos = lex_pos + value.rindex(lexer.newline_char)
                    lex_pos += len(value)
                    break
            else:
                if lex_pos < len(stream):
                    print("Allowed tokens:", lexer.tokens)
                    raise UnexpectedInput(stream, lex_pos, line, lex_pos - col_start_pos)
                break

<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Run timing test (GPU) scaled over number of LPUs.
"""

import numpy as np

import csv
import re
import subprocess
import sys

script_name = 'timing_demo_gpu.py'

w = csv.writer(sys.stdout)
for lpus in xrange(2, 9):
    average_step_sync_time_list = []
    average_throughput_list = []
    total_throughput_list = []
    runtime_all_list = []
    runtime_main_list = []
    runtime_loop_list = []
    for i in xrange(2):
        out = subprocess.check_output(['srun', '-n', '1', '-c', str(lpus),
                                       '--gres=gpu:%i' % lpus,
                                       'python', script_name,
                                       '-u', str(lpus), '-s', '1000', '-g', '0',
                                       '-m', '50'])
        average_step_sync_time, average_throughput, total_throughput, \
            runtime_all, runtime_main, runtime_loop = out.strip('()\n\"').split(', ')
        average_step_sync_time_list.append(float(average_step_sync_time))
        average_throughput_list.append(float(average_throughput))
        total_throughput_list.append(float(total_throughput))
        runtime_all_list.append(float(runtime_all))
        runtime_main_list.append(float(runtime_main))
        runtime_loop_list.append(float(runtime_loop))
    w.writerow([lpus,
                np.average(average_step_sync_time_list),
                np.average(average_throughput_list),
                np.average(total_throughput_list),
                np.average(runtime_all_list),
                np.average(runtime_main_list),
                np.average(runtime_loop_list)])
<commit_msg>Fix total number of ports per LPU used in scaling over number of LPUs benchmark.<commit_after>#!/usr/bin/env python

"""
Run timing test (GPU) scaled over number of LPUs.
"""

import numpy as np

import csv
import re
import subprocess
import sys

script_name = 'timing_demo_gpu.py'

w = csv.writer(sys.stdout)
total_spike = 1000
for lpus in xrange(2, 9):
    average_step_sync_time_list = []
    average_throughput_list = []
    total_throughput_list = []
    runtime_all_list = []
    runtime_main_list = []
    runtime_loop_list = []
    for i in xrange(3):
        out = subprocess.check_output(['srun', '-n', '1', '-c', str(lpus),
                                       '--gres=gpu:%i' % lpus,
                                       'python', script_name,
                                       '-u', str(lpus),
                                       '-s', str(total_spike/(lpus-1)), '-g', '0',
                                       '-m', '50'])
        average_step_sync_time, average_throughput, total_throughput, \
            runtime_all, runtime_main, runtime_loop = out.strip('()\n\"').split(', ')
        average_step_sync_time_list.append(float(average_step_sync_time))
        average_throughput_list.append(float(average_throughput))
        total_throughput_list.append(float(total_throughput))
        runtime_all_list.append(float(runtime_all))
        runtime_main_list.append(float(runtime_main))
        runtime_loop_list.append(float(runtime_loop))
    w.writerow([lpus,
                np.average(average_step_sync_time_list),
                np.average(average_throughput_list),
                np.average(total_throughput_list),
                np.average(runtime_all_list),
                np.average(runtime_main_list),
                np.average(runtime_loop_list)])
<|endoftext|>