<commit_before>from collections import defaultdict
import time

import Shapes
import Colors
import lsdisplay

# TODO: Custom error handlers

def validateFrame(frame):
   # print(repr(frame)) # Debugging
    frameLen = len(frame) - 1
    if frameLen % 3 is not 0:
        return False
    cells = frameLen/3
    if (cells/frame[0]).is_integer() is False:
        return False
    if all(i <= 128 for i in frame[1:]) is False:
        return False
    return True

class LSAnimation:

    def __init__(self):
        self._frames = list()

    def addFrame(self, frame):
        self.insertFrame(len(self._frames), frame)
        return True

    def insertFrame(self, index, frame):
    # Inserts before given frame
        if validateFrame(frame) is False:
            print("Error frame is invalid")
            raise Exception
        if self._checkIndex(index) is True:
            self._frames.insert(index,frame)
        return True

    def deleteFrame(self, index):
        index -= 1
        if self._checkIndex(index) is True:
            frames = self._frames[0:index]
            frames.extend(self._frames[index+1:])
            self._frames = frames
        return True

    def showFrames(self):
        Frame = self.nextFrame()
        i = 0
        while True:
            i += 1
            try:
                print("Frame {:d}: {:s}".format(i, str(next(Frame))))
            except StopIteration:
                print("Out of frames")
                break

    def nextFrame(self):
        if len(self._frames) is 0:
            print("No frames!")
        else:
            for frame in self._frames:
                yield(frame)

    def play(self, display, frameRate = 30):
        if frameRate < 0:
            raise ValueError("Please specify a non-negative frame rate")
        print("Starting animation ({:d} frames at {:d} fps)".format(len(self._frames), frameRate))
        Frame = self.nextFrame()
        i = 0
        for frame in Frame:
            i+=1
            if frameRate is 0:
                input("Press any key to advance the animation...\n")
                print("Frame {:d}: {:s}".format(i, str(frame)))
            stime = time.time()
            display.floor.renderFrame(frame)
            display.heartbeat()
            renderTime = time.time() - stime

            if frameRate is not 0:
                fps = 1.0/renderTime
                if fps < frameRate:
                    print("[Animation]{0:.4f} FPS".format(1.0/renderTime), end="\r")
                else:
                    time.sleep((1.0/frameRate) - renderTime)
        if frameRate is 0:
            print("Animation ended.")
        print(" " * 22, end = "\r")

    def _checkIndex(self, index):
        if index not in range(-1,len(self._frames)+1):
            print("Error frame index out of range")
            raise Exception
        return True

class LSFrameGen:

    def __init__(self, rows, cols):
        self.rows = rows
        self.cols = cols
        self.frame = defaultdict(lambda: defaultdict(int))

    # Allows you to edit an existing frame structure, if no colormask is set
    # then the tile will keep its current colormask
    def edit(self, row, col, colormask):
        if row > self.rows or col > self.cols:
            print("Edit error, index out of range")
            raise Exception
        self.frame[row][col] = [colormask]

    def fill(self, colormask):
        for row in range(0, self.rows):
            for col in range(0, self.cols):
                self.frame[row][col] = [colormask]

    def shiftLeft (self):
        for col in range(0, self.cols-1):
            for row in range(0, self.rows):
                self.frame[row][col] = self.frame[row][col+1]
        for row in range(0, self.rows):
            self.frame[row][self.cols-1] = [(0,0,0)]

    def shiftRight (self):
        for col in reversed(range(0, self.cols)):
            for row in range(0, self.rows):
                self.frame[row][col] = self.frame[row][col-1]
        for row in range(0, self.rows):
            self.frame[row][0] = [(0,0,0)]

    def shiftUp (self):
        for row in range(0, self.rows-1):
            for col in range(0, self.cols):
                self.frame[row][col] = self.frame[row+1][col]
        for col in range(0, self.cols):
            self.frame[self.rows-1][col] = [(0,0,0)]

    def shiftDown (self):
        for row in reversed(range(0, self.rows)):
            for col in range(0, self.cols):
                self.frame[row][col] = self.frame[row-1][col]
        for col in range(0, self.cols):
            self.frame[0][col] = [(0,0,0)]

#TODO: shiftUp and shiftDown (shiftDiag?)

    def print(self):
        for row in self.frame:
            print([self.frame[row][col] for col in self.frame[row]])

    def get(self):
        frameOut = list()
        frameOut.append(self.cols)
        for row in range(0, self.rows):
            for col in range(0, self.cols):
                try:
                    cell = self.frame[row][col][0]
                    frameOut.append(cell[0])
                    frameOut.append(cell[1])
                    frameOut.append(cell[2])
                except:
                    print("Warning: ({:d},{:d}) has no update".format(row,col))
                    frameOut.append(128)
                    frameOut.append(128)
                    frameOut.append(128)
        return(frameOut)  # frameOut is a list consisting of the number of columns in the frame
                          # followed by a repeating pattern of 3 integers, each representing a
                          # subsequent tile's red, green, and blue colormasks

def main():
    print("Importing LSDisplay")
    import lsdisplay

    d = lsdisplay.LSDisplay(initScreen=False)

    colormask = (Shapes.ZERO, 0,0)
    diffmask = (Shapes.ONE, Shapes.TWO, Shapes.THREE)
    redZero = (Shapes.ZERO, Shapes.OFF, Shapes.OFF)
    greenZero = (0, Shapes.ZERO, 0)
    blueZero = (0, 0, 126)

    ourAnimation = LSAnimation()

    frame = LSFrameGen(3,d.cols)

    for _ in range(0,100):
        for i in range(d.cols):
            frame.edit(0,i,redZero)
            frame.edit(1,i,greenZero)
            frame.edit(2,i,blueZero)
        ourAnimation.addFrame(frame.get())
        for i in range(0,d.cols):
            frame.edit(1,i,redZero)
            frame.edit(2,i,greenZero)
            frame.edit(0,i,blueZero)
        ourAnimation.addFrame(frame.get())
        for i in range(0,d.cols):
            frame.edit(2,i,redZero)
            frame.edit(0,i,greenZero)
            frame.edit(1,i,blueZero)
        ourAnimation.addFrame(frame.get())

   # ourAnimation.deleteFrame(1)

  #  ourAnimation.showFrames()

    ourAnimation.play(d, frameRate=0)

if __name__ == '__main__':
    main()
<commit_msg>Added ScrollingText class<commit_after>from collections import defaultdict
import time

import Shapes
import Colors
import lsdisplay

# TODO: Custom error handlers

def validateFrame(frame):
   # print(repr(frame)) # Debugging
    frameLen = len(frame) - 1
    if frameLen % 3 is not 0:
        return False
    cells = frameLen/3
    if (cells/frame[0]).is_integer() is False:
        return False
    if all(i <= 128 for i in frame[1:]) is False:
        return False
    return True

class LSAnimation:

    def __init__(self):
        self._frames = list()

    def addFrame(self, frame):
        self.insertFrame(len(self._frames), frame)
        return True

    def insertFrame(self, index, frame):
    # Inserts before given frame
        if validateFrame(frame) is False:
            print("Error frame is invalid")
            raise Exception
        if self._checkIndex(index) is True:
            self._frames.insert(index,frame)
        return True

    def deleteFrame(self, index):
        index -= 1
        if self._checkIndex(index) is True:
            frames = self._frames[0:index]
            frames.extend(self._frames[index+1:])
            self._frames = frames
        return True

    def showFrames(self):
        Frame = self.nextFrame()
        i = 0
        while True:
            i += 1
            try:
                print("Frame {:d}: {:s}".format(i, str(next(Frame))))
            except StopIteration:
                print("Out of frames")
                break

    def nextFrame(self):
        if len(self._frames) is 0:
            print("No frames!")
        else:
            for frame in self._frames:
                yield(frame)

    def play(self, display, frameRate = 30):
        if frameRate < 0:
            raise ValueError("Please specify a non-negative frame rate")
        print("Starting animation ({:d} frames at {:d} fps)".format(len(self._frames), frameRate))
        Frame = self.nextFrame()
        i = 0
        for frame in Frame:
            i+=1
            if frameRate is 0:
                input("Press any key to advance the animation...\n")
                print("Frame {:d}: {:s}".format(i, str(frame)))
            stime = time.time()
            display.floor.renderFrame(frame)
            display.heartbeat()
            renderTime = time.time() - stime

            if frameRate is not 0:
                fps = 1.0/renderTime
                if fps < frameRate:
                    print("[Animation]{0:.4f} FPS".format(1.0/renderTime), end="\r")
                else:
                    time.sleep((1.0/frameRate) - renderTime)
        if frameRate is 0:
            print("Animation ended.")
        print(" " * 22, end = "\r")

    def _checkIndex(self, index):
        if index not in range(-1,len(self._frames)+1):
            print("Error frame index out of range")
            raise Exception
        return True

class ScrollingText(LSAnimation):
    def __init__(self, text, color=Colors.WHITE, height=1, width=None):
        super().__init__()
        self.charString = Shapes.stringToShapes(text)
        self.direction="left"
        self.height = height
        if width is None:
            self.width = len(charString)
        else:
            self.width = width
        self._buildAnimation()

    def setDirection(self, direction):
        if "left" not in direction.lower() and "right" not in direction.lower():
            raise NotImplementedError("Text must go 'left' or 'right'")
        else:
            self.direction = direction
        self._buildAnimation()

    def _buildAnimation (self):
        self._frames = list()  # TODO: A hack! LSAnimation should implement a dropFrames option
        self.frame = LSFrameGen(self.height, self.width)
        cs = self.charString[:]
        if "right" in self.direction.lower():
            cs = reversed(cs)
        if self.height is 1:
            for char in cs:
                charR = charG = charB = char                 # TODO: implement colors besides white
                endCap = lambda x: self.width-1 if "left" in x.lower() else 0
                self.frame.edit(0, endCap(self.direction), (charR, charG, charB))
                self.addFrame(self.frame.get())
                shiftFunc = self._pickShift()
                shiftFunc()
        else:
            raise NotImplementedError("height cannot be more than 1")

    def _pickShift(self):
        lowerD = self.direction.lower()
        if "left" in lowerD:
            return self.frame.shiftLeft
        elif "right" in lowerD:
            return self.frame.shiftRight

class LSFrameGen:

    def __init__(self, rows, cols):
        self.rows = rows
        self.cols = cols
        self.frame = defaultdict(lambda: defaultdict(int))

    # Allows you to edit an existing frame structure, if no colormask is set
    # then the tile will keep its current colormask
    def edit(self, row, col, colormask):
        if row > self.rows or col > self.cols:
            print("Edit error, index out of range")
            raise Exception
        self.frame[row][col] = [colormask]

    def fill(self, colormask):
        for row in range(0, self.rows):
            for col in range(0, self.cols):
                self.frame[row][col] = [colormask]

    def shiftLeft (self):
        for col in range(0, self.cols-1):
            for row in range(0, self.rows):
                self.frame[row][col] = self.frame[row][col+1]
        for row in range(0, self.rows):
            self.frame[row][self.cols-1] = [(0,0,0)]

    def shiftRight (self):
        for col in reversed(range(0, self.cols)):
            for row in range(0, self.rows):
                self.frame[row][col] = self.frame[row][col-1]
        for row in range(0, self.rows):
            self.frame[row][0] = [(0,0,0)]

    def shiftUp (self):
        for row in range(0, self.rows-1):
            for col in range(0, self.cols):
                self.frame[row][col] = self.frame[row+1][col]
        for col in range(0, self.cols):
            self.frame[self.rows-1][col] = [(0,0,0)]

    def shiftDown (self):
        for row in reversed(range(0, self.rows)):
            for col in range(0, self.cols):
                self.frame[row][col] = self.frame[row-1][col]
        for col in range(0, self.cols):
            self.frame[0][col] = [(0,0,0)]

#TODO: shiftUp and shiftDown (shiftDiag?)

    def print(self):
        for row in self.frame:
            print([self.frame[row][col] for col in self.frame[row]])

    def get(self):
        frameOut = list()
        frameOut.append(self.cols)
        for row in range(0, self.rows):
            for col in range(0, self.cols):
                try:
                    cell = self.frame[row][col][0]
                    frameOut.append(cell[0])
                    frameOut.append(cell[1])
                    frameOut.append(cell[2])
                except:
                    print("Warning: ({:d},{:d}) has no update".format(row,col))
                    frameOut.append(128)
                    frameOut.append(128)
                    frameOut.append(128)
        return(frameOut)  # frameOut is a list consisting of the number of columns in the frame
                          # followed by a repeating pattern of 3 integers, each representing a
                          # subsequent tile's red, green, and blue colormasks

def main():
    print("Importing LSDisplay")
    import lsdisplay

    d = lsdisplay.LSDisplay(initScreen=False)

    colormask = (Shapes.ZERO, 0,0)
    diffmask = (Shapes.ONE, Shapes.TWO, Shapes.THREE)
    redZero = (Shapes.ZERO, Shapes.OFF, Shapes.OFF)
    greenZero = (0, Shapes.ZERO, 0)
    blueZero = (0, 0, 126)

    ourAnimation = LSAnimation()

    frame = LSFrameGen(3,d.cols)

    for _ in range(0,100):
        for i in range(d.cols):
            frame.edit(0,i,redZero)
            frame.edit(1,i,greenZero)
            frame.edit(2,i,blueZero)
        ourAnimation.addFrame(frame.get())
        for i in range(0,d.cols):
            frame.edit(1,i,redZero)
            frame.edit(2,i,greenZero)
            frame.edit(0,i,blueZero)
        ourAnimation.addFrame(frame.get())
        for i in range(0,d.cols):
            frame.edit(2,i,redZero)
            frame.edit(0,i,greenZero)
            frame.edit(1,i,blueZero)
        ourAnimation.addFrame(frame.get())

   # ourAnimation.deleteFrame(1)

  #  ourAnimation.showFrames()

    ourAnimation.play(d, frameRate=0)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
from __future__ import print_function, division

from job import Joblist
from host import Hostlist
from utility import color

import sys
import os
import argparse
import re

def main_raising():
    global args
    parser = argparse.ArgumentParser(
        description="More comprehensive version of bjobs.",
        epilog="Any non-listed arguments are passed to bjobs.")
    parser.add_argument(
        "-l", "--long",
        help="long job description",
        action="store_true",
    )
    parser.add_argument(
        "-w", "--wide",
        help="don't shorten strings",
        action="store_true",
    )
    exg = parser.add_mutually_exclusive_group()
    exg.add_argument(
        "-p", "--pending",
        help="show pending jobs with reasons and potential hosts",
        action="store_true",
    )
    exg.add_argument(
        "--group",
        help="group jobs by attribute",
        metavar="BY",
    )
    parser.add_argument(
        "-aices",
        help="short for -G p_aices",
        action="store_true",
    )
    parser.add_argument_group("further arguments",
                              description="are passed to bjobs")

    args, bjobsargs = parser.parse_known_args()

    if args.pending:
        args.group = "PENDING REASONS"
    if args.aices:
        bjobsargs = ["-G", "p_aices"] + bjobsargs

    whoami = os.getenv("USER")

    print("Reading job list from LSF ...", end="\r")
    sys.stdout.flush()
    joblist = Joblist(bjobsargs)
    print("                             ", end="\r")
    if args.pending:
        joblists = joblist.groupby("Status")
        if "PEND" in joblists:
            joblist = joblists["PEND"]
        else:
            joblist = Joblist()
    joblists = joblist.groupby(args.group)

    if not args.pending:
        for group, joblist in joblists.items():
            if group:
                groupn = group
                if args.group == "User":
                    groupn = joblist[0]["Userstr"]
                title = "{} = {} [{}]".format(args.group, groupn, len(joblist))
            else:
                title = None
            joblist.display(args.long, args.wide, title)
        return
    for reasons in sorted(joblists.keys(), key=len):
        pendjobs = joblists[reasons]
        if len(reasons) == 1 and reasons[0][1] is True:
            title = "{} [{}]".format(reasons[0][0], len(pendjobs))
            pendjobs.display(args.long, args.wide, title)
            continue
        lists = {}
        resgrouped = pendjobs.groupby("Requested Resources")
        for res, rlist in resgrouped.iteritems():
            hostgrouped = rlist.groupby("Specified Hosts")
            for hosts, hlist in hostgrouped.iteritems():
                lists[res, hosts] = hlist
        for case, casejobs in lists.iteritems():
            title = "[{}]".format(len(casejobs))
            casejobs.display(args.long, args.wide, title)
            singlenode = "span[hosts=1]" in case[0]
            minprocs = min(job["Processors Requested"] for job in casejobs)
            print()
            print("Pending reasons:")
            cs = {
                "Job's requirement for exclusive execution not satisfied": "y",
                "An exclusive job has reserved the host": "y",
                "Not enough slots or resources "
                "for whole duration of the job": "r",
            }
            for reason, count in reasons:
                s = reason
                if reason in cs:
                    s = color(reason, cs[reason])
                print("\t" + str(count).ljust(8) + s)
            if case[1]:
                req = [case[1]]
            else:
                req = case[0]
                req = re.sub(" && \(hpcwork\)", "", req)
                req = re.sub(" && \(hostok\)", "", req)
                req = re.sub(" && \(mem>\d+\)", "", req)
                req = ["-R", req]
            print("Reading host list from LSF ...", end="\r")
            sys.stdout.flush()
            hl = {h["HOST"]: h for h in Hostlist(req)}
            if singlenode:
                hl = {n: h for n, h in hl.iteritems() if h["MAX"] >= minprocs}
            print("Reading job list from LSF ... ", end="\r")
            sys.stdout.flush()
            jl = Joblist(["-m", " ".join(hl.keys()), "-u", "all"])
            byproc = jl.groupby("Processors")
            print("Potential hosts:              ")
            for hostname in sorted(hl.keys()):
                host = hl[hostname]
                freeslots = host["MAX"] - host["RUN"] - host["RSV"]
                if hostname in byproc:
                    if byproc[hostname][0]["Exclusive Execution"]:
                        freeslots = 0
                if freeslots == 0:
                    c = "r"
                elif freeslots == host["MAX"]:
                    c = "g"
                else:
                    c = "y"
                freeslots = color("{:>2}*free".format(freeslots), c)
                print("\t{}: {}".format(host, freeslots), end="")
                if host["RSV"] > 0:
                    rsv = "{:>2}*".format(host["RSV"]) + color("reserved", "y")
                    print("\t" + rsv, end="")
                if hostname in byproc:
                    jobs = byproc[hostname]
                    users = []
                    for job in jobs:
                        if job["Exclusive Execution"]:
                            js = "-x "
                        else:
                            js = str(job["Processors"][hostname]).rjust(2)
                            js += "*"
                        if job["User"] == whoami:
                            js += color(job["User"], "g")
                        else:
                            js += job["User"]
                        users.append(js)
                    print("\t{}".format("\t".join(users)), end="")
                print()
            if len(jl):
                print("conflicting users:")
                for user, jobs in jl.groupby("User").items():
                    procs = {}
                    for job in jobs:
                        idx = "Processors" if args.wide else "Hostgroups"
                        for p, c in job[idx].iteritems():
                            if not p in procs:
                                procs[p] = 0
                            if job["Exclusive Execution"]:
                                procs[p] += job["Hosts"][0]["MAX"]
                            else:
                                procs[p] += c
                    procsstr = "\t".join("{:>3}*{}".format(c, p)
                                         for p, c in procs.iteritems())
                    hosts = set(sum((job["Hosts"] for job in jobs), []))
                    if jobs[0]["User"] == whoami:
                        c = "g"
                    else:
                        c = 0
                    ustr = jobs[0]["Userstr"].ljust(40)
                    print("\t" + color(ustr, c) + procsstr)

def main():
    try:
        main_raising()
    except KeyboardInterrupt:
        pass
    except:
        print(color("ERROR -- probably a job status changed while " +
                    sys.argv[0] + " processed it", "r"), file=sys.stderr)

if __name__ == "__main__":
    main()
<commit_msg>shortened -h<commit_after>#!/usr/bin/env python
from __future__ import print_function, division

from job import Joblist
from host import Hostlist
from utility import color

import sys
import os
import argparse
import re

def main_raising():
    global args
    parser = argparse.ArgumentParser(
        description="More comprehensive version of bjobs."
    )
    parser.add_argument(
        "-l", "--long",
        help="long job description",
        action="store_true",
    )
    parser.add_argument(
        "-w", "--wide",
        help="don't shorten strings",
        action="store_true",
    )
    exg = parser.add_mutually_exclusive_group()
    exg.add_argument(
        "-p", "--pending",
        help="show pending jobs with reasons and potential hosts",
        action="store_true",
    )
    exg.add_argument(
        "--group",
        help="group jobs by attribute",
        metavar="BY",
    )
    parser.add_argument(
        "-aices",
        help="short for -G p_aices",
        action="store_true",
    )
    parser.add_argument_group("further arguments",
                              description="are passed to bjobs")

    args, bjobsargs = parser.parse_known_args()

    if args.pending:
        args.group = "PENDING REASONS"
    if args.aices:
        bjobsargs = ["-G", "p_aices"] + bjobsargs

    whoami = os.getenv("USER")

    print("Reading job list from LSF ...", end="\r")
    sys.stdout.flush()
    joblist = Joblist(bjobsargs)
    print("                             ", end="\r")
    if args.pending:
        joblists = joblist.groupby("Status")
        if "PEND" in joblists:
            joblist = joblists["PEND"]
        else:
            joblist = Joblist()
    joblists = joblist.groupby(args.group)

    if not args.pending:
        for group, joblist in joblists.items():
            if group:
                groupn = group
                if args.group == "User":
                    groupn = joblist[0]["Userstr"]
                title = "{} = {} [{}]".format(args.group, groupn, len(joblist))
            else:
                title = None
            joblist.display(args.long, args.wide, title)
        return
    for reasons in sorted(joblists.keys(), key=len):
        pendjobs = joblists[reasons]
        if len(reasons) == 1 and reasons[0][1] is True:
            title = "{} [{}]".format(reasons[0][0], len(pendjobs))
            pendjobs.display(args.long, args.wide, title)
            continue
        lists = {}
        resgrouped = pendjobs.groupby("Requested Resources")
        for res, rlist in resgrouped.iteritems():
            hostgrouped = rlist.groupby("Specified Hosts")
            for hosts, hlist in hostgrouped.iteritems():
                lists[res, hosts] = hlist
        for case, casejobs in lists.iteritems():
            title = "[{}]".format(len(casejobs))
            casejobs.display(args.long, args.wide, title)
            singlenode = "span[hosts=1]" in case[0]
            minprocs = min(job["Processors Requested"] for job in casejobs)
            print()
            print("Pending reasons:")
            cs = {
                "Job's requirement for exclusive execution not satisfied": "y",
                "An exclusive job has reserved the host": "y",
                "Not enough slots or resources "
                "for whole duration of the job": "r",
            }
            for reason, count in reasons:
                s = reason
                if reason in cs:
                    s = color(reason, cs[reason])
                print("\t" + str(count).ljust(8) + s)
            if case[1]:
                req = [case[1]]
            else:
                req = case[0]
                req = re.sub(" && \(hpcwork\)", "", req)
                req = re.sub(" && \(hostok\)", "", req)
                req = re.sub(" && \(mem>\d+\)", "", req)
                req = ["-R", req]
            print("Reading host list from LSF ...", end="\r")
            sys.stdout.flush()
            hl = {h["HOST"]: h for h in Hostlist(req)}
            if singlenode:
                hl = {n: h for n, h in hl.iteritems() if h["MAX"] >= minprocs}
            print("Reading job list from LSF ... ", end="\r")
            sys.stdout.flush()
            jl = Joblist(["-m", " ".join(hl.keys()), "-u", "all"])
            byproc = jl.groupby("Processors")
            print("Potential hosts:              ")
            for hostname in sorted(hl.keys()):
                host = hl[hostname]
                freeslots = host["MAX"] - host["RUN"] - host["RSV"]
                if hostname in byproc:
                    if byproc[hostname][0]["Exclusive Execution"]:
                        freeslots = 0
                if freeslots == 0:
                    c = "r"
                elif freeslots == host["MAX"]:
                    c = "g"
                else:
                    c = "y"
                freeslots = color("{:>2}*free".format(freeslots), c)
                print("\t{}: {}".format(host, freeslots), end="")
                if host["RSV"] > 0:
                    rsv = "{:>2}*".format(host["RSV"]) + color("reserved", "y")
                    print("\t" + rsv, end="")
                if hostname in byproc:
                    jobs = byproc[hostname]
                    users = []
                    for job in jobs:
                        if job["Exclusive Execution"]:
                            js = "-x "
                        else:
                            js = str(job["Processors"][hostname]).rjust(2)
                            js += "*"
                        if job["User"] == whoami:
                            js += color(job["User"], "g")
                        else:
                            js += job["User"]
                        users.append(js)
                    print("\t{}".format("\t".join(users)), end="")
                print()
            if len(jl):
                print("conflicting users:")
                for user, jobs in jl.groupby("User").items():
                    procs = {}
                    for job in jobs:
                        idx = "Processors" if args.wide else "Hostgroups"
                        for p, c in job[idx].iteritems():
                            if not p in procs:
                                procs[p] = 0
                            if job["Exclusive Execution"]:
                                procs[p] += job["Hosts"][0]["MAX"]
                            else:
                                procs[p] += c
                    procsstr = "\t".join("{:>3}*{}".format(c, p)
                                         for p, c in procs.iteritems())
                    hosts = set(sum((job["Hosts"] for job in jobs), []))
                    if jobs[0]["User"] == whoami:
                        c = "g"
                    else:
                        c = 0
                    ustr = jobs[0]["Userstr"].ljust(40)
                    print("\t" + color(ustr, c) + procsstr)

def main():
    try:
        main_raising()
    except KeyboardInterrupt:
        pass
    except:
        print(color("ERROR -- probably a job status changed while " +
                    sys.argv[0] + " processed it", "r"), file=sys.stderr)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import math
import random

from operator import itemgetter
from collections import defaultdict

from lsh.model import LSHItem
from lsh.readers import LSHReader
from lsh.writers import LSHWriter

"""

KNNQuery needs to run in two modes:
    INDEX   items are given to the object for storage
    QUERY   given a LSHItem id, return the k-nearest neighbours in the index, with counts populated

"""

def compute_sim(lshash1, lshash2, hash_length):
    diff_bits = lshash1 ^ lshash2
    while diff_bits:
        diff_bits &= diff_bits - 1 # Clear the least significant bit set
        hash_length -= 1
    return hash_length

def cosine_sim(bow1, bow2):
    dot = sum(x * y for x, y in zip(bow1, bow2))
    l = math.sqrt(sum(x**2 for x in bow1)) * math.sqrt(sum(x**2 for x in bow2))
    if l==0:
        return 0
    else:
        return float(dot) / l

class KNNQuery(object):
    def __init__(self, perm_num, sig_length, window_size, n_hashes, prefix_length):
        self.generate_perms(perm_num, sig_length)
        self.window_size = window_size
        self.items = []
        self.sig_length = sig_length
        self.n_hashes = n_hashes
        self.prefix_length = prefix_length

    def find_all_neighbours(self, query_item, k, correct=None):
        if correct is None:
            correct = {}
        neighbours = {}
        candidates = defaultdict(lambda: defaultdict(int))
        for perm in self.perms:
            buckets = defaultdict(list)
            prefixes = {}

            # do the hashing
            for item in self.items:
                h = item.signature
                h.lrotate(perm)
                prefix = h.get_prefix(self.prefix_length)
                prefixes[item.id] = prefix
                buckets[prefix].append(item)

            # do the lookup-ing
            for item in self.items:
                query_prefix = prefixes[item.id]
                self.add_candidates(buckets, candidates[item.id], item, query_prefix, correct)

        for id in candidates:
            candidates[id] = sorted(candidates[id].items(), key=itemgetter(1), reverse=True)

        if correct:
            self.print_debug(query_item, candidates, correct)
        return candidates

    def print_debug(self, query_item, sorted_candidates, correct=None):
        for candidate in sorted_candidates:
            cosine = cosine_sim(query_item.vector, candidate[0].vector)
            if cosine > 0.1:
                print("XOR similarity:", compute_sim(query_item.signature.get_first_working(), candidate[0].signature.get_first_working(), self.sig_length), "Cosine similarity:", cosine)

    def add_candidates(self, buckets, candidates, query_item, query_prefix, correct=None):
        if correct is None:
            correct = []
        ncandidates = self.window_size * 2
        nadded = 0
        for item in buckets[query_prefix]:
            if item.id != query_item.id:
                candidates[item] += 1
                nadded += 1
        prefix_index = 0
        while nadded <= ncandidates and prefix_index < self.prefix_length:
            prefix = query_prefix ^ (1 << prefix_index)
            for item in buckets[prefix]:
                candidates[item] += 1
                nadded += 1
            prefix_index += 1

    def add_items_to_index(self, filelike):
        r = LSHReader()
        for item in r.process_file(filelike):
            self.add_item_to_index(item)

    def add_item_to_index(self, item):
        self.items.append(item)

    def generate_perms(self, perm_num, sig_length):
        self.perms = list(random.randint(1, sig_length - 1) for i in range(perm_num))
        random.shuffle(self.perms)

if __name__ == '__main__':
    TESTFILE = './schwa-py-lsh/test/test_data_1000.txt'
    q = KNNQuery(perm_num=10, sig_length=100, window_size=10)
    q.add_items_to_index(open(TESTFILE, 'r'))
    for item in q.items:
        print(str(item))
        print('\n'.join(str(n) for n in q.find_neighbours(item, 10)))
        print()
<commit_msg> * adding timing information for each permutation<commit_after>#!/usr/bin/env python3

import math
import random
from datetime import datetime

from operator import itemgetter
from collections import defaultdict

from lsh.model import LSHItem
from lsh.readers import LSHReader
from lsh.writers import LSHWriter

"""

KNNQuery needs to run in two modes:
    INDEX   items are given to the object for storage
    QUERY   given a LSHItem id, return the k-nearest neighbours in the index, with counts populated

"""

def compute_sim(lshash1, lshash2, hash_length):
    diff_bits = lshash1 ^ lshash2
    while diff_bits:
        diff_bits &= diff_bits - 1 # Clear the least significant bit set
        hash_length -= 1
    return hash_length

def cosine_sim(bow1, bow2):
    dot = sum(x * y for x, y in zip(bow1, bow2))
    l = math.sqrt(sum(x**2 for x in bow1)) * math.sqrt(sum(x**2 for x in bow2))
    if l==0:
        return 0
    else:
        return float(dot) / l

class KNNQuery(object):
    def __init__(self, perm_num, sig_length, window_size, n_hashes, prefix_length):
        self.generate_perms(perm_num, sig_length)
        self.window_size = window_size
        self.items = []
        self.sig_length = sig_length
        self.n_hashes = n_hashes
        self.prefix_length = prefix_length

    def find_all_neighbours(self, query_item, k, correct=None):
        if correct is None:
            correct = {}
        neighbours = {}
        candidates = defaultdict(lambda: defaultdict(int))
        start = datetime.now()
        for i, perm in enumerate(self.perms):
            buckets = defaultdict(list)
            prefixes = {}
            if i and i % 100 == 0:
                print("Processed {} permutations, Total time: {}".format(i, datetime.now() - start))

            # do the hashing
            for item in self.items:
                h = item.signature
                h.lrotate(perm)
                prefix = h.get_prefix(self.prefix_length)
                prefixes[item.id] = prefix
                buckets[prefix].append(item)

            # do the lookup-ing
            for item in self.items:
                query_prefix = prefixes[item.id]
                self.add_candidates(buckets, candidates[item.id], item, query_prefix, correct)

        for id in candidates:
            candidates[id] = sorted(candidates[id].items(), key=itemgetter(1), reverse=True)

        if correct:
            self.print_debug(query_item, candidates, correct)
        return candidates

    def print_debug(self, query_item, sorted_candidates, correct=None):
        for candidate in sorted_candidates:
            cosine = cosine_sim(query_item.vector, candidate[0].vector)
            if cosine > 0.1:
                print("XOR similarity:", compute_sim(query_item.signature.get_first_working(), candidate[0].signature.get_first_working(), self.sig_length), "Cosine similarity:", cosine)

    def add_candidates(self, buckets, candidates, query_item, query_prefix, correct=None):
        if correct is None:
            correct = []
        ncandidates = self.window_size * 2
        nadded = 0
        for item in buckets[query_prefix]:
            if item.id != query_item.id:
                candidates[item] += 1
                nadded += 1
        prefix_index = 0
        while nadded <= ncandidates and prefix_index < self.prefix_length:
            prefix = query_prefix ^ (1 << prefix_index)
            for item in buckets[prefix]:
                candidates[item] += 1
                nadded += 1
            prefix_index += 1

    def add_items_to_index(self, filelike):
        r = LSHReader()
        for item in r.process_file(filelike):
            self.add_item_to_index(item)

    def add_item_to_index(self, item):
        self.items.append(item)

    def generate_perms(self, perm_num, sig_length):
        self.perms = list(random.randint(1, sig_length - 1) for i in range(perm_num))
        random.shuffle(self.perms)

if __name__ == '__main__':
    TESTFILE = './schwa-py-lsh/test/test_data_1000.txt'
    q = KNNQuery(perm_num=10, sig_length=100, window_size=10)
    q.add_items_to_index(open(TESTFILE, 'r'))
    for item in q.items:
        print(str(item))
        print('\n'.join(str(n) for n in q.find_neighbours(item, 10)))
        print()
<|endoftext|>


<commit_before>from django.conf.urls import patterns, include, url
from django.conf import settings
from django.conf.urls.static import static
from django.views.generic import TemplateView

#import lsst.settings
from django.conf import settings

from lsst import views as lsstmon_views
#import lsst.views as lsstmon_views
import core.pandajob.views_support as core_lsstmon_support_views
#import core.pandajob.views as core_lsstmon_views

urlpatterns = patterns('',
    url(r'^$', lsstmon_views.mainPage, name='index'),
    url(r'^$', lsstmon_views.mainPage, name='mainPage'),
    url(r'^jobs/$', lsstmon_views.jobList, name='jobList'),
    url(r'^jobs/(.*)/$', lsstmon_views.jobList, name='jobList'),
    url(r'^jobs/(.*)/(.*)/$', lsstmon_views.jobList, name='jobList'),
    url(r'^job$', lsstmon_views.jobInfo, name='jobInfo'),
    url(r'^job/(.*)/$', lsstmon_views.jobInfo, name='jobInfo'),
    url(r'^job/(.*)/(.*)/$', lsstmon_views.jobInfo, name='jobInfo'),
    url(r'^users/$', lsstmon_views.userList, name='userList'),
    url(r'^user/(?P<user>.*)/$', lsstmon_views.userInfo, name='userInfo'),
    url(r'^user/$', lsstmon_views.userInfo, name='userInfo'),
    url(r'^sites/$', lsstmon_views.siteList, name='siteList'),
    url(r'^site/(?P<site>.*)/$', lsstmon_views.siteInfo, name='siteInfo'),
    url(r'^site/$', lsstmon_views.siteInfo, name='siteInfo'),
    url(r'^wns/(?P<site>.*)/$', lsstmon_views.wnInfo, name='wnInfo'),
    url(r'^wn/(?P<site>.*)/(?P<wnname>.*)/$', lsstmon_views.wnInfo, name='wnInfo'),
    url(r'^tasks/$', lsstmon_views.taskList, name='taskList'),
    url(r'^task$', lsstmon_views.taskInfo, name='taskInfo'),
    url(r'^task/$', lsstmon_views.taskInfo, name='taskInfo'),
    url(r'^errors/$', lsstmon_views.errorSummary, name='errorSummary'),
    url(r'^task/(?P<jeditaskid>.*)/$', lsstmon_views.taskInfo, name='taskInfo'),
    url(r'^dash/$', lsstmon_views.dashboard, name='dashboard'),
    url(r'^dash/analysis/$', lsstmon_views.dashAnalysis, name='dashAnalysis'),
    url(r'^dash/production/$', lsstmon_views.dashProduction, name='dashProduction'),

    ### support views for LSST
    url(r'^support/$', core_lsstmon_support_views.maxpandaid, name='supportRoot'),
    url(r'^support/maxpandaid/$', core_lsstmon_support_views.maxpandaid, name='supportMaxpandaid'),

    ### robots.txt
    url('^robots\.txt$', TemplateView.as_view(template_name='robots.txt', content_type='text/plain')),

) + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)

#urlpatterns += common_patterns
#urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)
<commit_msg>try to recover<commit_after>from django.conf.urls import patterns, include, url
from django.conf import settings
from django.conf.urls.static import static
from django.views.generic import TemplateView

#import lsst.settings
from django.conf import settings

from lsst import views as lsstmon_views
#import lsst.views as lsstmon_views
import core.pandajob.views_support as core_lsstmon_support_views
#import core.pandajob.views as core_lsstmon_views

urlpatterns = patterns('',
    url(r'^$', lsstmon_views.mainPage, name='mainPage'),
    url(r'^$', lsstmon_views.mainPage, name='index'),
    url(r'^jobs/$', lsstmon_views.jobList, name='jobList'),
    url(r'^jobs/(.*)/$', lsstmon_views.jobList, name='jobList'),
    url(r'^jobs/(.*)/(.*)/$', lsstmon_views.jobList, name='jobList'),
    url(r'^job$', lsstmon_views.jobInfo, name='jobInfo'),
    url(r'^job/(.*)/$', lsstmon_views.jobInfo, name='jobInfo'),
    url(r'^job/(.*)/(.*)/$', lsstmon_views.jobInfo, name='jobInfo'),
    url(r'^users/$', lsstmon_views.userList, name='userList'),
    url(r'^user/(?P<user>.*)/$', lsstmon_views.userInfo, name='userInfo'),
    url(r'^user/$', lsstmon_views.userInfo, name='userInfo'),
    url(r'^sites/$', lsstmon_views.siteList, name='siteList'),
    url(r'^site/(?P<site>.*)/$', lsstmon_views.siteInfo, name='siteInfo'),
    url(r'^site/$', lsstmon_views.siteInfo, name='siteInfo'),
    url(r'^wns/(?P<site>.*)/$', lsstmon_views.wnInfo, name='wnInfo'),
    url(r'^wn/(?P<site>.*)/(?P<wnname>.*)/$', lsstmon_views.wnInfo, name='wnInfo'),
    url(r'^tasks/$', lsstmon_views.taskList, name='taskList'),
    url(r'^task$', lsstmon_views.taskInfo, name='taskInfo'),
    url(r'^task/$', lsstmon_views.taskInfo, name='taskInfo'),
    url(r'^errors/$', lsstmon_views.errorSummary, name='errorSummary'),
    url(r'^task/(?P<jeditaskid>.*)/$', lsstmon_views.taskInfo, name='taskInfo'),
    url(r'^dash/$', lsstmon_views.dashboard, name='dashboard'),
    url(r'^dash/analysis/$', lsstmon_views.dashAnalysis, name='dashAnalysis'),
    url(r'^dash/production/$', lsstmon_views.dashProduction, name='dashProduction'),

    ### support views for LSST
    url(r'^support/$', core_lsstmon_support_views.maxpandaid, name='supportRoot'),
    url(r'^support/maxpandaid/$', core_lsstmon_support_views.maxpandaid, name='supportMaxpandaid'),

    ### robots.txt
    url('^robots\.txt$', TemplateView.as_view(template_name='robots.txt', content_type='text/plain')),

) + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)

#urlpatterns += common_patterns
#urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)
<|endoftext|>


<commit_before># From The Algorithmic Beauty of Plants
# Chapter 1 - Graphical modeling using L-system
# 1.3 Turtle interpretation of strings
# Edge rewriting L-system

from PIL import Image, ImageDraw

import numpy as np

black = (  0,   0,   0, 255)
white = (255, 255, 255, 255)
size  = (1000, 1000)

iterations = 4
angel = 22.5 # In gradus
start_point = (500, 1000)
width_point = 2
step_length = 1
axiom = 'F'
productions = {
    'F': 'F[FF+F][F]+F-F'
}

def rule_production(axiom, productions, iterations):
    productions['+'] = '+'
    productions['-'] = '-'
    productions['['] = '['
    productions[']'] = ']'
    rule = axiom
    for i in range(iterations):
        rule = ''.join([productions[s] for s in rule])
    return rule

class LSystem2D:
    def __init__(self, axiom, productions, iterations, angel):
        self.axiom = axiom
        self.productions = productions
        self.iterations = iterations
        self.angel = self.ang_to_rad(angel)

    def ang_to_rad(self, deg):
        return np.pi/180*deg

    def make_rule(self):
        #self.rule = rule_production(self.axiom, self.prodictions, self.iterations)
        rule = self.axiom
        productions = self.productions
        productions['+'] = '+'
        productions['-'] = '-'
        productions['['] = '['
        productions[']'] = ']'
        for i in range(self.iterations):
            rule = ''.join([productions[s] for s in rule])
        self.rule = rule

figure = LSystem2D(axiom, productions, iterations, angel)
figure.make_rule()

#rule = rule_production(axiom, productions, iterations)

rule = figure.rule

def rotate(point, phi):
    rotation_matrix = [
        [ np.cos(phi), np.sin(phi)],
        [-np.sin(phi), np.cos(phi)]
    ]
    return point.dot(rotation_matrix)

def rad_to_euc(r, phi):
    x, y = rotate(np.array([r, r]), phi)
    return x, y

def draw_koch_islands(angel, rule, start_point, width_point, step_length):
    '''Draw figure'''
    canvas = Image.new('RGBA', size, white)
    draw = ImageDraw.Draw(canvas)
    xy_angel = np.pi
    save_angel = []
    for r in rule:
        if r == 'F':
            end_point = start_point + np.array(rad_to_euc(step_length, xy_angel))
            draw.line([tuple(start_point), tuple(end_point)], fill=black, width=width_point)
            start_point = end_point
        elif r == 'f':
            start_point += np.array(rad_to_euc(step_length, curr_angel))
        elif r == '-':
            xy_angel -= angel
        elif r == '+':
            xy_angel += angel
        elif r == '[':
            save_angel.append((xy_angel, start_point.copy()))
        elif r == ']':
            xy_angel, start_point = save_angel.pop()
        else:
            pass
    return canvas

def make_dimensions(angel, rule, size):
    '''Count step length and start point to fit in canvas size'''
    x_dim = y_dim = np.array((0, 0))
    xy_point = np.array((0,0), dtype='float64')
    curr_angel = np.pi
    save_angel = []
    for r in rule:
        if r in 'fF':
            xy_point += np.array(rad_to_euc(step_length, curr_angel))
            x_dim = min(x_dim[0], xy_point[0]), max(x_dim[1], xy_point[0])
            y_dim = min(y_dim[0], xy_point[1]), max(y_dim[1], xy_point[1])
        elif r == '-':
            curr_angel -= angel
        elif r == '+':
            curr_angel += angel
        elif r == '[':
            save_angel.append((curr_angel, xy_point.copy()))
        elif r == ']':
            curr_angel, xy_point = save_angel.pop()
        else:
            pass

    x_width = sum([abs(x) for x in x_dim])
    y_width = sum([abs(y) for y in y_dim])
    step = np.floor(min(size[0]/(x_width), size[1]/(y_width)))
    step = step if step > 1 else 1

    center = np.array((abs(x_dim[0])*step + (size[0] - x_width * step)/2,
                       abs(y_dim[0])*step + (size[1] - y_width * step)/2))

    return center, step

def deg_to_rad(deg):
    return np.pi/180*deg

# Count step length and start point to fit in canvas size
start_point, step_length = make_dimensions(deg_to_rad(angel), rule, size)

# Draw image
canvas = draw_koch_islands(deg_to_rad(angel), rule, start_point, width_point, step_length)
canvas.show()<commit_msg>Delete rule_production function<commit_after># From The Algorithmic Beauty of Plants
# Chapter 1 - Graphical modeling using L-system
# 1.3 Turtle interpretation of strings
# Edge rewriting L-system

from PIL import Image, ImageDraw

import numpy as np

black = (  0,   0,   0, 255)
white = (255, 255, 255, 255)
size  = (1000, 1000)

iterations = 4
angel = 22.5 # In gradus
start_point = (500, 1000)
width_point = 2
step_length = 1
axiom = 'F'
productions = {
    'F': 'F[FF+F][F]+F-F'
}

class LSystem2D:
    def __init__(self, axiom, productions, iterations, angel):
        self.axiom = axiom
        self.productions = productions
        self.iterations = iterations
        self.angel = self.ang_to_rad(angel)

    def ang_to_rad(self, deg):
        return np.pi/180*deg

    def make_rule(self):
        rule = self.axiom
        productions = self.productions
        productions['+'] = '+'
        productions['-'] = '-'
        productions['['] = '['
        productions[']'] = ']'
        for i in range(self.iterations):
            rule = ''.join([productions[s] for s in rule])
        self.rule = rule

figure = LSystem2D(axiom, productions, iterations, angel)
figure.make_rule()

rule = figure.rule

def rotate(point, phi):
    rotation_matrix = [
        [ np.cos(phi), np.sin(phi)],
        [-np.sin(phi), np.cos(phi)]
    ]
    return point.dot(rotation_matrix)

def rad_to_euc(r, phi):
    x, y = rotate(np.array([r, r]), phi)
    return x, y

def draw_koch_islands(angel, rule, start_point, width_point, step_length):
    '''Draw figure'''
    canvas = Image.new('RGBA', size, white)
    draw = ImageDraw.Draw(canvas)
    xy_angel = np.pi
    save_angel = []
    for r in rule:
        if r == 'F':
            end_point = start_point + np.array(rad_to_euc(step_length, xy_angel))
            draw.line([tuple(start_point), tuple(end_point)], fill=black, width=width_point)
            start_point = end_point
        elif r == 'f':
            start_point += np.array(rad_to_euc(step_length, curr_angel))
        elif r == '-':
            xy_angel -= angel
        elif r == '+':
            xy_angel += angel
        elif r == '[':
            save_angel.append((xy_angel, start_point.copy()))
        elif r == ']':
            xy_angel, start_point = save_angel.pop()
        else:
            pass
    return canvas

def make_dimensions(angel, rule, size):
    '''Count step length and start point to fit in canvas size'''
    x_dim = y_dim = np.array((0, 0))
    xy_point = np.array((0,0), dtype='float64')
    curr_angel = np.pi
    save_angel = []
    for r in rule:
        if r in 'fF':
            xy_point += np.array(rad_to_euc(step_length, curr_angel))
            x_dim = min(x_dim[0], xy_point[0]), max(x_dim[1], xy_point[0])
            y_dim = min(y_dim[0], xy_point[1]), max(y_dim[1], xy_point[1])
        elif r == '-':
            curr_angel -= angel
        elif r == '+':
            curr_angel += angel
        elif r == '[':
            save_angel.append((curr_angel, xy_point.copy()))
        elif r == ']':
            curr_angel, xy_point = save_angel.pop()
        else:
            pass

    x_width = sum([abs(x) for x in x_dim])
    y_width = sum([abs(y) for y in y_dim])
    step = np.floor(min(size[0]/(x_width), size[1]/(y_width)))
    step = step if step > 1 else 1

    center = np.array((abs(x_dim[0])*step + (size[0] - x_width * step)/2,
                       abs(y_dim[0])*step + (size[1] - y_width * step)/2))

    return center, step

def deg_to_rad(deg):
    return np.pi/180*deg

# Count step length and start point to fit in canvas size
start_point, step_length = make_dimensions(deg_to_rad(angel), rule, size)

# Draw image
canvas = draw_koch_islands(deg_to_rad(angel), rule, start_point, width_point, step_length)
canvas.show()<|endoftext|>


<commit_before>#!/usr/bin/env python3
"""Merge all translation sources into a single JSON file."""
import json
import os
import pathlib
import re
import subprocess
from typing import Dict, List, Union

from .const import CLI_2_DOCKER_IMAGE, CORE_PROJECT_ID
from .error import ExitApp
from .util import get_lokalise_token

FILENAME_FORMAT = re.compile(r"strings\.(?P<suffix>\w+)\.json")
DOWNLOAD_DIR = pathlib.Path("build/translations-download").absolute()

def run_download_docker():
    """Run the Docker image to download the translations."""
    print("Running Docker to download latest translations.")
    run = subprocess.run(
        [
            "docker",
            "run",
            "-v",
            f"{DOWNLOAD_DIR}:/opt/dest/locale",
            "--rm",
            f"lokalise/lokalise-cli-2:{CLI_2_DOCKER_IMAGE}",
            # Lokalise command
            "lokalise2",
            "--token",
            get_lokalise_token(),
            "--project-id",
            CORE_PROJECT_ID,
            "file",
            "download",
            CORE_PROJECT_ID,
            "--original-filenames=false",
            "--replace-breaks=false",
            "--export-empty-as",
            "skip",
            "--format",
            "json",
            "--unzip-to",
            "/opt/dest",
        ]
    )
    print()

    if run.returncode != 0:
        raise ExitApp("Failed to download translations")

def save_json(filename: str, data: Union[List, Dict]):
    """Save JSON data to a file.

    Returns True on success.
    """
    data = json.dumps(data, sort_keys=True, indent=4)
    with open(filename, "w", encoding="utf-8") as fdesc:
        fdesc.write(data)
        return True
    return False

def get_component_path(lang, component):
    """Get the component translation path."""
    if os.path.isdir(os.path.join("homeassistant", "components", component)):
        return os.path.join(
            "homeassistant", "components", component, "translations", f"{lang}.json"
        )
    raise ExitApp(f"Integration {component} not found under homeassistant/components/")

def get_platform_path(lang, component, platform):
    """Get the platform translation path."""
    if os.path.isdir(os.path.join("homeassistant", "components", component, platform)):
        return os.path.join(
            "homeassistant",
            "components",
            component,
            platform,
            "translations",
            f"{lang}.json",
        )
    else:
        return os.path.join(
            "homeassistant",
            "components",
            component,
            "translations",
            f"{platform}.{lang}.json",
        )

def get_component_translations(translations):
    """Get the component level translations."""
    translations = translations.copy()
    translations.pop("platform", None)

    return translations

def save_language_translations(lang, translations):
    """Distribute the translations for this language."""
    components = translations.get("component", {})
    for component, component_translations in components.items():
        base_translations = get_component_translations(component_translations)
        if base_translations:
            path = get_component_path(lang, component)
            os.makedirs(os.path.dirname(path), exist_ok=True)
            save_json(path, base_translations)

        for platform, platform_translations in component_translations.get(
            "platform", {}
        ).items():
            path = get_platform_path(lang, component, platform)
            os.makedirs(os.path.dirname(path), exist_ok=True)
            save_json(path, platform_translations)

def write_integration_translations():
    """Write integration translations."""
    for lang_file in DOWNLOAD_DIR.glob("*.json"):
        lang = lang_file.stem
        translations = json.loads(lang_file.read_text())
        save_language_translations(lang, translations)

def run():
    """Run the script."""
    DOWNLOAD_DIR.mkdir(parents=True, exist_ok=True)

    run_download_docker()

    write_integration_translations()

    return 0
<commit_msg>Fix translations download (#36770)<commit_after>#!/usr/bin/env python3
"""Merge all translation sources into a single JSON file."""
import json
import os
import pathlib
import re
import subprocess
from typing import Dict, List, Union

from .const import CLI_2_DOCKER_IMAGE, CORE_PROJECT_ID, INTEGRATIONS_DIR
from .error import ExitApp
from .util import get_lokalise_token

FILENAME_FORMAT = re.compile(r"strings\.(?P<suffix>\w+)\.json")
DOWNLOAD_DIR = pathlib.Path("build/translations-download").absolute()

def run_download_docker():
    """Run the Docker image to download the translations."""
    print("Running Docker to download latest translations.")
    run = subprocess.run(
        [
            "docker",
            "run",
            "-v",
            f"{DOWNLOAD_DIR}:/opt/dest/locale",
            "--rm",
            f"lokalise/lokalise-cli-2:{CLI_2_DOCKER_IMAGE}",
            # Lokalise command
            "lokalise2",
            "--token",
            get_lokalise_token(),
            "--project-id",
            CORE_PROJECT_ID,
            "file",
            "download",
            CORE_PROJECT_ID,
            "--original-filenames=false",
            "--replace-breaks=false",
            "--export-empty-as",
            "skip",
            "--format",
            "json",
            "--unzip-to",
            "/opt/dest",
        ]
    )
    print()

    if run.returncode != 0:
        raise ExitApp("Failed to download translations")

def save_json(filename: str, data: Union[List, Dict]):
    """Save JSON data to a file.

    Returns True on success.
    """
    data = json.dumps(data, sort_keys=True, indent=4)
    with open(filename, "w", encoding="utf-8") as fdesc:
        fdesc.write(data)
        return True
    return False

def get_component_path(lang, component):
    """Get the component translation path."""
    if os.path.isdir(os.path.join("homeassistant", "components", component)):
        return os.path.join(
            "homeassistant", "components", component, "translations", f"{lang}.json"
        )
    raise ExitApp(f"Integration {component} not found under homeassistant/components/")

def get_platform_path(lang, component, platform):
    """Get the platform translation path."""
    return os.path.join(
        "homeassistant",
        "components",
        component,
        "translations",
        f"{platform}.{lang}.json",
    )

def get_component_translations(translations):
    """Get the component level translations."""
    translations = translations.copy()
    translations.pop("platform", None)

    return translations

def save_language_translations(lang, translations):
    """Distribute the translations for this language."""
    components = translations.get("component", {})
    for component, component_translations in components.items():
        base_translations = get_component_translations(component_translations)
        if base_translations:
            path = get_component_path(lang, component)
            os.makedirs(os.path.dirname(path), exist_ok=True)
            save_json(path, base_translations)

        if "platform" not in component_translations:
            continue

        for platform, platform_translations in component_translations[
            "platform"
        ].items():
            path = get_platform_path(lang, component, platform)
            os.makedirs(os.path.dirname(path), exist_ok=True)
            save_json(path, platform_translations)

def write_integration_translations():
    """Write integration translations."""
    for lang_file in DOWNLOAD_DIR.glob("*.json"):
        lang = lang_file.stem
        translations = json.loads(lang_file.read_text())
        save_language_translations(lang, translations)

def delete_old_translations():
    """Delete old translations."""
    for fil in INTEGRATIONS_DIR.glob("*/translations/*"):
        fil.unlink()

def run():
    """Run the script."""
    DOWNLOAD_DIR.mkdir(parents=True, exist_ok=True)

    run_download_docker()

    delete_old_translations()

    write_integration_translations()

    return 0
<|endoftext|>


<commit_before>import atexit
import hashlib
import inspect
import os
import signal
import sys
import subprocess
import time
from typing import List, Callable, Any, Dict, Union

from mediawords.util.log import create_logger
from mediawords.util.paths import lock_file, McLockFileException, unlock_file, McUnlockFileException
from mediawords.util.perl import decode_object_from_bytes_if_needed

log = create_logger(__name__)

def process_with_pid_is_running(pid: int) -> bool:
    """Return True if process with PID is running."""
    try:
        os.kill(pid, 0)
    except OSError:
        return False
    else:
        return True

class McRunCommandInForegroundException(subprocess.SubprocessError):
    pass

def run_command_in_foreground(command: List[str], env: Union[Dict[str, str], None] = None, cwd: str = None) -> None:
    """Run command in foreground, raise McRunCommandInForegroundException if it fails."""
    log.debug("Running command: %s" % ' '.join(command))

    if len(command) == 0:
        raise McRunCommandInForegroundException('Command is empty.')

    command = decode_object_from_bytes_if_needed(command)

    # Add some more PATHs to look into
    process_env = os.environ.copy()
    process_env['PATH'] = '/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin:' + process_env['PATH']

    if env is not None:
        process_env.update(env)

    # noinspection PyBroadException
    try:
        if sys.platform.lower() == 'darwin':
            # OS X -- requires some crazy STDOUT / STDERR buffering
            line_buffered = 1
            process = subprocess.Popen(command,
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.STDOUT,
                                       bufsize=line_buffered,
                                       env=process_env,
                                       cwd=cwd)
            while True:
                output = process.stdout.readline()
                if len(output) == 0 and process.poll() is not None:
                    break
                log.info(output.strip())
            rc = process.poll()
            if rc > 0:
                raise McRunCommandInForegroundException("Process returned non-zero exit code %d" % rc)
        else:
            # assume Ubuntu
            subprocess.check_call(command, env=process_env, cwd=cwd)
    except subprocess.CalledProcessError as ex:
        raise McRunCommandInForegroundException("Process returned non-zero exit code %d" % ex.returncode)
    except Exception as ex:
        raise McRunCommandInForegroundException("Error while running command: %s" % str(ex))

class McGracefullyKillChildProcessException(Exception):
    pass

def gracefully_kill_child_process(child_pid: int, sigkill_timeout: int = 60) -> None:
    """Try to kill child process gracefully with SIGKILL, then abruptly with SIGTERM."""
    if child_pid is None:
        raise McGracefullyKillChildProcessException("Child PID is unset.")

    if not process_with_pid_is_running(pid=child_pid):
        log.warning("Child process with PID %d is not running, maybe it's dead already?" % child_pid)
    else:
        log.info("Sending SIGKILL to child process with PID %d..." % child_pid)

        try:
            os.kill(child_pid, signal.SIGKILL)
        except OSError as e:
            # Might be already killed
            log.warning("Unable to send SIGKILL to child PID %d: %s" % (child_pid, str(e)))

        for retry in range(sigkill_timeout):
            if process_with_pid_is_running(pid=child_pid):
                log.info("Child with PID %d is still up (retry %d)." % (child_pid, retry))
                time.sleep(1)
            else:
                break

        if process_with_pid_is_running(pid=child_pid):
            log.warning("SIGKILL didn't work child process with PID %d, sending SIGTERM..." % child_pid)

            try:
                os.kill(child_pid, signal.SIGTERM)
            except OSError as e:
                # Might be already killed
                log.warning("Unable to send SIGTERM to child PID %d: %s" % (child_pid, str(e)))

            time.sleep(3)

        if process_with_pid_is_running(pid=child_pid):
            log.warning("Even SIGKILL didn't do anything, kill child process with PID %d manually!" % child_pid)

class McRunAloneException(Exception):
    """Exception on run_alone()."""
    pass

class McUnableToDetermineCaller(McRunAloneException):
    """Exception thrown when run_alone() can not determine caller."""
    pass

class McScriptInstanceIsAlreadyRunning(McRunAloneException):
    """Exception thrown when another instance of the caller script is already running."""
    pass

__run_alone_function_lock_file = None

def run_alone(isolated_function: Callable, *args, **kwargs) -> Any:
    """Run function while making sure that only a single instance of it is running."""

    global __run_alone_function_lock_file

    def __function_unique_id(func: Callable) -> str:
        """Return unique signature of the function, consisting of its absolute path and a name.

        Retry locking for 5 seconds before giving up."""

        function_module = inspect.getmodule(func)
        function_signature = inspect.signature(func)

        module_path = function_module.__file__
        if not os.path.isfile(module_path):
            raise Exception("Module '%s' for function '%s' does not exist." % (module_path, str(func)))

        function_name = func.__qualname__
        if function_name is None or len(function_name) == 0:
            raise Exception("Unable to determine function name: %s" % str(func))

        parameters_type = str(function_signature.parameters)
        return_value_type = str(function_signature.return_annotation)

        unique_id = '%(module_path)s:%(function_name)s:%(parameters)s:%(return_value)s' % {
            'module_path': module_path,
            'function_name': function_name,
            'parameters': parameters_type,
            'return_value': return_value_type,
        }
        return unique_id

    # noinspection PyUnusedLocal
    def __remove_run_alone_lock_file(signum: int = None, frame: int = None) -> None:
        global __run_alone_function_lock_file

        if __run_alone_function_lock_file is not None:
            log.info("Caught SIGTERM, unlocking '%s'..." % __run_alone_function_lock_file)
            try:
                unlock_file(__run_alone_function_lock_file)
            except McUnlockFileException as exception:
                # Not critical, the lock file might have been removed by some other process
                log.warning("Unlocking file failed: %s" % str(exception))
        else:
            log.debug("Nothing to unlock.")

        sys.exit(signum)

    try:
        function_unique_id = __function_unique_id(isolated_function)
    except Exception as ex:
        raise McUnableToDetermineCaller("Unable to determine caller script: %s" % str(ex))

    timeout = 5

    log.debug("Function unique ID: %s" % function_unique_id)

    function_unique_id_hash = hashlib.sha256(bytes(function_unique_id, 'utf-8')).hexdigest()
    log.debug("Function unique ID hash: %s" % function_unique_id_hash)

    # Catch SIGINTs and SIGTERMs while running the function to be able to remove lock file afterwards
    original_sigint_handler = signal.getsignal(signal.SIGINT)
    original_sigterm_handler = signal.getsignal(signal.SIGTERM)
    signal.signal(signal.SIGINT, __remove_run_alone_lock_file)
    signal.signal(signal.SIGTERM, __remove_run_alone_lock_file)
    atexit.register(__remove_run_alone_lock_file)

    if sys.platform.lower() == 'darwin':
        # OS X -- /var/run is not world-writable by default
        lock_file_path = '/var/tmp'
    else:
        # Linux -- keep lock files in '/var/run/lock' as they will be removed after reboot
        lock_file_path = '/var/run/lock'

    if not os.path.exists(lock_file_path):
        raise McRunAloneException(
            'Lock file location "%s" does not exist.' % lock_file_path
        )
    if not os.access(lock_file_path, os.W_OK):
        raise McRunAloneException(
            'Lock file location "%s" exists but is not writable.' % lock_file_path
        )

    function_lock_file = os.path.join(lock_file_path, function_unique_id_hash)

    try:
        lock_file(path=function_lock_file, timeout=timeout)
        __run_alone_function_lock_file = function_lock_file
    except McLockFileException as ex:
        raise McScriptInstanceIsAlreadyRunning(
            "Instance of %s is already running: %s" % (str(isolated_function), str(ex))
        )

    # noinspection PyCallingNonCallable
    return_value = isolated_function(*args, **kwargs)

    try:
        unlock_file(__run_alone_function_lock_file)
    except McUnlockFileException as exc:
        # Not critical, the lock file might have been removed by some other process
        log.warning("Unlocking file failed: %s" % str(exc))

    # Reset signal handlers
    atexit.unregister(__remove_run_alone_lock_file)
    signal.signal(signal.SIGINT, original_sigint_handler)
    signal.signal(signal.SIGTERM, original_sigterm_handler)

    __run_alone_function_lock_file = None

    return return_value
<commit_msg>Exit without exception in run_alone<commit_after>import atexit
import hashlib
import inspect
import os
import signal
import sys
import subprocess
import time
from typing import List, Callable, Any, Dict, Union

from mediawords.util.log import create_logger
from mediawords.util.paths import lock_file, McLockFileException, unlock_file, McUnlockFileException
from mediawords.util.perl import decode_object_from_bytes_if_needed

log = create_logger(__name__)

def process_with_pid_is_running(pid: int) -> bool:
    """Return True if process with PID is running."""
    try:
        os.kill(pid, 0)
    except OSError:
        return False
    else:
        return True

class McRunCommandInForegroundException(subprocess.SubprocessError):
    pass

def run_command_in_foreground(command: List[str], env: Union[Dict[str, str], None] = None, cwd: str = None) -> None:
    """Run command in foreground, raise McRunCommandInForegroundException if it fails."""
    log.debug("Running command: %s" % ' '.join(command))

    if len(command) == 0:
        raise McRunCommandInForegroundException('Command is empty.')

    command = decode_object_from_bytes_if_needed(command)

    # Add some more PATHs to look into
    process_env = os.environ.copy()
    process_env['PATH'] = '/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin:' + process_env['PATH']

    if env is not None:
        process_env.update(env)

    # noinspection PyBroadException
    try:
        if sys.platform.lower() == 'darwin':
            # OS X -- requires some crazy STDOUT / STDERR buffering
            line_buffered = 1
            process = subprocess.Popen(command,
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.STDOUT,
                                       bufsize=line_buffered,
                                       env=process_env,
                                       cwd=cwd)
            while True:
                output = process.stdout.readline()
                if len(output) == 0 and process.poll() is not None:
                    break
                log.info(output.strip())
            rc = process.poll()
            if rc > 0:
                raise McRunCommandInForegroundException("Process returned non-zero exit code %d" % rc)
        else:
            # assume Ubuntu
            subprocess.check_call(command, env=process_env, cwd=cwd)
    except subprocess.CalledProcessError as ex:
        raise McRunCommandInForegroundException("Process returned non-zero exit code %d" % ex.returncode)
    except Exception as ex:
        raise McRunCommandInForegroundException("Error while running command: %s" % str(ex))

class McGracefullyKillChildProcessException(Exception):
    pass

def gracefully_kill_child_process(child_pid: int, sigkill_timeout: int = 60) -> None:
    """Try to kill child process gracefully with SIGKILL, then abruptly with SIGTERM."""
    if child_pid is None:
        raise McGracefullyKillChildProcessException("Child PID is unset.")

    if not process_with_pid_is_running(pid=child_pid):
        log.warning("Child process with PID %d is not running, maybe it's dead already?" % child_pid)
    else:
        log.info("Sending SIGKILL to child process with PID %d..." % child_pid)

        try:
            os.kill(child_pid, signal.SIGKILL)
        except OSError as e:
            # Might be already killed
            log.warning("Unable to send SIGKILL to child PID %d: %s" % (child_pid, str(e)))

        for retry in range(sigkill_timeout):
            if process_with_pid_is_running(pid=child_pid):
                log.info("Child with PID %d is still up (retry %d)." % (child_pid, retry))
                time.sleep(1)
            else:
                break

        if process_with_pid_is_running(pid=child_pid):
            log.warning("SIGKILL didn't work child process with PID %d, sending SIGTERM..." % child_pid)

            try:
                os.kill(child_pid, signal.SIGTERM)
            except OSError as e:
                # Might be already killed
                log.warning("Unable to send SIGTERM to child PID %d: %s" % (child_pid, str(e)))

            time.sleep(3)

        if process_with_pid_is_running(pid=child_pid):
            log.warning("Even SIGKILL didn't do anything, kill child process with PID %d manually!" % child_pid)

class McRunAloneException(Exception):
    """Exception on run_alone()."""
    pass

class McUnableToDetermineCaller(McRunAloneException):
    """Exception thrown when run_alone() can not determine caller."""
    pass

class McScriptInstanceIsAlreadyRunning(McRunAloneException):
    """Exception thrown when another instance of the caller script is already running."""
    pass

__run_alone_function_lock_file = None

def run_alone(isolated_function: Callable, *args, **kwargs) -> Any:
    """Run function while making sure that only a single instance of it is running."""

    global __run_alone_function_lock_file

    def __function_unique_id(func: Callable) -> str:
        """Return unique signature of the function, consisting of its absolute path and a name.

        Retry locking for 5 seconds before giving up."""

        function_module = inspect.getmodule(func)
        function_signature = inspect.signature(func)

        module_path = function_module.__file__
        if not os.path.isfile(module_path):
            raise Exception("Module '%s' for function '%s' does not exist." % (module_path, str(func)))

        function_name = func.__qualname__
        if function_name is None or len(function_name) == 0:
            raise Exception("Unable to determine function name: %s" % str(func))

        parameters_type = str(function_signature.parameters)
        return_value_type = str(function_signature.return_annotation)

        unique_id = '%(module_path)s:%(function_name)s:%(parameters)s:%(return_value)s' % {
            'module_path': module_path,
            'function_name': function_name,
            'parameters': parameters_type,
            'return_value': return_value_type,
        }
        return unique_id

    # noinspection PyUnusedLocal
    def __remove_run_alone_lock_file(signum: int = None,
                                     frame: int = None,
                                     no_exception: bool = False) -> None:
        global __run_alone_function_lock_file

        if __run_alone_function_lock_file is not None:
            log.info("Caught SIGTERM, unlocking '%s'..." % __run_alone_function_lock_file)
            try:
                unlock_file(__run_alone_function_lock_file)
            except McUnlockFileException as exception:
                # Not critical, the lock file might have been removed by some other process
                log.warning("Unlocking file failed: %s" % str(exception))
        else:
            log.debug("Nothing to unlock.")

        if no_exception:
            os._exit(signum)
        else:
            sys.exit(signum)

    try:
        function_unique_id = __function_unique_id(isolated_function)
    except Exception as ex:
        raise McUnableToDetermineCaller("Unable to determine caller script: %s" % str(ex))

    timeout = 5

    log.debug("Function unique ID: %s" % function_unique_id)

    function_unique_id_hash = hashlib.sha256(bytes(function_unique_id, 'utf-8')).hexdigest()
    log.debug("Function unique ID hash: %s" % function_unique_id_hash)

    # Catch SIGINTs and SIGTERMs while running the function to be able to remove lock file afterwards
    original_sigint_handler = signal.getsignal(signal.SIGINT)
    original_sigterm_handler = signal.getsignal(signal.SIGTERM)
    signal.signal(signal.SIGINT, __remove_run_alone_lock_file)
    signal.signal(signal.SIGTERM, __remove_run_alone_lock_file)
    atexit.register(__remove_run_alone_lock_file, signum=0, no_exception=True)

    if sys.platform.lower() == 'darwin':
        # OS X -- /var/run is not world-writable by default
        lock_file_path = '/var/tmp'
    else:
        # Linux -- keep lock files in '/var/run/lock' as they will be removed after reboot
        lock_file_path = '/var/run/lock'

    if not os.path.exists(lock_file_path):
        raise McRunAloneException(
            'Lock file location "%s" does not exist.' % lock_file_path
        )
    if not os.access(lock_file_path, os.W_OK):
        raise McRunAloneException(
            'Lock file location "%s" exists but is not writable.' % lock_file_path
        )

    function_lock_file = os.path.join(lock_file_path, function_unique_id_hash)

    try:
        lock_file(path=function_lock_file, timeout=timeout)
        __run_alone_function_lock_file = function_lock_file
    except McLockFileException as ex:
        raise McScriptInstanceIsAlreadyRunning(
            "Instance of %s is already running: %s" % (str(isolated_function), str(ex))
        )

    # noinspection PyCallingNonCallable
    return_value = isolated_function(*args, **kwargs)

    try:
        unlock_file(__run_alone_function_lock_file)
    except McUnlockFileException as exc:
        # Not critical, the lock file might have been removed by some other process
        log.warning("Unlocking file failed: %s" % str(exc))

    # Reset signal handlers
    atexit.unregister(__remove_run_alone_lock_file)
    signal.signal(signal.SIGINT, original_sigint_handler)
    signal.signal(signal.SIGTERM, original_sigterm_handler)

    __run_alone_function_lock_file = None

    return return_value
<|endoftext|>


<commit_before>import discord
import aiohttp
import json
from __main__ import send_cmd_help
from .utils import chat_formatting as chat
from discord.ext import commands

BASEURL = 'http://dnd5eapi.co/api/'

class DND:
    '''D&D Lookup Stuff'''

    def __init__(self, bot):
        self.bot = bot

    @commands.group(pass_context=True)
    async def dnd(self, ctx):
        if ctx.invoked_subcommand is None:
            await send_cmd_help(ctx)
            return

    @dnd.command(name='spells')
    async def lookup_spells(self, spell=None):
        '''Lookup Spells'''
        if spell is not None:
            print('initiate name query')
        url = '{}{}'.format(BASEURL, 'spells')
        print(url)
        await self.bot.say('URL lookup: '+url)
        await _present_list(self, url)
        #Your code will go here
        await self.bot.say("Lookup Spells initiated.")

            # em=discord.Embed(color=discord.Color.red(),title='Spells',description='{} found'.format(count))
            # em.add_field(name='Name',value='\n'.join(r['name'] for r in results))
            # await self.bot.say(embed=em)

    @dnd.command(name='classes')
    async def lookup_classes(self, klass=None):
        '''Lookup Classes'''
        baseurl = BASEURL+'classes'
        #Your code will go here
        await self.bot.say("Lookup Classes initiated.")
        await self.bot.say("<{}>".format(baseurl))

    @dnd.command(name='monsters')
    async def lookup_monsters(self, monster=None):
        '''Lookup Monsters'''
        baseurl = BASEURL+'monsters'
        #Your code will go here
        await self.bot.say("Lookup Monsters initiated.")
        await self.bot.say("<{}>".format(baseurl))

    @dnd.command(name='equipment')
    async def lookup_equipment(self, equiped=None):
        '''Lookup Equpiment'''
        baseurl = BASEURL+'equipment'
        #Your code will go here
        await self.bot.say("Lookup Spells initiated.")
        await self.bot.say("<{}>".format(baseurl))

async def _get_file(url):
    async with aiohttp.ClientSession() as session:
        async with session.get(url) as response:
            print('_get_file('+url+')')
            json_file = await response.json()
            if json_file is not None:
                return json_file

async def _present_list(self, url):
    json_file = await _get_file(url)
    urllength = len(url)
    print(urllength)
    await self.bot.say('{}'.format(urllength))
    if json_file is not None:
        count=json_file['count']
        results = json_file['results']
        await self.bot.say('count: {}'.format(count))
        package = []
        for i, r in enumerate(results):
            package.append('{} {}'.format(i, r['name']))

        pages = pagify(package, delims=['\n'])
        for page in pages:
            await self.bot.say(chat.box(page))
    return

def setup(bot):
    bot.add_cog(DND(bot))
<commit_msg><commit_after>import discord
import aiohttp
import json
from __main__ import send_cmd_help
from .utils import chat_formatting as chat
from discord.ext import commands

BASEURL = 'http://dnd5eapi.co/api/'

class DND:
    '''D&D Lookup Stuff'''

    def __init__(self, bot):
        self.bot = bot

    @commands.group(pass_context=True)
    async def dnd(self, ctx):
        if ctx.invoked_subcommand is None:
            await send_cmd_help(ctx)
            return

    @dnd.command(name='spells')
    async def lookup_spells(self, spell=None):
        '''Lookup Spells'''
        if spell is not None:
            print('initiate name query')
        url = '{}{}'.format(BASEURL, 'spells')
        print(url)
        await self.bot.say('URL lookup: '+url)
        await _present_list(self, url)
        #Your code will go here
        await self.bot.say("Lookup Spells initiated.")

            # em=discord.Embed(color=discord.Color.red(),title='Spells',description='{} found'.format(count))
            # em.add_field(name='Name',value='\n'.join(r['name'] for r in results))
            # await self.bot.say(embed=em)

    @dnd.command(name='classes')
    async def lookup_classes(self, klass=None):
        '''Lookup Classes'''
        baseurl = BASEURL+'classes'
        #Your code will go here
        await self.bot.say("Lookup Classes initiated.")
        await self.bot.say("<{}>".format(baseurl))

    @dnd.command(name='monsters')
    async def lookup_monsters(self, monster=None):
        '''Lookup Monsters'''
        baseurl = BASEURL+'monsters'
        #Your code will go here
        await self.bot.say("Lookup Monsters initiated.")
        await self.bot.say("<{}>".format(baseurl))

    @dnd.command(name='equipment')
    async def lookup_equipment(self, equiped=None):
        '''Lookup Equpiment'''
        baseurl = BASEURL+'equipment'
        #Your code will go here
        await self.bot.say("Lookup Spells initiated.")
        await self.bot.say("<{}>".format(baseurl))

async def _get_file(url):
    async with aiohttp.ClientSession() as session:
        async with session.get(url) as response:
            print('_get_file('+url+')')
            json_file = await response.json()
            if json_file is not None:
                return json_file

async def _present_list(self, url):
    json_file = await _get_file(url)
    urllength = len(url)
    print(urllength)
    await self.bot.say('{}'.format(urllength))
    if json_file is not None:
        count=json_file['count']
        results = json_file['results']
        await self.bot.say('count: {}'.format(count))
        package = []
        for i, r in enumerate(results):
            package.append('{} {}'.format(i, r['name']))

        pages = chat.pagify(package, delims=['\n'])
        for page in pages:
            await self.bot.say(chat.box(page))
    return

def setup(bot):
    bot.add_cog(DND(bot))
<|endoftext|>


<commit_before>__author__ = 'maroun'

import os

from nose.plugins import Plugin

class DocString(Plugin):
    """
    This plugin enables you to display attributes next to the original
    docstring.

    Usage examples:
        > python main.py --with-docstring --prefix=platform,id --suffix=type
        > python main.py --with-docstring --replace=a,A --suffix=id,section
    """

    args = dict()

    def describeTest(self, running_test):

        # meta information about running test
        self.test = running_test.test.__dict__['test']

        prefix = self._get_affix('prefix')
        suffix = self._get_affix('suffix')

        docstring = self._get_docstring()

        return '({}) {} ({})'.format(prefix, docstring, suffix)

    def options(self, parser, env=os.environ):
        super(DocString, self).options(parser, env)

        parser.add_option(
            '--prefix', default='id,platform',
            help='Append to this flag list of attributes you want to be printed'
                 'before the original docstring, separated by COMMA')
        parser.add_option(
            '--suffix', default='section,type,module',
            help='Append to this flag list of attributes you want to be printed'
                 'after the original docstring, separated by COMMA')
        parser.add_option(
            '--replace',
            help='Replace characters in original docstring, for example:'
                 '--replace(a, A)'
        )

    def configure(self, options, conf):
        super(DocString, self).configure(options, conf)

        self.args['prefix'] = options.prefix.split(',')
        self.args['suffix'] = options.suffix.split(',')
        if options.replace:
            self.args['replace'] = options.replace.split(',')

    def _get_affix(self, affix_type):
        """
        Returns list containing affixes that should be appendind to docstring.
        :param affix_type: 'suffix' or 'prefix'
        :type affix_type: str
        :param keywords: contains meta information about running test
        :return: a list containing wanted affixes depending on 'affix_type'
        """
        affix = list()

        for key in self.args[affix_type]:
            affix.append(self.test.keywords.get(key, ''))

        # remove empty strings resulted from unexisting keys
        affix = filter(len, map(str, affix))
        return ', '.join(affix)

    def _get_docstring(self):
        """
        :return: modified docstring if --replace is toggled, original otherwise.
        """
        docstring = self.test.__doc__
        if 'replace' in self.args.keys() and len(self.args['replace']) == 2:
            return docstring.replace(self.args['replace'][0],
                                     self.args['replace'][1])
        return docstring<commit_msg>Made a class member a variablea<commit_after>__author__ = 'maroun'

import os

from nose.plugins import Plugin

class DocString(Plugin):
    """
    This plugin enables you to display attributes next to the original
    docstring.

    Usage examples:
        > python main.py --with-docstring --prefix=platform,id --suffix=type
        > python main.py --with-docstring --replace=a,A --suffix=id,section
    """

    args = dict()

    def describeTest(self, running_test):

        # meta information about running test
        test = running_test.test.__dict__['test']

        prefix = self._get_affix('prefix', test)
        suffix = self._get_affix('suffix', test)

        docstring = self._get_docstring(test)

        return '({}) {} ({})'.format(prefix, docstring, suffix)

    def options(self, parser, env=os.environ):
        super(DocString, self).options(parser, env)

        parser.add_option(
            '--prefix', default='id,platform',
            help='Append to this flag list of attributes you want to be printed'
                 'before the original docstring, separated by COMMA')
        parser.add_option(
            '--suffix', default='section,type,module',
            help='Append to this flag list of attributes you want to be printed'
                 'after the original docstring, separated by COMMA')
        parser.add_option(
            '--replace',
            help='Replace characters in original docstring, for example:'
                 '--replace(a, A)'
        )

    def configure(self, options, conf):
        super(DocString, self).configure(options, conf)

        self.args['prefix'] = options.prefix.split(',')
        self.args['suffix'] = options.suffix.split(',')
        if options.replace:
            self.args['replace'] = options.replace.split(',')

    def _get_affix(self, affix_type, running_test):
        """
        Returns list containing affixes that should be appendind to docstring.
        :param affix_type: 'suffix' or 'prefix'
        :type affix_type: str
        :param running_test: contains meta information about the current test
        :return: a list containing wanted affixes depending on 'affix_type'
        """
        affix = list()

        for key in self.args[affix_type]:
            affix.append(running_test.keywords.get(key, ''))

        # remove empty strings resulted from unexisting keys
        affix = filter(len, map(str, affix))
        return ', '.join(affix)

    def _get_docstring(self, running_test):
        """
        :return: modified docstring if --replace is toggled, original otherwise.
        """
        docstring = running_test.__doc__
        if 'replace' in self.args.keys() and len(self.args['replace']) == 2:
            return docstring.replace(self.args['replace'][0],
                                     self.args['replace'][1])
        return docstring<|endoftext|>


<commit_before>import os

from mock.mock import patch

from data import pickle_cache
from spec.mamba import *

pickle_patch = patch('data.pickle_cache.pickle')
pickle_path_patch = patch.object(pickle_cache, '_PICKLE_PATH', 'patched')
open_patch = patch.object(pickle_cache, '_open_pkl_path')
os_patch = patch('data.pickle_cache.os')

with description('pickle_cache'):
  with before.each:
    self.patched_pickle = pickle_patch.start()
    pickle_path_patch.start()
    self.patched_open = open_patch.start()
    original_join = os.path.join
    original_dirname = os.path.dirname
    self.patched_os = os_patch.start()
    self.patched_os.path.join.side_effect = original_join
    self.patched_os.path.dirname.side_effect = original_dirname

  with after.each:
    pickle_patch.stop()
    pickle_path_patch.stop()
    open_patch.stop()
    os_patch.stop()
    self.patched_os.stop()

  with description('_sanitize'):
    with it('sanitizes empty input'):
      expect(pickle_cache._sanitize('')).to(equal('_empty_str'))

    with it('sanitizes bool'):
      expect(pickle_cache._sanitize(True)).to(equal('True'))
      expect(pickle_cache._sanitize(False)).to(equal('False'))

    with it('sanitizes hazardous text'):
      expect(pickle_cache._sanitize('../../../')).to(equal('_306eec36b'))

    with it('sanitizes hazardous text uniquely'):
      expect(pickle_cache._sanitize('../../../')).not_to(equal(
          pickle_cache._sanitize('../../../../')))

  with description('_path_for_prefix_and_args'):
    with it('runs on empty input'):
      expect(pickle_cache._path_for_prefix_and_args('pre', [], {})).to(equal(
          'patched/pre/_.pkl'
      ))

    with it('runs for simple input'):
      expect(pickle_cache._path_for_prefix_and_args(
          'pre', ['a1', 'a2'], {'zzz': 'z', 'aaa': 'a', 'ccc': 'c'})).to(equal(
          'patched/pre/a1,a2/aaa-a,ccc-c,zzz-z.pkl'
      ))

  with description('cache'):
    with before.all:
      @pickle_cache.cache('prefix')
      def fn(*args, **kwargs):
        return args, kwargs

      self.fn = fn

    with it('creates a directory if none present'):
      self.patched_os.path.exists.return_value = False
      self.fn()
      expect(self.patched_os.makedirs).to(have_been_called_with(
          'patched/prefix', exist_ok=True))

    with it('loads pkl if present'):
      self.patched_os.path.exists.return_value = True
      self.patched_open.return_value = '<opened pkl>'
      self.fn()
      expect(self.patched_open).to(have_been_called_with(
          'patched/prefix/_.pkl', 'rb'))
      expect(self.patched_pickle.load).to(have_been_called_with('<opened pkl>'))

    with it('creates pkl if not present'):
      self.patched_os.path.exists.return_value = False
      self.patched_open.return_value = '<opened pkl>'
      self.fn(1, key='value')
      expect(self.patched_open).to(have_been_called_with(
          'patched/prefix/1/key-value.pkl', 'wb'))
      expect(self.patched_pickle.dump).to(have_been_called_with(
          ((1,), {'key': 'value'}), '<opened pkl>'))

  with description('cache_from_file'):
    with before.each:
      self.pkl_time = 0
      self.src_time = 0
      def stub_getmtime(path):
        if path.endswith('.pkl'):
          return self.pkl_time
        return self.src_time

      self.patched_os.path.getmtime.side_effect = stub_getmtime

      @pickle_cache.cache_from_file('prefix')
      def fn(path):
        return path

      self.fn = fn

    with it('ignores pkl if src is newer'):
      self.pkl_time = 0
      self.src_time = 20170101
      self.fn('file.py')
      expect(self.patched_pickle.load).not_to(have_been_called)

    with it('uses pkl if pkl is newer than src'):
      self.pkl_time = 201701010
      self.src_time = 0
      self.fn('file.py')
      expect(self.patched_pickle.load).to(have_been_called)
<commit_msg>Update test for pickle protocol change.<commit_after>import os

from mock.mock import patch

from data import pickle_cache
from spec.mamba import *

pickle_patch = patch('data.pickle_cache.pickle')
pickle_path_patch = patch.object(pickle_cache, '_PICKLE_PATH', 'patched')
open_patch = patch.object(pickle_cache, '_open_pkl_path')
os_patch = patch('data.pickle_cache.os')

with description('pickle_cache'):
  with before.each:
    self.patched_pickle = pickle_patch.start()
    pickle_path_patch.start()
    self.patched_open = open_patch.start()
    original_join = os.path.join
    original_dirname = os.path.dirname
    self.patched_os = os_patch.start()
    self.patched_os.path.join.side_effect = original_join
    self.patched_os.path.dirname.side_effect = original_dirname

  with after.each:
    pickle_patch.stop()
    pickle_path_patch.stop()
    open_patch.stop()
    os_patch.stop()
    self.patched_os.stop()

  with description('_sanitize'):
    with it('sanitizes empty input'):
      expect(pickle_cache._sanitize('')).to(equal('_empty_str'))

    with it('sanitizes bool'):
      expect(pickle_cache._sanitize(True)).to(equal('True'))
      expect(pickle_cache._sanitize(False)).to(equal('False'))

    with it('sanitizes hazardous text'):
      expect(pickle_cache._sanitize('../../../')).to(equal('_306eec36b'))

    with it('sanitizes hazardous text uniquely'):
      expect(pickle_cache._sanitize('../../../')).not_to(equal(
          pickle_cache._sanitize('../../../../')))

  with description('_path_for_prefix_and_args'):
    with it('runs on empty input'):
      expect(pickle_cache._path_for_prefix_and_args('pre', [], {})).to(equal(
          'patched/pre/_.pkl'
      ))

    with it('runs for simple input'):
      expect(pickle_cache._path_for_prefix_and_args(
          'pre', ['a1', 'a2'], {'zzz': 'z', 'aaa': 'a', 'ccc': 'c'})).to(equal(
          'patched/pre/a1,a2/aaa-a,ccc-c,zzz-z.pkl'
      ))

  with description('cache'):
    with before.all:
      @pickle_cache.cache('prefix')
      def fn(*args, **kwargs):
        return args, kwargs

      self.fn = fn

    with it('creates a directory if none present'):
      self.patched_os.path.exists.return_value = False
      self.fn()
      expect(self.patched_os.makedirs).to(have_been_called_with(
          'patched/prefix', exist_ok=True))

    with it('loads pkl if present'):
      self.patched_os.path.exists.return_value = True
      self.patched_open.return_value = '<opened pkl>'
      self.fn()
      expect(self.patched_open).to(have_been_called_with(
          'patched/prefix/_.pkl', 'rb'))
      expect(self.patched_pickle.load).to(have_been_called_with('<opened pkl>'))

    with it('creates pkl if not present'):
      self.patched_os.path.exists.return_value = False
      self.patched_open.return_value = '<opened pkl>'
      self.fn(1, key='value')
      expect(self.patched_open).to(have_been_called_with(
          'patched/prefix/1/key-value.pkl', 'wb'))
      expect(self.patched_pickle.dump).to(have_been_called_with(
          ((1,), {'key': 'value'}), '<opened pkl>', protocol=mock.ANY))

  with description('cache_from_file'):
    with before.each:
      self.pkl_time = 0
      self.src_time = 0
      def stub_getmtime(path):
        if path.endswith('.pkl'):
          return self.pkl_time
        return self.src_time

      self.patched_os.path.getmtime.side_effect = stub_getmtime

      @pickle_cache.cache_from_file('prefix')
      def fn(path):
        return path

      self.fn = fn

    with it('ignores pkl if src is newer'):
      self.pkl_time = 0
      self.src_time = 20170101
      self.fn('file.py')
      expect(self.patched_pickle.load).not_to(have_been_called)

    with it('uses pkl if pkl is newer than src'):
      self.pkl_time = 201701010
      self.src_time = 0
      self.fn('file.py')
      expect(self.patched_pickle.load).to(have_been_called)
<|endoftext|>