<commit_before>import numpy.matlib as np

class Posor(object):
    def __init__(self, pos, theta, phi, psi):
        self.pos = pos
        self.theta = theta
        self.phi = phi
        self.psi = psi

# Take 2 numpy arrays, xs and ys, along with the view, represented as an object
# with minx, maxx, miny, maxy, xangle, yangle, and two objects camerapos, with
# pos, theta, phi, and psi, and laserpos, with pos, theta, and phi.
# Return 3 numpy arrays, giving the corresponding points in absolute coördinates

# Laser starts out as the plane with normal y = z = 0.  Orientation is created
# in the following way: Start by looking along the positive x axis, with z up.
# Rotate theta radians clockwise along the z axis, towards positive y.  Rotate
# phi radians upwards along the new y axis, towards positive z.  Finally, rotate
# psi radians counterclockwise along the new x axis, towards positive z.

# The transformation from camera coördinates to absolute coördinates is then
# given by the matrix product:
# [ cos th -sin th 0 ] [ cos ph 0 -sin ph ] [ 1      0       0 ]
# [ sin th  cos th 0 ] [      0 1       0 ] [ 0 cos ps -sin ps ]
# [      0       0 1 ] [ sin ph 0  cos ph ] [ 0 sin ps  cos ps ]
def 3dize(xs, ys, view, cameraposor, laserposor):
    plane = calc_plane(laserposor)
    rays = calc_rays(xs, ys, view)
    rot_rays = rotate(rays, cameraposor)
    3d_points = intersect(plane, cameraposor, rot_rays)
    return 3d_points

class Plane(object):
    def __init__(self, pos, normal):
        self.pos = pos
        self.normal = normal

# Take a posor object and return another object with point x, y, z and normal
# dx, dy, dz
def calc_plane(posor):
    normal = np.mat([[1, 0, 0]], dtype=np.float).transpose()
    rot_normal = rotate(normal, posor)
    return Plane(posor.pos, rot_normal)

# points is a matrix
def rotate(points, posor):
    rot_matrix = calc_rot_matrix(posor)
    return rot_matrix * points
<commit_msg>Different view<commit_after>import numpy.matlib as np

class Posor(object):
    def __init__(self, pos, theta, phi, psi):
        self.pos = pos
        self.theta = theta
        self.phi = phi
        self.psi = psi

# Take 2 numpy arrays, xs and ys, along with the view, represented as an object
# with centerx, centery, dist, angle, and two objects camerapos, with
# pos, theta, phi, and psi, and laserpos, with pos, theta, and phi.
# Return 3 numpy arrays, giving the corresponding points in absolute coördinates

# Laser starts out as the plane with normal y = z = 0.  Orientation is created
# in the following way: Start by looking along the positive x axis, with z up.
# Rotate theta radians clockwise along the z axis, towards positive y.  Rotate
# phi radians upwards along the new y axis, towards positive z.  Finally, rotate
# psi radians counterclockwise along the new x axis, towards positive z.

# The transformation from camera coördinates to absolute coördinates is then
# given by the matrix product:
# [ cos th -sin th 0 ] [ cos ph 0 -sin ph ] [ 1      0       0 ]
# [ sin th  cos th 0 ] [      0 1       0 ] [ 0 cos ps -sin ps ]
# [      0       0 1 ] [ sin ph 0  cos ph ] [ 0 sin ps  cos ps ]
def 3dize(xs, ys, view, cameraposor, laserposor):
    plane = calc_plane(laserposor)
    rays = calc_rays(xs, ys, view)
    rot_rays = rotate(rays, cameraposor)
    3d_points = intersect(plane, cameraposor, rot_rays)
    return 3d_points

class Plane(object):
    def __init__(self, pos, normal):
        self.pos = pos
        self.normal = normal

# Take a posor object and return another object with point x, y, z and normal
# dx, dy, dz
def calc_plane(posor):
    normal = np.mat([[1, 0, 0]], dtype=np.float).transpose()
    rot_normal = rotate(normal, posor)
    return Plane(posor.pos, rot_normal)

# points is a matrix
def rotate(points, posor):
    rot_matrix = calc_rot_matrix(posor)
    return rot_matrix * points
<|endoftext|>


<commit_before>#!/usr/bin/python
# coding=utf-8
#
# 3dpct.py by Florian Schiessl (3.12.2013)
#
# 3d printer control toolchain
#
# This script provides simple communication with a reprap machine.
#
# Usage:
# use -h option ;)
#
import serial, os, sys, re, argparse, string, signal

class gparser:
    def __init__(self):
        # Matches G-Code comments
        self.re_comments = re.compile('\s*;.*$')
        # Matches empty lines
        self.re_empty = re.compile('^\s*$')
        # Matches comment-only lines and BOM
        self.re_comment_only = re.compile('(^\s*;.*$)|(\xef\xbb\xbf*)')
        self.do_checksum = True
        self.parsed = []

    def readfile(self,fname):
        fcontent = []
        if fname == "-":
            fh = sys.stdin
        else:
            fh = open(fname)
        for line in fh:
            fcontent.append(line.rstrip())
        fh.close
        self.fcontent = fcontent

    def checksum(self,line):
        cs = 0
        for i in range(0,len(line)):
            cs ^= ord(line[i]) & 0xff
        cs &= 0xff
        return str(cs)

    def csline(self,line):
        return line+"*"+self.checksum(line)

    def prepare_checksum(self):
        self.parsed = []
        return self.csline("M110 N0")

    def parse_file(self):
        if self.do_checksum:
            self.prepare_checksum()
        for line in self.fcontent:
            self.parse(line)

    def parse(self, line):
        if not self.re_empty.match(line) and not self.re_comment_only.match(line):
            line = self.re_comments.sub("",line)
            if self.do_checksum:
                line = "N"+str(len(self.parsed)+1)+" "+line
                line = self.csline(line)
            self.parsed.append(line)
            return line
        return ""

    def get_parsed(self):
        return self.parsed

    def dump_parsed(self):
        for i in self.parsed:
            print i

class dddprinter:
    def __init__(self):
        self.parser = gparser()
        # Matches Emergency Condition
        self.re_emergency = re.compile('!!')
        # Matches resend query with (optional leading n)
        self.re_resend = re.compile('(^rs [Nn]?)|(^Resend:)')
        # Matches lines which should be ignored
        self.re_ignore = re.compile('(^echo:)|(^[Mm]arlin)|(^start)|(^Error:checksum)|(^T:)')
        self.max_resends = 10
        self.resend_counter = 0
        self.currentln = 1
        self.silent = False # output Printer communication
        self.dir_mark = True # Show direction marks
        self.gcode = [""]

    def loadgcodestring(self,gcodestr):
        self.loadgcode = gcodestr.split(gcodestr, "\n")

    def loadgcode(self,gcode):
        self.gcode = [""] # For correct line Numbering
        self.gcode.extend(gcode)

    def connect(self,tty,baud):
        self.ser = serial.Serial(tty, baud, timeout=1)
        #self.cswrite("M110 N0") # Reset Line Counter

    def cswrite(self,line):
        self.write(parser.csline(line))

    def ldm(self,line):
        if self.silent: return
        if self.dir_mark: print "<~: "+line
        else: print line

    def rdm(self,line):
        if self.silent: return
        if self.dir_mark: print "~>: "+line
        else: print line

    def rprpt(self):
        if self.silent: return ""
        if self.dir_mark: return "<-: "
        else: return ""

    def write(self,line):
        self.ldm(line)
        self.ser.write(line+"\n")

    def read(self):
        result = self.ser.readline().rstrip()
        self.rdm(result)
        return result

    def read_flush(self):
        result = self.read()
        while not result == "":
            result = self.read()
        print "--- EOF ---"

    def resend(self,linenumber):
        self.write(self.gcode[linenumber])
        if self.resend_counter <= self.max_resends:
            self.waitforok()
            self.currentln = linenumber # Re-Send all after Checksum-Error
        else:
            self.panic("too many resends!")

    def waitforok(self):
        result = self.read().rstrip()
        while (len(result) == 0 or
                self.re_ignore.match(result) or
                parser.re_empty.match(result)):
            result = self.read().rstrip() # Ignore empty lines, unuseful infos, 0-lenght
        if self.re_emergency.match(result):
            # Emergency
            self.panic("received EMERGENCY CONDITION!")
        elif self.re_resend.match(result):
            # Printer queried a resend
            self.read_flush()
            result = self.re_resend.sub("",result) # removing request
            self.resend(int(result))
        elif result == "ok":
            # ok
            self.resend_counter = 0
        else:
            self.panic("UNKNOWN:"+result+":"+str(len(result)))
        return

    def _print(self):
        while self.currentln < len(self.gcode):
            self.write(self.gcode[self.currentln])
            self.waitforok()
            self.currentln += 1
        print "Print successfull :D"
        print "programm succeeded."
        sys.exit(0)

    def panic(self,message):
        print message
        print "flushing input:"
        self.read_flush()
        print "programm stopped."
        sys.exit(1)

    def start_cli(self):
        if self.parser.do_checksum:
            self.write(self.parser.prepare_checksum())
            self.waitforok()
        while(1):
            output = self.parser.parse(raw_input(self.rprpt()))
            if not output == "":
                self.write(output)
                self.waitforok()

if __name__ == "__main__":
    parser = gparser()
    printer = dddprinter()
    argp = argparse.ArgumentParser(
        description="3dpct - a 3d printer control toolchain",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    argp.add_argument(
        "-f", "--file",
        metavar="FILE",
        action="append",
        default=argparse.SUPPRESS,
        help="load gcode from given file name or use '-' for STDIN",
        nargs="?")
    argp.add_argument(
        "-o", "--output",
        action='store_true',
        default=argparse.SUPPRESS,
        help="output filtered gcode to STDOUT")
    argp.add_argument(
        "-p","--print",
        action='store_true',
        default=argparse.SUPPRESS,
        help="use 3d printer to print gcode")
    argp.add_argument(
        "-c", "--cli",
        action='store_true',
        default=argparse.SUPPRESS,
        help="start a cli for manual printer control using gcodes")
    argp.add_argument(
        "-n","--no-direction-mark",
        action='store_true',
        default=argparse.SUPPRESS,
        help="don't show direction marks and prompt (eg: <~: ,~>: ,<-: )")
    argp.add_argument(
        "-e", "--execute",
        action='store',
        default=argparse.SUPPRESS,
        nargs=argparse.REMAINDER,
        help="execute a single gcode line (must be last argument)")
    argp.add_argument(
        "--no-checksum",
        action='store_true',
        default=argparse.SUPPRESS,
        help="don't generate line numbers and checksums")
    argp.add_argument(
        "--silent",
        action="store_true",
        default=argparse.SUPPRESS,
        help="don't output communication while printing")
    argp.add_argument(
        "-d","--device",
        metavar="DEVICE",
        default="/dev/ttyUSB0",
        help="the device to use for printing",
        nargs="?")
    argp.add_argument(
        "-b","--baud",
        default=250000,
        help="the baudrate to use on device",
        nargs="?",
        type=int)
    args = vars(argp.parse_args())

    # Catching Signals
    def signal_handler(signal, frame):
        print ""
        sys.exit(0)
    signal.signal(signal.SIGINT, signal_handler)

    # parsing options
    done = False
    if 'no_checksum' in args:
        parser.do_checksum = False
        printer.parser.do_checksum = False
    if 'no_direction_mark' in args:
        printer.dir_mark = False
    if 'silent' in args:
        printer.silent = True
    if 'file' in args:
        parser.readfile(args['file'][0])
        parser.parse_file()
        if 'output' in args:
            parser.dump_parsed()
            done = True
        if 'print' in args:
            print "Using device "+args['device']+" with "+str(args['baud'])+" Baud."
            printer.connect(args['device'], args['baud'])
            if printer.parser.do_checksum:
                printer.write(printer.parser.prepare_checksum())
                printer.waitforok()
            printer.loadgcode(parser.get_parsed())
            printer._print()
            done = True
    elif 'cli' in args:
        printer.connect(args['device'], args['baud'])
        printer.start_cli()
        done = True
    elif 'execute' in args:
        printer.connect(args['device'], args['baud'])
        if printer.parser.do_checksum:
            printer.write(printer.parser.prepare_checksum())
            printer.waitforok()
        printer.write(printer.parser.parse(" ".join(args['execute'])))
        printer.waitforok()
        done = True

    if not done:
        argp.print_help()
<commit_msg>Fixes for issue #1. New style classes. Format changes.<commit_after>#!/usr/bin/python
# coding=utf-8
#
# 3dpct.py by Florian Schiessl (3.12.2013)
#
# 3d printer control toolchain
#
# This script provides simple communication with a reprap machine.
#
# Usage:
# use -h option ;)
#
import serial, os, sys, re, argparse, string, signal

class gparser(object):
    def __init__(self):
        # Matches G-Code comments
        self.re_comments = re.compile('\s*;.*$')
        # Matches empty lines
        self.re_empty = re.compile('^\s*$')
        # Matches comment-only lines and BOM
        self.re_comment_only = re.compile('(^\s*;.*$)|(\xef\xbb\xbf*)')
        self.do_checksum = True
        self.parsed = []

    def readfile(self,fname):
        fcontent = []
        if fname == "-":
            fh = sys.stdin
        else:
            fh = open(fname)
        for line in fh:
            fcontent.append(line.rstrip())
        fh.close
        self.fcontent = fcontent

    def checksum(self,line):
        cs = 0
        for i in range(0,len(line)):
            cs ^= ord(line[i]) & 0xff
        cs &= 0xff
        return str(cs)

    def csline(self,line):
        return line+"*"+self.checksum(line)

    def prepare_checksum(self):
        self.parsed = []
        return "M110 N0"

    def parse_file(self):
        if self.do_checksum:
            self.prepare_checksum()
        for line in self.fcontent:
            self.parse(line)

    def parse(self, line):
        if not self.re_empty.match(line) and
            not self.re_comment_only.match(line):
            line = self.re_comments.sub("",line)
            if self.do_checksum:
                line = "N"+str(len(self.parsed)+1)+" "+line
                line = self.csline(line)
            self.parsed.append(line)
            return line
        return ""

    def get_parsed(self):
        return self.parsed

    def dump_parsed(self):
        for i in self.parsed:
            print i

class dddprinter(object):
    def __init__(self):
        self.parser = gparser()
        # Matches Emergency Condition
        self.re_emergency = re.compile('!!')
        # Matches resend query with (optional leading n)
        self.re_resend = re.compile('(^rs [Nn]?)|(^Resend:)')
        # Matches lines which should be ignored
        self.re_ignore = re.compile('(^echo:)|(^[Mm]arlin)|(^start)|'
            '(^Error:checksum)|(^\D:)')
        self.max_resends = 10
        self.resend_counter = 0
        self.currentln = 1
        self.silent = False # output Printer communication
        self.dir_mark = True # Show direction marks
        self.gcode = [""]

    def loadgcodestring(self,gcodestr):
        self.loadgcode = gcodestr.split(gcodestr, "\n")

    def loadgcode(self,gcode):
        self.gcode = [""] # For correct line Numbering
        self.gcode.extend(gcode)

    def connect(self,tty,baud):
        self.ser = serial.Serial(tty, baud, timeout=1)

    def cswrite(self,line):
        self.write(parser.csline(line))

    def ldm(self,line):
        if self.silent: return
        if self.dir_mark: print "<~: "+line
        else: print line

    def rdm(self,line):
        if self.silent: return
        if self.dir_mark: print "~>: "+line
        else: print line

    def rprpt(self):
        if self.silent: return ""
        if self.dir_mark: return "<-: "
        else: return ""

    def write(self,line):
        self.ldm(line)
        self.ser.write(line+"\n")

    def read(self):
        result = self.ser.readline().strip()
        self.rdm(result)
        return result

    def read_flush(self):
        result = self.read()
        while not result == "":
            result = self.read()
        print "--- EOF ---"

    def resend(self,linenumber):
        self.write(self.gcode[linenumber])
        if self.resend_counter <= self.max_resends:
            self.waitforok()
            self.currentln = linenumber # Re-Send all after Checksum-Error
        else:
            self.panic("too many resends!")

    def waitforok(self):
        result = self.read()
        while (len(result) == 0 or
                self.re_ignore.match(result) or
                parser.re_empty.match(result)):
            result = self.read() # Ignore empty lines, unuseful infos, 0-lenght
        if self.re_emergency.match(result):
            # Emergency
            self.panic("received EMERGENCY CONDITION!")
        elif self.re_resend.match(result):
            # Printer queried a resend
            self.read_flush()
            result = self.re_resend.sub("",result) # removing request
            self.resend(int(result))
        elif result == "ok":
            # ok
            self.resend_counter = 0
        else:
            self.panic("UNKNOWN:"+result+":"+str(len(result)))
        return

    def _print(self):
        while self.currentln < len(self.gcode):
            self.write(self.gcode[self.currentln])
            self.waitforok()
            self.currentln += 1
        print "Print successfull :D"
        print "programm succeeded."
        sys.exit(0)

    def panic(self,message):
        print message
        print "flushing input:"
        self.read_flush()
        print "programm stopped."
        sys.exit(1)

    def start_cli(self):
        if self.parser.do_checksum:
            self.write(self.parser.prepare_checksum())
            self.waitforok()
        while(1):
            output = self.parser.parse(raw_input(self.rprpt()))
            if not output == "":
                self.write(output)
                self.waitforok()

if __name__ == "__main__":
    parser = gparser()
    printer = dddprinter()
    argp = argparse.ArgumentParser(
        description="3dpct - a 3d printer control toolchain",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    argp.add_argument(
        "-f", "--file",
        metavar="FILE",
        action="append",
        default=argparse.SUPPRESS,
        help="load gcode from given file name or use '-' for STDIN",
        nargs="?")
    argp.add_argument(
        "-o", "--output",
        action='store_true',
        default=argparse.SUPPRESS,
        help="output filtered gcode to STDOUT")
    argp.add_argument(
        "-p","--print",
        action='store_true',
        default=argparse.SUPPRESS,
        help="use 3d printer to print gcode")
    argp.add_argument(
        "-c", "--cli",
        action='store_true',
        default=argparse.SUPPRESS,
        help="start a cli for manual printer control using gcodes")
    argp.add_argument(
        "-n","--no-direction-mark",
        action='store_true',
        default=argparse.SUPPRESS,
        help="don't show direction marks and prompt (eg: <~: ,~>: ,<-: )")
    argp.add_argument(
        "-e", "--execute",
        action='store',
        default=argparse.SUPPRESS,
        nargs=argparse.REMAINDER,
        help="execute a single gcode line (must be last argument)")
    argp.add_argument(
        "--no-checksum",
        action='store_true',
        default=argparse.SUPPRESS,
        help="don't generate line numbers and checksums")
    argp.add_argument(
        "--silent",
        action="store_true",
        default=argparse.SUPPRESS,
        help="don't output communication while printing")
    argp.add_argument(
        "-d","--device",
        metavar="DEVICE",
        default="/dev/ttyUSB0",
        help="the device to use for printing",
        nargs="?")
    argp.add_argument(
        "-b","--baud",
        default=250000,
        help="the baudrate to use on device",
        nargs="?",
        type=int)
    args = vars(argp.parse_args())

    # Catching Signals
    def signal_handler(signal, frame):
        print ""
        sys.exit(0)
    signal.signal(signal.SIGINT, signal_handler)

    # parsing options
    done = False
    if 'no_checksum' in args:
        parser.do_checksum = False
        printer.parser.do_checksum = False
    if 'no_direction_mark' in args:
        printer.dir_mark = False
    if 'silent' in args:
        printer.silent = True
    if 'file' in args:
        parser.readfile(args['file'][0])
        parser.parse_file()
        if 'output' in args:
            parser.dump_parsed()
            done = True
        if 'print' in args:
            print ("Using device "+args['device']+" with "+str(args['baud'])+
                " Baud.")
            printer.connect(args['device'], args['baud'])
            if printer.parser.do_checksum:
                printer.write(printer.parser.prepare_checksum())
                printer.waitforok()
            printer.loadgcode(parser.get_parsed())
            printer._print()
            done = True
    elif 'cli' in args:
        printer.connect(args['device'], args['baud'])
        printer.start_cli()
        done = True
    elif 'execute' in args:
        printer.connect(args['device'], args['baud'])
        if printer.parser.do_checksum:
            printer.write(printer.parser.prepare_checksum())
            printer.waitforok()
        printer.write(printer.parser.parse(" ".join(args['execute'])))
        printer.waitforok()
        done = True

    if not done:
        argp.print_help()
<|endoftext|>


<commit_before>import random
import unittest
import copy
import numpy as np
import tt
from copy import deepcopy
import tt.riemannian

class TestTTLearning(unittest.TestCase):

    def setUp(self):
        # Reproducibility.
        random.seed(0)
        np.random.seed(2)

    def test_projector_splitting_add(self):
        Y = tt.rand([5, 2, 3], 3, [1, 2, 3, 1])
        my_res = riemannian.projector_splitting_add(Y.copy(), Y.copy(), debug=True)
        np.testing.assert_array_almost_equal(2 * Y.full(), my_res.full())

    def test_project(self):
        def random_tanget_space_point(X):
            coresX = tt.tensor.to_list(X)
            point = 0 * tt.ones(X.n)
            for dim in range(X.d):
                curr = deepcopy(coresX)
                curr[dim] = np.random.rand(curr[dim].shape[0], curr[dim].shape[1], curr[dim].shape[2])
                point += tt.tensor.from_list(curr)
            return point

        X = tt.rand([4, 4, 4], 3, [1, 4, 4, 1])
        Z = random_tanget_space_point(X)
        PZ = riemannian.project(X, Z, debug=True)
        np.testing.assert_array_almost_equal(Z.full(), PZ.full())

        X = tt.rand([2, 3, 4], 3, [1, 5, 4, 1])
        Z = random_tanget_space_point(X)
        PZ = riemannian.project(X, Z, debug=True)
        np.testing.assert_array_almost_equal(Z.full(), PZ.full())

    def test_project_sum(self):

        X = tt.rand([4, 4, 4], 3, [1, 4, 4, 1])
        Z = [0] * 15
        Z[0] = tt.rand([4, 4, 4], 3, [1, 4, 4, 1])
        Z[1] = tt.rand([4, 4, 4], 3, [1, 4, 3, 1])
        Z[2] = tt.rand([4, 4, 4], 3, [1, 2, 3, 1])
        for idx in range(3, 15):
            Z[idx] = tt.rand([4, 4, 4], 3, [1, 2, 2, 1])
        project_sum = riemannian.project(X, Z, debug=True)

        sum_project = X * 0
        for idx in range(len(Z)):
            sum_project += riemannian.project(X, Z[idx], debug=True)
        np.testing.assert_array_almost_equal(sum_project.full(), project_sum.full())

if __name__ == '__main__':
    unittest.main()
<commit_msg>Test jit and non-jit versions; debug and non-debug versions<commit_after>import random
import unittest
import copy
import numpy as np
import tt
from copy import deepcopy
import riemannian

class TestTTLearning(unittest.TestCase):

    def setUp(self):
        # Reproducibility.
        random.seed(0)
        np.random.seed(2)

    def test_projector_splitting_add(self):
        for debug_mode in [False, True]:
            Y = tt.rand([5, 2, 3], 3, [1, 2, 3, 1])
            my_res = riemannian.projector_splitting_add(Y.copy(), Y.copy(),
                                                        debug=debug_mode)
            np.testing.assert_array_almost_equal(2 * Y.full(), my_res.full())

    def test_project(self):
        def random_tanget_space_point(X):
            coresX = tt.tensor.to_list(X)
            point = 0 * tt.ones(X.n)
            for dim in range(X.d):
                curr = deepcopy(coresX)
                curr[dim] = np.random.rand(curr[dim].shape[0],
                                           curr[dim].shape[1],
                                           curr[dim].shape[2])
                point += tt.tensor.from_list(curr)
            return point

        for debug_mode in [False, True]:
            for use_jit in [False, True]:
                X = tt.rand([4, 4, 4], 3, [1, 4, 4, 1])
                Z = random_tanget_space_point(X)
                PZ = riemannian.project(X, Z, use_jit=use_jit,
                                        debug=debug_mode)
                np.testing.assert_array_almost_equal(Z.full(), PZ.full())

                X = tt.rand([2, 3, 4], 3, [1, 5, 4, 1])
                Z = random_tanget_space_point(X)
                PZ = riemannian.project(X, Z, use_jit=use_jit,
                                        debug=debug_mode)
                np.testing.assert_array_almost_equal(Z.full(), PZ.full())

    def test_project_sum_equal_ranks(self):
        for debug_mode in [False, True]:
            for use_jit in [False, True]:
                X = tt.rand([4, 4, 4], 3, [1, 4, 4, 1])
                Z = [0] * 7
                for idx in range(7):
                    Z[idx] = tt.rand([4, 4, 4], 3, [1, 2, 3, 1])
                project_sum = riemannian.project(X, Z, use_jit=use_jit,
                                                 debug=debug_mode)

                sum_project = X * 0
                for idx in range(len(Z)):
                    sum_project += riemannian.project(X, Z[idx],
                                                      use_jit=use_jit,
                                                      debug=debug_mode)
                np.testing.assert_array_almost_equal(sum_project.full(),
                                                     project_sum.full())

    def test_project_sum_different_ranks(self):
        for debug_mode in [False, True]:
            for use_jit in [False, True]:
                X = tt.rand([4, 4, 4], 3, [1, 4, 4, 1])
                Z = [0] * 7
                Z[0] = tt.rand([4, 4, 4], 3, [1, 4, 4, 1])
                Z[1] = tt.rand([4, 4, 4], 3, [1, 4, 3, 1])
                Z[2] = tt.rand([4, 4, 4], 3, [1, 2, 3, 1])
                for idx in range(3, 7):
                    Z[idx] = tt.rand([4, 4, 4], 3, [1, 2, 2, 1])
                project_sum = riemannian.project(X, Z, use_jit=use_jit,
                                                 debug=debug_mode)

                sum_project = X * 0
                for idx in range(len(Z)):
                    sum_project += riemannian.project(X, Z[idx],
                                                      use_jit=use_jit,
                                                      debug=debug_mode)
                np.testing.assert_array_almost_equal(sum_project.full(),
                                                     project_sum.full())

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import unittest

from rdflib.graph import ConjunctiveGraph as Graph
from rdflib.namespace import Namespace as NS

from rdflib.sparql import Algebra

from StringIO import StringIO

class TestSparqlASK(unittest.TestCase):
    def setUp(self):
        self.graph = Graph()

        io = StringIO("""
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix : <http://goonmill.org/2007/skill.n3#> .

:Foo a rdfs:Class .

:bar a :Foo .
""")

        self.graph.load(io, format='n3')

        self.compliance_setting, Algebra.DAWG_DATASET_COMPLIANCE = Algebra.DAWG_DATASET_COMPLIANCE, False

    def tearDown(self):
        Algebra.DAWG_DATASET_COMPLIANCE = self.compliance_setting

    def test_ask_true(self):
        """
        Ask for a triple that exists, assert that the response is True.
        """
        res = self.graph.query('ASK { <http://goonmill.org/2007/skill.n3#bar> a <http://goonmill.org/2007/skill.n3#Foo> } ')
        self.assertEquals(res.askAnswer, [True], "The answer should have been that the triple was found")

    def test_ask_false(self):
        """
        Ask for a triple that does not exist, assert that the response is False.
        """
        res = self.graph.query('ASK { <http://goonmill.org/2007/skill.n3#baz> a <http://goonmill.org/2007/skill.n3#Foo> } ')
        self.assertEquals(res.askAnswer, [False], "The answer should have been that the triple was not found")

class TestSparqlASKWithCompliance(TestSparqlASK):
    def setUp(self):
        TestSparqlASK.setUp(self)
        Algebra.DAWG_DATASET_COMPLIANCE = True

if __name__ == "__main__":
    unittest.main()

<commit_msg>marked as known issue<commit_after>import unittest

from rdflib.graph import ConjunctiveGraph as Graph
from rdflib.namespace import Namespace as NS

from rdflib.sparql import Algebra

from StringIO import StringIO

class TestSparqlASK(unittest.TestCase):

    known_issue = True

    def setUp(self):
        self.graph = Graph()

        io = StringIO("""
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix : <http://goonmill.org/2007/skill.n3#> .

:Foo a rdfs:Class .

:bar a :Foo .
""")

        self.graph.load(io, format='n3')

        self.compliance_setting, Algebra.DAWG_DATASET_COMPLIANCE = Algebra.DAWG_DATASET_COMPLIANCE, False

    def tearDown(self):
        Algebra.DAWG_DATASET_COMPLIANCE = self.compliance_setting

    def test_ask_true(self):
        """
        Ask for a triple that exists, assert that the response is True.
        """
        res = self.graph.query('ASK { <http://goonmill.org/2007/skill.n3#bar> a <http://goonmill.org/2007/skill.n3#Foo> } ')
        self.assertEquals(res.askAnswer, [True], "The answer should have been that the triple was found")

    def test_ask_false(self):
        """
        Ask for a triple that does not exist, assert that the response is False.
        """
        res = self.graph.query('ASK { <http://goonmill.org/2007/skill.n3#baz> a <http://goonmill.org/2007/skill.n3#Foo> } ')
        self.assertEquals(res.askAnswer, [False], "The answer should have been that the triple was not found")

class TestSparqlASKWithCompliance(TestSparqlASK):
    def setUp(self):
        TestSparqlASK.setUp(self)
        Algebra.DAWG_DATASET_COMPLIANCE = True

if __name__ == "__main__":
    unittest.main()

<|endoftext|>


<commit_before>from twisted.words.protocols import irc
from txircd.modbase import Command
from txircd.utils import irc_lower
from fnmatch import fnmatch

# As per the IRCv3.2 METADATA spec, numerics 760-769 must be reserved for
# metadata: http://ircv3.atheme.org/specification/metadata-3.2
# The defined numerics are as follows:
irc.RPL_WHOISKEYVALUE = "760"
irc.RPL_KEYVALUE = "761"
irc.RPL_METADATAEND = "762"
irc.ERR_TARGETINVALID = "765"
irc.ERR_NOMATCHINGKEYS = "766"
irc.ERR_KEYINVALID = "767"
irc.ERR_KEYNOTSET = "768"
irc.ERR_KEYNOPERMISSION = "769"

class MetadataCommand(Command):
    def onUse(self, user, data):
        subcmd = data["subcmd"]
        target = data["target"]

        if subcmd == "LIST":
            namespaceList = ["server", "client", "user", "ext"]
            if "o" in user.mode:
                namespaceList.append("private")
            if data["filter"]:
                filter = data["filter"]
                encounteredItem = False
                for namespace in namespaceList:
                    for key, value in target.metadata[namespace].iteritems():
                        if fnmatch("{}.{}".format(namespace, key), filter):
                            encounteredItem = True
                            if value:
                                user.sendMessage(irc.RPL_KEYVALUE, "{}.{}".format(namespace, key), ":{}".format(value))
                            else:
                                user.sendMessage(irc.RPL_KEYVALUE, "{}.{}".format(namespace, key))
                if encounteredItem:
                    user.sendMessage(irc.RPL_METADATAEND, ":End of metadata")
                else:
                    user.sendMessage(irc.ERR_NOMATCHINGKEYS, filter, ":no matching keys")
            else:
                encounteredItem = False
                for namespace in namespaceList:
                    for key, value in target.metadata[namespace].iteritems():
                        encounteredItem = True
                        if value:
                            user.sendMessage(irc.RPL_KEYVALUE, "{}.{}".format(namespace, key), ":{}".format(value))
                        else:
                            user.sendMessage(irc.RPL_KEYVALUE, "{}.{}".format(namespace, key))
                if encounteredItem:
                    user.sendMessage(irc.RPL_METADATAEND, ":End of metadata")
                else:
                    user.sendMessage(irc.ERR_NOMATCHINGKEYS, "*", ":no matching keys")
        elif subcmd == "SET":
            namespace, key = data["key"].split(".", 1)
            if namespace not in ["user", "client"]:
                user.sendMessage(irc.ERR_KEYNOPERMISSION, data["targetname"], key, ":permission denied")
                return
            # This may be restricted later to only keys in the key registry; for now, I don't think allowing
            # whatever keys in the user-settable namespaces is a major problem. //EA
            if data["value"]:
                target.setMetadata(namespace, key, data["value"])
                user.sendMessage(irc.RPL_KEYVALUE, "{}.{}".format(namespace, key), ":{}".format(data["value"]))
            else:
                target.delMetadata(namespace, key)
                user.sendMessage(irc.RPL_KEYVALUE, "{}.{}".format(namespace, key))
            user.sendMessage(irc.RPL_METADATAEND, ":end of metadata")
        elif subcmd == "CLEAR":
            for namespace in ["client", "user"]:
                removeKeys = target.metadata[namespace].keys()
                for key in removeKeys:
                    target.delMetadata(namespace, key)
                    user.sendMessage(irc.RPL_KEYVALUE, data["targetname"], "{}.{}".format(namespace, key))
            user.sendMessage(irc.RPL_METADATAEND, ":end of metadata")

    def processParams(self, user, params):
        if user.registered > 0:
            user.sendMessage(irc.ERR_NOTYETREGISTERED, "METADATA", ":You have not registered")
            return {}
        if not params or len(params) < 2:
            user.sendMessage(irc.ERR_NEEDMOREPARAMS, "METADATA", ":Not enough parameters")
            return {}
        target = None
        if params[0] in self.ircd.channels:
            target = self.ircd.channels[params[0]]
        elif params[0] in self.ircd.users:
            target = self.ircd.users[params[0]]
        else:
            user.sendMessage(irc.ERR_TARGETINVALID, params[0], ":invalid metadata target")
            return {}
        subcmd = params[1].upper()
        if subcmd == "LIST":
            return {
                "user": user,
                "targetname": params[0],
                "target": target,
                "subcmd": "LIST",
                "filter": params[2] if len(params) >= 3 else None
            }
        if subcmd == "SET":
            if len(params) < 3:
                user.sendMessage(irc.ERR_NEEDMOREPARAMS, "METADATA", ":Not enough parameters")
                return {}
            if "o" not in user.mode:
                try:
                    if not user.hasAccess(target.name, self.ircd.servconfig["channel_minimum_level"]["METADATA"]):
                        user.sendMessage(irc.ERR_CHANOPRIVSNEEDED, target.name, ":You do not have access to set metadata on this channel")
                        return {}
                except AttributeError: # in this case, it's a user, not a channel
                    if user != target:
                        user.sendMessage(irc.ERR_NOPRIVILEGES, ":Permission denied - You do not have the correct operator privileges")
                        return {}
            if "." not in params[2]:
                user.sendMessage(irc.ERR_KEYINVALID, params[2], ":invalid metadata key")
                return {}
            return {
                "user": user,
                "targetname": params[0],
                "target": target,
                "subcmd": "SET",
                "key": params[2],
                "value": params[3] if len(params) >= 4 else None
            }
        if subcmd == "CLEAR":
            if "o" not in user.mode and user != target:
                user.sendMessage(irc.ERR_NOPRIVILEGES, ":Permission denied - You do not have the correct operator privileges")
                return {}
            return {
                "user": user,
                "targetname": params[0],
                "target": target,
                "subcmd": "CLEAR"
            }
        user.sendMessage(irc.ERR_NEEDMOREPARAMS, "METADATA", ":Incorrect metadata subcommand") # borrowing this numeric is fine
        return {}

    def capRequest(self, user, capability):
        return True

    def capAcknowledge(self, user, capability):
        return False

    def capRequestRemove(self, user, capability):
        return True

    def capAcknowledgeRemove(self, user, capability):
        return False

    def capClear(self, user, capability):
        return True

    def whoisSendMetadata(self, cmd, data):
        if cmd != "WHOIS":
            return
        user = data["user"]
        target = data["targetuser"]
        namespaceList = ["server", "client", "user", "ext"]
        if "o" in user.mode:
            namespaceList.append("private")
        for namespace in namespaceList:
            for key, value in target.metadata[namespace].iteritems():
                user.sendMessage(irc.RPL_WHOISKEYVALUE, key, ":{}".format(value))

    def notify(self, target, namespace, key, oldValue, value):
        try:
            if target.nickname is None:
                return # An unregistered user isn't being monitored
        except AttributeError: # don't process channels
            return
        source = None
        if namespace in ["client", "user"]:
            source = target.nickname
        elif namespace in ["server", "ext"]:
            source = self.ircd.name
        else:
            return
        lowerNick = irc_lower(target.nickname)
        if "monitorwatchedby" in self.ircd.module_data_cache and lowerNick in self.ircd.module_data_cache["monitorwatchedby"]:
            watcherList = self.ircd.module_data_cache["monitorwatchedby"][lowerNick]
        else:
            watcherList = []
        watchers = set(watcherList)
        watchers.add(target)
        if not value and key not in target.metadata[namespace]:
            for u in watchers:
                if "cap" in u.cache and "metadata-notify" in u.cache["cap"]:
                    u.sendMessage("METADATA", source, target.nickname, "{}.{}".format(namespace, key), to=None, prefix=None)
        else:
            for u in watchers:
                if "cap" in u.cache and "metadata-notify" in u.cache["cap"]:
                    u.sendMessage("METADATA", source, target.nickname, "{}.{}".format(namespace, key), ":{}".format(value), to=None, prefix=None)

class Spawner(object):
    def __init__(self, ircd):
        self.ircd = ircd
        self.metadata_cmd = None

    def spawn(self):
        if "channel_minimum_level" not in self.ircd.servconfig:
            self.ircd.servconfing["channel_minimum_level"] = {}
        if "METADATA" not in self.ircd.servconfig["channel_minimum_level"]:
            self.ircd.servconfig["channel_minimum_level"]["METADATA"] = "o"
        self.metadata_cmd = MetadataCommand()
        if "cap" not in self.ircd.module_data_cache:
            self.ircd.module_data_cache["cap"] = {}
        self.ircd.module_data_cache["cap"]["metadata-notify"] = self.metadata_cmd
        self.ircd.isupport["METADATA"] = None
        return {
            "commands": {
                "METADATA": self.metadata_cmd
            },
            "actions": {
                "metadataupdate": [self.metadata_cmd.notify]
            }
        }

    def cleanup(self):
        del self.ircd.commands["METADATA"]
        del self.ircd.isupport["METADATA"]
        del self.ircd.module_data_cache["cap"]["metadata-notify"]
        self.ircd.actions["metadataupdate"].remove(self.metadata_cmd.notify)<commit_msg>Fix metadata not being reported in WHOIS<commit_after>from twisted.words.protocols import irc
from txircd.modbase import Command
from txircd.utils import irc_lower
from fnmatch import fnmatch

# As per the IRCv3.2 METADATA spec, numerics 760-769 must be reserved for
# metadata: http://ircv3.atheme.org/specification/metadata-3.2
# The defined numerics are as follows:
irc.RPL_WHOISKEYVALUE = "760"
irc.RPL_KEYVALUE = "761"
irc.RPL_METADATAEND = "762"
irc.ERR_TARGETINVALID = "765"
irc.ERR_NOMATCHINGKEYS = "766"
irc.ERR_KEYINVALID = "767"
irc.ERR_KEYNOTSET = "768"
irc.ERR_KEYNOPERMISSION = "769"

class MetadataCommand(Command):
    def onUse(self, user, data):
        subcmd = data["subcmd"]
        target = data["target"]

        if subcmd == "LIST":
            namespaceList = ["server", "client", "user", "ext"]
            if "o" in user.mode:
                namespaceList.append("private")
            if data["filter"]:
                filter = data["filter"]
                encounteredItem = False
                for namespace in namespaceList:
                    for key, value in target.metadata[namespace].iteritems():
                        if fnmatch("{}.{}".format(namespace, key), filter):
                            encounteredItem = True
                            if value:
                                user.sendMessage(irc.RPL_KEYVALUE, "{}.{}".format(namespace, key), ":{}".format(value))
                            else:
                                user.sendMessage(irc.RPL_KEYVALUE, "{}.{}".format(namespace, key))
                if encounteredItem:
                    user.sendMessage(irc.RPL_METADATAEND, ":End of metadata")
                else:
                    user.sendMessage(irc.ERR_NOMATCHINGKEYS, filter, ":no matching keys")
            else:
                encounteredItem = False
                for namespace in namespaceList:
                    for key, value in target.metadata[namespace].iteritems():
                        encounteredItem = True
                        if value:
                            user.sendMessage(irc.RPL_KEYVALUE, "{}.{}".format(namespace, key), ":{}".format(value))
                        else:
                            user.sendMessage(irc.RPL_KEYVALUE, "{}.{}".format(namespace, key))
                if encounteredItem:
                    user.sendMessage(irc.RPL_METADATAEND, ":End of metadata")
                else:
                    user.sendMessage(irc.ERR_NOMATCHINGKEYS, "*", ":no matching keys")
        elif subcmd == "SET":
            namespace, key = data["key"].split(".", 1)
            if namespace not in ["user", "client"]:
                user.sendMessage(irc.ERR_KEYNOPERMISSION, data["targetname"], key, ":permission denied")
                return
            # This may be restricted later to only keys in the key registry; for now, I don't think allowing
            # whatever keys in the user-settable namespaces is a major problem. //EA
            if data["value"]:
                target.setMetadata(namespace, key, data["value"])
                user.sendMessage(irc.RPL_KEYVALUE, "{}.{}".format(namespace, key), ":{}".format(data["value"]))
            else:
                target.delMetadata(namespace, key)
                user.sendMessage(irc.RPL_KEYVALUE, "{}.{}".format(namespace, key))
            user.sendMessage(irc.RPL_METADATAEND, ":end of metadata")
        elif subcmd == "CLEAR":
            for namespace in ["client", "user"]:
                removeKeys = target.metadata[namespace].keys()
                for key in removeKeys:
                    target.delMetadata(namespace, key)
                    user.sendMessage(irc.RPL_KEYVALUE, data["targetname"], "{}.{}".format(namespace, key))
            user.sendMessage(irc.RPL_METADATAEND, ":end of metadata")

    def processParams(self, user, params):
        if user.registered > 0:
            user.sendMessage(irc.ERR_NOTYETREGISTERED, "METADATA", ":You have not registered")
            return {}
        if not params or len(params) < 2:
            user.sendMessage(irc.ERR_NEEDMOREPARAMS, "METADATA", ":Not enough parameters")
            return {}
        target = None
        if params[0] in self.ircd.channels:
            target = self.ircd.channels[params[0]]
        elif params[0] in self.ircd.users:
            target = self.ircd.users[params[0]]
        else:
            user.sendMessage(irc.ERR_TARGETINVALID, params[0], ":invalid metadata target")
            return {}
        subcmd = params[1].upper()
        if subcmd == "LIST":
            return {
                "user": user,
                "targetname": params[0],
                "target": target,
                "subcmd": "LIST",
                "filter": params[2] if len(params) >= 3 else None
            }
        if subcmd == "SET":
            if len(params) < 3:
                user.sendMessage(irc.ERR_NEEDMOREPARAMS, "METADATA", ":Not enough parameters")
                return {}
            if "o" not in user.mode:
                try:
                    if not user.hasAccess(target.name, self.ircd.servconfig["channel_minimum_level"]["METADATA"]):
                        user.sendMessage(irc.ERR_CHANOPRIVSNEEDED, target.name, ":You do not have access to set metadata on this channel")
                        return {}
                except AttributeError: # in this case, it's a user, not a channel
                    if user != target:
                        user.sendMessage(irc.ERR_NOPRIVILEGES, ":Permission denied - You do not have the correct operator privileges")
                        return {}
            if "." not in params[2]:
                user.sendMessage(irc.ERR_KEYINVALID, params[2], ":invalid metadata key")
                return {}
            return {
                "user": user,
                "targetname": params[0],
                "target": target,
                "subcmd": "SET",
                "key": params[2],
                "value": params[3] if len(params) >= 4 else None
            }
        if subcmd == "CLEAR":
            if "o" not in user.mode and user != target:
                user.sendMessage(irc.ERR_NOPRIVILEGES, ":Permission denied - You do not have the correct operator privileges")
                return {}
            return {
                "user": user,
                "targetname": params[0],
                "target": target,
                "subcmd": "CLEAR"
            }
        user.sendMessage(irc.ERR_NEEDMOREPARAMS, "METADATA", ":Incorrect metadata subcommand") # borrowing this numeric is fine
        return {}

    def capRequest(self, user, capability):
        return True

    def capAcknowledge(self, user, capability):
        return False

    def capRequestRemove(self, user, capability):
        return True

    def capAcknowledgeRemove(self, user, capability):
        return False

    def capClear(self, user, capability):
        return True

    def whoisSendMetadata(self, cmd, data):
        if cmd != "WHOIS":
            return
        user = data["user"]
        target = data["targetuser"]
        namespaceList = ["server", "client", "user", "ext"]
        if "o" in user.mode:
            namespaceList.append("private")
        for namespace in namespaceList:
            for key, value in target.metadata[namespace].iteritems():
                user.sendMessage(irc.RPL_WHOISKEYVALUE, key, ":{}".format(value))

    def notify(self, target, namespace, key, oldValue, value):
        try:
            if target.nickname is None:
                return # An unregistered user isn't being monitored
        except AttributeError: # don't process channels
            return
        source = None
        if namespace in ["client", "user"]:
            source = target.nickname
        elif namespace in ["server", "ext"]:
            source = self.ircd.name
        else:
            return
        lowerNick = irc_lower(target.nickname)
        if "monitorwatchedby" in self.ircd.module_data_cache and lowerNick in self.ircd.module_data_cache["monitorwatchedby"]:
            watcherList = self.ircd.module_data_cache["monitorwatchedby"][lowerNick]
        else:
            watcherList = []
        watchers = set(watcherList)
        watchers.add(target)
        if not value and key not in target.metadata[namespace]:
            for u in watchers:
                if "cap" in u.cache and "metadata-notify" in u.cache["cap"]:
                    u.sendMessage("METADATA", source, target.nickname, "{}.{}".format(namespace, key), to=None, prefix=None)
        else:
            for u in watchers:
                if "cap" in u.cache and "metadata-notify" in u.cache["cap"]:
                    u.sendMessage("METADATA", source, target.nickname, "{}.{}".format(namespace, key), ":{}".format(value), to=None, prefix=None)

class Spawner(object):
    def __init__(self, ircd):
        self.ircd = ircd
        self.metadata_cmd = None

    def spawn(self):
        if "channel_minimum_level" not in self.ircd.servconfig:
            self.ircd.servconfing["channel_minimum_level"] = {}
        if "METADATA" not in self.ircd.servconfig["channel_minimum_level"]:
            self.ircd.servconfig["channel_minimum_level"]["METADATA"] = "o"
        self.metadata_cmd = MetadataCommand()
        if "cap" not in self.ircd.module_data_cache:
            self.ircd.module_data_cache["cap"] = {}
        self.ircd.module_data_cache["cap"]["metadata-notify"] = self.metadata_cmd
        self.ircd.isupport["METADATA"] = None
        return {
            "commands": {
                "METADATA": self.metadata_cmd
            },
            "actions": {
                "metadataupdate": [self.metadata_cmd.notify],
                "commandextra": [self.metadata_cmd.whoisSendMetadata]
            }
        }

    def cleanup(self):
        del self.ircd.commands["METADATA"]
        del self.ircd.isupport["METADATA"]
        del self.ircd.module_data_cache["cap"]["metadata-notify"]
        self.ircd.actions["metadataupdate"].remove(self.metadata_cmd.notify)
        self.ircd.actions["commandextra"].remove(self.metadata_cmd.whoisSendMetadata)<|endoftext|>


<commit_before>#!/usr/bin/env python3

import json
import os.path
import re
import sys
import urllib.request

import requests

def scrape(thread_url):
	m = re.search(r'4chan.org/(.*?)/thread/(\d*)(:?/.*)?', thread_url)
	if not m:
		print("[FAIL] Invalid thread url, skipping:", thread_url, file=sys.stderr)
		return None

	board, thread_id = m.group(1, 2)

	r = requests.get(thread_url+'.json')
	r = requests.get('https://a.4cdn.org/{board}/thread/{thread_id}.json'.format(**locals()))

	if r.status_code == 404:
		print("[FAIL] Thread 404'd, skipping:", thread_url, file=sys.stderr)
		return None

	r.raise_for_status() # Throw at other errors.
	print("[ OK ]", thread_url, file=sys.stderr)

	data = r.json()
	posts = data['posts']

	thread_name = posts[0]['semantic_url'].replace('/', '_') # Don't blindly trust data from internet enough to mess up with path separator.
	with open('thread.{board}.{thread_id}.{thread_name}.json'.format(**locals()), 'w') as thread_f:
		json.dump(data, thread_f,
			sort_keys=True,
			ensure_ascii=False,
			indent='\t')
		thread_f.write('\n')

	return board, thread_id, posts

def download(url, filename):
	if not os.path.exists(filename):
		sys.stderr.write(filename+" ")
		sys.stderr.flush()
		urllib.request.urlretrieve(url, filename)
		sys.stderr.write("OK\n")

def process_thread(board, thread_id, posts):
	for post in posts:
		try:
			filename = str(post['tim']) + post['ext']
		except KeyError: # No image in post
			continue

		img_url = 'http://images.4chan.org/{board}/src/{filename}'.format(**locals())
		download(img_url, filename)

def main(urls):
	if not urls:
		urls = [ line.strip() for line in sys.stdin if not line.isspace() ]

	for thread_url in urls:
		thread = scrape(thread_url)
		if thread:
			process_thread(*thread)

if __name__ == "__main__":
	main(sys.argv[1:])
<commit_msg>Fix typo in regex (wrongly written non-capturing group)<commit_after>#!/usr/bin/env python3

import json
import os.path
import re
import sys
import urllib.request

import requests

def scrape(thread_url):
	m = re.search(r'4chan.org/(.*?)/thread/(\d*)(?:/.*)?', thread_url)
	if not m:
		print("[FAIL] Invalid thread url, skipping:", thread_url, file=sys.stderr)
		return None

	board, thread_id = m.group(1, 2)

	r = requests.get(thread_url+'.json')
	r = requests.get('https://a.4cdn.org/{board}/thread/{thread_id}.json'.format(**locals()))

	if r.status_code == 404:
		print("[FAIL] Thread 404'd, skipping:", thread_url, file=sys.stderr)
		return None

	r.raise_for_status() # Throw at other errors.
	print("[ OK ]", thread_url, file=sys.stderr)

	data = r.json()
	posts = data['posts']

	thread_name = posts[0]['semantic_url'].replace('/', '_') # Don't blindly trust data from internet enough to mess up with path separator.
	with open('thread.{board}.{thread_id}.{thread_name}.json'.format(**locals()), 'w') as thread_f:
		json.dump(data, thread_f,
			sort_keys=True,
			ensure_ascii=False,
			indent='\t')
		thread_f.write('\n')

	return board, thread_id, posts

def download(url, filename):
	if not os.path.exists(filename):
		sys.stderr.write(filename+" ")
		sys.stderr.flush()
		urllib.request.urlretrieve(url, filename)
		sys.stderr.write("OK\n")

def process_thread(board, thread_id, posts):
	for post in posts:
		try:
			filename = str(post['tim']) + post['ext']
		except KeyError: # No image in post
			continue

		img_url = 'http://images.4chan.org/{board}/src/{filename}'.format(**locals())
		download(img_url, filename)

def main(urls):
	if not urls:
		urls = [ line.strip() for line in sys.stdin if not line.isspace() ]

	for thread_url in urls:
		thread = scrape(thread_url)
		if thread:
			process_thread(*thread)

if __name__ == "__main__":
	main(sys.argv[1:])
<|endoftext|>


<commit_before>import sys
from itertools import islice, imap, ifilter, count
import hashlib

class Cracker:
    def __init__(self):
        self.pw1 = ""
        self.pw2 = ['X'] * 8

    def update(self, (i, c)):
        if len(self.pw1) < 8:
            self.pw1 = self.pw1 + c
        if i < '8' and self.pw2[int(i)] == 'X':
            self.pw2[int(i)] = c
        self.progress()

    def progress(self):
        print "part1: %s, part2: %s" % (self.pw1, ''.join(self.pw2))

    def ready(self):
        return len(self.pw1) == 8 and not 'X' in self.pw2

def door_hash(s):
    return hashlib.md5(s).hexdigest()

def valid(s):
    return s.startswith('00000')

def password_part(s):
    return (s[5], s[6])

def password(salt):
    for i in count(0):
        s = salt + str(i)
        candidate = door_hash(s)
        if valid(candidate):
            yield password_part(candidate)

if __name__ == '__main__':
    door = sys.stdin.next().rstrip()
    passwords = Cracker()
    for update in password(door):
        passwords.update(update)
        if passwords.ready():
            break
<commit_msg>Remove obsolete imports<commit_after>import sys
from itertools import count
import hashlib

class Cracker:
    def __init__(self):
        self.pw1 = ""
        self.pw2 = ['X'] * 8

    def update(self, (i, c)):
        if len(self.pw1) < 8:
            self.pw1 = self.pw1 + c
        if i < '8' and self.pw2[int(i)] == 'X':
            self.pw2[int(i)] = c
        self.progress()

    def progress(self):
        print "part1: %s, part2: %s" % (self.pw1, ''.join(self.pw2))

    def ready(self):
        return len(self.pw1) == 8 and not 'X' in self.pw2

def door_hash(s):
    return hashlib.md5(s).hexdigest()

def valid(s):
    return s.startswith('00000')

def password_part(s):
    return (s[5], s[6])

def password(salt):
    for i in count(0):
        s = salt + str(i)
        candidate = door_hash(s)
        if valid(candidate):
            yield password_part(candidate)

if __name__ == '__main__':
    door = sys.stdin.next().rstrip()
    passwords = Cracker()
    for update in password(door):
        passwords.update(update)
        if passwords.ready():
            break
<|endoftext|>


<commit_before># Copyright 2013-2014 The Meson development team

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import re
import typing as T

from . import compilers
from . import coredata
from . import mesonlib
from . import mparser
from . import mlog
from .interpreterbase import FeatureNew, typed_pos_args, typed_kwargs, ContainerTypeInfo, KwargInfo, permittedKwargs
if T.TYPE_CHECKING:
    from .interpreterbase import TV_func, TYPE_var, TYPE_kwargs
    from typing_extensions import TypedDict
    FuncOptionArgs = TypedDict('FuncOptionArgs', {
        'type': str,
        'description': str,
        'yield': bool,
        'choices': T.Optional[T.List[str]],
        'value': object,
        'min': T.Optional[int],
        'max': T.Optional[int],
        'deprecated': T.Union[bool, T.Dict[str, str], T.List[str]],
        })
    ParserArgs = TypedDict('ParserArgs', {
        'yield': bool,
        'choices': T.Optional[T.List[str]],
        'value': object,
        'min': T.Optional[int],
        'max': T.Optional[int],
        })

forbidden_option_names = set(coredata.BUILTIN_OPTIONS.keys())
forbidden_prefixes = [lang + '_' for lang in compilers.all_languages] + ['b_', 'backend_']
reserved_prefixes = ['cross_']

def is_invalid_name(name: str, *, log: bool = True) -> bool:
    if name in forbidden_option_names:
        return True
    pref = name.split('_')[0] + '_'
    if pref in forbidden_prefixes:
        return True
    if pref in reserved_prefixes:
        if log:
            mlog.deprecation('Option uses prefix "%s", which is reserved for Meson. This will become an error in the future.' % pref)
    return False

class OptionException(mesonlib.MesonException):
    pass

optname_regex = re.compile('[^a-zA-Z0-9_-]')

class OptionInterpreter:
    def __init__(self, subproject: str) -> None:
        self.options: 'coredata.KeyedOptionDictType' = {}
        self.subproject = subproject
        self.option_types = {'string': self.string_parser,
                             'boolean': self.boolean_parser,
                             'combo': self.combo_parser,
                             'integer': self.integer_parser,
                             'array': self.string_array_parser,
                             'feature': self.feature_parser,
                             }

    def process(self, option_file: str) -> None:
        try:
            with open(option_file, encoding='utf-8') as f:
                ast = mparser.Parser(f.read(), option_file).parse()
        except mesonlib.MesonException as me:
            me.file = option_file
            raise me
        if not isinstance(ast, mparser.CodeBlockNode):
            e = OptionException('Option file is malformed.')
            e.lineno = ast.lineno()
            e.file = option_file
            raise e
        for cur in ast.lines:
            try:
                self.current_node = cur
                self.evaluate_statement(cur)
            except mesonlib.MesonException as e:
                e.lineno = cur.lineno
                e.colno = cur.colno
                e.file = option_file
                raise e
            except Exception as e:
                raise mesonlib.MesonException(
                    str(e), lineno=cur.lineno, colno=cur.colno, file=option_file)

    def reduce_single(self, arg: T.Union[str, mparser.BaseNode]) -> 'TYPE_var':
        if isinstance(arg, str):
            return arg
        elif isinstance(arg, (mparser.StringNode, mparser.BooleanNode,
                              mparser.NumberNode)):
            return arg.value
        elif isinstance(arg, mparser.ArrayNode):
            return [self.reduce_single(curarg) for curarg in arg.args.arguments]
        elif isinstance(arg, mparser.DictNode):
            d = {}
            for k, v in arg.args.kwargs.items():
                if not isinstance(k, mparser.StringNode):
                    raise OptionException('Dictionary keys must be a string literal')
                d[k.value] = self.reduce_single(v)
            return d
        elif isinstance(arg, mparser.UMinusNode):
            res = self.reduce_single(arg.value)
            if not isinstance(res, (int, float)):
                raise OptionException('Token after "-" is not a number')
            FeatureNew.single_use('negative numbers in meson_options.txt', '0.54.1', self.subproject)
            return -res
        elif isinstance(arg, mparser.NotNode):
            res = self.reduce_single(arg.value)
            if not isinstance(res, bool):
                raise OptionException('Token after "not" is not a a boolean')
            FeatureNew.single_use('negation ("not") in meson_options.txt', '0.54.1', self.subproject)
            return not res
        elif isinstance(arg, mparser.ArithmeticNode):
            l = self.reduce_single(arg.left)
            r = self.reduce_single(arg.right)
            if not (arg.operation == 'add' and isinstance(l, str) and isinstance(r, str)):
                raise OptionException('Only string concatenation with the "+" operator is allowed')
            FeatureNew.single_use('string concatenation in meson_options.txt', '0.55.0', self.subproject)
            return l + r
        else:
            raise OptionException('Arguments may only be string, int, bool, or array of those.')

    def reduce_arguments(self, args: mparser.ArgumentNode) -> T.Tuple['TYPE_var', 'TYPE_kwargs']:
        if args.incorrect_order():
            raise OptionException('All keyword arguments must be after positional arguments.')
        reduced_pos = [self.reduce_single(arg) for arg in args.arguments]
        reduced_kw = {}
        for key in args.kwargs.keys():
            if not isinstance(key, mparser.IdNode):
                raise OptionException('Keyword argument name is not a string.')
            a = args.kwargs[key]
            reduced_kw[key.value] = self.reduce_single(a)
        return reduced_pos, reduced_kw

    def evaluate_statement(self, node: mparser.BaseNode) -> None:
        if not isinstance(node, mparser.FunctionNode):
            raise OptionException('Option file may only contain option definitions')
        func_name = node.func_name
        if func_name != 'option':
            raise OptionException('Only calls to option() are allowed in option files.')
        (posargs, kwargs) = self.reduce_arguments(node.args)
        self.func_option(posargs, kwargs)

    @typed_kwargs('option',
                  KwargInfo('type', str, required=True),
                  KwargInfo('description', str, default=''),
                  KwargInfo('yield', bool, default=coredata.default_yielding, since='0.45.0'),
                  KwargInfo('choices', (ContainerTypeInfo(list, str), type(None))),
                  KwargInfo('value', object),
                  KwargInfo('min', (int, type(None))),
                  KwargInfo('max', (int, type(None))),
                  KwargInfo('deprecated', (bool, ContainerTypeInfo(dict, str), ContainerTypeInfo(list, str)),
                            default=False, since='0.60.0')
                  )
    @typed_pos_args('option', str)
    def func_option(self, args: T.Tuple[str], kwargs: 'FuncOptionArgs') -> None:
        opt_name = args[0]
        if optname_regex.search(opt_name) is not None:
            raise OptionException('Option names can only contain letters, numbers or dashes.')
        if is_invalid_name(opt_name):
            raise OptionException('Option name %s is reserved.' % opt_name)

        opt_type = kwargs['type']
        parser = self.option_types.get(opt_type)
        if not parser:
            raise OptionException(f'Unknown type {opt_type}.')
        description = kwargs['description'] or opt_name

        # Only keep in kwargs arguments that are used by option type's parser
        # because they use @permittedKwargs().
        known_parser_kwargs = {'value', 'choices', 'yield', 'min', 'max'}
        parser_kwargs = {k: v for k, v in kwargs.items() if k in known_parser_kwargs and v is not None}
        opt = parser(description, T.cast('ParserArgs', parser_kwargs))
        opt.deprecated = kwargs['deprecated']

        key = mesonlib.OptionKey(opt_name, self.subproject)
        if key in self.options:
            raise OptionException(f'Option {opt_name} already exists')
        self.options[key] = opt

    @permittedKwargs({'value', 'yield'})
    def string_parser(self, description: str, kwargs: 'ParserArgs') -> coredata.UserOption:
        value = kwargs.get('value', '')
        return coredata.UserStringOption(description, value, kwargs['yield'])

    @permittedKwargs({'value', 'yield'})
    def boolean_parser(self, description: str, kwargs: 'ParserArgs') -> coredata.UserOption:
        value = kwargs.get('value', True)
        return coredata.UserBooleanOption(description, value, kwargs['yield'])

    @permittedKwargs({'value', 'yield', 'choices'})
    def combo_parser(self, description: str, kwargs: 'ParserArgs') -> coredata.UserOption:
        choices = kwargs.get('choices')
        if not choices:
            raise OptionException('Combo option missing "choices" keyword.')
        value = kwargs.get('value', choices[0])
        return coredata.UserComboOption(description, choices, value, kwargs['yield'])

    @permittedKwargs({'value', 'min', 'max', 'yield'})
    def integer_parser(self, description: str, kwargs: 'ParserArgs') -> coredata.UserOption:
        value = kwargs.get('value')
        if value is None:
            raise OptionException('Integer option must contain value argument.')
        inttuple = (kwargs.get('min'), kwargs.get('max'), value)
        return coredata.UserIntegerOption(description, inttuple, kwargs['yield'])

    @permittedKwargs({'value', 'yield', 'choices'})
    def string_array_parser(self, description: str, kwargs: 'ParserArgs') -> coredata.UserOption:
        choices = kwargs.get('choices', [])
        value = kwargs.get('value', choices)
        if not isinstance(value, list):
            raise OptionException('Array choices must be passed as an array.')
        return coredata.UserArrayOption(description, value,
                                        choices=choices,
                                        yielding=kwargs['yield'])

    @permittedKwargs({'value', 'yield'})
    def feature_parser(self, description: str, kwargs: 'ParserArgs') -> coredata.UserOption:
        value = kwargs.get('value', 'auto')
        return coredata.UserFeatureOption(description, value, kwargs['yield'])
<commit_msg>optinterpreter: Downgrade error to deprecation<commit_after># Copyright 2013-2014 The Meson development team

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import re
import typing as T

from . import compilers
from . import coredata
from . import mesonlib
from . import mparser
from . import mlog
from .interpreterbase import FeatureNew, typed_pos_args, typed_kwargs, ContainerTypeInfo, KwargInfo, permittedKwargs
if T.TYPE_CHECKING:
    from .interpreterbase import TV_func, TYPE_var, TYPE_kwargs
    from typing_extensions import TypedDict
    FuncOptionArgs = TypedDict('FuncOptionArgs', {
        'type': str,
        'description': str,
        'yield': bool,
        'choices': T.Optional[T.List[str]],
        'value': object,
        'min': T.Optional[int],
        'max': T.Optional[int],
        'deprecated': T.Union[bool, T.Dict[str, str], T.List[str]],
        })
    ParserArgs = TypedDict('ParserArgs', {
        'yield': bool,
        'choices': T.Optional[T.List[str]],
        'value': object,
        'min': T.Optional[int],
        'max': T.Optional[int],
        })

forbidden_option_names = set(coredata.BUILTIN_OPTIONS.keys())
forbidden_prefixes = [lang + '_' for lang in compilers.all_languages] + ['b_', 'backend_']
reserved_prefixes = ['cross_']

def is_invalid_name(name: str, *, log: bool = True) -> bool:
    if name in forbidden_option_names:
        return True
    pref = name.split('_')[0] + '_'
    if pref in forbidden_prefixes:
        return True
    if pref in reserved_prefixes:
        if log:
            mlog.deprecation('Option uses prefix "%s", which is reserved for Meson. This will become an error in the future.' % pref)
    return False

class OptionException(mesonlib.MesonException):
    pass

optname_regex = re.compile('[^a-zA-Z0-9_-]')

class OptionInterpreter:
    def __init__(self, subproject: str) -> None:
        self.options: 'coredata.KeyedOptionDictType' = {}
        self.subproject = subproject
        self.option_types = {'string': self.string_parser,
                             'boolean': self.boolean_parser,
                             'combo': self.combo_parser,
                             'integer': self.integer_parser,
                             'array': self.string_array_parser,
                             'feature': self.feature_parser,
                             }

    def process(self, option_file: str) -> None:
        try:
            with open(option_file, encoding='utf-8') as f:
                ast = mparser.Parser(f.read(), option_file).parse()
        except mesonlib.MesonException as me:
            me.file = option_file
            raise me
        if not isinstance(ast, mparser.CodeBlockNode):
            e = OptionException('Option file is malformed.')
            e.lineno = ast.lineno()
            e.file = option_file
            raise e
        for cur in ast.lines:
            try:
                self.current_node = cur
                self.evaluate_statement(cur)
            except mesonlib.MesonException as e:
                e.lineno = cur.lineno
                e.colno = cur.colno
                e.file = option_file
                raise e
            except Exception as e:
                raise mesonlib.MesonException(
                    str(e), lineno=cur.lineno, colno=cur.colno, file=option_file)

    def reduce_single(self, arg: T.Union[str, mparser.BaseNode]) -> 'TYPE_var':
        if isinstance(arg, str):
            return arg
        elif isinstance(arg, (mparser.StringNode, mparser.BooleanNode,
                              mparser.NumberNode)):
            return arg.value
        elif isinstance(arg, mparser.ArrayNode):
            return [self.reduce_single(curarg) for curarg in arg.args.arguments]
        elif isinstance(arg, mparser.DictNode):
            d = {}
            for k, v in arg.args.kwargs.items():
                if not isinstance(k, mparser.StringNode):
                    raise OptionException('Dictionary keys must be a string literal')
                d[k.value] = self.reduce_single(v)
            return d
        elif isinstance(arg, mparser.UMinusNode):
            res = self.reduce_single(arg.value)
            if not isinstance(res, (int, float)):
                raise OptionException('Token after "-" is not a number')
            FeatureNew.single_use('negative numbers in meson_options.txt', '0.54.1', self.subproject)
            return -res
        elif isinstance(arg, mparser.NotNode):
            res = self.reduce_single(arg.value)
            if not isinstance(res, bool):
                raise OptionException('Token after "not" is not a a boolean')
            FeatureNew.single_use('negation ("not") in meson_options.txt', '0.54.1', self.subproject)
            return not res
        elif isinstance(arg, mparser.ArithmeticNode):
            l = self.reduce_single(arg.left)
            r = self.reduce_single(arg.right)
            if not (arg.operation == 'add' and isinstance(l, str) and isinstance(r, str)):
                raise OptionException('Only string concatenation with the "+" operator is allowed')
            FeatureNew.single_use('string concatenation in meson_options.txt', '0.55.0', self.subproject)
            return l + r
        else:
            raise OptionException('Arguments may only be string, int, bool, or array of those.')

    def reduce_arguments(self, args: mparser.ArgumentNode) -> T.Tuple['TYPE_var', 'TYPE_kwargs']:
        if args.incorrect_order():
            raise OptionException('All keyword arguments must be after positional arguments.')
        reduced_pos = [self.reduce_single(arg) for arg in args.arguments]
        reduced_kw = {}
        for key in args.kwargs.keys():
            if not isinstance(key, mparser.IdNode):
                raise OptionException('Keyword argument name is not a string.')
            a = args.kwargs[key]
            reduced_kw[key.value] = self.reduce_single(a)
        return reduced_pos, reduced_kw

    def evaluate_statement(self, node: mparser.BaseNode) -> None:
        if not isinstance(node, mparser.FunctionNode):
            raise OptionException('Option file may only contain option definitions')
        func_name = node.func_name
        if func_name != 'option':
            raise OptionException('Only calls to option() are allowed in option files.')
        (posargs, kwargs) = self.reduce_arguments(node.args)
        self.func_option(posargs, kwargs)

    @typed_kwargs('option',
                  KwargInfo('type', str, required=True),
                  KwargInfo('description', str, default=''),
                  KwargInfo('yield', bool, default=coredata.default_yielding, since='0.45.0'),
                  KwargInfo('choices', (ContainerTypeInfo(list, str), type(None))),
                  KwargInfo('value', object),
                  KwargInfo('min', (int, type(None))),
                  KwargInfo('max', (int, type(None))),
                  KwargInfo('deprecated', (bool, ContainerTypeInfo(dict, str), ContainerTypeInfo(list, str)),
                            default=False, since='0.60.0')
                  )
    @typed_pos_args('option', str)
    def func_option(self, args: T.Tuple[str], kwargs: 'FuncOptionArgs') -> None:
        opt_name = args[0]
        if optname_regex.search(opt_name) is not None:
            raise OptionException('Option names can only contain letters, numbers or dashes.')
        if is_invalid_name(opt_name):
            raise OptionException('Option name %s is reserved.' % opt_name)

        opt_type = kwargs['type']
        parser = self.option_types.get(opt_type)
        if not parser:
            raise OptionException(f'Unknown type {opt_type}.')
        description = kwargs['description'] or opt_name

        # Only keep in kwargs arguments that are used by option type's parser
        # because they use @permittedKwargs().
        known_parser_kwargs = {'value', 'choices', 'yield', 'min', 'max'}
        parser_kwargs = {k: v for k, v in kwargs.items() if k in known_parser_kwargs and v is not None}
        opt = parser(description, T.cast('ParserArgs', parser_kwargs))
        opt.deprecated = kwargs['deprecated']

        key = mesonlib.OptionKey(opt_name, self.subproject)
        if key in self.options:
            mlog.deprecation(f'Option {opt_name} already exists.')
        self.options[key] = opt

    @permittedKwargs({'value', 'yield'})
    def string_parser(self, description: str, kwargs: 'ParserArgs') -> coredata.UserOption:
        value = kwargs.get('value', '')
        return coredata.UserStringOption(description, value, kwargs['yield'])

    @permittedKwargs({'value', 'yield'})
    def boolean_parser(self, description: str, kwargs: 'ParserArgs') -> coredata.UserOption:
        value = kwargs.get('value', True)
        return coredata.UserBooleanOption(description, value, kwargs['yield'])

    @permittedKwargs({'value', 'yield', 'choices'})
    def combo_parser(self, description: str, kwargs: 'ParserArgs') -> coredata.UserOption:
        choices = kwargs.get('choices')
        if not choices:
            raise OptionException('Combo option missing "choices" keyword.')
        value = kwargs.get('value', choices[0])
        return coredata.UserComboOption(description, choices, value, kwargs['yield'])

    @permittedKwargs({'value', 'min', 'max', 'yield'})
    def integer_parser(self, description: str, kwargs: 'ParserArgs') -> coredata.UserOption:
        value = kwargs.get('value')
        if value is None:
            raise OptionException('Integer option must contain value argument.')
        inttuple = (kwargs.get('min'), kwargs.get('max'), value)
        return coredata.UserIntegerOption(description, inttuple, kwargs['yield'])

    @permittedKwargs({'value', 'yield', 'choices'})
    def string_array_parser(self, description: str, kwargs: 'ParserArgs') -> coredata.UserOption:
        choices = kwargs.get('choices', [])
        value = kwargs.get('value', choices)
        if not isinstance(value, list):
            raise OptionException('Array choices must be passed as an array.')
        return coredata.UserArrayOption(description, value,
                                        choices=choices,
                                        yielding=kwargs['yield'])

    @permittedKwargs({'value', 'yield'})
    def feature_parser(self, description: str, kwargs: 'ParserArgs') -> coredata.UserOption:
        value = kwargs.get('value', 'auto')
        return coredata.UserFeatureOption(description, value, kwargs['yield'])
<|endoftext|>