<commit_before>#!/usr/bin/env python
##########################################################################
#
# Copyright 2009 VMware, Inc.
# Copyright 2008 Tungsten Graphics, Inc., Cedar Park, Texas.
# All Rights Reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sub license, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice (including the
# next paragraph) shall be included in all copies or substantial portions
# of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
# IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
# ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#
##########################################################################

from gallium import *
from base import *

def lods(*dims):
    size = max(dims)
    lods = 0
    while size:
        lods += 1
        size >>= 1
    return lods

def minify(dims, level = 1):
    return [max(dim>>level, 1) for dim in dims]

def tex_coords(texture, face, level, zslice):
    st = [
        [0.0, 0.0],
        [1.0, 0.0],
        [1.0, 1.0],
        [0.0, 1.0],
    ]

    if texture.target == PIPE_TEXTURE_2D:
        return [[s, t, 0.0] for s, t in st]
    elif texture.target == PIPE_TEXTURE_3D:
        depth = texture.get_depth(level)
        if depth > 1:
            r = float(zslice)/float(depth - 1)
        else:
            r = 0.0
        return [[s, t, r] for s, t in st]
    elif texture.target == PIPE_TEXTURE_CUBE:
        result = []
        for s, t in st:
            # See http://developer.nvidia.com/object/cube_map_ogl_tutorial.html
            sc = 2.0*s - 1.0
            tc = 2.0*t - 1.0
            if face == PIPE_TEX_FACE_POS_X:
                rx = 1.0
                ry = -tc
                rz = -sc
            if face == PIPE_TEX_FACE_NEG_X:
                rx = -1.0
                ry = -tc
                rz = sc
            if face == PIPE_TEX_FACE_POS_Y:
                rx = sc
                ry = 1.0
                rz = tc
            if face == PIPE_TEX_FACE_NEG_Y:
                rx = sc
                ry = -1.0
                rz = -tc
            if face == PIPE_TEX_FACE_POS_Z:
                rx = sc
                ry = -tc
                rz = 1.0
            if face == PIPE_TEX_FACE_NEG_Z:
                rx = -sc
                ry = -tc
                rz = -1.0
            result.append([rx, ry, rz])
        return result

def is_pot(n):
    return n & (n - 1) == 0

class TextureTest(TestCase):

    tags = (
        'target',
        'format',
        'width',
        'height',
        'depth',
        'last_level',
        'face',
        'level',
        'zslice',
    )

    def test(self):
        dev = self.dev

        target = self.target
        format = self.format
        width = self.width
        height = self.height
        depth = self.depth
        last_level = self.last_level
        face = self.face
        level = self.level
        zslice = self.zslice

        tex_usage = PIPE_TEXTURE_USAGE_SAMPLER
        geom_flags = 0
        if width != height:
            geom_flags |= PIPE_TEXTURE_GEOM_NON_SQUARE
        if not is_pot(width) or not is_pot(height) or not is_pot(depth):
            geom_flags |= PIPE_TEXTURE_GEOM_NON_POWER_OF_TWO

        if not dev.is_format_supported(format, target, tex_usage, geom_flags):
            raise TestSkip

        ctx = self.dev.context_create()

        # disabled blending/masking
        blend = Blend()
        blend.rgb_src_factor = PIPE_BLENDFACTOR_ONE
        blend.alpha_src_factor = PIPE_BLENDFACTOR_ONE
        blend.rgb_dst_factor = PIPE_BLENDFACTOR_ZERO
        blend.alpha_dst_factor = PIPE_BLENDFACTOR_ZERO
        blend.colormask = PIPE_MASK_RGBA
        ctx.set_blend(blend)

        # no-op depth/stencil/alpha
        depth_stencil_alpha = DepthStencilAlpha()
        ctx.set_depth_stencil_alpha(depth_stencil_alpha)

        # rasterizer
        rasterizer = Rasterizer()
        rasterizer.front_winding = PIPE_WINDING_CW
        rasterizer.cull_mode = PIPE_WINDING_NONE
        rasterizer.bypass_vs_clip_and_viewport = 1
        ctx.set_rasterizer(rasterizer)

        # samplers
        sampler = Sampler()
        sampler.wrap_s = PIPE_TEX_WRAP_CLAMP_TO_EDGE
        sampler.wrap_t = PIPE_TEX_WRAP_CLAMP_TO_EDGE
        sampler.wrap_r = PIPE_TEX_WRAP_CLAMP_TO_EDGE
        sampler.min_mip_filter = PIPE_TEX_MIPFILTER_NEAREST
        sampler.min_img_filter = PIPE_TEX_MIPFILTER_NEAREST
        sampler.mag_img_filter = PIPE_TEX_MIPFILTER_NEAREST
        sampler.normalized_coords = 1
        sampler.min_lod = 0
        sampler.max_lod = PIPE_MAX_TEXTURE_LEVELS - 1
        ctx.set_sampler(0, sampler)

        #  texture
        texture = dev.texture_create(
            target = target,
            format = format,
            width = width,
            height = height,
            depth = depth,
            last_level = last_level,
            tex_usage = tex_usage,
        )

        expected_rgba = FloatArray(height*width*4)
        texture.get_surface(
            face = face,
            level = level,
            zslice = zslice,
        ).sample_rgba(expected_rgba)

        ctx.set_sampler_texture(0, texture)

        #  framebuffer
        cbuf_tex = dev.texture_create(
            PIPE_FORMAT_A8R8G8B8_UNORM,
            width,
            height,
            tex_usage = PIPE_TEXTURE_USAGE_RENDER_TARGET,
        )

        cbuf = cbuf_tex.get_surface()
        fb = Framebuffer()
        fb.width = width
        fb.height = height
        fb.nr_cbufs = 1
        fb.set_cbuf(0, cbuf)
        ctx.set_framebuffer(fb)
        rgba = FloatArray(4);
        rgba[0] = 0.5
        rgba[1] = 0.5
        rgba[2] = 0.5
        rgba[3] = 0.5
        ctx.clear(PIPE_CLEAR_COLOR, rgba, 0.0, 0)
        del fb

        # vertex shader
        vs = Shader('''
            VERT1.1
            DCL IN[0], POSITION, CONSTANT
            DCL IN[1], GENERIC, CONSTANT
            DCL OUT[0], POSITION, CONSTANT
            DCL OUT[1], GENERIC, CONSTANT
            0:MOV OUT[0], IN[0]
            1:MOV OUT[1], IN[1]
            2:END
        ''')
        #vs.dump()
        ctx.set_vertex_shader(vs)

        # fragment shader
        op = {
            PIPE_TEXTURE_1D: "1D",
            PIPE_TEXTURE_2D: "2D",
            PIPE_TEXTURE_3D: "3D",
            PIPE_TEXTURE_CUBE: "CUBE",
        }[target]
        fs = Shader('''
            FRAG1.1
            DCL IN[0], GENERIC[0], LINEAR
            DCL OUT[0], COLOR, CONSTANT
            DCL SAMP[0], CONSTANT
            0:TEX OUT[0], IN[0], SAMP[0], %s
            1:END
        ''' % op)
        #fs.dump()
        ctx.set_fragment_shader(fs)

        nverts = 4
        nattrs = 2
        verts = FloatArray(nverts * nattrs * 4)

        x = 0
        y = 0
        w, h = minify((width, height), level)

        pos = [
            [x, y],
            [x+w, y],
            [x+w, y+h],
            [x, y+h],
        ]

        tex = tex_coords(texture, face, level, zslice)

        for i in range(0, 4):
            j = 8*i
            verts[j + 0] = pos[i][0] # x
            verts[j + 1] = pos[i][1] # y
            verts[j + 2] = 0.0 # z
            verts[j + 3] = 1.0 # w
            verts[j + 4] = tex[i][0] # s
            verts[j + 5] = tex[i][1] # r
            verts[j + 6] = tex[i][2] # q
            verts[j + 7] = 1.0

        ctx.draw_vertices(PIPE_PRIM_TRIANGLE_FAN,
                          nverts,
                          nattrs,
                          verts)

        ctx.flush()

        cbuf = cbuf_tex.get_surface()

        self.assert_rgba(cbuf, x, y, w, h, expected_rgba, 4.0/256, 0.85)

def main():
    dev = Device()
    suite = TestSuite()

    targets = [
        PIPE_TEXTURE_2D,
        PIPE_TEXTURE_CUBE,
        PIPE_TEXTURE_3D,
    ]

    formats = [
        PIPE_FORMAT_A8R8G8B8_UNORM,
        PIPE_FORMAT_X8R8G8B8_UNORM,
        #PIPE_FORMAT_A8R8G8B8_SRGB,
        PIPE_FORMAT_R5G6B5_UNORM,
        PIPE_FORMAT_A1R5G5B5_UNORM,
        PIPE_FORMAT_A4R4G4B4_UNORM,
        #PIPE_FORMAT_Z32_UNORM,
        #PIPE_FORMAT_Z24S8_UNORM,
        #PIPE_FORMAT_Z24X8_UNORM,
        #PIPE_FORMAT_Z16_UNORM,
        #PIPE_FORMAT_S8_UNORM,
        PIPE_FORMAT_A8_UNORM,
        PIPE_FORMAT_L8_UNORM,
        PIPE_FORMAT_YCBCR,
        PIPE_FORMAT_DXT1_RGB,
        #PIPE_FORMAT_DXT1_RGBA,
        #PIPE_FORMAT_DXT3_RGBA,
        #PIPE_FORMAT_DXT5_RGBA,
    ]

    sizes = [64, 32, 16, 8, 4, 2, 1]
    #sizes = [1020, 508, 252, 62, 30, 14, 6, 3]
    #sizes = [64]
    #sizes = [63]

    faces = [
        PIPE_TEX_FACE_POS_X,
        PIPE_TEX_FACE_NEG_X,
        PIPE_TEX_FACE_POS_Y,
        PIPE_TEX_FACE_NEG_Y,
        PIPE_TEX_FACE_POS_Z,
        PIPE_TEX_FACE_NEG_Z,
    ]

    for target in targets:
        for format in formats:
            for size in sizes:
                if target == PIPE_TEXTURE_3D:
                    depth = size
                else:
                    depth = 1
                for face in faces:
                    if target != PIPE_TEXTURE_CUBE and face:
                        continue
                    levels = lods(size)
                    for last_level in range(levels):
                        for level in range(0, last_level + 1):
                            zslice = 0
                            while zslice < depth >> level:
                                test = TextureTest(
                                    dev = dev,
                                    target = target,
                                    format = format,
                                    width = size,
                                    height = size,
                                    depth = depth,
                                    last_level = last_level,
                                    face = face,
                                    level = level,
                                    zslice = zslice,
                                )
                                suite.add_test(test)
                                zslice = (zslice + 1)*2 - 1
    suite.run()

if __name__ == '__main__':
    main()
<commit_msg>python/tests: Test sampling from a depth texture.<commit_after>#!/usr/bin/env python
##########################################################################
#
# Copyright 2009 VMware, Inc.
# Copyright 2008 Tungsten Graphics, Inc., Cedar Park, Texas.
# All Rights Reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sub license, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice (including the
# next paragraph) shall be included in all copies or substantial portions
# of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
# IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
# ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#
##########################################################################

from gallium import *
from base import *

def lods(*dims):
    size = max(dims)
    lods = 0
    while size:
        lods += 1
        size >>= 1
    return lods

def minify(dims, level = 1):
    return [max(dim>>level, 1) for dim in dims]

def tex_coords(texture, face, level, zslice):
    st = [
        [0.0, 0.0],
        [1.0, 0.0],
        [1.0, 1.0],
        [0.0, 1.0],
    ]

    if texture.target == PIPE_TEXTURE_2D:
        return [[s, t, 0.0] for s, t in st]
    elif texture.target == PIPE_TEXTURE_3D:
        depth = texture.get_depth(level)
        if depth > 1:
            r = float(zslice)/float(depth - 1)
        else:
            r = 0.0
        return [[s, t, r] for s, t in st]
    elif texture.target == PIPE_TEXTURE_CUBE:
        result = []
        for s, t in st:
            # See http://developer.nvidia.com/object/cube_map_ogl_tutorial.html
            sc = 2.0*s - 1.0
            tc = 2.0*t - 1.0
            if face == PIPE_TEX_FACE_POS_X:
                rx = 1.0
                ry = -tc
                rz = -sc
            if face == PIPE_TEX_FACE_NEG_X:
                rx = -1.0
                ry = -tc
                rz = sc
            if face == PIPE_TEX_FACE_POS_Y:
                rx = sc
                ry = 1.0
                rz = tc
            if face == PIPE_TEX_FACE_NEG_Y:
                rx = sc
                ry = -1.0
                rz = -tc
            if face == PIPE_TEX_FACE_POS_Z:
                rx = sc
                ry = -tc
                rz = 1.0
            if face == PIPE_TEX_FACE_NEG_Z:
                rx = -sc
                ry = -tc
                rz = -1.0
            result.append([rx, ry, rz])
        return result

def is_pot(n):
    return n & (n - 1) == 0

class TextureColorSampleTest(TestCase):

    tags = (
        'target',
        'format',
        'width',
        'height',
        'depth',
        'last_level',
        'face',
        'level',
        'zslice',
    )

    def test(self):
        dev = self.dev

        target = self.target
        format = self.format
        width = self.width
        height = self.height
        depth = self.depth
        last_level = self.last_level
        face = self.face
        level = self.level
        zslice = self.zslice

        tex_usage = PIPE_TEXTURE_USAGE_SAMPLER
        geom_flags = 0
        if width != height:
            geom_flags |= PIPE_TEXTURE_GEOM_NON_SQUARE
        if not is_pot(width) or not is_pot(height) or not is_pot(depth):
            geom_flags |= PIPE_TEXTURE_GEOM_NON_POWER_OF_TWO

        if not dev.is_format_supported(format, target, tex_usage, geom_flags):
            raise TestSkip

        ctx = self.dev.context_create()

        # disabled blending/masking
        blend = Blend()
        blend.rgb_src_factor = PIPE_BLENDFACTOR_ONE
        blend.alpha_src_factor = PIPE_BLENDFACTOR_ONE
        blend.rgb_dst_factor = PIPE_BLENDFACTOR_ZERO
        blend.alpha_dst_factor = PIPE_BLENDFACTOR_ZERO
        blend.colormask = PIPE_MASK_RGBA
        ctx.set_blend(blend)

        # no-op depth/stencil/alpha
        depth_stencil_alpha = DepthStencilAlpha()
        ctx.set_depth_stencil_alpha(depth_stencil_alpha)

        # rasterizer
        rasterizer = Rasterizer()
        rasterizer.front_winding = PIPE_WINDING_CW
        rasterizer.cull_mode = PIPE_WINDING_NONE
        rasterizer.bypass_vs_clip_and_viewport = 1
        ctx.set_rasterizer(rasterizer)

        # samplers
        sampler = Sampler()
        sampler.wrap_s = PIPE_TEX_WRAP_CLAMP_TO_EDGE
        sampler.wrap_t = PIPE_TEX_WRAP_CLAMP_TO_EDGE
        sampler.wrap_r = PIPE_TEX_WRAP_CLAMP_TO_EDGE
        sampler.min_mip_filter = PIPE_TEX_MIPFILTER_NEAREST
        sampler.min_img_filter = PIPE_TEX_MIPFILTER_NEAREST
        sampler.mag_img_filter = PIPE_TEX_MIPFILTER_NEAREST
        sampler.normalized_coords = 1
        sampler.min_lod = 0
        sampler.max_lod = PIPE_MAX_TEXTURE_LEVELS - 1
        ctx.set_sampler(0, sampler)

        #  texture
        texture = dev.texture_create(
            target = target,
            format = format,
            width = width,
            height = height,
            depth = depth,
            last_level = last_level,
            tex_usage = tex_usage,
        )

        expected_rgba = FloatArray(height*width*4)
        texture.get_surface(
            face = face,
            level = level,
            zslice = zslice,
        ).sample_rgba(expected_rgba)

        ctx.set_sampler_texture(0, texture)

        #  framebuffer
        cbuf_tex = dev.texture_create(
            PIPE_FORMAT_A8R8G8B8_UNORM,
            width,
            height,
            tex_usage = PIPE_TEXTURE_USAGE_RENDER_TARGET,
        )

        cbuf = cbuf_tex.get_surface()
        fb = Framebuffer()
        fb.width = width
        fb.height = height
        fb.nr_cbufs = 1
        fb.set_cbuf(0, cbuf)
        ctx.set_framebuffer(fb)
        rgba = FloatArray(4);
        rgba[0] = 0.5
        rgba[1] = 0.5
        rgba[2] = 0.5
        rgba[3] = 0.5
        ctx.clear(PIPE_CLEAR_COLOR, rgba, 0.0, 0)
        del fb

        # vertex shader
        vs = Shader('''
            VERT1.1
            DCL IN[0], POSITION, CONSTANT
            DCL IN[1], GENERIC, CONSTANT
            DCL OUT[0], POSITION, CONSTANT
            DCL OUT[1], GENERIC, CONSTANT
            0:MOV OUT[0], IN[0]
            1:MOV OUT[1], IN[1]
            2:END
        ''')
        #vs.dump()
        ctx.set_vertex_shader(vs)

        # fragment shader
        op = {
            PIPE_TEXTURE_1D: "1D",
            PIPE_TEXTURE_2D: "2D",
            PIPE_TEXTURE_3D: "3D",
            PIPE_TEXTURE_CUBE: "CUBE",
        }[target]
        fs = Shader('''
            FRAG1.1
            DCL IN[0], GENERIC[0], LINEAR
            DCL OUT[0], COLOR, CONSTANT
            DCL SAMP[0], CONSTANT
            0:TEX OUT[0], IN[0], SAMP[0], %s
            1:END
        ''' % op)
        #fs.dump()
        ctx.set_fragment_shader(fs)

        nverts = 4
        nattrs = 2
        verts = FloatArray(nverts * nattrs * 4)

        x = 0
        y = 0
        w, h = minify((width, height), level)

        pos = [
            [x, y],
            [x+w, y],
            [x+w, y+h],
            [x, y+h],
        ]

        tex = tex_coords(texture, face, level, zslice)

        for i in range(0, 4):
            j = 8*i
            verts[j + 0] = pos[i][0] # x
            verts[j + 1] = pos[i][1] # y
            verts[j + 2] = 0.0 # z
            verts[j + 3] = 1.0 # w
            verts[j + 4] = tex[i][0] # s
            verts[j + 5] = tex[i][1] # r
            verts[j + 6] = tex[i][2] # q
            verts[j + 7] = 1.0

        ctx.draw_vertices(PIPE_PRIM_TRIANGLE_FAN,
                          nverts,
                          nattrs,
                          verts)

        ctx.flush()

        cbuf = cbuf_tex.get_surface()

        self.assert_rgba(cbuf, x, y, w, h, expected_rgba, 4.0/256, 0.85)

class TextureDepthSampleTest(TestCase):

    tags = (
        'target',
        'format',
        'width',
        'height',
        'depth',
        'last_level',
        'face',
        'level',
        'zslice',
    )

    def test(self):
        dev = self.dev

        target = self.target
        format = self.format
        width = self.width
        height = self.height
        depth = self.depth
        last_level = self.last_level
        face = self.face
        level = self.level
        zslice = self.zslice

        tex_usage = PIPE_TEXTURE_USAGE_SAMPLER
        geom_flags = 0
        if width != height:
            geom_flags |= PIPE_TEXTURE_GEOM_NON_SQUARE
        if not is_pot(width) or not is_pot(height) or not is_pot(depth):
            geom_flags |= PIPE_TEXTURE_GEOM_NON_POWER_OF_TWO

        if not dev.is_format_supported(format, target, tex_usage, geom_flags):
            raise TestSkip

        ctx = self.dev.context_create()

        # disabled blending/masking
        blend = Blend()
        blend.rgb_src_factor = PIPE_BLENDFACTOR_ONE
        blend.alpha_src_factor = PIPE_BLENDFACTOR_ONE
        blend.rgb_dst_factor = PIPE_BLENDFACTOR_ZERO
        blend.alpha_dst_factor = PIPE_BLENDFACTOR_ZERO
        blend.colormask = PIPE_MASK_RGBA
        ctx.set_blend(blend)

        # depth/stencil/alpha
        depth_stencil_alpha = DepthStencilAlpha()
        depth_stencil_alpha.depth.enabled = 1
        depth_stencil_alpha.depth.writemask = 1
        depth_stencil_alpha.depth.func = PIPE_FUNC_LESS
        ctx.set_depth_stencil_alpha(depth_stencil_alpha)

        # rasterizer
        rasterizer = Rasterizer()
        rasterizer.front_winding = PIPE_WINDING_CW
        rasterizer.cull_mode = PIPE_WINDING_NONE
        rasterizer.bypass_vs_clip_and_viewport = 1
        ctx.set_rasterizer(rasterizer)

        # samplers
        sampler = Sampler()
        sampler.wrap_s = PIPE_TEX_WRAP_CLAMP_TO_EDGE
        sampler.wrap_t = PIPE_TEX_WRAP_CLAMP_TO_EDGE
        sampler.wrap_r = PIPE_TEX_WRAP_CLAMP_TO_EDGE
        sampler.min_mip_filter = PIPE_TEX_MIPFILTER_NEAREST
        sampler.min_img_filter = PIPE_TEX_MIPFILTER_NEAREST
        sampler.mag_img_filter = PIPE_TEX_MIPFILTER_NEAREST
        sampler.normalized_coords = 1
        sampler.min_lod = 0
        sampler.max_lod = PIPE_MAX_TEXTURE_LEVELS - 1
        ctx.set_sampler(0, sampler)

        #  texture
        texture = dev.texture_create(
            target = target,
            format = format,
            width = width,
            height = height,
            depth = depth,
            last_level = last_level,
            tex_usage = tex_usage,
        )

        expected_rgba = FloatArray(height*width*4)
        texture.get_surface(
            face = face,
            level = level,
            zslice = zslice,
        ).sample_rgba(expected_rgba)

        ctx.set_sampler_texture(0, texture)

        #  framebuffer
        cbuf_tex = dev.texture_create(
            PIPE_FORMAT_A8R8G8B8_UNORM,
            width,
            height,
            tex_usage = PIPE_TEXTURE_USAGE_RENDER_TARGET,
        )

        zsbuf_tex = dev.texture_create(
            PIPE_FORMAT_Z24X8_UNORM,
            width,
            height,
            tex_usage = PIPE_TEXTURE_USAGE_RENDER_TARGET,
        )

        cbuf = cbuf_tex.get_surface()
        zsbuf = zsbuf_tex.get_surface()
        fb = Framebuffer()
        fb.width = width
        fb.height = height
        fb.nr_cbufs = 1
        fb.set_cbuf(0, cbuf)
        fb.set_zsbuf(zsbuf)
        ctx.set_framebuffer(fb)
        rgba = FloatArray(4);
        rgba[0] = 0.5
        rgba[1] = 0.5
        rgba[2] = 0.5
        rgba[3] = 0.5
        ctx.clear(PIPE_CLEAR_DEPTHSTENCIL, rgba, 1.0, 0)
        del fb

        # vertex shader
        vs = Shader('''
            VERT1.1
            DCL IN[0], POSITION, CONSTANT
            DCL IN[1], GENERIC, CONSTANT
            DCL OUT[0], POSITION, CONSTANT
            DCL OUT[1], GENERIC, CONSTANT
            0:MOV OUT[0], IN[0]
            1:MOV OUT[1], IN[1]
            2:END
        ''')
        #vs.dump()
        ctx.set_vertex_shader(vs)

        # fragment shader
        op = {
            PIPE_TEXTURE_1D: "1D",
            PIPE_TEXTURE_2D: "2D",
            PIPE_TEXTURE_3D: "3D",
            PIPE_TEXTURE_CUBE: "CUBE",
        }[target]
        fs = Shader('''
            FRAG1.1
            DCL IN[0], GENERIC[0], LINEAR
            DCL SAMP[0], CONSTANT
            DCL OUT[0].z, POSITION
            0:TEX OUT[0].z, IN[0], SAMP[0], %s
            1:END
        ''' % op)
        #fs.dump()
        ctx.set_fragment_shader(fs)

        nverts = 4
        nattrs = 2
        verts = FloatArray(nverts * nattrs * 4)

        x = 0
        y = 0
        w, h = minify((width, height), level)

        pos = [
            [x, y],
            [x+w, y],
            [x+w, y+h],
            [x, y+h],
        ]

        tex = tex_coords(texture, face, level, zslice)

        for i in range(0, 4):
            j = 8*i
            verts[j + 0] = pos[i][0] # x
            verts[j + 1] = pos[i][1] # y
            verts[j + 2] = 0.0 # z
            verts[j + 3] = 1.0 # w
            verts[j + 4] = tex[i][0] # s
            verts[j + 5] = tex[i][1] # r
            verts[j + 6] = tex[i][2] # q
            verts[j + 7] = 1.0

        ctx.draw_vertices(PIPE_PRIM_TRIANGLE_FAN,
                          nverts,
                          nattrs,
                          verts)

        ctx.flush()

        zsbuf = zsbuf_tex.get_surface()

        self.assert_rgba(zsbuf, x, y, w, h, expected_rgba, 4.0/256, 0.85)

def main():
    dev = Device()
    suite = TestSuite()

    targets = [
        PIPE_TEXTURE_2D,
        PIPE_TEXTURE_CUBE,
        PIPE_TEXTURE_3D,
    ]

    formats = [
        PIPE_FORMAT_A8R8G8B8_UNORM,
        PIPE_FORMAT_X8R8G8B8_UNORM,
        #PIPE_FORMAT_A8R8G8B8_SRGB,
        PIPE_FORMAT_R5G6B5_UNORM,
        PIPE_FORMAT_A1R5G5B5_UNORM,
        PIPE_FORMAT_A4R4G4B4_UNORM,
        PIPE_FORMAT_Z32_UNORM,
        PIPE_FORMAT_Z24S8_UNORM,
        PIPE_FORMAT_Z24X8_UNORM,
        PIPE_FORMAT_Z16_UNORM,
        PIPE_FORMAT_A8_UNORM,
        PIPE_FORMAT_L8_UNORM,
        PIPE_FORMAT_YCBCR,
        PIPE_FORMAT_DXT1_RGB,
        #PIPE_FORMAT_DXT1_RGBA,
        #PIPE_FORMAT_DXT3_RGBA,
        #PIPE_FORMAT_DXT5_RGBA,
    ]

    sizes = [64, 32, 16, 8, 4, 2, 1]
    #sizes = [1020, 508, 252, 62, 30, 14, 6, 3]
    #sizes = [64]
    #sizes = [63]

    faces = [
        PIPE_TEX_FACE_POS_X,
        PIPE_TEX_FACE_NEG_X,
        PIPE_TEX_FACE_POS_Y,
        PIPE_TEX_FACE_NEG_Y,
        PIPE_TEX_FACE_POS_Z,
        PIPE_TEX_FACE_NEG_Z,
    ]

    for target in targets:
        for format in formats:
            for size in sizes:
                if target == PIPE_TEXTURE_3D:
                    depth = size
                else:
                    depth = 1
                for face in faces:
                    if target != PIPE_TEXTURE_CUBE and face:
                        continue
                    levels = lods(size)
                    for last_level in range(levels):
                        for level in range(0, last_level + 1):
                            zslice = 0
                            while zslice < depth >> level:
                                if format in (
                                    PIPE_FORMAT_Z32_UNORM,
                                    PIPE_FORMAT_Z24S8_UNORM,
                                    PIPE_FORMAT_Z24X8_UNORM,
                                    PIPE_FORMAT_Z16_UNORM,
                                ):
                                    klass = TextureDepthSampleTest
                                else:
                                    klass = TextureColorSampleTest

                                test = klass(
                                    dev = dev,
                                    target = target,
                                    format = format,
                                    width = size,
                                    height = size,
                                    depth = depth,
                                    last_level = last_level,
                                    face = face,
                                    level = level,
                                    zslice = zslice,
                                )
                                suite.add_test(test)
                                zslice = (zslice + 1)*2 - 1
    suite.run()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># coding=utf-8

from webapp2 import RequestHandler
import urllib
from google.appengine.api.urlfetch import fetch
from google.appengine.api import taskqueue
import hmac
from hashlib import sha1
from base64 import b64encode
from time import time
from random import getrandbits
import json
import logging

api_urls = {
    'request_token_url': 'http://twitter.com/oauth/request_token',
    'access_token_url' : 'http://twitter.com/oauth/access_token',
    'authorize_url'    : 'http://twitter.com/oauth/authorize',
    'tweet_url'        : 'https://api.twitter.com/1.1/statuses/update.json',
    'github_commits'   : 'https://api.github.com/repos/defeo/defeo.github.io/commits/',
    'blog_base'        : 'http://defeo.lu',
    'blog_posts'       : '/api/posts.json'
    }

# Configuration variables
entry_url   = '/tweet-ghblog'
worker_url  = '/tweet-ghblog/commit'
def w_config(m):
    def wrap(self):
        self._post_prefix = self.app.config['tweet_blog'].get('post_prefix') or '[POST]'
        self._post_dir    = self.app.config['tweet_blog'].get('post_dir')    or '_posts/'
        m(self)
    return wrap

class CheckCommits(RequestHandler):
    '''
    This hook repsonds to post requests from the github Webhook
    service. It looks for new post commits inside the commits posted
    by github.
    '''
    @w_config
    def post(self):
        commits = json.loads(self.request.get('payload'))
        # Fetch json summary of posts from the blog
        posts = fetch(api_urls['blog_base'] + api_urls['blog_posts'])
        if posts.status_code != 200:
            logging.error('Cannot fetch post list. %s' % post.content)
            self.abort(500, detail='Cannot fetch post list.')
        else:
            # Examine each commit in the payload of the request
            count = 0
            for c in commits["commits"]:
                if c['message'].startswith(self._post_prefix):
                    # If commit starts with the right prefix, launch a task to examine it
                    taskqueue.add(url=worker_url, params={'commit-id': c['id'], 'posts': posts.content})
                    count += 1

            logging.info('Considering %d commits.' % count)
            self.response.content_type = 'application/json'
            self.response.write('{"commits": %d}' % count)

class Tweet(RequestHandler):
    '''
    This hook examines a single commit, looks for newly added files,
    and tweets any new post found.
    '''
    @w_config
    def post(self):
        # Parse post list
        posts = json.loads(self.request.get('posts'))

        # Fetch commit data from github
        commit = fetch(api_urls['github_commits'] + self.request.get('commit-id'),
                       validate_certificate=True)
        if commit.status_code != 200:
            logging.error('Cannot fetch commit data. %s' % commit.content)
            self.abort(500, 'Cannot fetch commid data.')

        # Parse commit data
        commit = json.loads(commit.content)
        if commit['commit']['message'].startswith(self._post_prefix):
            # If commit starts with right prefix, look for new files in the post directory
            count = 0
            for f in commit['files']:
                if f['status'] == 'added' and f['filename'].startswith(self._post_dir):
                    count += 1
                    # Tweet post announcement
                    post = posts.get(f['filename'])
                    if post is None:
                        logging.error('Post %s missing in list.' % f['filename'])
                        self.abort(500, 'Post seems to be missing in list.')

                    tweet = u'#Blog post %s.%s %s%s' % (post['title'],
                                                        ''.join(' #' + t for t in post['tags'][:2]),
                                                         api_urls['blog_base'], post['url'])

                    res = self._sign(api_urls['tweet_url'], 'POST', {'status': tweet})

                    if res.status_code != 200:
                        res = json.loads(res.content)
                        logging.error('Error while tweeting. %s ' % repr(res['errors']))
                        self.abort(500, 'Error while tweeting.')
                    else:
                        logging.info('Tweeted %s' % tweet)
            self.response.content_type = 'application/json'
            self.response.write('{"posts": %d}' % count)
        else:
            logging.info('Ignoring commit %s: badly formatted message.' % self.request.get('commit-id'))
            self.response.content_type = 'application/json'
            self.response.write('{"error":"Ignoring badly formatted commit message."}')

    def _sign(self, url, method='GET', data=None):
        def quote(s):
            'Percent encoding following RFC 3986'
            return urllib.quote(s.encode('utf-8'), '-._~')

        data = data.items() if data is not None else []

        # OAuth header
        oauth = [
            ('oauth_consumer_key'     , self.app.config['tweet_blog']['consumer_key']),
            ('oauth_nonce'            , b64encode('%0x' % getrandbits(256))[:32]),
            ('oauth_signature_method' , 'HMAC-SHA1'),
            ('oauth_timestamp'        , str(int(time()))),
            ('oauth_token'            , self.app.config['tweet_blog']['access_token']),
            ('oauth_version'          , '1.0')
            ]
        # OAuth signature
        sig = '%s&%s&%s' % (
            method.upper(),
            quote(url),
            quote("&".join('%s=%s' % tuple(map(quote, p))
                           for p in sorted(oauth + data,
                                           key=lambda(k,_):k))))
        key = '%s&%s' % (quote(self.app.config['tweet_blog']['consumer_secret']),
                         quote(self.app.config['tweet_blog']['access_token_secret']))
        oauth.append(('oauth_signature',
                      b64encode(hmac.new(key,sig,sha1).digest())))

        # form the OAuth header
        oauth_h = 'OAuth ' + ', '.join('%s="%s"' % tuple(map(quote, o))
                                       for o in oauth)

        # Build the request
        return fetch(url,
                     '&'.join('%s=%s' % tuple(map(quote, d)) for d in data),
                     method, {'Authorization': oauth_h},
                     validate_certificate=True)

routes = [(entry_url, CheckCommits),
          (worker_url, Tweet)]
<commit_msg>Moved blog api fetch to avoid timing issues<commit_after># coding=utf-8

from webapp2 import RequestHandler
import urllib
from google.appengine.api.urlfetch import fetch
from google.appengine.api import taskqueue
import hmac
from hashlib import sha1
from base64 import b64encode
from time import time
from random import getrandbits
import json
import logging

api_urls = {
    'request_token_url': 'http://twitter.com/oauth/request_token',
    'access_token_url' : 'http://twitter.com/oauth/access_token',
    'authorize_url'    : 'http://twitter.com/oauth/authorize',
    'tweet_url'        : 'https://api.twitter.com/1.1/statuses/update.json',
    'github_commits'   : 'https://api.github.com/repos/defeo/defeo.github.io/commits/',
    'blog_base'        : 'http://defeo.lu',
    'blog_posts'       : '/api/posts.json'
    }

# Configuration variables
entry_url   = '/tweet-ghblog'
worker_url  = '/tweet-ghblog/commit'
def w_config(m):
    def wrap(self):
        self._post_prefix = self.app.config['tweet_blog'].get('post_prefix') or '[POST]'
        self._post_dir    = self.app.config['tweet_blog'].get('post_dir')    or '_posts/'
        m(self)
    return wrap

class CheckCommits(RequestHandler):
    '''
    This hook repsonds to post requests from the github Webhook
    service. It looks for new post commits inside the commits posted
    by github.
    '''
    @w_config
    def post(self):
        commits = json.loads(self.request.get('payload'))
        # Examine each commit in the payload of the request
        count = 0
        for c in commits["commits"]:
            if c['message'].startswith(self._post_prefix):
                # If commit starts with the right prefix, launch a task to examine it
                taskqueue.add(url=worker_url, params={'commit-id': c['id']})
                count += 1

        logging.info('Considering %d commits.' % count)
        self.response.content_type = 'application/json'
        self.response.write('{"commits": %d}' % count)

class Tweet(RequestHandler):
    '''
    This hook examines a single commit, looks for newly added files,
    and tweets any new post found.
    '''
    @w_config
    def post(self):
        # Fetch and parse json summary of posts from the blog
        posts = fetch(api_urls['blog_base'] + api_urls['blog_posts'])
        if posts.status_code != 200:
            logging.error('Cannot fetch post list. %s' % post.content)
            self.abort(500, detail='Cannot fetch post list.')
        posts = json.loads(posts.content)

        # Fetch and parse commit data from github
        commit = fetch(api_urls['github_commits'] + self.request.get('commit-id'),
                       validate_certificate=True)
        if commit.status_code != 200:
            logging.error('Cannot fetch commit data. %s' % commit.content)
            self.abort(500, 'Cannot fetch commit data.')
        commit = json.loads(commit.content)

        if commit['commit']['message'].startswith(self._post_prefix):
            # If commit starts with right prefix, look for new files in the post directory
            count = 0
            for f in commit['files']:
                if f['status'] == 'added' and f['filename'].startswith(self._post_dir):
                    count += 1
                    # Tweet post announcement
                    post = posts.get(f['filename'])
                    if post is None:
                        logging.error('Post %s missing in list.' % f['filename'])
                        self.abort(500, 'Post seems to be missing in list.')

                    tweet = u'#Blog post %s.%s %s%s' % (post['title'],
                                                        ''.join(' #' + t for t in post['tags'][:2]),
                                                         api_urls['blog_base'], post['url'])

                    res = self._sign(api_urls['tweet_url'], 'POST', {'status': tweet})

                    if res.status_code != 200:
                        res = json.loads(res.content)
                        logging.error('Error while tweeting. %s ' % repr(res['errors']))
                        self.abort(500, 'Error while tweeting.')
                    else:
                        logging.info('Tweeted %s' % tweet)
            self.response.content_type = 'application/json'
            self.response.write('{"posts": %d}' % count)
        else:
            logging.info('Ignoring commit %s: badly formatted message.' % self.request.get('commit-id'))
            self.response.content_type = 'application/json'
            self.response.write('{"error":"Ignoring badly formatted commit message."}')

    def _sign(self, url, method='GET', data=None):
        def quote(s):
            'Percent encoding following RFC 3986'
            return urllib.quote(s.encode('utf-8'), '-._~')

        data = data.items() if data is not None else []

        # OAuth header
        oauth = [
            ('oauth_consumer_key'     , self.app.config['tweet_blog']['consumer_key']),
            ('oauth_nonce'            , b64encode('%0x' % getrandbits(256))[:32]),
            ('oauth_signature_method' , 'HMAC-SHA1'),
            ('oauth_timestamp'        , str(int(time()))),
            ('oauth_token'            , self.app.config['tweet_blog']['access_token']),
            ('oauth_version'          , '1.0')
            ]
        # OAuth signature
        sig = '%s&%s&%s' % (
            method.upper(),
            quote(url),
            quote("&".join('%s=%s' % tuple(map(quote, p))
                           for p in sorted(oauth + data,
                                           key=lambda(k,_):k))))
        key = '%s&%s' % (quote(self.app.config['tweet_blog']['consumer_secret']),
                         quote(self.app.config['tweet_blog']['access_token_secret']))
        oauth.append(('oauth_signature',
                      b64encode(hmac.new(key,sig,sha1).digest())))

        # form the OAuth header
        oauth_h = 'OAuth ' + ', '.join('%s="%s"' % tuple(map(quote, o))
                                       for o in oauth)

        # Build the request
        return fetch(url,
                     '&'.join('%s=%s' % tuple(map(quote, d)) for d in data),
                     method, {'Authorization': oauth_h},
                     validate_certificate=True)

routes = [(entry_url, CheckCommits),
          (worker_url, Tweet)]
<|endoftext|>


<commit_before>#!/usr/bin/python

# Author: Disa Mhembere, Johns Hopkins University
# Separated: 10/2/2012
# Plot all .np arrays in a common dir on the same axis & save
# 1 indexed

import matplotlib
matplotlib.use("Agg")

import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages
import pylab as pl

import numpy as np
import os
import sys
from glob import glob
import argparse
import scipy
from scipy import interpolate

# Issues: Done nothing with MAD

def plotInvDist(invDir, pngName, numBins =100):
  # ClustCoeff  Degree  Eigen  MAD  numEdges.npy  ScanStat  Triangle
  MADdir = "MAD"
  ccDir = "ClustCoeff"
  DegDir = "Degree"
  EigDir = "Eigen"
  SS1dir = "ScanStat"
  triDir = "Triangle"

  invDirs = [triDir, ccDir, SS1dir, DegDir ]
  '''
  if not os.path.exists(invDir):
    print "%s does not exist" % invDir
    sys.exit(1)

  pl.figure(2)
  fig_gl, axes = pl.subplots(nrows=3, ncols=2)
  fig_gl.tight_layout()

  for idx, drcty in enumerate (invDirs):
    for arrfn in glob(os.path.join(invDir, drcty,'*.npy')):
      try:
        arr = np.load(arrfn)
        arr = np.log(arr[arr.nonzero()])
        print "Processing %s..." % arrfn
      except:
        print "Ivariant file not found %s"  % arrfn
      pl.figure(1)
      n, bins, patches = pl.hist(arr, bins=numBins , range=None, normed=False, weights=None, cumulative=False, \
               bottom=None, histtype='stepfilled', align='mid', orientation='vertical', \
               rwidth=None, log=False, color=None, label=None, hold=None)

      n = np.append(n,0)
      n = n/float(sum(n))

      fig = pl.figure(2)
      fig.subplots_adjust(hspace=.5)

      ax = pl.subplot(3,2,idx+1)

      if idx == 0:
        plt.axis([0, 35, 0, 0.04])
        ax.set_yticks(scipy.arange(0,0.04,0.01))
      if idx == 1 or idx == 2:
        ax.set_yticks(scipy.arange(0,0.03,0.01))
      if idx == 3:
        ax.set_yticks(scipy.arange(0,0.04,0.01))

      # Interpolation
      f = interpolate.interp1d(bins, n, kind='cubic')

      x = np.arange(bins[0],bins[-1],0.03) # vary linspc

      interp = f(x)
      ltz = interp < 0
      interp[ltz] = 0
      pl.plot(x, interp,color ='grey' ,linewidth=1)

    if idx == 0:
      pl.ylabel('Probability (%)')
      pl.xlabel('log number of local triangles')
    if idx == 1:
      #pl.ylabel('Probability') #**
      pl.xlabel('log local clustering coefficient')
    if idx == 2:
      pl.ylabel('Probability (%)')
      pl.xlabel('log scan1 statistic')
    if idx == 3:
      #pl.ylabel('Probability') #**
      pl.xlabel('log local degree')
  '''

  ''' Eigenvalues '''
  ax = pl.subplot(3,2,5)
  ax.set_yticks(scipy.arange(0,180000,40000))
  for eigValInstance in glob(os.path.join(invDir, EigDir,"*.npy")):
    try:
      eigv = np.load(eigValInstance)
    except:
      print "Eigenvalue array"

    n = len(eigv)
    pl.plot(range(1,n+1), (np.sort(eigv)[::-1])/40000 , color='grey')
    pl.ylabel('Magnitude')
    pl.xlabel('Eigenvalue rank in top 100')

  ''' Edges '''
  arrfn = os.path.join(invDir, 'numEdges.npy')
  try:
    arr = np.load(arrfn)
    arr = np.log(arr[arr.nonzero()])
    print "Processing %s..." % arrfn
  except:
    print "Ivariant file not found %s"  % arrfn
  pl.figure(1)
  n, bins, patches = pl.hist(arr, bins=10 , range=None, normed=False, weights=None, cumulative=False, \
           bottom=None, histtype='stepfilled', align='mid', orientation='vertical', \
           rwidth=None, log=False, color=None, label=None, hold=None)

  n = np.append(n,0)

  fig = pl.figure(2)
  ax = pl.subplot(3,2,6)
  ax.set_xticks(scipy.arange(17.2,18.1,0.2))

  f = interpolate.interp1d(bins, n, kind='cubic')
  x = np.arange(bins[0],bins[-1],0.01) # vary linspc

  interp = f(x)
  ltz = interp < 0
  interp[ltz] = 0
  pl.plot(x, interp,color ='grey' ,linewidth=1)
  pl.ylabel('Frequency')
  pl.xlabel('log global edge number')

  pp = PdfPages(pngName+'.pdf')
  pp.savefig()
  pl.savefig(pngName+'F.pdf')
  #pl.savefig(os.path.join(toDir, "CombinedTriangles.png"))

def main():

    parser = argparse.ArgumentParser(description='Plot distribution of invariant arrays of several graphs')
    parser.add_argument('invDir', action='store',help='The full path of directory containing .npy invariant arrays')
    parser.add_argument('pngName', action='store', help='Full path of directory of resulting png file')
    parser.add_argument('numBins', type = int, action='store', help='Number of bins')

    result = parser.parse_args()

    plotInvDist(result.invDir, result.pngName, result.numBins)

if __name__ == '__main__':
  main()

<commit_msg>Minor<commit_after>#!/usr/bin/python

# Author: Disa Mhembere, Johns Hopkins University
# Separated: 10/2/2012
# Plot all .np arrays in a common dir on the same axis & save
# 1 indexed

import matplotlib
matplotlib.use("Agg")

import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages
import pylab as pl

import numpy as np
import os
import sys
from glob import glob
import argparse
import scipy
from scipy import interpolate

# Issues: Done nothing with MAD

def plotInvDist(invDir, pngName, numBins =100):
  # ClustCoeff  Degree  Eigen  MAD  numEdges.npy  ScanStat  Triangle
  MADdir = "MAD"
  ccDir = "ClustCoeff"
  DegDir = "Degree"
  EigDir = "Eigen"
  SS1dir = "ScanStat"
  triDir = "Triangle"

  invDirs = [triDir, ccDir, SS1dir, DegDir ]

  if not os.path.exists(invDir):
    print "%s does not exist" % invDir
    sys.exit(1)

  pl.figure(2)
  fig_gl, axes = pl.subplots(nrows=3, ncols=2)
  fig_gl.tight_layout()

  for idx, drcty in enumerate (invDirs):
    for arrfn in glob(os.path.join(invDir, drcty,'*.npy')):
      try:
        arr = np.load(arrfn)
        arr = np.log(arr[arr.nonzero()])
        print "Processing %s..." % arrfn
      except:
        print "Ivariant file not found %s"  % arrfn
      pl.figure(1)
      n, bins, patches = pl.hist(arr, bins=numBins , range=None, normed=False, weights=None, cumulative=False, \
               bottom=None, histtype='stepfilled', align='mid', orientation='vertical', \
               rwidth=None, log=False, color=None, label=None, hold=None)

      n = np.append(n,0)
      n = n/float(sum(n))

      fig = pl.figure(2)
      fig.subplots_adjust(hspace=.5)

      ax = pl.subplot(3,2,idx+1)

      if idx == 0:
        plt.axis([0, 35, 0, 0.04])
        ax.set_yticks(scipy.arange(0,0.04,0.01))
      if idx == 1 or idx == 2:
        ax.set_yticks(scipy.arange(0,0.03,0.01))
      if idx == 3:
        ax.set_yticks(scipy.arange(0,0.04,0.01))

      # Interpolation
      f = interpolate.interp1d(bins, n, kind='cubic')

      x = np.arange(bins[0],bins[-1],0.03) # vary linspc

      interp = f(x)
      ltz = interp < 0
      interp[ltz] = 0
      pl.plot(x, interp,color ='grey' ,linewidth=1)

    if idx == 0:
      pl.ylabel('Probability (%)')
      pl.xlabel('log number of local triangles')
    if idx == 1:
      #pl.ylabel('Probability') #**
      pl.xlabel('log local clustering coefficient')
    if idx == 2:
      pl.ylabel('Probability (%)')
      pl.xlabel('log scan1 statistic')
    if idx == 3:
      #pl.ylabel('Probability') #**
      pl.xlabel('log local degree')

  ''' Eigenvalues '''
  ax = pl.subplot(3,2,5)
  ax.set_yticks(scipy.arange(0,180000,40000))
  for eigValInstance in glob(os.path.join(invDir, EigDir,"*.npy")):
    try:
      eigv = np.load(eigValInstance)
    except:
      print "Eigenvalue array"

    n = len(eigv)
    pl.plot(range(1,n+1), (np.sort(eigv)[::-1])/40000 , color='grey')
    pl.ylabel('Magnitude')
    pl.xlabel('Eigenvalue rank in top 100')

  ''' Edges '''
  arrfn = os.path.join(invDir, 'numEdges.npy')
  try:
    arr = np.load(arrfn)
    arr = np.log(arr[arr.nonzero()])
    print "Processing %s..." % arrfn
  except:
    print "Ivariant file not found %s"  % arrfn
  pl.figure(1)
  n, bins, patches = pl.hist(arr, bins=10 , range=None, normed=False, weights=None, cumulative=False, \
           bottom=None, histtype='stepfilled', align='mid', orientation='vertical', \
           rwidth=None, log=False, color=None, label=None, hold=None)

  n = np.append(n,0)

  fig = pl.figure(2)
  ax = pl.subplot(3,2,6)
  ax.set_xticks(scipy.arange(17.2,18.1,0.2))

  f = interpolate.interp1d(bins, n, kind='cubic')
  x = np.arange(bins[0],bins[-1],0.01) # vary linspc

  interp = f(x)
  ltz = interp < 0
  interp[ltz] = 0
  pl.plot(x, interp,color ='grey' ,linewidth=1)
  pl.ylabel('Frequency')
  pl.xlabel('log global edge number')

  #pp = PdfPages(pngName+'.pdf')
  #pp.savefig()
  pl.savefig(pngName+'.pdf')
  #pl.savefig(os.path.join(toDir, "CombinedTriangles.png"))

def main():

    parser = argparse.ArgumentParser(description='Plot distribution of invariant arrays of several graphs')
    parser.add_argument('invDir', action='store',help='The full path of directory containing .npy invariant arrays')
    parser.add_argument('pngName', action='store', help='Full path of directory of resulting png file')
    parser.add_argument('numBins', type = int, action='store', help='Number of bins')

    result = parser.parse_args()

    plotInvDist(result.invDir, result.pngName, result.numBins)

if __name__ == '__main__':
  main()

<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# aiocouchdb documentation build configuration file, created by
# sphinx-quickstart on Fri Jun 27 20:44:37 2014.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.doctest',
    'sphinx.ext.intersphinx',
    'sphinx.ext.coverage',
    'sphinx.ext.viewcode',
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = 'aiocouchdb'
copyright = '2014, Alexander Shorin'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.0'
# The full version, including alpha/beta/rc tags.
release = '0.0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'default'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'aiocouchdbdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
  ('index', 'aiocouchdb.tex', 'aiocouchdb Documentation',
   'Alexander Shorin', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'aiocouchdb', 'aiocouchdb Documentation',
     ['Alexander Shorin'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  ('index', 'aiocouchdb', 'aiocouchdb Documentation',
   'Alexander Shorin', 'aiocouchdb', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {
    'http://docs.python.org/': None,
    'http://docs.couchdb.org/en/latest': None
}
<commit_msg>Update version for sphinx docs<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# aiocouchdb documentation build configuration file, created by
# sphinx-quickstart on Fri Jun 27 20:44:37 2014.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import aiocouchdb
import sys
import os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.doctest',
    'sphinx.ext.intersphinx',
    'sphinx.ext.coverage',
    'sphinx.ext.viewcode',
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = 'aiocouchdb'
copyright = '2014, Alexander Shorin'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = aiocouchdb.__version__.split('-')[0]
# The full version, including alpha/beta/rc tags.
release = aiocouchdb.__version__

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'default'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'aiocouchdbdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
  ('index', 'aiocouchdb.tex', 'aiocouchdb Documentation',
   'Alexander Shorin', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'aiocouchdb', 'aiocouchdb Documentation',
     ['Alexander Shorin'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  ('index', 'aiocouchdb', 'aiocouchdb Documentation',
   'Alexander Shorin', 'aiocouchdb', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {
    'http://docs.python.org/': None,
    'http://docs.couchdb.org/en/latest': None
}
<|endoftext|>


<commit_before>#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

import functools
import inspect
import sys
from typing import Any
from typing import Callable
from typing import Dict
from typing import List
from typing import Optional
from typing import Union

import pandas as pd

from apache_beam.dataframe import expressions
from apache_beam.dataframe import partitionings

# pylint: disable=deprecated-method
if sys.version_info < (3, ):
  _getargspec = inspect.getargspec

  def _unwrap(func):
    while hasattr(func, '__wrapped__'):
      func = func.__wrapped__
    return func
else:
  _getargspec = inspect.getfullargspec
  _unwrap = inspect.unwrap

class DeferredBase(object):

  _pandas_type_map = {}  # type: Dict[Union[type, None], type]

  def __init__(self, expr):
    self._expr = expr

  @classmethod
  def _register_for(cls, pandas_type):
    def wrapper(deferred_type):
      cls._pandas_type_map[pandas_type] = deferred_type
      return deferred_type

    return wrapper

  @classmethod
  def wrap(cls, expr):
    proxy_type = type(expr.proxy())
    if proxy_type in cls._pandas_type_map:
      wrapper_type = cls._pandas_type_map[proxy_type]
    else:
      if expr.requires_partition_by() != partitionings.Singleton():
        raise ValueError(
            'Scalar expression %s partitoned by non-singleton %s' %
            (expr, expr.requires_partition_by()))
      wrapper_type = _DeferredScalar
    return wrapper_type(expr)

  def _elementwise(self, func, name=None, other_args=(), inplace=False):
    return _elementwise_function(func, name, inplace=inplace)(self, *other_args)

  def __reduce__(self):
    return UnusableUnpickledDeferredBase, (str(self), )

class UnusableUnpickledDeferredBase(object):
  """Placeholder object used to break the transitive pickling chain in case a
  DeferredBase accidentially gets pickled (e.g. as part of globals).

  Trying to use this object after unpickling is a bug and will result in an
  error.
  """
  def __init__(self, name):
    self._name = name

  def __repr__(self):
    return 'UnusablePickledDeferredBase(%r)' % self.name

class DeferredFrame(DeferredBase):
  @property
  def dtypes(self):
    return self._expr.proxy().dtypes

class _DeferredScalar(DeferredBase):
  pass

DeferredBase._pandas_type_map[None] = _DeferredScalar

def name_and_func(method):
  if isinstance(method, str):
    return method, lambda df, *args, **kwargs: getattr(df, method)(*args, **
                                                                   kwargs)
  else:
    return method.__name__, method

def _elementwise_method(func, name=None, restrictions=None, inplace=False):
  return _proxy_method(
      func,
      name,
      restrictions,
      inplace,
      requires_partition_by=partitionings.Nothing(),
      preserves_partition_by=partitionings.Singleton())

def _proxy_method(
    func,
    name=None,
    restrictions=None,
    inplace=False,
    requires_partition_by=partitionings.Singleton(),
    preserves_partition_by=partitionings.Nothing()):
  if name is None:
    name, func = name_and_func(func)
  if restrictions is None:
    restrictions = {}
  return _proxy_function(
      func,
      name,
      restrictions,
      inplace,
      requires_partition_by,
      preserves_partition_by)

def _elementwise_function(func, name=None, restrictions=None, inplace=False):
  return _proxy_function(
      func,
      name,
      restrictions,
      inplace,
      requires_partition_by=partitionings.Nothing(),
      preserves_partition_by=partitionings.Singleton())

def _proxy_function(
      func,  # type: Union[Callable, str]
      name=None,  # type: Optional[str]
      restrictions=None,  # type: Optional[Dict[str, Union[Any, List[Any]]]]
      inplace=False,  # type: bool
      requires_partition_by=partitionings.Singleton(),  # type: partitionings.Partitioning
      preserves_partition_by=partitionings.Nothing(),  # type: partitionings.Partitioning
):

  if name is None:
    if isinstance(func, str):
      name = func
    else:
      name = func.__name__
  if restrictions is None:
    restrictions = {}

  def wrapper(*args, **kwargs):
    for key, values in ():  #restrictions.items():
      if key in kwargs:
        value = kwargs[key]
      else:
        try:
          ix = _getargspec(func).args.index(key)
        except ValueError:
          # TODO: fix for delegation?
          continue
        if len(args) <= ix:
          continue
        value = args[ix]
      if not isinstance(values, list):
        values = [values]
      if value not in values:
        raise NotImplementedError(
            '%s=%s not supported for %s' % (key, value, name))
    deferred_arg_indices = []
    deferred_arg_exprs = []
    constant_args = [None] * len(args)
    for ix, arg in enumerate(args):
      if isinstance(arg, DeferredBase):
        deferred_arg_indices.append(ix)
        deferred_arg_exprs.append(arg._expr)
      elif isinstance(arg, pd.core.generic.NDFrame):
        deferred_arg_indices.append(ix)
        deferred_arg_exprs.append(expressions.ConstantExpression(arg, arg[0:0]))
      else:
        constant_args[ix] = arg

    if inplace:
      actual_func = copy_and_mutate(func)
    else:
      actual_func = func

    def apply(*actual_args):
      full_args = list(constant_args)
      for ix, arg in zip(deferred_arg_indices, actual_args):
        full_args[ix] = arg
      return actual_func(*full_args, **kwargs)

    result_expr = expressions.ComputedExpression(
        name,
        apply,
        deferred_arg_exprs,
        requires_partition_by=requires_partition_by,
        preserves_partition_by=preserves_partition_by)
    if inplace:
      args[0]._expr = result_expr

    else:
      return DeferredFrame.wrap(result_expr)

  return wrapper

def _agg_method(func):
  def wrapper(self, *args, **kwargs):
    return self.agg(func, *args, **kwargs)

  return wrapper

def wont_implement_method(msg):
  def wrapper(self, *args, **kwargs):
    raise WontImplementError(msg)

  return wrapper

def copy_and_mutate(func):
  def wrapper(self, *args, **kwargs):
    copy = self.copy()
    func(copy, *args, **kwargs)
    return copy

  return wrapper

def maybe_inplace(func):
  @functools.wraps(func)
  def wrapper(self, inplace=False, **kwargs):
    result = func(self, **kwargs)
    if inplace:
      self._expr = result._expr
    else:
      return result

  return wrapper

def args_to_kwargs(base_type):
  def wrap(func):
    arg_names = _getargspec(_unwrap(getattr(base_type, func.__name__))).args

    @functools.wraps(func)
    def wrapper(*args, **kwargs):
      for name, value in zip(arg_names, args):
        if name in kwargs:
          raise TypeError(
              "%s() got multiple values for argument '%s'" %
              (func.__name__, name))
        kwargs[name] = value
      return func(**kwargs)

    return wrapper

  return wrap

def populate_defaults(base_type):
  def wrap(func):
    base_argspec = _getargspec(_unwrap(getattr(base_type, func.__name__)))
    if not base_argspec.defaults:
      return func

    arg_to_default = dict(
        zip(
            base_argspec.args[-len(base_argspec.defaults):],
            base_argspec.defaults))

    unwrapped_func = _unwrap(func)
    # args that do not have defaults in func, but do have defaults in base
    func_argspec = _getargspec(unwrapped_func)
    num_non_defaults = len(func_argspec.args) - len(func_argspec.defaults or ())
    defaults_to_populate = set(
        func_argspec.args[:num_non_defaults]).intersection(
            arg_to_default.keys())

    @functools.wraps(func)
    def wrapper(**kwargs):
      for name in defaults_to_populate:
        if name not in kwargs:
          kwargs[name] = arg_to_default[name]
      return func(**kwargs)

    return wrapper

  return wrap

class WontImplementError(NotImplementedError):
  """An subclass of NotImplementedError to raise indicating that implementing
  the given method is infeasible.

  Raising this error will also prevent this doctests from being validated
  when run with the beam dataframe validation doctest runner.
  """
  pass
<commit_msg>Use index for partitioning of elementwise operations with multiple DataFrame inputs.<commit_after>#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

import functools
import inspect
import sys
from typing import Any
from typing import Callable
from typing import Dict
from typing import List
from typing import Optional
from typing import Union

import pandas as pd

from apache_beam.dataframe import expressions
from apache_beam.dataframe import partitionings

# pylint: disable=deprecated-method
if sys.version_info < (3, ):
  _getargspec = inspect.getargspec

  def _unwrap(func):
    while hasattr(func, '__wrapped__'):
      func = func.__wrapped__
    return func
else:
  _getargspec = inspect.getfullargspec
  _unwrap = inspect.unwrap

class DeferredBase(object):

  _pandas_type_map = {}  # type: Dict[Union[type, None], type]

  def __init__(self, expr):
    self._expr = expr

  @classmethod
  def _register_for(cls, pandas_type):
    def wrapper(deferred_type):
      cls._pandas_type_map[pandas_type] = deferred_type
      return deferred_type

    return wrapper

  @classmethod
  def wrap(cls, expr):
    proxy_type = type(expr.proxy())
    if proxy_type in cls._pandas_type_map:
      wrapper_type = cls._pandas_type_map[proxy_type]
    else:
      if expr.requires_partition_by() != partitionings.Singleton():
        raise ValueError(
            'Scalar expression %s partitoned by non-singleton %s' %
            (expr, expr.requires_partition_by()))
      wrapper_type = _DeferredScalar
    return wrapper_type(expr)

  def _elementwise(self, func, name=None, other_args=(), inplace=False):
    return _elementwise_function(func, name, inplace=inplace)(self, *other_args)

  def __reduce__(self):
    return UnusableUnpickledDeferredBase, (str(self), )

class UnusableUnpickledDeferredBase(object):
  """Placeholder object used to break the transitive pickling chain in case a
  DeferredBase accidentially gets pickled (e.g. as part of globals).

  Trying to use this object after unpickling is a bug and will result in an
  error.
  """
  def __init__(self, name):
    self._name = name

  def __repr__(self):
    return 'UnusablePickledDeferredBase(%r)' % self.name

class DeferredFrame(DeferredBase):
  @property
  def dtypes(self):
    return self._expr.proxy().dtypes

class _DeferredScalar(DeferredBase):
  pass

DeferredBase._pandas_type_map[None] = _DeferredScalar

def name_and_func(method):
  if isinstance(method, str):
    return method, lambda df, *args, **kwargs: getattr(df, method)(*args, **
                                                                   kwargs)
  else:
    return method.__name__, method

def _elementwise_method(func, name=None, restrictions=None, inplace=False):
  return _proxy_method(
      func,
      name,
      restrictions,
      inplace,
      requires_partition_by=partitionings.Nothing(),
      preserves_partition_by=partitionings.Singleton())

def _proxy_method(
    func,
    name=None,
    restrictions=None,
    inplace=False,
    requires_partition_by=partitionings.Singleton(),
    preserves_partition_by=partitionings.Nothing()):
  if name is None:
    name, func = name_and_func(func)
  if restrictions is None:
    restrictions = {}
  return _proxy_function(
      func,
      name,
      restrictions,
      inplace,
      requires_partition_by,
      preserves_partition_by)

def _elementwise_function(func, name=None, restrictions=None, inplace=False):
  return _proxy_function(
      func,
      name,
      restrictions,
      inplace,
      requires_partition_by=partitionings.Nothing(),
      preserves_partition_by=partitionings.Singleton())

def _proxy_function(
      func,  # type: Union[Callable, str]
      name=None,  # type: Optional[str]
      restrictions=None,  # type: Optional[Dict[str, Union[Any, List[Any]]]]
      inplace=False,  # type: bool
      requires_partition_by=partitionings.Singleton(),  # type: partitionings.Partitioning
      preserves_partition_by=partitionings.Nothing(),  # type: partitionings.Partitioning
):

  if name is None:
    if isinstance(func, str):
      name = func
    else:
      name = func.__name__
  if restrictions is None:
    restrictions = {}

  def wrapper(*args, **kwargs):
    for key, values in ():  #restrictions.items():
      if key in kwargs:
        value = kwargs[key]
      else:
        try:
          ix = _getargspec(func).args.index(key)
        except ValueError:
          # TODO: fix for delegation?
          continue
        if len(args) <= ix:
          continue
        value = args[ix]
      if not isinstance(values, list):
        values = [values]
      if value not in values:
        raise NotImplementedError(
            '%s=%s not supported for %s' % (key, value, name))
    deferred_arg_indices = []
    deferred_arg_exprs = []
    constant_args = [None] * len(args)
    for ix, arg in enumerate(args):
      if isinstance(arg, DeferredBase):
        deferred_arg_indices.append(ix)
        deferred_arg_exprs.append(arg._expr)
      elif isinstance(arg, pd.core.generic.NDFrame):
        deferred_arg_indices.append(ix)
        deferred_arg_exprs.append(expressions.ConstantExpression(arg, arg[0:0]))
      else:
        constant_args[ix] = arg

    if inplace:
      actual_func = copy_and_mutate(func)
    else:
      actual_func = func

    def apply(*actual_args):
      full_args = list(constant_args)
      for ix, arg in zip(deferred_arg_indices, actual_args):
        full_args[ix] = arg
      return actual_func(*full_args, **kwargs)

    if any(isinstance(arg.proxy(), pd.core.generic.NDFrame)
           for arg in deferred_arg_exprs
           ) and not requires_partition_by.is_subpartitioning_of(
               partitionings.Index()):
      # Implicit join on index.
      actual_requires_partition_by = partitionings.Index()
    else:
      actual_requires_partition_by = requires_partition_by

    result_expr = expressions.ComputedExpression(
        name,
        apply,
        deferred_arg_exprs,
        requires_partition_by=actual_requires_partition_by,
        preserves_partition_by=preserves_partition_by)
    if inplace:
      args[0]._expr = result_expr

    else:
      return DeferredFrame.wrap(result_expr)

  return wrapper

def _agg_method(func):
  def wrapper(self, *args, **kwargs):
    return self.agg(func, *args, **kwargs)

  return wrapper

def wont_implement_method(msg):
  def wrapper(self, *args, **kwargs):
    raise WontImplementError(msg)

  return wrapper

def copy_and_mutate(func):
  def wrapper(self, *args, **kwargs):
    copy = self.copy()
    func(copy, *args, **kwargs)
    return copy

  return wrapper

def maybe_inplace(func):
  @functools.wraps(func)
  def wrapper(self, inplace=False, **kwargs):
    result = func(self, **kwargs)
    if inplace:
      self._expr = result._expr
    else:
      return result

  return wrapper

def args_to_kwargs(base_type):
  def wrap(func):
    arg_names = _getargspec(_unwrap(getattr(base_type, func.__name__))).args

    @functools.wraps(func)
    def wrapper(*args, **kwargs):
      for name, value in zip(arg_names, args):
        if name in kwargs:
          raise TypeError(
              "%s() got multiple values for argument '%s'" %
              (func.__name__, name))
        kwargs[name] = value
      return func(**kwargs)

    return wrapper

  return wrap

def populate_defaults(base_type):
  def wrap(func):
    base_argspec = _getargspec(_unwrap(getattr(base_type, func.__name__)))
    if not base_argspec.defaults:
      return func

    arg_to_default = dict(
        zip(
            base_argspec.args[-len(base_argspec.defaults):],
            base_argspec.defaults))

    unwrapped_func = _unwrap(func)
    # args that do not have defaults in func, but do have defaults in base
    func_argspec = _getargspec(unwrapped_func)
    num_non_defaults = len(func_argspec.args) - len(func_argspec.defaults or ())
    defaults_to_populate = set(
        func_argspec.args[:num_non_defaults]).intersection(
            arg_to_default.keys())

    @functools.wraps(func)
    def wrapper(**kwargs):
      for name in defaults_to_populate:
        if name not in kwargs:
          kwargs[name] = arg_to_default[name]
      return func(**kwargs)

    return wrapper

  return wrap

class WontImplementError(NotImplementedError):
  """An subclass of NotImplementedError to raise indicating that implementing
  the given method is infeasible.

  Raising this error will also prevent this doctests from being validated
  when run with the beam dataframe validation doctest runner.
  """
  pass
<|endoftext|>