<commit_before># Copyright (c) 2012 Roberto Alsina y otros.

# Permission is hereby granted, free of charge, to any
# person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the
# Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the
# Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice
# shall be included in all copies or substantial portions of
# the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

"""Implementation of compile_html based on markdown."""

import codecs
import os
import re

try:
    from markdown import markdown
except ImportError:
    markdown = None

from nikola.plugin_categories import PageCompiler

class CompileMarkdown(PageCompiler):
    """Compile markdown into HTML."""

    name = "markdown"

    def compile_html(self, source, dest):
        if markdown is None:
            raise Exception('To build this site, you need to install the "markdown" package.')
        try:
            os.makedirs(os.path.dirname(dest))
        except:
            pass
        with codecs.open(dest, "w+", "utf8") as out_file:
            with codecs.open(source, "r", "utf8") as in_file:
                data = in_file.read()
            output = markdown(data, ['fenced_code', 'codehilite'])
            # remove the H1 because there is "title" h1.
            output = re.sub(r'<h1>.*</h1>', '', output)
            # python-markdown's highlighter uses the class 'codehilite' to wrap
            # code, # instead of the standard 'code'. None of the standard
            # pygments stylesheets use this class, so swap it to be 'code'
            output = re.sub(r'(<div[^>]+class="[^"]*)codehilite([^>]+)',
                            r'\1code\2', output)
            out_file.write(output)

    def create_post(self, path, onefile=False, title="", slug="", date="", tags=""):
        with codecs.open(path, "wb+", "utf8") as fd:
            if onefile:
                fd.write('<!-- \n')
                fd.write('.. title: %s\n' % title)
                fd.write('.. slug: %s\n' % slug)
                fd.write('.. date: %s\n' % date)
                fd.write('.. tags: %s\n' % tags)
                fd.write('.. link: \n')
                fd.write('.. description: \n')
                fd.write('-->\n\n')
            fd.write("\nWrite your post here.")
<commit_msg>Increment header levels in markdown compiler instead of stripping h1's<commit_after># Copyright (c) 2012 Roberto Alsina y otros.

# Permission is hereby granted, free of charge, to any
# person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the
# Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the
# Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice
# shall be included in all copies or substantial portions of
# the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

"""Implementation of compile_html based on markdown."""

import codecs
import os
import re

try:
    from markdown import markdown
except ImportError:
    markdown = None

from nikola.plugin_categories import PageCompiler

class CompileMarkdown(PageCompiler):
    """Compile markdown into HTML."""

    name = "markdown"

    def compile_html(self, source, dest):
        if markdown is None:
            raise Exception('To build this site, you need to install the "markdown" package.')
        try:
            os.makedirs(os.path.dirname(dest))
        except:
            pass
        with codecs.open(dest, "w+", "utf8") as out_file:
            with codecs.open(source, "r", "utf8") as in_file:
                data = in_file.read()
            output = markdown(data, ['fenced_code', 'codehilite'])
            # h1 is reserved for the title so increment all header levels
            for n in reversed(range(1,9)):
                output = re.sub('<h%i>' % n, '<h%i>' % (n + 1), output)
                output = re.sub('</h%i>' % n, '</h%i>' % (n + 1), output)
            # python-markdown's highlighter uses the class 'codehilite' to wrap
            # code, # instead of the standard 'code'. None of the standard
            # pygments stylesheets use this class, so swap it to be 'code'
            output = re.sub(r'(<div[^>]+class="[^"]*)codehilite([^>]+)',
                            r'\1code\2', output)
            out_file.write(output)

    def create_post(self, path, onefile=False, title="", slug="", date="", tags=""):
        with codecs.open(path, "wb+", "utf8") as fd:
            if onefile:
                fd.write('<!-- \n')
                fd.write('.. title: %s\n' % title)
                fd.write('.. slug: %s\n' % slug)
                fd.write('.. date: %s\n' % date)
                fd.write('.. tags: %s\n' % tags)
                fd.write('.. link: \n')
                fd.write('.. description: \n')
                fd.write('-->\n\n')
            fd.write("\nWrite your post here.")
<|endoftext|>


<commit_before>import numpy as np
import numpy.linalg as lg
from scipy.spatial.distance import euclidean

def gen_points_random(num_pts, ndim, r=3):
    points = np.random.uniform(-r,r,size=(num_pts,ndim))
    points = np.array(points)
    return points

# Returns a grid of points with a square or cube root of the num_pts requests
def gen_points_grid(num_pts, ndim, r=3):
    assert ndim <= 3, "gen_points_grid in data_generation has not been implemented yet for %d dimensions" % (ndim)
    x = np.linspace(-r,r, int(np.power(num_pts, 1./ndim)))
    points = np.array(np.meshgrid(*([x]*ndim)))
    points = points.reshape(ndim,int(np.power(num_pts, 1./ndim))**ndim).T
    return points

def gen_basestations(num_bases, ndim, r=4, bs_type="unit"):
    # point on the circle with radius r
    #random point and random angle
    if bs_type=="random":
        t = np.random.uniform(0, 2*np.pi, num_bases)
        v = np.random.uniform(0, 2*np.pi, num_bases)
        angles = np.random.uniform(0,180,size=(num_bases,ndim-1)).tolist()
        if ndim == 2:
            bases = zip(zip(r*np.cos(t), r*np.sin(t)), angles)
        elif ndim == 3:
            bases = zip(zip(r*np.cos(t)*np.sin(v), r*np.sin(t)*np.cos(v), r*np.cos(v)), angles)

    elif bs_type=="unit":
        t  = np.arange(0,2*np.pi, (2*np.pi)/num_bases)
        v  = np.arange(0,2*np.pi, (2*np.pi)/num_bases)
        if ndim == 2:
            base_pts  = zip(r*np.cos(t), r*np.sin(t))
            angles = [[np.degrees(np.sign(y)*np.arccos(x/np.linalg.norm([x,y])))+90.] for x, y in base_pts]
        elif ndim == 3:
            # TODO: currently we are assuming z is always 0, may want to change
            base_pts = zip(r*np.cos(t), r*np.sin(t), np.zeros((t.shape[0])))
            # TOD: currently we are assuming 90 degrees for z angle, may want to change

            angles = [[np.degrees(np.sign(y)*np.arccos(x/np.linalg.norm([x,y])))+90, 90.] for x, y, z in base_pts]

        bases = zip(base_pts, angles)

    elif bs_type=="colinear":
        #bases = [((4,0), [90.]), ((-4,0), [90.]), ((0,4), [180.])]
        bases = [((4,0), [90.]), ((-4,0), [270.]), ((0,4), [180.])]
        # bases = [((-4,0), [90.]), ((0,4), [180.])]
    elif bs_type=="outdoor":
        #bases = [((4,0), [90.]), ((-4,0), [90.]), ((0,4), [180.])]
        bases = [((-3.58,0), [90.]),((0, 2.726), [180.]), ((3.58,0), [270.])]
        # bases = [((-4,0), [90.]), ((0,4), [180.])]
    elif bs_type=="colinear-3D":
        bases = [((4,0,0), [90.,90.]), ((-4,0,0), [90.,90.]), ((0,4,0), [180.,90.])]
    elif bs_type=="structured":
        #bases = [[((4,0), [90.]), ((0,-4), [0.])],[((4,0), [90.]), ((0,4), [0.])]]
        bases = [((4,0), [90.]), ((-4,0), [270.]), ((0,4), [180.])]
        #bases = [((4,0), [90.]), ((0,4), [0.]), ((-4,0), [90.]), ((0,4), [0.])]
    elif bs_type=="structured-3D":
        bases = [((4,0,0), [90.,90.]), ((-4,0,0), [90.,90.]), ((0,4,0), [180.,90.])]
    elif bs_type=="faraway":
        bases = [((-3,18), [180.]), ((3,18), [180.])]
    elif bs_type=="faraway_but_closer":
        bases = [((-3,4), [180.]), ((3,4), [180.])]

    return bases

def generate_from_real_data(parsed_data):
    base_stations = [((-3.58,0), [90.]),((0, 2.726), [180.]), ((3.58,0), [270.])]
    virt_x=np.tile(np.linspace(-2.5,2.5, 6), 5)
    virt_y=np.repeat(np.linspace(-1.6,1.6, 5), 6)

    mobiles = np.vstack((virt_x, virt_y)).T
    mobiles = np.tile(mobiles,(parsed_data.shape[1],1))
    # true_angles = get_mobile_angles(base_stations, mobiles, 2)
    # true_angles %= 360

    angles = []
    for run_number in range(parsed_data.shape[0]):
        # xx= 180. - parsed_data[1,run_number,:,:].T.ravel()
        # print xx.shape
        # print xx
        # assert False
        this_run_angles = np.vstack(([180. - parsed_data[i,run_number,:,:].T.ravel() for i in range(parsed_data.shape[0])]))
        angles.append(this_run_angles.T)

    angles = np.vstack(angles)
    # angles = angles.T
    # print angles
    # print angles.shape
    # print mobiles
    # assert False

    # angles = np.vstack(([np.hstack(([180. - parsed_data[i,j,:,:].T.ravel() for i in range(parsed_data.shape[0])]))\
                                                                           # for j in range(parsed_data.shape[1])]))

    # angles = angles.T

    angles %= 360
    angles /= 360.

    return mobiles, base_stations, angles

    # print angles.shape
    # print mobiles.shape
    # assert False, 'Snapp, I cant believe this worked'

def get_angle(mobile_loc, base_loc, base_theta):
    y = mobile_loc[1] -  base_loc[1]
    x = mobile_loc[0] -  base_loc[0]
    # print "ML: ", mobile_loc
    # print "BL: ", base_loc
    hyp = euclidean(mobile_loc, base_loc)

    if y < 0:
        k = -1
    else:
        k = 1
    angle = np.degrees(k*np.arccos(x/hyp)) % 360
    return angle+base_theta

#mobile_loc - (x,y,z)
# base_lco - (x,y,z)
# base_angles - (alpha, beta, gamma)
def get3D_angles(mobile_loc, base_loc, base_angles):

    pos_idx = range(3)

    mobile_arr = np.array(mobile_loc)
    base_arr = np.array(base_loc)
    base_angles_arr = np.array(base_angles)
    idxs = [[0,1], [1,2], [0,2]]

    return [get_angle(mobile_arr[list(idx)], base_arr[list(idx)], base_angles_arr[idx[0]]) for idx in idxs]

def get_mobile_angles(bases, mobiles, ndim):
    if ndim == 2:
        mobile_angles = []
        # for bases in base_set:
        #   assert mobiles.shape[1]==2, "Mobiles must have 2 columns!"
        #   mobile_angles.append([[get_angle(mobile_loc, base[0], base[1][0]) for base in bases] for mobile_loc in mobiles])
        assert mobiles.shape[1]==2, "Mobiles must have 2 columns!"
        mobile_angles.append([[get_angle(mobile_loc, base[0], base[1][0]) for base in bases] for mobile_loc in mobiles])
        #t1
        angles_output = np.vstack(mobile_angles)
        #t2
        # angles_output = np.hstack(mobile_angles)
        # t3
        # angles_output = mobile_angles
        # print "AO: ", angles_output.shape
        # assert False
    elif ndim == 3:
        mobile_angles = [[get3D_angles(mobile_loc, base[0], base[1]) for base in bases] for mobile_loc in mobiles]

        mobile_angles = np.array(mobile_angles)
        angles_output = mobile_angles.reshape(mobile_angles.shape[0], -1)

    return angles_output

def generate_data(num_pts, num_stations, ndim, pts_r=3, bs_r=4, bs_type="random", points_type="random"):
    if points_type == "random":
        mobiles = gen_points_random(num_pts, ndim, r=pts_r)
    elif points_type == "grid":
        mobiles = gen_points_grid(num_pts, ndim, r=pts_r)
    else:
        assert False, "This pattern of point generation has not been implemented yet in data_generation"
    bases = gen_basestations(num_stations, ndim, r=bs_r, bs_type=bs_type)
    angles = get_mobile_angles(bases, mobiles, ndim)
    # angles = np.abs(90. - angles)
    angles %= 360
    angles /= 360.
    angles = np.nan_to_num(angles)
    return mobiles, bases, angles

def replicate_data(data, ndim,  base_idxs):

    if len(base_idxs) == 0:
        return data
    else:
        # using 1 angle for each base station (alpha)
        if ndim == 2:
            rep_data = []
            for idxs in base_idxs:
                print idxs
                rep_data.append(data[:,idxs])
            data = np.hstack(rep_data)
        # using 3 angles for each base station (alpha beta gamma)
        elif ndim == 3:
            rep_data = []
            for idxs in base_idxs:
                col_idxs = []
                for i in idxs:
                    col_idxs += [i*ndim, i*ndim+1, i*ndim+2]
                rep_data.append(data[:,col_idxs])
            data = np.hstack(rep_data)

    return data

# TODO: maaaaaayyyyyyyybbbbeeeee some assert tests, but everything **looks** great
def test_datagen():
    bs_type = "random"
    ndims = [2,3]
    num_pts = 4
    num_stations = 5
    for ndim in ndims:
        mobiles = gen_points_random(num_pts, ndim, r=3)
        bases = gen_basestations(num_stations, ndim, bs_type=bs_type)
        angles = get_mobile_angles(bases, mobiles, ndim)
        print "pre noise: ", angles
        angles = add_noise(angles)

        print "************* NDIM=%d ***************" % (ndim)
        print "Mobiles: ", mobiles
        print "Bases: ", bases
        print "Angles: ", angles

#test_datagen()

<commit_msg>updated angle things for error<commit_after>import numpy as np
import numpy.linalg as lg
from scipy.spatial.distance import euclidean

def gen_points_random(num_pts, ndim, r=3):
    points = np.random.uniform(-r,r,size=(num_pts,ndim))
    points = np.array(points)
    return points

# Returns a grid of points with a square or cube root of the num_pts requests
def gen_points_grid(num_pts, ndim, r=3):
    assert ndim <= 3, "gen_points_grid in data_generation has not been implemented yet for %d dimensions" % (ndim)
    x = np.linspace(-r,r, int(np.power(num_pts, 1./ndim)))
    points = np.array(np.meshgrid(*([x]*ndim)))
    points = points.reshape(ndim,int(np.power(num_pts, 1./ndim))**ndim).T
    return points

def gen_basestations(num_bases, ndim, r=4, bs_type="unit"):
    # point on the circle with radius r
    #random point and random angle
    if bs_type=="random":
        t = np.random.uniform(0, 2*np.pi, num_bases)
        v = np.random.uniform(0, 2*np.pi, num_bases)
        angles = np.random.uniform(0,180,size=(num_bases,ndim-1)).tolist()
        if ndim == 2:
            bases = zip(zip(r*np.cos(t), r*np.sin(t)), angles)
        elif ndim == 3:
            bases = zip(zip(r*np.cos(t)*np.sin(v), r*np.sin(t)*np.cos(v), r*np.cos(v)), angles)

    elif bs_type=="unit":
        t  = np.arange(0,2*np.pi, (2*np.pi)/num_bases)
        v  = np.arange(0,2*np.pi, (2*np.pi)/num_bases)
        if ndim == 2:
            base_pts  = zip(r*np.cos(t), r*np.sin(t))
            angles = [[np.degrees(np.sign(y)*np.arccos(x/np.linalg.norm([x,y])))+90.] for x, y in base_pts]
        elif ndim == 3:
            # TODO: currently we are assuming z is always 0, may want to change
            base_pts = zip(r*np.cos(t), r*np.sin(t), np.zeros((t.shape[0])))
            # TOD: currently we are assuming 90 degrees for z angle, may want to change

            angles = [[np.degrees(np.sign(y)*np.arccos(x/np.linalg.norm([x,y])))+90, 90.] for x, y, z in base_pts]

        bases = zip(base_pts, angles)

    elif bs_type=="colinear":
        #bases = [((4,0), [90.]), ((-4,0), [90.]), ((0,4), [180.])]
        bases = [((4,0), [90.]), ((-4,0), [270.]), ((0,4), [180.])]
        # bases = [((-4,0), [90.]), ((0,4), [180.])]
    elif bs_type=="outdoor":
        #bases = [((4,0), [90.]), ((-4,0), [90.]), ((0,4), [180.])]
        bases = [((-3.58,0), [90.]),((0, 2.726), [180.]), ((3.58,0), [270.])]
        # bases = [((-4,0), [90.]), ((0,4), [180.])]
    elif bs_type=="colinear-3D":
        bases = [((4,0,0), [90.,90.]), ((-4,0,0), [90.,90.]), ((0,4,0), [180.,90.])]
    elif bs_type=="structured":
        #bases = [[((4,0), [90.]), ((0,-4), [0.])],[((4,0), [90.]), ((0,4), [0.])]]
        bases = [((4,0), [90.]), ((-4,0), [270.]), ((0,4), [180.])]
        #bases = [((4,0), [90.]), ((0,4), [0.]), ((-4,0), [90.]), ((0,4), [0.])]
    elif bs_type=="structured-3D":
        bases = [((4,0,0), [90.,90.]), ((-4,0,0), [90.,90.]), ((0,4,0), [180.,90.])]
    elif bs_type=="faraway":
        #bases = [((-3,18), [180.]), ((3,18), [180.])]
        #bases = [((-3,18), [0.]), ((3,18), [0.])]
        bases = [((-18,0), [90.]), ((18,0), [90.])]
        bases = [((-18,18), [90.]), ((18,18), [90.])]
        bases = [((-18,-18), [90.]), ((18,-18), [90.])]
    elif bs_type=="faraway_but_closer":
        bases = [((-3,4), [0.]), ((3,4), [0.])]

    return bases

def generate_from_real_data(parsed_data):
    base_stations = [((-3.58,0), [90.]),((0, 2.726), [180.]), ((3.58,0), [270.])]
    virt_x=np.tile(np.linspace(-2.5,2.5, 6), 5)
    virt_y=np.repeat(np.linspace(-1.6,1.6, 5), 6)

    mobiles = np.vstack((virt_x, virt_y)).T
    mobiles = np.tile(mobiles,(parsed_data.shape[1],1))
    # true_angles = get_mobile_angles(base_stations, mobiles, 2)
    # true_angles %= 360

    angles = []
    for run_number in range(parsed_data.shape[0]):
        # xx= 180. - parsed_data[1,run_number,:,:].T.ravel()
        # print xx.shape
        # print xx
        # assert False
        this_run_angles = np.vstack(([180. - parsed_data[i,run_number,:,:].T.ravel() for i in range(parsed_data.shape[0])]))
        angles.append(this_run_angles.T)

    angles = np.vstack(angles)
    # angles = angles.T
    # print angles
    # print angles.shape
    # print mobiles
    # assert False

    # angles = np.vstack(([np.hstack(([180. - parsed_data[i,j,:,:].T.ravel() for i in range(parsed_data.shape[0])]))\
                                                                           # for j in range(parsed_data.shape[1])]))

    # angles = angles.T

    angles %= 360
    angles /= 360.

    return mobiles, base_stations, angles

    # print angles.shape
    # print mobiles.shape
    # assert False, 'Snapp, I cant believe this worked'

def get_angle(mobile_loc, base_loc, base_theta):
    y = mobile_loc[1] -  base_loc[1]
    x = mobile_loc[0] -  base_loc[0]
    # print "ML: ", mobile_loc
    # print "BL: ", base_loc
    hyp = euclidean(mobile_loc, base_loc)

    if y < 0:
        k = -1
    else:
        k = 1
    theta = np.degrees(np.arccos(np.abs(x)/np.abs(hyp)))
    angle = theta

    if x > 0 and y > 0:
        angle = theta
    if x < 0 and y > 0:
        angle = 180. - theta
    if x < 0 and y < 0:
        angle = 180. + theta
    if x > 0 and y < 0:
        angle = 360. - theta
    return (angle+base_theta) % 360.

    #angle = np.degrees(k*np.arccos(x/hyp)) % 360
    #return angle+base_theta

#mobile_loc - (x,y,z)
# base_lco - (x,y,z)
# base_angles - (alpha, beta, gamma)
def get3D_angles(mobile_loc, base_loc, base_angles):

    pos_idx = range(3)

    mobile_arr = np.array(mobile_loc)
    base_arr = np.array(base_loc)
    base_angles_arr = np.array(base_angles)
    idxs = [[0,1], [1,2], [0,2]]

    return [get_angle(mobile_arr[list(idx)], base_arr[list(idx)], base_angles_arr[idx[0]]) for idx in idxs]

def get_mobile_angles(bases, mobiles, ndim):
    if ndim == 2:
        mobile_angles = []
        # for bases in base_set:
        #   assert mobiles.shape[1]==2, "Mobiles must have 2 columns!"
        #   mobile_angles.append([[get_angle(mobile_loc, base[0], base[1][0]) for base in bases] for mobile_loc in mobiles])
        assert mobiles.shape[1]==2, "Mobiles must have 2 columns!"
        mobile_angles.append([[get_angle(mobile_loc, base[0], base[1][0]) for base in bases] for mobile_loc in mobiles])
        #t1
        angles_output = np.vstack(mobile_angles)
        #t2
        # angles_output = np.hstack(mobile_angles)
        # t3
        # angles_output = mobile_angles
        # print "AO: ", angles_output.shape
        # assert False
    elif ndim == 3:
        mobile_angles = [[get3D_angles(mobile_loc, base[0], base[1]) for base in bases] for mobile_loc in mobiles]

        mobile_angles = np.array(mobile_angles)
        angles_output = mobile_angles.reshape(mobile_angles.shape[0], -1)

    return angles_output

def generate_data(num_pts, num_stations, ndim, pts_r=3, bs_r=4, bs_type="random", points_type="random"):
    if points_type == "random":
        mobiles = gen_points_random(num_pts, ndim, r=pts_r)
    elif points_type == "grid":
        mobiles = gen_points_grid(num_pts, ndim, r=pts_r)
    else:
        assert False, "This pattern of point generation has not been implemented yet in data_generation"
    bases = gen_basestations(num_stations, ndim, r=bs_r, bs_type=bs_type)
    angles = get_mobile_angles(bases, mobiles, ndim)
    # angles = np.abs(90. - angles)
    angles %= 360
    angles /= 360.
    angles = np.nan_to_num(angles)
    return mobiles, bases, angles

def replicate_data(data, ndim,  base_idxs):

    if len(base_idxs) == 0:
        return data
    else:
        # using 1 angle for each base station (alpha)
        if ndim == 2:
            rep_data = []
            for idxs in base_idxs:
                print idxs
                rep_data.append(data[:,idxs])
            data = np.hstack(rep_data)
        # using 3 angles for each base station (alpha beta gamma)
        elif ndim == 3:
            rep_data = []
            for idxs in base_idxs:
                col_idxs = []
                for i in idxs:
                    col_idxs += [i*ndim, i*ndim+1, i*ndim+2]
                rep_data.append(data[:,col_idxs])
            data = np.hstack(rep_data)

    return data

# TODO: maaaaaayyyyyyyybbbbeeeee some assert tests, but everything **looks** great

def test_angle_calculation():
    r = 4.
    bases = [((0.,0.), (0.))]

def test_datagen():
    bs_type = "random"
    ndims = [2,3]
    num_pts = 4
    num_stations = 5
    for ndim in ndims:
        mobiles = gen_points_random(num_pts, ndim, r=3)
        bases = gen_basestations(num_stations, ndim, bs_type=bs_type)
        angles = get_mobile_angles(bases, mobiles, ndim)
        print "pre noise: ", angles
        angles = add_noise(angles)

        print "************* NDIM=%d ***************" % (ndim)
        print "Mobiles: ", mobiles
        print "Bases: ", bases
        print "Angles: ", angles

#test_datagen()

<|endoftext|>


<commit_before>from django.conf import settings
from rest_framework.views import APIView
from rest_framework.authentication import TokenAuthentication
from rest_framework.permissions import IsAuthenticated
from sis_provisioner.views.admin import RESTDispatch
from sis_provisioner.dao.user import (
    get_person_by_netid, get_person_by_gmail_id, user_sis_id, user_email,
    user_fullname)
from sis_provisioner.exceptions import UserPolicyException
from restclients_core.exceptions import DataFailureException
from logging import getLogger

logger = getLogger(__name__)

class LoginValidationView(APIView):
    authentication_classes = [TokenAuthentication]
    permission_classes = [IsAuthenticated]

    def post(self, request, *args, **kwargs):
        try:
            login_data = request.data['logins']
        except KeyError as ex:
            return RESTDispatch.error_response(400, 'Missing list of logins')

        users = []
        for login in login_data:
            login = login.strip().lower()
            if not any(u.get('login') == login for u in users):
                try:
                    user = {}
                    try:
                        person = get_person_by_gmail_id(login)
                        user['login'] = person.login_id
                    except UserPolicyException:
                        login = self.strip_domain(login)
                        person = get_person_by_netid(login)
                        user['login'] = person.uwnetid

                    sis_id = user_sis_id(person)
                    if not any(u.get('sis_id') == sis_id for u in users):
                        user['sis_id'] = sis_id
                        user['email'] = user_email(person)
                        name = user_fullname(person)
                        if len(name) == 2:
                            user['first_name'] = name[0]
                            user['last_name'] = name[1]
                            user['full_name'] = ' '.join(name)
                        else:
                            user['first_name'] = None
                            user['last_name'] = None
                            user['full_name'] = name[0]
                        users.append(user)

                except DataFailureException as ex:
                    users.append({'login': login, 'error': ex.msg})

                except UserPolicyException as ex:
                    users.append({'login': login, 'error': '{}'.format(ex)})

        return RESTDispatch.json_response({'users': users})

    @staticmethod
    def strip_domain(login):
        try:
            (username, domain) = login.split('@')
            if domain in getattr(settings, 'ADD_USER_DOMAIN_WHITELIST', []):
                return username
        except ValueError:
            pass
        return login
<commit_msg>drops first, last name for login api<commit_after>from django.conf import settings
from rest_framework.views import APIView
from rest_framework.authentication import TokenAuthentication
from rest_framework.permissions import IsAuthenticated
from sis_provisioner.views.admin import RESTDispatch
from sis_provisioner.dao.user import (
    get_person_by_netid, get_person_by_gmail_id, user_sis_id, user_email,
    user_fullname)
from sis_provisioner.exceptions import UserPolicyException
from restclients_core.exceptions import DataFailureException
from logging import getLogger

logger = getLogger(__name__)

class LoginValidationView(APIView):
    authentication_classes = [TokenAuthentication]
    permission_classes = [IsAuthenticated]

    def post(self, request, *args, **kwargs):
        try:
            login_data = request.data['logins']
        except KeyError as ex:
            return RESTDispatch.error_response(400, 'Missing list of logins')

        users = []
        for login in login_data:
            login = login.strip().lower()
            if not any(u.get('login') == login for u in users):
                try:
                    user = {}
                    try:
                        person = get_person_by_gmail_id(login)
                        user['login'] = person.login_id
                    except UserPolicyException:
                        login = self.strip_domain(login)
                        person = get_person_by_netid(login)
                        user['login'] = person.uwnetid

                    sis_id = user_sis_id(person)
                    if not any(u.get('sis_id') == sis_id for u in users):
                        user['sis_id'] = sis_id
                        user['email'] = user_email(person)
                        user['full_name'] = ' '.join(user_fullname(person))
                        users.append(user)

                except DataFailureException as ex:
                    users.append({'login': login, 'error': ex.msg})

                except UserPolicyException as ex:
                    users.append({'login': login, 'error': '{}'.format(ex)})

        return RESTDispatch.json_response({'users': users})

    @staticmethod
    def strip_domain(login):
        try:
            (username, domain) = login.split('@')
            if domain in getattr(settings, 'ADD_USER_DOMAIN_WHITELIST', []):
                return username
        except ValueError:
            pass
        return login
<|endoftext|>


<commit_before>import numpy as np
from numba import cuda, float32, int32
from numba.core.errors import TypingError
from numba.cuda.testing import unittest, CUDATestCase
from numba.cuda.testing import skip_on_cudasim

GLOBAL_CONSTANT = 5
GLOBAL_CONSTANT_2 = 6
GLOBAL_CONSTANT_TUPLE = 5, 6

def udt_global_constants(A):
    sa = cuda.shared.array(shape=GLOBAL_CONSTANT, dtype=float32)
    i = cuda.grid(1)
    A[i] = sa[i]

def udt_global_build_tuple(A):
    sa = cuda.shared.array(shape=(GLOBAL_CONSTANT, GLOBAL_CONSTANT_2),
                           dtype=float32)
    i, j = cuda.grid(2)
    A[i, j] = sa[i, j]

def udt_global_build_list(A):
    sa = cuda.shared.array(shape=[GLOBAL_CONSTANT, GLOBAL_CONSTANT_2],
                           dtype=float32)
    i, j = cuda.grid(2)
    A[i, j] = sa[i, j]

def udt_global_constant_tuple(A):
    sa = cuda.shared.array(shape=GLOBAL_CONSTANT_TUPLE, dtype=float32)
    i, j = cuda.grid(2)
    A[i, j] = sa[i, j]

def udt_invalid_1(A):
    sa = cuda.shared.array(shape=A[0], dtype=float32)
    i = cuda.grid(1)
    A[i] = sa[i]

def udt_invalid_2(A):
    sa = cuda.shared.array(shape=(1, A[0]), dtype=float32)
    i, j = cuda.grid(2)
    A[i, j] = sa[i, j]

def udt_invalid_3(A):
    sa = cuda.shared.array(shape=(1, A[0]), dtype=float32)
    i = cuda.grid(1)
    A[i] = sa[i, 0]

class TestSharedMemoryCreation(CUDATestCase):
    def getarg(self):
        return np.array(100, dtype=np.float32, ndmin=1)

    def getarg2(self):
        return self.getarg().reshape(1,1)

    def test_global_constants(self):
        udt = cuda.jit((float32[:],))(udt_global_constants)
        udt[1, 1](self.getarg())

    def test_global_build_tuple(self):
        udt = cuda.jit((float32[:, :],))(udt_global_build_tuple)
        udt[1, 1](self.getarg2())

    @skip_on_cudasim('Simulator does not perform macro expansion')
    def test_global_build_list(self):
        with self.assertRaises(TypingError) as raises:
            cuda.jit((float32[:, :],))(udt_global_build_list)

        self.assertIn("No implementation of function "
                      "Function(<function shared.array",
                      str(raises.exception))
        self.assertIn("found for signature:\n \n "
                      ">>> array(shape=list(int64), dtype=class(float32)",
                      str(raises.exception))

    def test_global_constant_tuple(self):
        udt = cuda.jit((float32[:, :],))(udt_global_constant_tuple)
        udt[1, 1](self.getarg2())

    @skip_on_cudasim("Can't check for constants in simulator")
    def test_invalid_1(self):
        # Scalar shape cannot be a floating point value
        with self.assertRaises(TypingError) as raises:
            cuda.jit((float32[:],))(udt_invalid_1)

        self.assertIn("No implementation of function "
                      "Function(<function shared.array",
                      str(raises.exception))
        self.assertIn("found for signature:\n \n "
                      ">>> array(shape=float32, dtype=class(float32))",
                      str(raises.exception))

    @skip_on_cudasim("Can't check for constants in simulator")
    def test_invalid_2(self):
        # Tuple shape cannot contain a floating point value
        with self.assertRaises(TypingError) as raises:
            cuda.jit((float32[:, :],))(udt_invalid_2)

        self.assertIn("No implementation of function "
                      "Function(<function shared.array",
                      str(raises.exception))
        self.assertIn("found for signature:\n \n "
                      ">>> array(shape=Tuple(Literal[int](1), "
                      "array(float32, 1d, A)), dtype=class(float32))",
                      str(raises.exception))

    @skip_on_cudasim("Can't check for constants in simulator")
    def test_invalid_3(self):
        # Scalar shape must be literal
        with self.assertRaises(TypingError) as raises:
            cuda.jit((int32[:],))(udt_invalid_1)

        self.assertIn("No implementation of function "
                      "Function(<function shared.array",
                      str(raises.exception))
        self.assertIn("found for signature:\n \n "
                      ">>> array(shape=int32, dtype=class(float32))",
                      str(raises.exception))

    @skip_on_cudasim("Can't check for constants in simulator")
    def test_invalid_4(self):
        # Tuple shape must contain only literals
        with self.assertRaises(TypingError) as raises:
            cuda.jit((int32[:],))(udt_invalid_3)

        self.assertIn("No implementation of function "
                      "Function(<function shared.array",
                      str(raises.exception))
        self.assertIn("found for signature:\n \n "
                      ">>> array(shape=Tuple(Literal[int](1), int32), "
                      "dtype=class(float32))",
                      str(raises.exception))

if __name__ == '__main__':
    unittest.main()
<commit_msg>Fixes change of list spelling in a cuda test.<commit_after>import numpy as np
from numba import cuda, float32, int32
from numba.core.errors import TypingError
from numba.cuda.testing import unittest, CUDATestCase
from numba.cuda.testing import skip_on_cudasim

GLOBAL_CONSTANT = 5
GLOBAL_CONSTANT_2 = 6
GLOBAL_CONSTANT_TUPLE = 5, 6

def udt_global_constants(A):
    sa = cuda.shared.array(shape=GLOBAL_CONSTANT, dtype=float32)
    i = cuda.grid(1)
    A[i] = sa[i]

def udt_global_build_tuple(A):
    sa = cuda.shared.array(shape=(GLOBAL_CONSTANT, GLOBAL_CONSTANT_2),
                           dtype=float32)
    i, j = cuda.grid(2)
    A[i, j] = sa[i, j]

def udt_global_build_list(A):
    sa = cuda.shared.array(shape=[GLOBAL_CONSTANT, GLOBAL_CONSTANT_2],
                           dtype=float32)
    i, j = cuda.grid(2)
    A[i, j] = sa[i, j]

def udt_global_constant_tuple(A):
    sa = cuda.shared.array(shape=GLOBAL_CONSTANT_TUPLE, dtype=float32)
    i, j = cuda.grid(2)
    A[i, j] = sa[i, j]

def udt_invalid_1(A):
    sa = cuda.shared.array(shape=A[0], dtype=float32)
    i = cuda.grid(1)
    A[i] = sa[i]

def udt_invalid_2(A):
    sa = cuda.shared.array(shape=(1, A[0]), dtype=float32)
    i, j = cuda.grid(2)
    A[i, j] = sa[i, j]

def udt_invalid_3(A):
    sa = cuda.shared.array(shape=(1, A[0]), dtype=float32)
    i = cuda.grid(1)
    A[i] = sa[i, 0]

class TestSharedMemoryCreation(CUDATestCase):
    def getarg(self):
        return np.array(100, dtype=np.float32, ndmin=1)

    def getarg2(self):
        return self.getarg().reshape(1,1)

    def test_global_constants(self):
        udt = cuda.jit((float32[:],))(udt_global_constants)
        udt[1, 1](self.getarg())

    def test_global_build_tuple(self):
        udt = cuda.jit((float32[:, :],))(udt_global_build_tuple)
        udt[1, 1](self.getarg2())

    @skip_on_cudasim('Simulator does not perform macro expansion')
    def test_global_build_list(self):
        with self.assertRaises(TypingError) as raises:
            cuda.jit((float32[:, :],))(udt_global_build_list)

        self.assertIn("No implementation of function "
                      "Function(<function shared.array",
                      str(raises.exception))
        self.assertIn("found for signature:\n \n "
                      ">>> array(shape=list(int64)<iv=[5, 6]>, "
                      "dtype=class(float32)",
                      str(raises.exception))

    def test_global_constant_tuple(self):
        udt = cuda.jit((float32[:, :],))(udt_global_constant_tuple)
        udt[1, 1](self.getarg2())

    @skip_on_cudasim("Can't check for constants in simulator")
    def test_invalid_1(self):
        # Scalar shape cannot be a floating point value
        with self.assertRaises(TypingError) as raises:
            cuda.jit((float32[:],))(udt_invalid_1)

        self.assertIn("No implementation of function "
                      "Function(<function shared.array",
                      str(raises.exception))
        self.assertIn("found for signature:\n \n "
                      ">>> array(shape=float32, dtype=class(float32))",
                      str(raises.exception))

    @skip_on_cudasim("Can't check for constants in simulator")
    def test_invalid_2(self):
        # Tuple shape cannot contain a floating point value
        with self.assertRaises(TypingError) as raises:
            cuda.jit((float32[:, :],))(udt_invalid_2)

        self.assertIn("No implementation of function "
                      "Function(<function shared.array",
                      str(raises.exception))
        self.assertIn("found for signature:\n \n "
                      ">>> array(shape=Tuple(Literal[int](1), "
                      "array(float32, 1d, A)), dtype=class(float32))",
                      str(raises.exception))

    @skip_on_cudasim("Can't check for constants in simulator")
    def test_invalid_3(self):
        # Scalar shape must be literal
        with self.assertRaises(TypingError) as raises:
            cuda.jit((int32[:],))(udt_invalid_1)

        self.assertIn("No implementation of function "
                      "Function(<function shared.array",
                      str(raises.exception))
        self.assertIn("found for signature:\n \n "
                      ">>> array(shape=int32, dtype=class(float32))",
                      str(raises.exception))

    @skip_on_cudasim("Can't check for constants in simulator")
    def test_invalid_4(self):
        # Tuple shape must contain only literals
        with self.assertRaises(TypingError) as raises:
            cuda.jit((int32[:],))(udt_invalid_3)

        self.assertIn("No implementation of function "
                      "Function(<function shared.array",
                      str(raises.exception))
        self.assertIn("found for signature:\n \n "
                      ">>> array(shape=Tuple(Literal[int](1), int32), "
                      "dtype=class(float32))",
                      str(raises.exception))

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before># Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""ResNet50 model for Keras adapted from tf.keras.applications.ResNet50.

# Reference:
- [Deep Residual Learning for Image Recognition](
    https://arxiv.org/abs/1512.03385)
Adapted from code contributed by BigMoyan.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import warnings

import tensorflow as tf

BATCH_NORM_DECAY = 0.9
BATCH_NORM_EPSILON = 1e-5
L2_WEIGHT_DECAY = 1e-4

def _obtain_input_shape(input_shape,
                        default_size,
                        data_format):
  """Internal utility to compute/validate a model's input shape.

  Arguments:
    input_shape: Either None (will return the default network input shape),
        or a user-provided shape to be validated.
    default_size: Default input width/height for the model.
    data_format: Image data format to use.

  Returns:
    An integer shape tuple (may include None entries).

  Raises:
    ValueError: In case of invalid argument values.
  """
  if input_shape and len(input_shape) == 3:
    if data_format == 'channels_first':
      if input_shape[0] not in {1, 3}:
        warnings.warn(
            'This model usually expects 1 or 3 input channels. '
            'However, it was passed an input_shape with ' +
            str(input_shape[0]) + ' input channels.')
      default_shape = (input_shape[0], default_size, default_size)
    else:
      if input_shape[-1] not in {1, 3}:
        warnings.warn(
            'This model usually expects 1 or 3 input channels. '
            'However, it was passed an input_shape with ' +
            str(input_shape[-1]) + ' input channels.')
      default_shape = (default_size, default_size, input_shape[-1])

  return input_shape

def identity_building_block(input_tensor,
                            kernel_size,
                            filters,
                            stage,
                            block,
                            training=None):
  """The identity block is the block that has no conv layer at shortcut.

  Arguments:
    input_tensor: input tensor
    kernel_size: default 3, the kernel size of
        middle conv layer at main path
    filters: list of integers, the filters of 3 conv layer at main path
    stage: integer, current stage label, used for generating layer names
    block: 'a','b'..., current block label, used for generating layer names
    training: Only used if training keras model with Estimator.  In other
      scenarios it is handled automatically.

  Returns:
    Output tensor for the block.
  """
  filters1, filters2 = filters
  if tf.keras.backend.image_data_format() == 'channels_last':
    bn_axis = 3
  else:
    bn_axis = 1
  conv_name_base = 'res' + str(stage) + block + '_branch'
  bn_name_base = 'bn' + str(stage) + block + '_branch'

  x = tf.keras.layers.Conv2D(filters1, kernel_size,
                             padding='same',
                             kernel_initializer='he_normal',
                             kernel_regularizer=
                             tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
                             bias_regularizer=
                             tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
                             name=conv_name_base + '2a')(input_tensor)
  x = tf.keras.layers.BatchNormalization(axis=bn_axis,
                                         name=bn_name_base + '2a',
                                         momentum=BATCH_NORM_DECAY,
                                         epsilon=BATCH_NORM_EPSILON)(
                                             x, training=training)
  x = tf.keras.layers.Activation('relu')(x)

  x = tf.keras.layers.Conv2D(filters2, kernel_size,
                             padding='same',
                             kernel_initializer='he_normal',
                             kernel_regularizer=
                             tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
                             bias_regularizer=
                             tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
                             name=conv_name_base + '2b')(x)
  x = tf.keras.layers.BatchNormalization(axis=bn_axis,
                                         name=bn_name_base + '2b',
                                         momentum=BATCH_NORM_DECAY,
                                         epsilon=BATCH_NORM_EPSILON)(
                                             x, training=training)

  x = tf.keras.layers.add([x, input_tensor])
  x = tf.keras.layers.Activation('relu')(x)
  return x

def conv_building_block(input_tensor,
                        kernel_size,
                        filters,
                        stage,
                        block,
                        strides=(2, 2),
                        training=None):
  """A block that has a conv layer at shortcut.

  Arguments:
    input_tensor: input tensor
    kernel_size: default 3, the kernel size of
        middle conv layer at main path
    filters: list of integers, the filters of 3 conv layer at main path
    stage: integer, current stage label, used for generating layer names
    block: 'a','b'..., current block label, used for generating layer names
    strides: Strides for the first conv layer in the block.
    training: Only used if training keras model with Estimator.  In other
      scenarios it is handled automatically.

  Returns:
    Output tensor for the block.

  Note that from stage 3,
  the first conv layer at main path is with strides=(2, 2)
  And the shortcut should have strides=(2, 2) as well
  """
  filters1, filters2 = filters
  if tf.keras.backend.image_data_format() == 'channels_last':
    bn_axis = 3
  else:
    bn_axis = 1
  conv_name_base = 'res' + str(stage) + block + '_branch'
  bn_name_base = 'bn' + str(stage) + block + '_branch'

  x = tf.keras.layers.Conv2D(filters1, kernel_size, strides=strides,
                             padding='same',
                             kernel_initializer='he_normal',
                             kernel_regularizer=
                             tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
                             bias_regularizer=
                             tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
                             name=conv_name_base + '2a')(input_tensor)
  x = tf.keras.layers.BatchNormalization(axis=bn_axis,
                                         name=bn_name_base + '2a',
                                         momentum=BATCH_NORM_DECAY,
                                         epsilon=BATCH_NORM_EPSILON)(
                                             x, training=training)
  x = tf.keras.layers.Activation('relu')(x)

  x = tf.keras.layers.Conv2D(filters2, kernel_size, padding='same',
                             kernel_initializer='he_normal',
                             kernel_regularizer=
                             tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
                             bias_regularizer=
                             tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
                             name=conv_name_base + '2b')(x)
  x = tf.keras.layers.BatchNormalization(axis=bn_axis,
                                         name=bn_name_base + '2b',
                                         momentum=BATCH_NORM_DECAY,
                                         epsilon=BATCH_NORM_EPSILON)(
                                             x, training=training)

  shortcut = tf.keras.layers.Conv2D(filters2, (1, 1), strides=strides,
                                    kernel_initializer='he_normal',
                                    kernel_regularizer=
                                    tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
                                    bias_regularizer=
                                    tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
                                    name=conv_name_base + '1')(input_tensor)
  shortcut = tf.keras.layers.BatchNormalization(
      axis=bn_axis, name=bn_name_base + '1',
      momentum=BATCH_NORM_DECAY, epsilon=BATCH_NORM_EPSILON)(
          shortcut, training=training)

  x = tf.keras.layers.add([x, shortcut])
  x = tf.keras.layers.Activation('relu')(x)
  return x

def resnet56(input_shape=None, classes=100, training=None):
  """Instantiates the ResNet56 architecture.

  Arguments:
      input_shape: optional shape tuple
      classes: optional number of classes to classify images into
      training: Only used if training keras model with Estimator.  In other
      scenarios it is handled automatically.

  Returns:
      A Keras model instance.
  """
  # Determine proper input shape
  input_shape = _obtain_input_shape(
      input_shape,
      default_size=32,
      data_format=tf.keras.backend.image_data_format())

  img_input = tf.keras.layers.Input(shape=input_shape)
  if tf.keras.backend.image_data_format() == 'channels_last':
    bn_axis = 3
  else:
    bn_axis = 1

  x = tf.keras.layers.ZeroPadding2D(padding=(1, 1), name='conv1_pad')(img_input)
  x = tf.keras.layers.Conv2D(16, (3, 3),
                             strides=(1, 1),
                             padding='valid',
                             kernel_initializer='he_normal',
                             kernel_regularizer=
                             tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
                             bias_regularizer=
                             tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
                             name='conv1')(x)
  x = tf.keras.layers.BatchNormalization(axis=bn_axis, name='bn_conv1',
                                         momentum=BATCH_NORM_DECAY,
                                         epsilon=BATCH_NORM_EPSILON)(
                                             x, training=training)
  x = tf.keras.layers.Activation('relu')(x)

  x = conv_building_block(x, 3, [16, 16], stage=2, block='a', strides=(1, 1),
                          training=training)
  x = identity_building_block(x, 3, [16, 16], stage=2, block='b',
                              training=training)
  x = identity_building_block(x, 3, [16, 16], stage=2, block='c',
                              training=training)
  x = identity_building_block(x, 3, [16, 16], stage=2, block='d',
                              training=training)
  x = identity_building_block(x, 3, [16, 16], stage=2, block='e',
                              training=training)
  x = identity_building_block(x, 3, [16, 16], stage=2, block='f',
                              training=training)
  x = identity_building_block(x, 3, [16, 16], stage=2, block='g',
                              training=training)
  x = identity_building_block(x, 3, [16, 16], stage=2, block='h',
                              training=training)
  x = identity_building_block(x, 3, [16, 16], stage=2, block='i',
                              training=training)

  x = conv_building_block(x, 3, [32, 32], stage=3, block='a',
                          training=training)
  x = identity_building_block(x, 3, [32, 32], stage=3, block='b',
                              training=training)
  x = identity_building_block(x, 3, [32, 32], stage=3, block='c',
                              training=training)
  x = identity_building_block(x, 3, [32, 32], stage=3, block='d',
                              training=training)
  x = identity_building_block(x, 3, [32, 32], stage=3, block='e',
                              training=training)
  x = identity_building_block(x, 3, [32, 32], stage=3, block='f',
                              training=training)
  x = identity_building_block(x, 3, [32, 32], stage=3, block='g',
                              training=training)
  x = identity_building_block(x, 3, [32, 32], stage=3, block='h',
                              training=training)
  x = identity_building_block(x, 3, [32, 32], stage=3, block='i',
                              training=training)

  x = conv_building_block(x, 3, [64, 64], stage=4, block='a',
                          training=training)
  x = identity_building_block(x, 3, [64, 64], stage=4, block='b',
                              training=training)
  x = identity_building_block(x, 3, [64, 64], stage=4, block='c',
                              training=training)
  x = identity_building_block(x, 3, [64, 64], stage=4, block='d',
                              training=training)
  x = identity_building_block(x, 3, [64, 64], stage=4, block='e',
                              training=training)
  x = identity_building_block(x, 3, [64, 64], stage=4, block='f',
                              training=training)
  x = identity_building_block(x, 3, [64, 64], stage=4, block='g',
                              training=training)
  x = identity_building_block(x, 3, [64, 64], stage=4, block='h',
                              training=training)
  x = identity_building_block(x, 3, [64, 64], stage=4, block='i',
                              training=training)

  x = tf.keras.layers.GlobalAveragePooling2D(name='avg_pool')(x)
  x = tf.keras.layers.Dense(classes, activation='softmax',
                            kernel_initializer='he_normal',
                            kernel_regularizer=
                            tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
                            bias_regularizer=
                            tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
                            name='fc10')(x)

  inputs = img_input
  # Create model.
  model = tf.keras.models.Model(inputs, x, name='resnet56')

  return model
<commit_msg>Fix a comment<commit_after># Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""ResNet56 model for Keras adapted from tf.keras.applications.ResNet50.

# Reference:
- [Deep Residual Learning for Image Recognition](
    https://arxiv.org/abs/1512.03385)
Adapted from code contributed by BigMoyan.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import warnings

import tensorflow as tf

BATCH_NORM_DECAY = 0.9
BATCH_NORM_EPSILON = 1e-5
L2_WEIGHT_DECAY = 1e-4

def _obtain_input_shape(input_shape,
                        default_size,
                        data_format):
  """Internal utility to compute/validate a model's input shape.

  Arguments:
    input_shape: Either None (will return the default network input shape),
        or a user-provided shape to be validated.
    default_size: Default input width/height for the model.
    data_format: Image data format to use.

  Returns:
    An integer shape tuple (may include None entries).

  Raises:
    ValueError: In case of invalid argument values.
  """
  if input_shape and len(input_shape) == 3:
    if data_format == 'channels_first':
      if input_shape[0] not in {1, 3}:
        warnings.warn(
            'This model usually expects 1 or 3 input channels. '
            'However, it was passed an input_shape with ' +
            str(input_shape[0]) + ' input channels.')
      default_shape = (input_shape[0], default_size, default_size)
    else:
      if input_shape[-1] not in {1, 3}:
        warnings.warn(
            'This model usually expects 1 or 3 input channels. '
            'However, it was passed an input_shape with ' +
            str(input_shape[-1]) + ' input channels.')
      default_shape = (default_size, default_size, input_shape[-1])

  return input_shape

def identity_building_block(input_tensor,
                            kernel_size,
                            filters,
                            stage,
                            block,
                            training=None):
  """The identity block is the block that has no conv layer at shortcut.

  Arguments:
    input_tensor: input tensor
    kernel_size: default 3, the kernel size of
        middle conv layer at main path
    filters: list of integers, the filters of 3 conv layer at main path
    stage: integer, current stage label, used for generating layer names
    block: 'a','b'..., current block label, used for generating layer names
    training: Only used if training keras model with Estimator.  In other
      scenarios it is handled automatically.

  Returns:
    Output tensor for the block.
  """
  filters1, filters2 = filters
  if tf.keras.backend.image_data_format() == 'channels_last':
    bn_axis = 3
  else:
    bn_axis = 1
  conv_name_base = 'res' + str(stage) + block + '_branch'
  bn_name_base = 'bn' + str(stage) + block + '_branch'

  x = tf.keras.layers.Conv2D(filters1, kernel_size,
                             padding='same',
                             kernel_initializer='he_normal',
                             kernel_regularizer=
                             tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
                             bias_regularizer=
                             tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
                             name=conv_name_base + '2a')(input_tensor)
  x = tf.keras.layers.BatchNormalization(axis=bn_axis,
                                         name=bn_name_base + '2a',
                                         momentum=BATCH_NORM_DECAY,
                                         epsilon=BATCH_NORM_EPSILON)(
                                             x, training=training)
  x = tf.keras.layers.Activation('relu')(x)

  x = tf.keras.layers.Conv2D(filters2, kernel_size,
                             padding='same',
                             kernel_initializer='he_normal',
                             kernel_regularizer=
                             tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
                             bias_regularizer=
                             tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
                             name=conv_name_base + '2b')(x)
  x = tf.keras.layers.BatchNormalization(axis=bn_axis,
                                         name=bn_name_base + '2b',
                                         momentum=BATCH_NORM_DECAY,
                                         epsilon=BATCH_NORM_EPSILON)(
                                             x, training=training)

  x = tf.keras.layers.add([x, input_tensor])
  x = tf.keras.layers.Activation('relu')(x)
  return x

def conv_building_block(input_tensor,
                        kernel_size,
                        filters,
                        stage,
                        block,
                        strides=(2, 2),
                        training=None):
  """A block that has a conv layer at shortcut.

  Arguments:
    input_tensor: input tensor
    kernel_size: default 3, the kernel size of
        middle conv layer at main path
    filters: list of integers, the filters of 3 conv layer at main path
    stage: integer, current stage label, used for generating layer names
    block: 'a','b'..., current block label, used for generating layer names
    strides: Strides for the first conv layer in the block.
    training: Only used if training keras model with Estimator.  In other
      scenarios it is handled automatically.

  Returns:
    Output tensor for the block.

  Note that from stage 3,
  the first conv layer at main path is with strides=(2, 2)
  And the shortcut should have strides=(2, 2) as well
  """
  filters1, filters2 = filters
  if tf.keras.backend.image_data_format() == 'channels_last':
    bn_axis = 3
  else:
    bn_axis = 1
  conv_name_base = 'res' + str(stage) + block + '_branch'
  bn_name_base = 'bn' + str(stage) + block + '_branch'

  x = tf.keras.layers.Conv2D(filters1, kernel_size, strides=strides,
                             padding='same',
                             kernel_initializer='he_normal',
                             kernel_regularizer=
                             tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
                             bias_regularizer=
                             tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
                             name=conv_name_base + '2a')(input_tensor)
  x = tf.keras.layers.BatchNormalization(axis=bn_axis,
                                         name=bn_name_base + '2a',
                                         momentum=BATCH_NORM_DECAY,
                                         epsilon=BATCH_NORM_EPSILON)(
                                             x, training=training)
  x = tf.keras.layers.Activation('relu')(x)

  x = tf.keras.layers.Conv2D(filters2, kernel_size, padding='same',
                             kernel_initializer='he_normal',
                             kernel_regularizer=
                             tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
                             bias_regularizer=
                             tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
                             name=conv_name_base + '2b')(x)
  x = tf.keras.layers.BatchNormalization(axis=bn_axis,
                                         name=bn_name_base + '2b',
                                         momentum=BATCH_NORM_DECAY,
                                         epsilon=BATCH_NORM_EPSILON)(
                                             x, training=training)

  shortcut = tf.keras.layers.Conv2D(filters2, (1, 1), strides=strides,
                                    kernel_initializer='he_normal',
                                    kernel_regularizer=
                                    tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
                                    bias_regularizer=
                                    tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
                                    name=conv_name_base + '1')(input_tensor)
  shortcut = tf.keras.layers.BatchNormalization(
      axis=bn_axis, name=bn_name_base + '1',
      momentum=BATCH_NORM_DECAY, epsilon=BATCH_NORM_EPSILON)(
          shortcut, training=training)

  x = tf.keras.layers.add([x, shortcut])
  x = tf.keras.layers.Activation('relu')(x)
  return x

def resnet56(input_shape=None, classes=100, training=None):
  """Instantiates the ResNet56 architecture.

  Arguments:
      input_shape: optional shape tuple
      classes: optional number of classes to classify images into
      training: Only used if training keras model with Estimator.  In other
      scenarios it is handled automatically.

  Returns:
      A Keras model instance.
  """
  # Determine proper input shape
  input_shape = _obtain_input_shape(
      input_shape,
      default_size=32,
      data_format=tf.keras.backend.image_data_format())

  img_input = tf.keras.layers.Input(shape=input_shape)
  if tf.keras.backend.image_data_format() == 'channels_last':
    bn_axis = 3
  else:
    bn_axis = 1

  x = tf.keras.layers.ZeroPadding2D(padding=(1, 1), name='conv1_pad')(img_input)
  x = tf.keras.layers.Conv2D(16, (3, 3),
                             strides=(1, 1),
                             padding='valid',
                             kernel_initializer='he_normal',
                             kernel_regularizer=
                             tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
                             bias_regularizer=
                             tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
                             name='conv1')(x)
  x = tf.keras.layers.BatchNormalization(axis=bn_axis, name='bn_conv1',
                                         momentum=BATCH_NORM_DECAY,
                                         epsilon=BATCH_NORM_EPSILON)(
                                             x, training=training)
  x = tf.keras.layers.Activation('relu')(x)

  x = conv_building_block(x, 3, [16, 16], stage=2, block='a', strides=(1, 1),
                          training=training)
  x = identity_building_block(x, 3, [16, 16], stage=2, block='b',
                              training=training)
  x = identity_building_block(x, 3, [16, 16], stage=2, block='c',
                              training=training)
  x = identity_building_block(x, 3, [16, 16], stage=2, block='d',
                              training=training)
  x = identity_building_block(x, 3, [16, 16], stage=2, block='e',
                              training=training)
  x = identity_building_block(x, 3, [16, 16], stage=2, block='f',
                              training=training)
  x = identity_building_block(x, 3, [16, 16], stage=2, block='g',
                              training=training)
  x = identity_building_block(x, 3, [16, 16], stage=2, block='h',
                              training=training)
  x = identity_building_block(x, 3, [16, 16], stage=2, block='i',
                              training=training)

  x = conv_building_block(x, 3, [32, 32], stage=3, block='a',
                          training=training)
  x = identity_building_block(x, 3, [32, 32], stage=3, block='b',
                              training=training)
  x = identity_building_block(x, 3, [32, 32], stage=3, block='c',
                              training=training)
  x = identity_building_block(x, 3, [32, 32], stage=3, block='d',
                              training=training)
  x = identity_building_block(x, 3, [32, 32], stage=3, block='e',
                              training=training)
  x = identity_building_block(x, 3, [32, 32], stage=3, block='f',
                              training=training)
  x = identity_building_block(x, 3, [32, 32], stage=3, block='g',
                              training=training)
  x = identity_building_block(x, 3, [32, 32], stage=3, block='h',
                              training=training)
  x = identity_building_block(x, 3, [32, 32], stage=3, block='i',
                              training=training)

  x = conv_building_block(x, 3, [64, 64], stage=4, block='a',
                          training=training)
  x = identity_building_block(x, 3, [64, 64], stage=4, block='b',
                              training=training)
  x = identity_building_block(x, 3, [64, 64], stage=4, block='c',
                              training=training)
  x = identity_building_block(x, 3, [64, 64], stage=4, block='d',
                              training=training)
  x = identity_building_block(x, 3, [64, 64], stage=4, block='e',
                              training=training)
  x = identity_building_block(x, 3, [64, 64], stage=4, block='f',
                              training=training)
  x = identity_building_block(x, 3, [64, 64], stage=4, block='g',
                              training=training)
  x = identity_building_block(x, 3, [64, 64], stage=4, block='h',
                              training=training)
  x = identity_building_block(x, 3, [64, 64], stage=4, block='i',
                              training=training)

  x = tf.keras.layers.GlobalAveragePooling2D(name='avg_pool')(x)
  x = tf.keras.layers.Dense(classes, activation='softmax',
                            kernel_initializer='he_normal',
                            kernel_regularizer=
                            tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
                            bias_regularizer=
                            tf.keras.regularizers.l2(L2_WEIGHT_DECAY),
                            name='fc10')(x)

  inputs = img_input
  # Create model.
  model = tf.keras.models.Model(inputs, x, name='resnet56')

  return model
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# License: BSD-3 (https://tldrlegal.com/license/bsd-3-clause-license-(revised))
# Copyright (c) 2016-2021, Cabral, Juan; Luczywo, Nadia
# Copyright (c) 2022, QuatroPe
# All rights reserved.

# =============================================================================
# DOCS
# =============================================================================

"""Framework for defining models with hyperparameters."""

# =============================================================================
# IMPORTS
# =============================================================================

from abc import ABCMeta

import attr

# =============================================================================
# CONSTANTS
# =============================================================================

HMODEL_METADATA_KEY = "__hmodel__"

HMODEL_CONFIG = "__hmodel_config__"

DEFAULT_HMODEL_CONFIG = {"repr": False, "frozen": False}

# =============================================================================
# FUNCTIONS
# =============================================================================

def hparam(default, **kwargs):
    """Create a hyper parameter.

    By design decision, hyper-parameter is required to have a sensitive default
    value.

    Parameters
    ----------
    default :
        Sensitive default value of the hyper-parameter.
    **kwargs :
        Additional keyword arguments are passed and are documented in
        ``attr.ib()``.

    Return
    ------
    Hyper parameter with a default value.

    Notes
    -----
    This function is a thin-wrapper over the attrs function ``attr.ib()``.
    """
    field_metadata = kwargs.pop("metadata", {})
    model_metadata = field_metadata.setdefault(HMODEL_METADATA_KEY, {})
    model_metadata["hparam"] = True
    return attr.field(
        default=default, metadata=field_metadata, kw_only=True, **kwargs
    )

def mproperty(**kwargs):
    """Create a internal property for the model.

    By design decision, hyper-parameter is required to have a sensitive default
    value.

    Parameters
    ----------
    default :
        Sensitive default value of the hyper-parameter.
    **kwargs :
        Additional keyword arguments are passed and are documented in
        ``attr.ib()``.

    Return
    ------
    Hyper parameter with a default value.

    Notes
    -----
    This function is a thin-wrapper over the attrs function ``attr.ib()``.

    """
    field_metadata = kwargs.pop("metadata", {})
    model_metadata = field_metadata.setdefault(HMODEL_METADATA_KEY, {})
    model_metadata["mproperty"] = True
    return attr.field(init=False, metadata=field_metadata, **kwargs)

@attr.define(repr=False)
class HModelABC(metaclass=ABCMeta):
    """Create a new models with hyperparameters."""

    def __init_subclass__(cls):
        """Initiate of subclasses.

        It ensures that every inherited class is decorated by ``attr.define()``
        and assigns as class configuration the parameters defined in the class
        variable `__hmodel_config__`.

        In other words it is slightly equivalent to:

        .. code-block:: python

            @attr.s(**HModelABC.__hmodel_config__)
            class Decomposer(HModelABC):
                pass

        """
        model_config = getattr(cls, HMODEL_CONFIG, DEFAULT_HMODEL_CONFIG)
        return attr.define(maybe_cls=cls, slots=False, **model_config)

    @classmethod
    def get_hparams(cls):
        """Return a tuple of available hyper parameters."""

        def flt(f):
            return f.metadata[HMODEL_METADATA_KEY].get("hparam", False)

        return tuple(f.name for f in attr.fields(cls) if flt(f))

    def __repr__(self):
        """x.__repr__() <==> repr(x)."""

        clsname = type(self).__name__

        hparams = self.get_hparams()
        selfd = attr.asdict(
            self,
            recurse=False,
            filter=lambda attr, _: attr.name in hparams and attr.repr,
        )

        attrs_str = ", ".join(
            [f"{k}={repr(v)}" for k, v in sorted(selfd.items())]
        )
        return f"{clsname}({attrs_str})"
<commit_msg>hmodel implementhed<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# License: BSD-3 (https://tldrlegal.com/license/bsd-3-clause-license-(revised))
# Copyright (c) 2016-2021, Cabral, Juan; Luczywo, Nadia
# Copyright (c) 2022, QuatroPe
# All rights reserved.

# =============================================================================
# DOCS
# =============================================================================

"""Framework for defining models with hyperparameters."""

# =============================================================================
# IMPORTS
# =============================================================================

from abc import ABCMeta

import attr

# =============================================================================
# CONSTANTS
# =============================================================================

HMODEL_METADATA_KEY = "__hmodel__"

HMODEL_CONFIG = "__hmodel_config__"

DEFAULT_HMODEL_CONFIG = {"repr": False, "frozen": False}

# =============================================================================
# FUNCTIONS
# =============================================================================

def hparam(default, **kwargs):
    """Create a hyper parameter.

    By design decision, hyper-parameter is required to have a sensitive default
    value.

    Parameters
    ----------
    default :
        Sensitive default value of the hyper-parameter.
    **kwargs :
        Additional keyword arguments are passed and are documented in
        ``attr.ib()``.

    Return
    ------
    Hyper parameter with a default value.

    Notes
    -----
    This function is a thin-wrapper over the attrs function ``attr.ib()``.
    """
    field_metadata = kwargs.pop("metadata", {})
    model_metadata = field_metadata.setdefault(HMODEL_METADATA_KEY, {})
    model_metadata["hparam"] = True
    return attr.field(
        default=default, metadata=field_metadata, kw_only=True, **kwargs
    )

def mproperty(**kwargs):
    """Create a internal property for the model.

    By design decision, hyper-parameter is required to have a sensitive default
    value.

    Parameters
    ----------
    default :
        Sensitive default value of the hyper-parameter.
    **kwargs :
        Additional keyword arguments are passed and are documented in
        ``attr.ib()``.

    Return
    ------
    Hyper parameter with a default value.

    Notes
    -----
    This function is a thin-wrapper over the attrs function ``attr.ib()``.

    """
    field_metadata = kwargs.pop("metadata", {})
    model_metadata = field_metadata.setdefault(HMODEL_METADATA_KEY, {})
    model_metadata["mproperty"] = True
    return attr.field(init=False, metadata=field_metadata, **kwargs)

@attr.define(repr=False)
class HModelABC(metaclass=ABCMeta):
    """Create a new models with hyperparameters."""

    def __init_subclass__(cls):
        """Initiate of subclasses.

        It ensures that every inherited class is decorated by ``attr.define()``
        and assigns as class configuration the parameters defined in the class
        variable `__hmodel_config__`.

        In other words it is slightly equivalent to:

        .. code-block:: python

            @attr.s(**HModelABC.__hmodel_config__)
            class Decomposer(HModelABC):
                pass

        """
        model_config = getattr(cls, HMODEL_CONFIG, DEFAULT_HMODEL_CONFIG)
        return attr.define(maybe_cls=cls, slots=False, **model_config)

    @classmethod
    def get_hparams(cls):
        """Return a tuple of available hyper parameters."""

        def flt(f):
            return f.metadata[HMODEL_METADATA_KEY].get("hparam", False)

        return tuple(f.name for f in attr.fields(cls) if flt(f))

    def __repr__(self):
        """x.__repr__() <==> repr(x)."""
        clsname = type(self).__name__

        hparams = self.get_hparams()
        selfd = attr.asdict(
            self,
            recurse=False,
            filter=lambda attr, _: attr.name in hparams and attr.repr,
        )

        attrs_str = ", ".join(
            [f"{k}={repr(v)}" for k, v in sorted(selfd.items())]
        )
        return f"{clsname}({attrs_str})"
<|endoftext|>


<commit_before>import html5lib
import re
import lxml.etree
import lxml.html
import lxml.html.clean
import re
import traceback
from copy import deepcopy
import sys
from skitai.lib import strutil

TABSSPACE = re.compile(r'[\s\t]+')
def innerTrim(value):
	if strutil.is_str_like (value):
		# remove tab and white space
		value = re.sub(TABSSPACE, ' ', value)
		value = ''.join(value.splitlines())
		return value.strip()
	return ''

class Parser:
	@classmethod
	def from_string (cls, html):
		if strutil.is_str_like (html) and html.startswith('<?'):
			html = re.sub(r'^\<\?.*?\?\>', '', html, flags=re.DOTALL)
		try:
			return lxml.html.fromstring(html)
		except Exception:
			traceback.print_exc()
			return None

	@classmethod
	def to_string (cls, node):
		return lxml.etree.tostring(node)

	def by_xpath (cls, node, expression):
		items = node.xpath(expression)
		return items

	@classmethod
	def by_xpath_re (cls, node, expression):
		regexp_namespace = "http://exslt.org/regular-expressions"
		items = node.xpath(expression, namespaces={'re': regexp_namespace})
		return items

	@classmethod
	def by_css (cls, node, selector):
		return node.cssselect(selector)

	@classmethod
	def by_id (cls, node, idd):
		selector = '//*[@id="%s"]' % idd
		elems = node.xpath(selector)
		if elems:
			return elems[0]
		return None

	@classmethod
	def by_tag_attr (
			cls, node, tag=None, attr=None, value=None, childs=False):
		NS = "http://exslt.org/regular-expressions"
		# selector = tag or '*'
		selector = 'descendant-or-self::%s' % (tag or '*')
		if attr and value:
			selector = '%s[re:test(@%s, "%s", "i")]' % (selector, attr, value)
		elems = node.xpath(selector, namespaces={"re": NS})
		# remove the root node
		# if we have a selection tag
		if node in elems and (tag or childs):
			elems.remove(node)
		return elems

	@classmethod
	def by_tag (cls, node, tag):
		return cls.by_tags (node, [tag])

	@classmethod
	def by_tags (cls, node, tags):
		selector = ','.join(tags)
		elems = cls.by_css (node, selector)
		# remove the root node
		# if we have a selection tag
		if node in elems:
			elems.remove(node)
		return elems

	@classmethod
	def prev_siblings (cls, node):
		nodes = []
		for c, n in enumerate (node.itersiblings(preceding=True)):
			nodes.append(n)
		return nodes

	@classmethod
	def prev_sibling (cls, node):
		nodes = []
		for c, n in enumerate (node.itersiblings(preceding=True)):
			nodes.append(n)
			if c == 0:
				break
		return nodes[0] if nodes else None

	@classmethod
	def next_siblings (cls, node):
		nodes = []
		for c, n in enumerate (node.itersiblings(preceding=False)):
			nodes.append(n)
		return nodes

	@classmethod
	def next_sibling (cls, node):
		nodes = []
		for c, n in enumerate (node.itersiblings(preceding=False)):
			nodes.append(n)
			if c == 0:
				break
		return nodes[0] if nodes else None

	@classmethod
	def get_attr (cls, node, attr=None):
		if attr:
			return node.attrib.get(attr, None)
		return node.attrib

	@classmethod
	def has_attr (cls, node, attr = None):
		if attr:
			return attr in node.attrib
		return len (node.attrib) != 0

	@classmethod
	def del_attr (cls, node, attr=None):
		if attr:
			_attr = node.attrib.get(attr, None)
			if _attr:
				del node.attrib[attr]

	@classmethod
	def set_attr (cls, node, attr=None, value=None):
		if attr and value:
			node.set(attr, value)

	@classmethod
	def append_child (cls, node, child):
		node.append(child)

	@classmethod
	def child_nodes (cls, node):
		return list(node)

	@classmethod
	def child_nodes_with_text (cls, node):
		root = node
		# create the first text node
		# if we have some text in the node
		if root.text:
			t = lxml.html.HtmlElement()
			t.text = root.text
			t.tag = 'text'
			root.text = None
			root.insert(0, t)
		# loop childs
		for c, n in enumerate(list(root)):
			idx = root.index(n)
			# don't process texts nodes
			if n.tag == 'text':
				continue
			# create a text node for tail
			if n.tail:
				t = cls.create_element(tag='text', text=n.tail, tail=None)
				root.insert(idx + 1, t)
		return list(root)

	@classmethod
	def get_children (cls, node):
		return node.getchildren()

	@classmethod
	def get_parent (cls, node):
		return node.getparent()

	@classmethod
	def get_text (cls, node):
		txts = [i for i in node.itertext()]
		return innerTrim(u' '.join(txts).strip())

	@classmethod
	def get_texts (cls, node, trim = True):
		if trim:
			return [i.strip () for i in node.itertext()]
		else:
			return [i for i in node.itertext()]

	@classmethod
	def is_text_node (cls, node):
		return True if node.tag == 'text' else False

	@classmethod
	def get_tag (cls, node):
		return node.tag

	@classmethod
	def replace_tag (cls, node, tag):
		node.tag = tag

	@classmethod
	def strip_tags (cls, node, *tags):
		lxml.etree.strip_tags(node, *tags)

	@classmethod
	def drop_node (cls, node):
		node.drop_tag ()

	@classmethod
	def drop_tree (cls, node):
		node.drop_tree ()

	@classmethod
	def create_element (cls, tag='p', text=None, tail=None):
		t = lxml.html.HtmlElement()
		t.tag = tag
		t.text = text
		t.tail = tail
		return t

	@classmethod
	def get_comments (cls, node):
		return node.xpath('//comment()')

	@classmethod
	def remove_comments (cls, node):
		for item in cls.get_comments (node):
			try:
				item.drop_tree ()
			except AssertionError: # out of root node
				pass

	@classmethod
	def text_to_para (cls, text):
		return cls.create_element ('p', text)

	@classmethod
	def outer_html (cls, node):
		e0 = node
		if e0.tail:
			e0 = deepcopy(e0)
			e0.tail = None
		return cls.to_string(e0)

	@classmethod
	def clean_html (cls, node):
		article_cleaner = lxml.html.clean.Cleaner()
		article_cleaner.javascript = True
		article_cleaner.style = True
		article_cleaner.allow_tags = [
			'a', 'span', 'p', 'br', 'strong', 'b',
			'em', 'i', 'tt', 'code', 'pre', 'blockquote', 'img', 'h1',
			'h2', 'h3', 'h4', 'h5', 'h6']
		article_cleaner.remove_unknown_tags = False
		return article_cleaner.clean_html (node)

	@classmethod
	def get_param (cls, node, attr, name):
		name = name.lower ()
		params = cls.get_attr (node, attr)
		for param in params.split (";"):
			param = param.strip ()
			if not param.lower ().startswith (name):
				continue

			val = param [len (name):].strip ()
			if not val: return ""
			if val [0] == "=":
				val = val [1:].strip ()
				if not val: return ""
			if val [0] in "\"'":
				return val [1:-1]
			return val

def remove_control_characters (html):
	def str_to_int(s, default, base=10):
		if int(s, base) < 0x10000:
			if strutil.PY_MAJOR_VERSION == 2:
				return unichr(int(s, base))
			else:
				return chr(int(s, base))
		return default

	html = re.sub(r"&#(\d+);?", lambda c: str_to_int(c.group(1), c.group(0)), html)
	html = re.sub(r"&#[xX]([0-9a-fA-F]+);?", lambda c: str_to_int(c.group(1), c.group(0), base=16), html)
	html = re.sub(r"[\x00-\x08\x0b\x0e-\x1f\x7f]", "", html)
	return html

def remove_non_asc (html):
	html = re.sub(br"&#(\d+);?", "", html)
	html = re.sub(br"&#[xX]([0-9a-fA-F]+);?", "", html)
	html = re.sub(br"[\x00-\x08\x80-\xff]", "", html)
	return html

RX_CAHRSET = re.compile (br"[\s;]+charset\s*=\s*['\"]?([-a-z0-9]+)", re.M) #"
RX_META = re.compile (br"<meta\s+.+?>", re.I|re.M)

def get_charset (html):
	encoding = None
	pos = 0
	while 1:
		match = RX_META.search (html, pos)
		if not match: break
		#print (match.group ())
		charset = RX_CAHRSET.findall (match.group ().lower ())
		if charset:
			encoding = charset [0].decode ("utf8")
			#print (encoding)
			break
		pos = match.end ()
	return encoding

def to_str (html, encoding):
	def try_generic_encoding (html):
		try:
			return html.decode ("utf8")
		except UnicodeDecodeError:
			return html.decode ("iso8859-1")

	if encoding is None:
		encoding = get_charset (html)

	try:
		if not encoding:
			html = try_generic_encoding (html)
		else:
			try:
				html = html.decode (encoding)
			except LookupError:
				html = try_generic_encoding (html)

	except UnicodeDecodeError:
		return remove_non_asc (html)

	else:
		return remove_control_characters (html)

def html (html, baseurl, encoding = None):
	# html5lib rebuilds possibly mal-formed html
	try:
		return lxml.html.fromstring (lxml.etree.tostring (html5lib.parse (html, encoding = encoding, treebuilder="lxml")), baseurl)
	except ValueError:
		return lxml.html.fromstring (lxml.etree.tostring (html5lib.parse (to_str (html, encoding), treebuilder="lxml")), baseurl)

def etree (html, encoding = None):
	try:
		return html5lib.parse (html, encoding = encoding, treebuilder="lxml")
	except ValueError:
		return html5lib.parse (to_str (html, encoding), treebuilder="lxml")

if __name__ == "__main__":
	from urllib.request import urlopen
	from contextlib import closing

	with closing(urlopen("http://www.drugandalcoholrehabhouston.com")) as f:
		build (f.read ())

<commit_msg>fix treebuilder<commit_after>import html5lib
import re
import lxml.etree
import lxml.html
import lxml.html.clean
import re
import traceback
from copy import deepcopy
import sys
from skitai.lib import strutil

TABSSPACE = re.compile(r'[\s\t]+')
def innerTrim(value):
	if strutil.is_str_like (value):
		# remove tab and white space
		value = re.sub(TABSSPACE, ' ', value)
		value = ''.join(value.splitlines())
		return value.strip()
	return ''

class Parser:
	@classmethod
	def from_string (cls, html):
		if strutil.is_str_like (html) and html.startswith('<?'):
			html = re.sub(r'^\<\?.*?\?\>', '', html, flags=re.DOTALL)
		try:
			return lxml.html.fromstring(html)
		except Exception:
			traceback.print_exc()
			return None

	@classmethod
	def to_string (cls, node, encoding = "utf8", method='html', doctype = "<!DOCTYPE html>"):
		return lxml.html.tostring(node, encoding)

	def by_xpath (cls, node, expression):
		items = node.xpath(expression)
		return items

	@classmethod
	def by_xpath_re (cls, node, expression):
		regexp_namespace = "http://exslt.org/regular-expressions"
		items = node.xpath(expression, namespaces={'re': regexp_namespace})
		return items

	@classmethod
	def by_css (cls, node, selector):
		return node.cssselect(selector)

	@classmethod
	def by_id (cls, node, idd):
		selector = '//*[@id="%s"]' % idd
		elems = node.xpath(selector)
		if elems:
			return elems[0]
		return None

	@classmethod
	def by_tag_attr (
			cls, node, tag=None, attr=None, value=None, childs=False):
		NS = "http://exslt.org/regular-expressions"
		# selector = tag or '*'
		selector = 'descendant-or-self::%s' % (tag or '*')
		if attr and value:
			selector = '%s[re:test(@%s, "%s", "i")]' % (selector, attr, value)
		elems = node.xpath(selector, namespaces={"re": NS})
		# remove the root node
		# if we have a selection tag
		if node in elems and (tag or childs):
			elems.remove(node)
		return elems

	@classmethod
	def by_tag (cls, node, tag):
		return cls.by_tags (node, [tag])

	@classmethod
	def by_tags (cls, node, tags):
		selector = ','.join(tags)
		elems = cls.by_css (node, selector)
		# remove the root node
		# if we have a selection tag
		if node in elems:
			elems.remove(node)
		return elems

	@classmethod
	def prev_siblings (cls, node):
		nodes = []
		for c, n in enumerate (node.itersiblings(preceding=True)):
			nodes.append(n)
		return nodes

	@classmethod
	def prev_sibling (cls, node):
		nodes = []
		for c, n in enumerate (node.itersiblings(preceding=True)):
			nodes.append(n)
			if c == 0:
				break
		return nodes[0] if nodes else None

	@classmethod
	def next_siblings (cls, node):
		nodes = []
		for c, n in enumerate (node.itersiblings(preceding=False)):
			nodes.append(n)
		return nodes

	@classmethod
	def next_sibling (cls, node):
		nodes = []
		for c, n in enumerate (node.itersiblings(preceding=False)):
			nodes.append(n)
			if c == 0:
				break
		return nodes[0] if nodes else None

	@classmethod
	def get_attr (cls, node, attr=None):
		if attr:
			return node.attrib.get(attr, None)
		return node.attrib

	@classmethod
	def has_attr (cls, node, attr = None):
		if attr:
			return attr in node.attrib
		return len (node.attrib) != 0

	@classmethod
	def del_attr (cls, node, attr=None):
		if attr:
			_attr = node.attrib.get(attr, None)
			if _attr:
				del node.attrib[attr]

	@classmethod
	def set_attr (cls, node, attr=None, value=None):
		if attr and value:
			node.set(attr, value)

	@classmethod
	def append_child (cls, node, child):
		node.append(child)

	@classmethod
	def child_nodes (cls, node):
		return list(node)

	@classmethod
	def child_nodes_with_text (cls, node):
		root = node
		# create the first text node
		# if we have some text in the node
		if root.text:
			t = lxml.html.HtmlElement()
			t.text = root.text
			t.tag = 'text'
			root.text = None
			root.insert(0, t)
		# loop childs
		for c, n in enumerate(list(root)):
			idx = root.index(n)
			# don't process texts nodes
			if n.tag == 'text':
				continue
			# create a text node for tail
			if n.tail:
				t = cls.create_element(tag='text', text=n.tail, tail=None)
				root.insert(idx + 1, t)
		return list(root)

	@classmethod
	def get_children (cls, node):
		return node.getchildren()

	@classmethod
	def get_parent (cls, node):
		return node.getparent()

	@classmethod
	def get_text (cls, node):
		txts = [i for i in node.itertext()]
		return innerTrim(u' '.join(txts).strip())

	@classmethod
	def get_texts (cls, node, trim = True):
		if trim:
			return [i.strip () for i in node.itertext()]
		else:
			return [i for i in node.itertext()]

	@classmethod
	def is_text_node (cls, node):
		return True if node.tag == 'text' else False

	@classmethod
	def get_tag (cls, node):
		return node.tag

	@classmethod
	def replace_tag (cls, node, tag):
		node.tag = tag

	@classmethod
	def strip_tags (cls, node, *tags):
		lxml.etree.strip_tags(node, *tags)

	@classmethod
	def drop_node (cls, node):
		node.drop_tag ()

	@classmethod
	def drop_tree (cls, node):
		node.drop_tree ()

	@classmethod
	def create_element (cls, tag='p', text=None, tail=None):
		t = lxml.html.HtmlElement()
		t.tag = tag
		t.text = text
		t.tail = tail
		return t

	@classmethod
	def get_comments (cls, node):
		return node.xpath('//comment()')

	@classmethod
	def remove_comments (cls, node):
		for item in cls.get_comments (node):
			try:
				item.drop_tree ()
			except AssertionError: # out of root node
				pass

	@classmethod
	def text_to_para (cls, text):
		return cls.create_element ('p', text)

	@classmethod
	def outer_html (cls, node):
		e0 = node
		if e0.tail:
			e0 = deepcopy(e0)
			e0.tail = None
		return cls.to_string(e0)

	@classmethod
	def clean_html (cls, node):
		article_cleaner = lxml.html.clean.Cleaner()
		article_cleaner.javascript = True
		article_cleaner.style = True
		article_cleaner.allow_tags = [
			'a', 'span', 'p', 'br', 'strong', 'b',
			'em', 'i', 'tt', 'code', 'pre', 'blockquote', 'img', 'h1',
			'h2', 'h3', 'h4', 'h5', 'h6']
		article_cleaner.remove_unknown_tags = False
		return article_cleaner.clean_html (node)

	@classmethod
	def get_param (cls, node, attr, name):
		name = name.lower ()
		params = cls.get_attr (node, attr)
		for param in params.split (";"):
			param = param.strip ()
			if not param.lower ().startswith (name):
				continue

			val = param [len (name):].strip ()
			if not val: return ""
			if val [0] == "=":
				val = val [1:].strip ()
				if not val: return ""
			if val [0] in "\"'":
				return val [1:-1]
			return val

def remove_control_characters (html):
	def str_to_int(s, default, base=10):
		if int(s, base) < 0x10000:
			if strutil.PY_MAJOR_VERSION == 2:
				return unichr(int(s, base))
			else:
				return chr(int(s, base))
		return default

	html = re.sub(r"&#(\d+);?", lambda c: str_to_int(c.group(1), c.group(0)), html)
	html = re.sub(r"&#[xX]([0-9a-fA-F]+);?", lambda c: str_to_int(c.group(1), c.group(0), base=16), html)
	html = re.sub(r"[\x00-\x08\x0b\x0e-\x1f\x7f]", "", html)
	return html

def remove_non_asc (html):
	html = re.sub(br"&#(\d+);?", "", html)
	html = re.sub(br"&#[xX]([0-9a-fA-F]+);?", "", html)
	html = re.sub(br"[\x00-\x08\x80-\xff]", "", html)
	return html

RX_CAHRSET = re.compile (br"[\s;]+charset\s*=\s*['\"]?([-a-z0-9]+)", re.M) #"
RX_META = re.compile (br"<meta\s+.+?>", re.I|re.M)

def get_charset (html):
	encoding = None
	pos = 0
	while 1:
		match = RX_META.search (html, pos)
		if not match: break
		#print (match.group ())
		charset = RX_CAHRSET.findall (match.group ().lower ())
		if charset:
			encoding = charset [0].decode ("utf8")
			#print (encoding)
			break
		pos = match.end ()
	return encoding

def to_str (body, encoding = None):
	def try_generic_encoding (html):
		try:
			return html.decode ("utf8")
		except UnicodeDecodeError:
			try:
				return html.decode ("iso8859-1")
			except UnicodeDecodeError:
				return remove_non_asc (html).decode ("utf8")

	if encoding:
		try:
			body = body.decode (encoding)
		except (UnicodeDecodeError, LookupError):
			inline_encoding = get_charset (body)
			if inline_encoding and encoding != inline_encoding:
				try:
					body = body.decode (inline_encoding)
				except (UnicodeDecodeError, LookupError):
					body = try_generic_encoding (body)
			else:
				body = try_generic_encoding (body)
	else:
		body = try_generic_encoding (body)

	return remove_control_characters (body)

def html (html, baseurl, encoding = None):
	# html5lib rebuilds possibly mal-formed html
	try:
		return lxml.html.fromstring (lxml.etree.tostring (html5lib.parse (html, encoding = encoding, treebuilder="lxml")), baseurl)
	except ValueError:
		return lxml.html.fromstring (lxml.etree.tostring (html5lib.parse (to_str (html, encoding), treebuilder="lxml")), baseurl)

def etree (html, encoding = None):
	try:
		return html5lib.parse (html, encoding = encoding, treebuilder="lxml")
	except ValueError:
		return html5lib.parse (to_str (html, encoding), treebuilder="lxml")

if __name__ == "__main__":
	from urllib.request import urlopen
	from contextlib import closing

	with closing(urlopen("http://www.drugandalcoholrehabhouston.com")) as f:
		build (f.read ())

<|endoftext|>


<commit_before>from .base import BasePlugin
from .manager import PluginManager

def cmd(admin_only=False, acl='*', aliases=None, while_ignored=False,
        reply_in_thread=False, reply_broadcast=False, *args, **kwargs):
    """
    Decorator to mark plugin functions as commands in the form of !<cmd_name>

    * admin_only - indicates only users in bot_admin are allowed to execute (only used if AuthManager is loaded)
    * acl - indicates which ACL to perform permission checks against (only used if AuthManager is loaded)
    * aliases - register function with additional commands (i.e. !alias1, !alias2, etc)
    * while_ignored - allows a command to be run, even if channel has been !sleep
    """
    def wrapper(func):
        func.is_cmd = True
        func.is_subcmd = len(func.__name__.split('_')) > 1
        func.cmd_name = func.__name__.replace('_', ' ')
        func.admin_only = admin_only
        func.acl = acl
        func.aliases = aliases
        func.while_ignored = while_ignored
        func.reply_in_thread = reply_in_thread
        func.reply_broadcast = reply_broadcast
        return func
    return wrapper

def webhook(*args, **kwargs):
    """
    Decorator to mark plugin functions as entry points for web calls

    * route - web route to register, uses Flask syntax
    * method - GET/POST, defaults to POST
    """
    def wrapper(func):
        func.is_webhook = True
        func.route = args[0]
        func.form_params = kwargs.get('form_params', [])
        func.method = kwargs.get('method', 'POST')
        return func
    return wrapper
<commit_msg>Update docstring<commit_after>from .base import BasePlugin
from .manager import PluginManager

def cmd(admin_only=False, acl='*', aliases=None, while_ignored=False,
        reply_in_thread=False, reply_broadcast=False, *args, **kwargs):
    """
    Decorator to mark plugin functions as commands in the form of !<cmd_name>

    * admin_only - indicates only users in bot_admin are allowed to execute (only used if AuthManager is loaded)
    * acl - indicates which ACL to perform permission checks against (only used if AuthManager is loaded)
    * aliases - register function with additional commands (i.e. !alias1, !alias2, etc)
    * while_ignored - allows a command to be run, even if channel has been !sleep
    * reply_in_thread - determines whether bot replies in the channel or a thread
    * reply_broadcast - if replying in a thread, whether to also send the message to the channel
    """
    def wrapper(func):
        func.is_cmd = True
        func.is_subcmd = len(func.__name__.split('_')) > 1
        func.cmd_name = func.__name__.replace('_', ' ')
        func.admin_only = admin_only
        func.acl = acl
        func.aliases = aliases
        func.while_ignored = while_ignored
        func.reply_in_thread = reply_in_thread
        func.reply_broadcast = reply_broadcast
        return func
    return wrapper

def webhook(*args, **kwargs):
    """
    Decorator to mark plugin functions as entry points for web calls

    * route - web route to register, uses Flask syntax
    * method - GET/POST, defaults to POST
    """
    def wrapper(func):
        func.is_webhook = True
        func.route = args[0]
        func.form_params = kwargs.get('form_params', [])
        func.method = kwargs.get('method', 'POST')
        return func
    return wrapper
<|endoftext|>