<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

__author__ = 'moonshawdo@gamil.com'

import re
import os
import subprocess
import sys
import urllib2
import math

current_path = os.path.dirname(os.path.abspath(__file__))
root_path = os.path.abspath( os.path.join(current_path, os.pardir, os.pardir))
data_path = os.path.abspath(os.path.join(root_path, os.pardir, os.pardir, 'data'))
data_gae_proxy_path = os.path.join(data_path, 'gae_proxy')
python_path = os.path.abspath( os.path.join(root_path, 'python27', '1.0'))

noarch_lib = os.path.abspath( os.path.join(python_path, 'lib', 'noarch'))
sys.path.append(noarch_lib)

from config import config
import ip_utils

# This code functions:
# read ip range string in code
# merge over lapped range
# filter out bad ip range

# Then reproduce good format file

# Support format:
# # Comment (#）
#
# range seperater:
# range can seperate by (,) or(|) or new line
#
# Single rang format: ：
# "xxx.xxx.xxx-xxx.xxx-xxx"
# "xxx.xxx.xxx."
# "xxx.xxx.xxx.xxx/xx"
# "xxx.xxx.xxx.xxx"

def PRINT(strlog):
    print (strlog)

def print_range_list(ip_range_list):
    for ip_range in ip_range_list:
        begin = ip_range[0]
        end = ip_range[1]
        print ip_utils.ip_num_to_string(begin), ip_utils.ip_num_to_string(end)

def parse_range_string(input_lines):
    ip_range_list = []

    ip_lines_list = re.split("\r|\n", input_lines)
    for raw_line in ip_lines_list:
        raw_s = raw_line.split("#")
        context_line = raw_s[0]

        context_line = context_line.replace(' ', '')

        ips = re.split(",|\|", context_line)
        for line in ips:
            if len(line) == 0:
                #print "non line:", line
                continue
            begin, end = ip_utils.split_ip(line)
            if ip_utils.check_ip_valid(begin) == 0 or ip_utils.check_ip_valid(end) == 0:
                PRINT("ip format is error,line:%s, begin: %s,end: %s" % (line, begin, end))
                continue
            nbegin = ip_utils.ip_string_to_num(begin)
            nend = ip_utils.ip_string_to_num(end)
            ip_range_list.append([nbegin,nend])
            #print begin, end

    ip_range_list.sort()

    return ip_range_list

def merge_range(input_ip_range_list):
    output_ip_range_list = []
    range_num = len(input_ip_range_list)

    last_begin = input_ip_range_list[0][0]
    last_end = input_ip_range_list[0][1]
    for i in range(1,range_num):
        ip_range = input_ip_range_list[i]
        begin = ip_range[0]
        end = ip_range[1]

        #print "now:",ip_utils.ip_num_to_string(begin), ip_utils.ip_num_to_string(end)

        if begin > last_end + 2:
            #print "add:",ip_utils.ip_num_to_string(begin), ip_utils.ip_num_to_string(end)
            output_ip_range_list.append([last_begin, last_end])
            last_begin = begin
            last_end = end
        else:
            print "merge:", ip_utils.ip_num_to_string(last_begin), ip_utils.ip_num_to_string(last_end), ip_utils.ip_num_to_string(begin), ip_utils.ip_num_to_string(end)
            if end > last_end:
                last_end = end

    output_ip_range_list.append([last_begin, last_end])

    return output_ip_range_list

def filter_ip_range(good_range, bad_range):
    out_good_range = []
    bad_i = 0

    bad_begin, bad_end = bad_range[bad_i]

    for good_begin, good_end in good_range:
        while True:
            if good_begin > good_end:
                PRINT("bad good ip range when filter:%s-%s"  % (ip_utils.ip_num_to_string(good_begin), ip_utils.ip_num_to_string(good_end)))
                assert(good_begin < good_end)
            if good_end < bad_begin:
                # case:
                #     [  good  ]
                #                   [  bad  ]
                out_good_range.append([good_begin, good_end])
                break
            elif bad_end < good_begin:
                # case:
                #                   [  good  ]
                #     [   bad   ]
                bad_i += 1
                bad_begin, bad_end = bad_range[bad_i]
                continue
            elif good_begin <= bad_begin and good_end <= bad_end:
                # case:
                #     [   good    ]
                #           [   bad   ]
                PRINT("cut bad ip case 1:%s - %s" % (ip_utils.ip_num_to_string(bad_begin), ip_utils.ip_num_to_string(good_end)))
                if bad_begin - 1 > good_begin:
                    out_good_range.append([good_begin, bad_begin - 1])
                break
            elif good_begin >= bad_begin and good_end == bad_end:
                # case:
                #           [   good   ]
                #     [      bad       ]
                PRINT("cut bad ip case 2:%s - %s" % (ip_utils.ip_num_to_string(good_begin), ip_utils.ip_num_to_string(bad_end)))

                bad_i += 1
                bad_begin, bad_end = bad_range[bad_i]
                break
            elif good_begin >= bad_begin and good_end > bad_end:
                # case:
                #           [   good   ]
                #     [    bad  ]
                PRINT("cut bad ip case 3:%s - %s" % (ip_utils.ip_num_to_string(good_begin), ip_utils.ip_num_to_string(bad_end)))
                good_begin = bad_end + 1
                bad_i += 1
                bad_begin, bad_end = bad_range[bad_i]
                continue
            elif good_begin <= bad_begin and good_end >= bad_end:
                # case:
                #     [     good     ]
                #         [  bad  ]
                out_good_range.append([good_begin, bad_begin - 1])
                PRINT("cut bad ip case 4:%s - %s" % (ip_utils.ip_num_to_string(bad_begin), ip_utils.ip_num_to_string(bad_end)))
                good_begin = bad_end + 1
                bad_i += 1
                bad_begin, bad_end = bad_range[bad_i]
                continue
            elif good_begin >= bad_begin and good_end <= bad_end:
                # case:
                #          [good]
                #      [    bad    ]
                PRINT("cut bad ip case 5:%s - %s" % (ip_utils.ip_num_to_string(good_begin), ip_utils.ip_num_to_string(good_end)))
                break
            else:
                PRINT("any case? good:%s-%s bad:%s-%s" % (ip_utils.ip_num_to_string(good_begin), ip_utils.ip_num_to_string(good_end),
                    ip_utils.ip_num_to_string(bad_begin), ip_utils.ip_num_to_string(bad_end)))
                assert( False )

    return out_good_range

def download_apic(filename):
    url = 'http://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-latest'
    try:
        data = subprocess.check_output(['wget', url, '-O-'])
    except (OSError, AttributeError):
        print >> sys.stderr, "Fetching data from apnic.net, "\
                             "it might take a few minutes, please wait..."
        data = urllib2.urlopen(url).read()

    with open(filename, "w") as f:
        f.write(data)
    return data

def generage_range_from_apnic(input):

    cnregex = re.compile(r'^apnic\|(?:cn)\|ipv4\|[\d\.]+\|\d+\|\d+\|a\w*$',
                         re.I | re.M )
    cndata = cnregex.findall(input)

    results = []

    for item in cndata:
        unit_items = item.split('|')
        starting_ip = unit_items[3]
        num_ip = int(unit_items[4])

        cidr = 32 - int(math.log(num_ip, 2))

        results.append("%s/%s" % (starting_ip, cidr))

    return "\n".join(results)

def load_bad_ip_range():
    file_name = "delegated-apnic-latest.txt"
    apnic_file = os.path.join(config.DATA_PATH, file_name)
    if not os.path.isfile(apnic_file):
        download_apic(apnic_file)

    with open(apnic_file, "r") as inf:
        apnic_lines = inf.read()

    bad_ip_range_lines = generage_range_from_apnic(apnic_lines)

    sepcial_bad_ip_range_lines  = """
    130.211.0.0/16          #Empty ip range, no route to it.
    255.255.255.255/32      #for algorithm
    """
    return bad_ip_range_lines + sepcial_bad_ip_range_lines

def generate_ip_range():
    # load input good ip range
    file_name = "ip_range.txt"
    input_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), file_name)
    with open(input_file, "r") as inf:
        input_good_range_lines = inf.read()

    ip_range_list = parse_range_string(input_good_range_lines)
    ip_range_list = merge_range(ip_range_list)
    PRINT("Good ip range:\n")
    print_range_list(ip_range_list)

    if False:
        input_bad_ip_range_lines = load_bad_ip_range()
        bad_range_list = parse_range_string(input_bad_ip_range_lines)
        bad_range_list = merge_range(bad_range_list)
        PRINT("Bad ip range:\n")
        print_range_list(ip_range_list)

        ip_range_list = filter_ip_range(ip_range_list, bad_range_list)
        PRINT("Output ip range:\n")
        print_range_list(ip_range_list)

    # write out
    output_file = os.path.join(config.DATA_PATH, file_name)
    fd = open(output_file, "w")
    for ip_range in ip_range_list:
        begin = ip_range[0]
        end = ip_range[1]
        #print ip_utils.ip_num_to_string(begin), ip_utils.ip_num_to_string(end)
        fd.write(ip_utils.ip_num_to_string(begin)+ "-" + ip_utils.ip_num_to_string(end)+"\n")

    fd.close()

def test_load():
    print("Begin test load ip_range.txt\n")
    file_name = os.path.join(config.DATA_PATH, "ip_range.txt")
    fd = open(file_name, "r")
    if not fd:
        print "open ip_range.txt fail."
        exit()

    amount = 0
    for line in fd.readlines():
        if len(line) == 0 or line[0] == '#':
            continue
        begin, end = ip_utils.split_ip(line)

        nbegin = ip_utils.ip_string_to_num(begin)
        nend = ip_utils.ip_string_to_num(end)

        num = nend - nbegin
        amount += num
        print ip_utils.ip_num_to_string(nbegin), ip_utils.ip_num_to_string(nend), num

    fd.close()
    print "amount ip:", amount

def main():
    generate_ip_range()
    test_load()

if __name__ == "__main__":
    main()
<commit_msg>Update generate_ip_range.py<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

__author__ = 'moonshawdo@gamil.com'

import re
import os
import subprocess
import sys
import urllib2
import math

current_path = os.path.dirname(os.path.abspath(__file__))
root_path = os.path.abspath( os.path.join(current_path, os.pardir, os.pardir))
data_path = os.path.abspath(os.path.join(root_path, os.pardir, os.pardir, 'data'))
data_gae_proxy_path = os.path.join(data_path, 'gae_proxy')
python_path = os.path.abspath( os.path.join(root_path, 'python27', '1.0'))

noarch_lib = os.path.abspath( os.path.join(python_path, 'lib', 'noarch'))
sys.path.append(noarch_lib)

from config import config
import ip_utils

# This code functions:
# read ip range string in code
# merge over lapped range
# filter out bad ip range

# Then reproduce good format file

# Support format:
# # Comment (#）
#
# range seperater:
# range can seperate by (,) or(|) or new line
#
# Single rang format: ：
# "xxx.xxx.xxx-xxx.xxx-xxx"
# "xxx.xxx.xxx."
# "xxx.xxx.xxx.xxx/xx"
# "xxx.xxx.xxx.xxx"

def PRINT(strlog):
    print (strlog)

def print_range_list(ip_range_list):
    for ip_range in ip_range_list:
        begin = ip_range[0]
        end = ip_range[1]
        print ip_utils.ip_num_to_string(begin), ip_utils.ip_num_to_string(end)

def parse_range_string(input_lines):
    ip_range_list = []

    ip_lines_list = re.split("\r|\n", input_lines)
    for raw_line in ip_lines_list:
        raw_s = raw_line.split("#")
        context_line = raw_s[0]

        context_line = context_line.replace(' ', '')

        ips = re.split(",|\|", context_line)
        for line in ips:
            if len(line) == 0:
                #print "non line:", line
                continue
            begin, end = ip_utils.split_ip(line)
            if ip_utils.check_ip_valid(begin) == 0 or ip_utils.check_ip_valid(end) == 0:
                PRINT("ip format is error,line:%s, begin: %s,end: %s" % (line, begin, end))
                continue
            nbegin = ip_utils.ip_string_to_num(begin)
            nend = ip_utils.ip_string_to_num(end)
            ip_range_list.append([nbegin,nend])
            #print begin, end

    ip_range_list.sort()

    return ip_range_list

def merge_range(input_ip_range_list):
    output_ip_range_list = []
    range_num = len(input_ip_range_list)

    last_begin = input_ip_range_list[0][0]
    last_end = input_ip_range_list[0][1]
    for i in range(1,range_num):
        ip_range = input_ip_range_list[i]
        begin = ip_range[0]
        end = ip_range[1]

        #print "now:",ip_utils.ip_num_to_string(begin), ip_utils.ip_num_to_string(end)

        if begin > last_end + 2:
            #print "add:",ip_utils.ip_num_to_string(begin), ip_utils.ip_num_to_string(end)
            output_ip_range_list.append([last_begin, last_end])
            last_begin = begin
            last_end = end
        else:
            print "merge:", ip_utils.ip_num_to_string(last_begin), ip_utils.ip_num_to_string(last_end), ip_utils.ip_num_to_string(begin), ip_utils.ip_num_to_string(end)
            if end > last_end:
                last_end = end

    output_ip_range_list.append([last_begin, last_end])

    return output_ip_range_list

def filter_ip_range(good_range, bad_range):
    out_good_range = []
    bad_i = 0

    bad_begin, bad_end = bad_range[bad_i]

    for good_begin, good_end in good_range:
        while True:
            if good_begin > good_end:
                PRINT("bad good ip range when filter:%s-%s"  % (ip_utils.ip_num_to_string(good_begin), ip_utils.ip_num_to_string(good_end)))
                assert(good_begin < good_end)
            if good_end < bad_begin:
                # case:
                #     [  good  ]
                #                   [  bad  ]
                out_good_range.append([good_begin, good_end])
                break
            elif bad_end < good_begin:
                # case:
                #                   [  good  ]
                #     [   bad   ]
                bad_i += 1
                bad_begin, bad_end = bad_range[bad_i]
                continue
            elif good_begin <= bad_begin and good_end <= bad_end:
                # case:
                #     [   good    ]
                #           [   bad   ]
                PRINT("cut bad ip case 1:%s - %s" % (ip_utils.ip_num_to_string(bad_begin), ip_utils.ip_num_to_string(good_end)))
                if bad_begin - 1 > good_begin:
                    out_good_range.append([good_begin, bad_begin - 1])
                break
            elif good_begin >= bad_begin and good_end == bad_end:
                # case:
                #           [   good   ]
                #     [      bad       ]
                PRINT("cut bad ip case 2:%s - %s" % (ip_utils.ip_num_to_string(good_begin), ip_utils.ip_num_to_string(bad_end)))

                bad_i += 1
                bad_begin, bad_end = bad_range[bad_i]
                break
            elif good_begin >= bad_begin and good_end > bad_end:
                # case:
                #           [   good   ]
                #     [    bad  ]
                PRINT("cut bad ip case 3:%s - %s" % (ip_utils.ip_num_to_string(good_begin), ip_utils.ip_num_to_string(bad_end)))
                good_begin = bad_end + 1
                bad_i += 1
                bad_begin, bad_end = bad_range[bad_i]
                continue
            elif good_begin <= bad_begin and good_end >= bad_end:
                # case:
                #     [     good     ]
                #         [  bad  ]
                out_good_range.append([good_begin, bad_begin - 1])
                PRINT("cut bad ip case 4:%s - %s" % (ip_utils.ip_num_to_string(bad_begin), ip_utils.ip_num_to_string(bad_end)))
                good_begin = bad_end + 1
                bad_i += 1
                bad_begin, bad_end = bad_range[bad_i]
                continue
            elif good_begin >= bad_begin and good_end <= bad_end:
                # case:
                #          [good]
                #      [    bad    ]
                PRINT("cut bad ip case 5:%s - %s" % (ip_utils.ip_num_to_string(good_begin), ip_utils.ip_num_to_string(good_end)))
                break
            else:
                PRINT("any case? good:%s-%s bad:%s-%s" % (ip_utils.ip_num_to_string(good_begin), ip_utils.ip_num_to_string(good_end),
                    ip_utils.ip_num_to_string(bad_begin), ip_utils.ip_num_to_string(bad_end)))
                assert( False )

    return out_good_range

def download_apic(filename):
    url = 'http://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-latest'
    try:
        data = subprocess.check_output(['wget', url, '-O-'])
    except (OSError, AttributeError):
        print >> sys.stderr, "Fetching data from apnic.net, "\
                             "it might take a few minutes, please wait..."
        data = urllib2.urlopen(url).read()

    with open(filename, "w") as f:
        f.write(data)
    return data

def generage_range_from_apnic(input):

    cnregex = re.compile(r'^apnic\|(?:cn)\|ipv4\|[\d\.]+\|\d+\|\d+\|a\w*$',
                         re.I | re.M )
    cndata = cnregex.findall(input)

    results = []

    for item in cndata:
        unit_items = item.split('|')
        starting_ip = unit_items[3]
        num_ip = int(unit_items[4])

        cidr = 32 - int(math.log(num_ip, 2))

        results.append("%s/%s" % (starting_ip, cidr))

    return "\n".join(results)

def load_bad_ip_range():
    file_name = "delegated-apnic-latest.txt"
    apnic_file = os.path.join(config.DATA_PATH, file_name)
    if not os.path.isfile(apnic_file):
        download_apic(apnic_file)

    with open(apnic_file, "r") as inf:
        apnic_lines = inf.read()

    bad_ip_range_lines = generage_range_from_apnic(apnic_lines)

    special_bad_ip_range_lines  = """
    130.211.0.0/16          #Empty ip range, no route to it.
    255.255.255.255/32      #for algorithm
    """
    return bad_ip_range_lines + special_bad_ip_range_lines

def generate_ip_range():
    # load input good ip range
    file_name = "ip_range.txt"
    input_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), file_name)
    with open(input_file, "r") as inf:
        input_good_range_lines = inf.read()

    ip_range_list = parse_range_string(input_good_range_lines)
    ip_range_list = merge_range(ip_range_list)
    PRINT("Good ip range:\n")
    print_range_list(ip_range_list)

    if False:
        input_bad_ip_range_lines = load_bad_ip_range()
        bad_range_list = parse_range_string(input_bad_ip_range_lines)
        bad_range_list = merge_range(bad_range_list)
        PRINT("Bad ip range:\n")
        print_range_list(ip_range_list)

        ip_range_list = filter_ip_range(ip_range_list, bad_range_list)
        PRINT("Output ip range:\n")
        print_range_list(ip_range_list)

    # write out
    output_file = os.path.join(config.DATA_PATH, file_name)
    fd = open(output_file, "w")
    for ip_range in ip_range_list:
        begin = ip_range[0]
        end = ip_range[1]
        #print ip_utils.ip_num_to_string(begin), ip_utils.ip_num_to_string(end)
        fd.write(ip_utils.ip_num_to_string(begin)+ "-" + ip_utils.ip_num_to_string(end)+"\n")

    fd.close()

def test_load():
    print("Begin test load ip_range.txt\n")
    file_name = os.path.join(config.DATA_PATH, "ip_range.txt")
    fd = open(file_name, "r")
    if not fd:
        print "open ip_range.txt fail."
        exit()

    amount = 0
    for line in fd.readlines():
        if len(line) == 0 or line[0] == '#':
            continue
        begin, end = ip_utils.split_ip(line)

        nbegin = ip_utils.ip_string_to_num(begin)
        nend = ip_utils.ip_string_to_num(end)

        num = nend - nbegin
        amount += num
        print ip_utils.ip_num_to_string(nbegin), ip_utils.ip_num_to_string(nend), num

    fd.close()
    print "amount ip:", amount

def main():
    generate_ip_range()
    test_load()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#! /usr/bin/python

# PyGreen
# Copyright (c) 2013, Nicolas Vanhoren
#
# Released under the MIT license
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
# Software, and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

from __future__ import unicode_literals

import bottle
import os.path
from mako.lookup import TemplateLookup
import os
import os.path
import wsgiref.handlers
import sys
import logging
import re
import argparse
import sys
import markdown

_logger = logging.getLogger(__name__)

class PyGreen:

    def __init__(self):
        # the Bottle application
        self.app = bottle.Bottle()
        # a set of strings that identifies the extension of the files
        # that should be processed using Mako
        self.template_exts = set(["html"])
        # the folder where the files to serve are located. Do not set
        # directly, use set_folder instead
        self.folder = "."
        # the TemplateLookup of Mako
        self.templates = TemplateLookup(directories=[self.folder],
            imports=["from markdown import markdown"],
            input_encoding='iso-8859-1',
            collection_size=100,
            )
        # A list of regular expression. Files whose the name match
        # one of those regular expressions will not be outputed when generating
        # a static version of the web site
        self.file_exclusion = [r".*\.mako", r"(^|.*\/)\..*"]
        def base_lister():
            files = []
            for dirpath, dirnames, filenames in os.walk(self.folder):
                for f in filenames:
                    absp = os.path.join(dirpath, f)
                    path = os.path.relpath(absp, self.folder)
                    good = True
                    for ex in self.file_exclusion:
                        if re.match(ex, path):
                            good = False
                            continue
                    if good:
                        files.append(path)
            return files
        # A list of function. Each function must return a list of paths
        # of files to export during the generation of the static web site.
        # The default one simply returns all the files contained in the folder.
        # It is necessary to define new listers when new routes are defined
        # in the Bottle application, or the static site generation routine
        # will not be able to detect the files to export.
        self.file_listers = [base_lister]

        def file_renderer(path):
            if path.split(".")[-1] in self.template_exts:
                t = self.templates.get_template(path)
                data = t.render_unicode(pygreen=pygreen)
                return data.encode(t.module._source_encoding)
            return bottle.static_file(path, root=self.folder)
        # The default function used to render files. Could be modified to change the way files are
        # generated, like using another template language or transforming css...
        self.file_renderer = file_renderer
        self.app.route('/', method=['GET', 'POST', 'PUT', 'DELETE'])(lambda: self.file_renderer('index.html'))
        self.app.route('/<path:path>', method=['GET', 'POST', 'PUT', 'DELETE'])(lambda path: self.file_renderer(path))

    def set_folder(self, folder):
        """
        Sets the folder where the files to serve are located.
        """
        self.folder = folder
        self.templates.directories[0] = folder

    def run(self, **kwargs):
        """
        Launch a development web server.
        """
        kwargs.setdefault("host", "0.0.0.0")
        bottle.run(self.app, server="waitress", **kwargs)

    def get(self, path):
        """
        Get the content of a file, indentified by its path relative to the folder configured
        in PyGreen. If the file extension is one of the extensions that should be processed
        through Mako, it will be processed.
        """
        handler = wsgiref.handlers.SimpleHandler(sys.stdin, sys.stdout, sys.stderr, {})
        handler.setup_environ()
        env = handler.environ
        env.update({'PATH_INFO': "/%s" % path, 'REQUEST_METHOD': "GET"})
        out = b"".join(pygreen.app(env, lambda *args: None))
        return out

    def gen_static(self, output_folder):
        """
        Generates a complete static version of the web site. It will stored in
        output_folder.
        """
        files = []
        for l in self.file_listers:
            files += l()
        for f in files:
            _logger.info("generating %s" % f)
            content = self.get(f)
            loc = os.path.join(output_folder, f)
            d = os.path.dirname(loc)
            if not os.path.exists(d):
                os.makedirs(d)
            with open(loc, "wb") as file_:
                file_.write(content)

    def cli(self, cmd_args=None):
        """
        The command line interface of PyGreen.
        """
        logging.basicConfig(level=logging.INFO, format='%(message)s')

        parser = argparse.ArgumentParser(description='PyGreen, micro web framework/static web site generator')
        subparsers = parser.add_subparsers(dest='action')

        parser_serve = subparsers.add_parser('serve', help='serve the web site')
        parser_serve.add_argument('-p', '--port', type=int, default=8080, help='folder containg files to serve')
        parser_serve.add_argument('-f', '--folder', default=".", help='folder containg files to serve')
        parser_serve.add_argument('-d', '--disable-templates', action='store_true', default=False, help='just serve static files, do not use invoke Mako')
        def serve():
            if args.disable_templates:
                self.template_exts = set([])
            self.run(port=args.port)
        parser_serve.set_defaults(func=serve)

        parser_gen = subparsers.add_parser('gen', help='generate a static version of the site')
        parser_gen.add_argument('output', help='folder to store the files')
        parser_gen.add_argument('-f', '--folder', default=".", help='folder containg files to serve')
        def gen():
            self.gen_static(args.output)
        parser_gen.set_defaults(func=gen)

        args = parser.parse_args(cmd_args)
        self.set_folder(args.folder)
        args.func()

pygreen = PyGreen()

if __name__ == "__main__":
    pygreen.cli()
<commit_msg>Added .py files to the file exclusion list<commit_after>#! /usr/bin/python

# PyGreen
# Copyright (c) 2013, Nicolas Vanhoren
#
# Released under the MIT license
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
# Software, and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

from __future__ import unicode_literals

import bottle
import os.path
from mako.lookup import TemplateLookup
import os
import os.path
import wsgiref.handlers
import sys
import logging
import re
import argparse
import sys
import markdown

_logger = logging.getLogger(__name__)

class PyGreen:

    def __init__(self):
        # the Bottle application
        self.app = bottle.Bottle()
        # a set of strings that identifies the extension of the files
        # that should be processed using Mako
        self.template_exts = set(["html"])
        # the folder where the files to serve are located. Do not set
        # directly, use set_folder instead
        self.folder = "."
        # the TemplateLookup of Mako
        self.templates = TemplateLookup(directories=[self.folder],
            imports=["from markdown import markdown"],
            input_encoding='iso-8859-1',
            collection_size=100,
            )
        # A list of regular expression. Files whose the name match
        # one of those regular expressions will not be outputed when generating
        # a static version of the web site
        self.file_exclusion = [r".*\.mako", r".*\.py", r"(^|.*\/)\..*"]
        def base_lister():
            files = []
            for dirpath, dirnames, filenames in os.walk(self.folder):
                for f in filenames:
                    absp = os.path.join(dirpath, f)
                    path = os.path.relpath(absp, self.folder)
                    good = True
                    for ex in self.file_exclusion:
                        if re.match(ex, path):
                            good = False
                            continue
                    if good:
                        files.append(path)
            return files
        # A list of function. Each function must return a list of paths
        # of files to export during the generation of the static web site.
        # The default one simply returns all the files contained in the folder.
        # It is necessary to define new listers when new routes are defined
        # in the Bottle application, or the static site generation routine
        # will not be able to detect the files to export.
        self.file_listers = [base_lister]

        def file_renderer(path):
            if path.split(".")[-1] in self.template_exts:
                t = self.templates.get_template(path)
                data = t.render_unicode(pygreen=pygreen)
                return data.encode(t.module._source_encoding)
            return bottle.static_file(path, root=self.folder)
        # The default function used to render files. Could be modified to change the way files are
        # generated, like using another template language or transforming css...
        self.file_renderer = file_renderer
        self.app.route('/', method=['GET', 'POST', 'PUT', 'DELETE'])(lambda: self.file_renderer('index.html'))
        self.app.route('/<path:path>', method=['GET', 'POST', 'PUT', 'DELETE'])(lambda path: self.file_renderer(path))

    def set_folder(self, folder):
        """
        Sets the folder where the files to serve are located.
        """
        self.folder = folder
        self.templates.directories[0] = folder

    def run(self, **kwargs):
        """
        Launch a development web server.
        """
        kwargs.setdefault("host", "0.0.0.0")
        bottle.run(self.app, server="waitress", **kwargs)

    def get(self, path):
        """
        Get the content of a file, indentified by its path relative to the folder configured
        in PyGreen. If the file extension is one of the extensions that should be processed
        through Mako, it will be processed.
        """
        handler = wsgiref.handlers.SimpleHandler(sys.stdin, sys.stdout, sys.stderr, {})
        handler.setup_environ()
        env = handler.environ
        env.update({'PATH_INFO': "/%s" % path, 'REQUEST_METHOD': "GET"})
        out = b"".join(pygreen.app(env, lambda *args: None))
        return out

    def gen_static(self, output_folder):
        """
        Generates a complete static version of the web site. It will stored in
        output_folder.
        """
        files = []
        for l in self.file_listers:
            files += l()
        for f in files:
            _logger.info("generating %s" % f)
            content = self.get(f)
            loc = os.path.join(output_folder, f)
            d = os.path.dirname(loc)
            if not os.path.exists(d):
                os.makedirs(d)
            with open(loc, "wb") as file_:
                file_.write(content)

    def cli(self, cmd_args=None):
        """
        The command line interface of PyGreen.
        """
        logging.basicConfig(level=logging.INFO, format='%(message)s')

        parser = argparse.ArgumentParser(description='PyGreen, micro web framework/static web site generator')
        subparsers = parser.add_subparsers(dest='action')

        parser_serve = subparsers.add_parser('serve', help='serve the web site')
        parser_serve.add_argument('-p', '--port', type=int, default=8080, help='folder containg files to serve')
        parser_serve.add_argument('-f', '--folder', default=".", help='folder containg files to serve')
        parser_serve.add_argument('-d', '--disable-templates', action='store_true', default=False, help='just serve static files, do not use invoke Mako')
        def serve():
            if args.disable_templates:
                self.template_exts = set([])
            self.run(port=args.port)
        parser_serve.set_defaults(func=serve)

        parser_gen = subparsers.add_parser('gen', help='generate a static version of the site')
        parser_gen.add_argument('output', help='folder to store the files')
        parser_gen.add_argument('-f', '--folder', default=".", help='folder containg files to serve')
        def gen():
            self.gen_static(args.output)
        parser_gen.set_defaults(func=gen)

        args = parser.parse_args(cmd_args)
        self.set_folder(args.folder)
        args.func()

pygreen = PyGreen()

if __name__ == "__main__":
    pygreen.cli()
<|endoftext|>


<commit_before>import sys, os
from object import Object, List, String, Symbol, Integer, BuiltinFunction, true, false, null
from reader import WontParse, PartialParse, read_file, read_source
from rpython.config.translationoption import get_combined_translation_config
from rpython.rlib.rtimer import read_timestamp

config = get_combined_translation_config(translating=True)

class Environment:
    def __init__(self, parent, global_scope):
        self.parent = parent
        self.namespace = {}
        self.global_scope = global_scope

    def lookup(self, name):
        if name in self.namespace:
            return self.namespace[name]
        if name in self.global_scope:
            return self.global_scope[name]
        if self.parent is not None:
            return self.parent.lookup(name)
        raise Exception(name + " not in scope")

class Return(Exception):
    def __init__(self, retval):
        self.retval = retval

class Closure(Object):
    def __init__(self, env, expr):
        self.env = env
        self.expr = expr
        self.arglist = []
        self.vararg = None
        assert isinstance(expr[1], List)
        for symbol in expr[1]:
            assert isinstance(symbol, Symbol)
            if symbol.string.endswith('...'):
                self.vararg = symbol.string[0:max(0, len(symbol.string)-3)]
            else:
                self.arglist.append(symbol.string)

    def invoke(self, argv):
        env = Environment(self.env, self.env.global_scope)
        c = len(self.arglist)
        b = min(c, len(argv))
        for i in range(0, b):
            env.namespace[self.arglist[i]] = argv[i]
        for i in range(b, c):
            env.namespace[self.arglist[i]] = null
        if self.vararg is not None:
            varg = []
            for i in range(c, len(argv)):
                varg.append(argv[i])
            env.namespace[self.vararg] = List(varg)
        try:
            for i in range(2, len(self.expr)):
                interpret(env, self.expr[i])
        except Return as ret:
            return ret.retval
        return null

    def repr(self):
        return self.expr.repr()

def pyl_print(argv):
    space = ''
    for arg in argv:
        if isinstance(arg, String):
            string = arg.string
        else:
            string = arg.repr()
        os.write(1, space + string)
        space = ' '
    os.write(1, '\n')
    return null

def pyl_apply(argv):
    N = len(argv) - 1
    assert N >= 1
    args = argv[1:N]
    varg = argv[N]
    assert isinstance(varg, List)
    return argv[0].invoke(args + varg.items)

def pyl_list(argv):
    return List(argv)

def pyl_getitem(argv):
    assert len(argv) == 2
    return argv[0].getitem(argv[1])

def pyl_setitem(argv):
    assert len(argv) == 3
    return argv[0].setitem(argv[1], argv[2])

def pyl_getattr(argv):
    assert len(argv) == 2
    name = argv[1]
    assert isinstance(name, String)
    return argv[0].getattr(name.string)

def pyl_setattr(argv):
    assert len(argv) == 3
    name = argv[1]
    assert isinstance(name, String)
    return argv[0].setattr(name.string, argv[2])

def pyl_callattr(argv):
    assert len(argv) >= 2
    name = argv[1]
    assert isinstance(name, String)
    return argv[0].callattr(name.string, argv[2:len(argv)])

global_scope = {
    "print": BuiltinFunction(pyl_print, "print"),
    "apply": BuiltinFunction(pyl_apply, "apply"),
    "list": BuiltinFunction(pyl_list, "list"),
    "[]": BuiltinFunction(pyl_getitem, "[]"),
    "[]=": BuiltinFunction(pyl_setitem, "[]="),
    "getattr": BuiltinFunction(pyl_getattr, "getattr"),
    "setattr": BuiltinFunction(pyl_setattr, "setattr"),
    "callattr": BuiltinFunction(pyl_callattr, "callattr"),
    "true": true,
    "false": false,
    "null": null,
}

def cond_macro(env, exprs):
    retval = null
    for i in range(1, len(exprs)):
        block = exprs[i]
        head = block[0]
        if not (isinstance(head, Symbol) and head.string == 'else'):
            if is_false(interpret(env, head)):
                continue
        for i in range(1, len(block)):
            retval = interpret(env, block[i])
        break
    return retval

def while_macro(env, exprs):
    assert len(exprs) > 2
    retval = null
    cond = exprs[1]
    while not is_false(interpret(env, cond)):
        for i in range(2, len(exprs)):
            retval = interpret(env, exprs[i])
    return retval

def letvar_macro(env, exprs):
    assert len(exprs) == 3
    lhs, rhs = exprs[1], exprs[2]
    assert isinstance(lhs, Symbol)
    retval = env.namespace[lhs.string] = interpret(env, rhs)
    return retval

def setvar_macro(env, exprs):
    assert len(exprs) == 3
    lhs, rhs = exprs[1], exprs[2]
    assert isinstance(lhs, Symbol)
    dst = env
    while dst is not None:
        if lhs.string in dst.namespace:
            retval = dst.namespace[lhs.string] = interpret(env, rhs)
            return retval
        dst = dst.parent
    raise Exception("no such variable: " + lhs.string)

def func_macro(env, exprs):
    assert len(exprs) > 2
    return Closure(env, exprs)

def return_macro(env, exprs):
    assert len(exprs) == 2
    raise Return(interpret(env, exprs[1]))

def is_false(flag):
    return flag is null or flag is false

macros = {
    'cond': cond_macro,
    'while': while_macro,
    '=': letvar_macro,
    ':=': setvar_macro,
    'func': func_macro,
    'return': return_macro,
}

def interpret(env, expr):
    if isinstance(expr, List):
        if len(expr) == 0:
            raise Exception("cannot evaluate ()")
        if isinstance(expr[0], Symbol) and expr[0].string in macros:
            macro = macros[expr[0].string]
            return macro(env, expr)
        callee = interpret(env, expr[0])
        args = []
        for i in range(1, len(expr)):
            args.append(interpret(env, expr[i]))
        return callee.invoke(args)
    elif isinstance(expr, Symbol):
        return env.lookup(expr.string)
    else:
        return expr

def entry_point(argv):
    env = Environment(None, global_scope)
    if len(argv) <= 1:
        prompt = "pyl> "
        os.write(1, prompt)
        source = os.read(0, 4096)
        while source != "":
            try:
                try:
                    lst = read_source(source)
                except PartialParse as exc:
                    os.write(1, " "*len(prompt))
                    source += os.read(0, 4096)
                else:
                    assert isinstance(lst, List)
                    for expr in lst:
                        retval = interpret(env, expr)
                        pyl_print([retval])
                    os.write(1, prompt)
                    source = os.read(0, 4096)
            except Exception as exc:
                os.write(1, exc.__class__.__name__ + ": " + str(exc) + "\n")
                os.write(1, prompt)
                source = os.read(0, 4096)
        if source == "":
            os.write(1, "\n")
    else:
        #now = read_timestamp()
        #print "[STAMP BEGIN] "
        lst = read_file(argv[1])
        assert isinstance(lst, List)
        for expr in lst:
            interpret(env, expr)
        #delta = read_timestamp() - now
        #print "[STAMP NOW] " + str(delta)
    return 0

def target(*args):
    return entry_point, None

if __name__=='__main__':
    sys.exit(entry_point(sys.argv))
<commit_msg>accessor symbols<commit_after>import sys, os
from object import Object, List, String, Symbol, Integer, BuiltinFunction, true, false, null
from reader import WontParse, PartialParse, read_file, read_source
from rpython.config.translationoption import get_combined_translation_config
from rpython.rlib.rtimer import read_timestamp

config = get_combined_translation_config(translating=True)

class Environment:
    def __init__(self, parent, global_scope):
        self.parent = parent
        self.namespace = {}
        self.global_scope = global_scope

    def lookup(self, name):
        if name in self.namespace:
            return self.namespace[name]
        if name in self.global_scope:
            return self.global_scope[name]
        if self.parent is not None:
            return self.parent.lookup(name)
        raise Exception(name + " not in scope")

class Return(Exception):
    def __init__(self, retval):
        self.retval = retval

class Closure(Object):
    def __init__(self, env, expr):
        self.env = env
        self.expr = expr
        self.arglist = []
        self.vararg = None
        assert isinstance(expr[1], List)
        for symbol in expr[1]:
            assert isinstance(symbol, Symbol)
            if symbol.string.endswith('...'):
                self.vararg = symbol.string[0:max(0, len(symbol.string)-3)]
            else:
                self.arglist.append(symbol.string)

    def invoke(self, argv):
        env = Environment(self.env, self.env.global_scope)
        c = len(self.arglist)
        b = min(c, len(argv))
        for i in range(0, b):
            env.namespace[self.arglist[i]] = argv[i]
        for i in range(b, c):
            env.namespace[self.arglist[i]] = null
        if self.vararg is not None:
            varg = []
            for i in range(c, len(argv)):
                varg.append(argv[i])
            env.namespace[self.vararg] = List(varg)
        try:
            for i in range(2, len(self.expr)):
                interpret(env, self.expr[i])
        except Return as ret:
            return ret.retval
        return null

    def repr(self):
        return self.expr.repr()

def pyl_print(argv):
    space = ''
    for arg in argv:
        if isinstance(arg, String):
            string = arg.string
        else:
            string = arg.repr()
        os.write(1, space + string)
        space = ' '
    os.write(1, '\n')
    return null

def pyl_apply(argv):
    N = len(argv) - 1
    assert N >= 1
    args = argv[1:N]
    varg = argv[N]
    assert isinstance(varg, List)
    return argv[0].invoke(args + varg.items)

def pyl_list(argv):
    return List(argv)

def pyl_getitem(argv):
    assert len(argv) == 2
    return argv[0].getitem(argv[1])

def pyl_setitem(argv):
    assert len(argv) == 3
    return argv[0].setitem(argv[1], argv[2])

def pyl_getattr(argv):
    assert len(argv) == 2
    name = argv[1]
    assert isinstance(name, String)
    return argv[0].getattr(name.string)

def pyl_setattr(argv):
    assert len(argv) == 3
    name = argv[1]
    assert isinstance(name, String)
    return argv[0].setattr(name.string, argv[2])

def pyl_callattr(argv):
    assert len(argv) >= 2
    name = argv[1]
    assert isinstance(name, String)
    return argv[0].callattr(name.string, argv[2:len(argv)])

class GetAccessor(Object):
    def __init__(self, name):
        self.name = name

    def invoke(self, argv):
        assert len(argv) == 1
        return argv[0].getattr(self.name)

    def repr(self):
        return "." + self.name

class SetAccessor(Object):
    def __init__(self, name):
        self.name = name

    def invoke(self, argv):
        assert len(argv) == 2
        return argv[0].setattr(self.name, argv[1])

    def repr(self):
        return "." + self.name + '='

class CallAccessor(Object):
    def __init__(self, name):
        self.name = name

    def invoke(self, argv):
        N = len(argv)
        assert N >= 1
        return argv[0].callattr(self.name, argv[1:N])

    def repr(self):
        return "." + self.name + '!'

global_scope = {
    "print": BuiltinFunction(pyl_print, "print"),
    "apply": BuiltinFunction(pyl_apply, "apply"),
    "list": BuiltinFunction(pyl_list, "list"),
    "[]": BuiltinFunction(pyl_getitem, "[]"),
    "[]=": BuiltinFunction(pyl_setitem, "[]="),
    "getattr": BuiltinFunction(pyl_getattr, "getattr"),
    "setattr": BuiltinFunction(pyl_setattr, "setattr"),
    "callattr": BuiltinFunction(pyl_callattr, "callattr"),
    "true": true,
    "false": false,
    "null": null,
}

def cond_macro(env, exprs):
    retval = null
    for i in range(1, len(exprs)):
        block = exprs[i]
        head = block[0]
        if not (isinstance(head, Symbol) and head.string == 'else'):
            if is_false(interpret(env, head)):
                continue
        for i in range(1, len(block)):
            retval = interpret(env, block[i])
        break
    return retval

def while_macro(env, exprs):
    assert len(exprs) > 2
    retval = null
    cond = exprs[1]
    while not is_false(interpret(env, cond)):
        for i in range(2, len(exprs)):
            retval = interpret(env, exprs[i])
    return retval

def letvar_macro(env, exprs):
    assert len(exprs) == 3
    lhs, rhs = exprs[1], exprs[2]
    assert isinstance(lhs, Symbol)
    retval = env.namespace[lhs.string] = interpret(env, rhs)
    return retval

def setvar_macro(env, exprs):
    assert len(exprs) == 3
    lhs, rhs = exprs[1], exprs[2]
    assert isinstance(lhs, Symbol)
    dst = env
    while dst is not None:
        if lhs.string in dst.namespace:
            retval = dst.namespace[lhs.string] = interpret(env, rhs)
            return retval
        dst = dst.parent
    raise Exception("no such variable: " + lhs.string)

def func_macro(env, exprs):
    assert len(exprs) > 2
    return Closure(env, exprs)

def return_macro(env, exprs):
    assert len(exprs) == 2
    raise Return(interpret(env, exprs[1]))

def is_false(flag):
    return flag is null or flag is false

macros = {
    'cond': cond_macro,
    'while': while_macro,
    '=': letvar_macro,
    ':=': setvar_macro,
    'func': func_macro,
    'return': return_macro,
}

def interpret(env, expr):
    if isinstance(expr, List):
        if len(expr) == 0:
            raise Exception("cannot evaluate ()")
        if isinstance(expr[0], Symbol) and expr[0].string in macros:
            macro = macros[expr[0].string]
            return macro(env, expr)
        callee = interpret(env, expr[0])
        args = []
        for i in range(1, len(expr)):
            args.append(interpret(env, expr[i]))
        return callee.invoke(args)
    elif isinstance(expr, Symbol):
        if expr.string.startswith('.'):
            N = len(expr.string)
            if expr.string.endswith('='):
                return SetAccessor(expr.string[1:max(1, N-1)])
            if expr.string.endswith('!'):
                return CallAccessor(expr.string[1:max(1, N-1)])
            return GetAccessor(expr.string[1:N])
        return env.lookup(expr.string)
    else:
        return expr

def entry_point(argv):
    env = Environment(None, global_scope)
    if len(argv) <= 1:
        prompt = "pyl> "
        os.write(1, prompt)
        source = os.read(0, 4096)
        while source != "":
            try:
                try:
                    lst = read_source(source)
                except PartialParse as exc:
                    os.write(1, " "*len(prompt))
                    source += os.read(0, 4096)
                else:
                    assert isinstance(lst, List)
                    for expr in lst:
                        retval = interpret(env, expr)
                        pyl_print([retval])
                    os.write(1, prompt)
                    source = os.read(0, 4096)
            except Exception as exc:
                os.write(1, exc.__class__.__name__ + ": " + str(exc) + "\n")
                os.write(1, prompt)
                source = os.read(0, 4096)
        if source == "":
            os.write(1, "\n")
    else:
        #now = read_timestamp()
        #print "[STAMP BEGIN] "
        lst = read_file(argv[1])
        assert isinstance(lst, List)
        for expr in lst:
            interpret(env, expr)
        #delta = read_timestamp() - now
        #print "[STAMP NOW] " + str(delta)
    return 0

def target(*args):
    return entry_point, None

if __name__=='__main__':
    sys.exit(entry_point(sys.argv))
<|endoftext|>


<commit_before>#!/usr/bin/python
# qbranch.py 1.0
# Checks an Azure error queue
# Author: Mike Perttula

import json
import string
import time
import datetime
import argparse
from azure import *
from azure.storage import QueueService

def checkQueue(aName,aKey,aQueue):
	while 1:
		try:
			queue_service = QueueService(account_name=aName, account_key=aKey)
		except:
			print "Could not connect to queue"
		else:
			try:
				queue_metadata = queue_service.get_queue_metadata(aQueue)
			except:
				print "Could not find queue"

			else:
				try:
					count = queue_metadata['x-ms-approximate-messages-count']
					print aQueue + "queue is empty"
					if int(count) > 0:
						print "Messages in " + aQueue + " queue: " + count
						messages = queue_service.get_messages(aQueue)
						for message in messages:
   							print "Message " + message.message_id + " : " + message.message_text
    							queue_service.delete_message(aQueue, message.message_id, message.pop_receipt)
				except:
					print "Could not read queue"

		time.sleep(15)

parser = argparse.ArgumentParser(prog='qbranch')
parser.add_argument('config',help="Queue config file")
args = parser.parse_args()

if args.config:
	with open(args.config) as configFile:
		config = json.load(configFile)
	checkQueue(config["account"],config["key"],config["queue"])
else:
	parser.print_help()
<commit_msg>Fixed message spacing<commit_after>#!/usr/bin/python
# qbranch.py 1.0
# Checks an Azure error queue
# Author: Mike Perttula

import json
import string
import time
import datetime
import argparse
from azure import *
from azure.storage import QueueService

def checkQueue(aName,aKey,aQueue):
	while 1:
		try:
			queue_service = QueueService(account_name=aName, account_key=aKey)
		except:
			print "Could not connect to queue"
		else:
			try:
				queue_metadata = queue_service.get_queue_metadata(aQueue)
			except:
				print "Could not find queue"

			else:
				try:
					count = queue_metadata['x-ms-approximate-messages-count']
					print aQueue + " queue is empty"
					if int(count) > 0:
						print "Messages in " + aQueue + " queue: " + count
						messages = queue_service.get_messages(aQueue)
						for message in messages:
   							print "Message " + message.message_id + " : " + message.message_text
    							queue_service.delete_message(aQueue, message.message_id, message.pop_receipt)
				except:
					print "Could not read queue"

		time.sleep(15)

parser = argparse.ArgumentParser(prog='qbranch')
parser.add_argument('config',help="Queue config file")
args = parser.parse_args()

if args.config:
	with open(args.config) as configFile:
		config = json.load(configFile)
	checkQueue(config["account"],config["key"],config["queue"])
else:
	parser.print_help()
<|endoftext|>


<commit_before>import codecs
import time

class Token(object):
	def __init__( self, line ):
		entries = line.split('\t')
		self.form = entries[0]
		self.pos = entries[1]

def sentences( filestream ):

	sentence = []
	for line in filestream:
		line = line.rstrip()
		if line:
			sentence.append(Token(line))
		elif sentence:
			yield sentence
			sentence = []
	if sentence:
		yield sentence

def evaluate(posDict):
	goldPosSet = set()
	predPosSet = set()
	uniqueTags = {}
	uniqueTagsScores = {}
	correctPredictions = 0

	for key in posDict:

		# creates sets of gold and prediction tags

		goldPosSet.add(posDict[key][0])
		predPosSet.add(posDict[key][1])

	for key in posDict:

		# creates a dictionary with every existing tag (either existing only in gold, only in
		# prediction, or both) the key and a dictionary each as the value containing TP, FN, FP
		# as the key and zero as the value.

		if posDict[key][0] not in uniqueTags.keys():
			uniqueTags[posDict[key][0]] = {'TP':0,'FN':0,'FP':0}
		if posDict[key][1] not in uniqueTags.keys():
			uniqueTags[posDict[key][1]] = {'TP':0,'FN':0,'FP':0}

	for key in posDict:

		# Computes TP, FN and FP for each unique tag.

		if posDict[key][0] == posDict[key][1]:
			correctPredictions+=1
			uniqueTags[posDict[key][0]]['TP']+=1
		else:
			uniqueTags[posDict[key][0]]['FN']+=1
			uniqueTags[posDict[key][1]]['FP']+=1

	for pos in uniqueTags.keys():

		# Computes Precision, Recall, Accuracy and F-Score for each Tag based on TP, FN, FP.

		uniqueTagsScores[pos]={'Precision':0.00, 'Recall':0.00, 'Accuracy':0.00, 'F-Score':0.00}
		if uniqueTags[pos]['TP']+uniqueTags[pos]['FP'] == 0:
			uniqueTagsScores[pos]['Precision'] = 0.00
		else:
			uniqueTagsScores[pos]['Precision']=round((float(uniqueTags[pos]['TP']))/(float(uniqueTags[pos]['TP'])+float(uniqueTags[pos]['FP']))*100.00, 2)

		if uniqueTags[pos]['TP']+uniqueTags[pos]['FN'] == 0:
			uniqueTagsScores[pos]['Recall'] = 0.00
		else: uniqueTagsScores[pos]['Recall']=round((float(uniqueTags[pos]['TP']))/(float(uniqueTags[pos]['TP'])+float(uniqueTags[pos]['FN']))*100.00, 2)

		if uniqueTags[pos]['TP']+uniqueTags[pos]['FP']+uniqueTags[pos]['FN'] == 0:
			uniqueTagsScores[pos]['Accuracy'] = 0.00
		else: uniqueTagsScores[pos]['Accuracy']=round(float(uniqueTags[pos]['TP'])/(float(uniqueTags[pos]['TP'])+float(uniqueTags[pos]['FN'])+float(uniqueTags[pos]['FP']))*100.00, 2)

		if uniqueTagsScores[pos]['Precision']+uniqueTagsScores[pos]['Recall'] == 0.00:
			uniqueTagsScores[pos]['F-Score'] = 0.00
		else: uniqueTagsScores[pos]['F-Score'] = round((2*float(uniqueTagsScores[pos]['Precision'])*float(uniqueTagsScores[pos]['Recall']))/(float(uniqueTagsScores[pos]['Precision'])+float(uniqueTagsScores[pos]['Recall'])), 2)

	precisionSum = 0.0
	recallSum = 0.0
	fscoreSum = 0.0
	predictionCount = len(posDict)

	for pos in uniqueTagsScores.keys():
		precisionSum += uniqueTagsScores[pos]['Precision']
		recallSum += uniqueTagsScores[pos]['Recall']
		fscoreSum += uniqueTagsScores[pos]['F-Score']

	overallPrecision = precisionSum/float(len(uniqueTagsScores))
	overallRecall = recallSum/float(len(uniqueTagsScores))
	overallFscore = fscoreSum/float(len(uniqueTagsScores))

	overallAccuracy = (float(correctPredictions)/float(predictionCount))*100
	falseTags = predictionCount-correctPredictions

	print "Total Predictions:\t"+str(predictionCount)
	print "Correct Predictions:\t"+str(correctPredictions)
	print "False Predictions:\t"+str(falseTags)
	print ""
	print "Overall Accuracy:\t"+str(overallAccuracy)
	print "Overall Precision:\t"+str(overallPrecision)
	print "Overall Recall:\t"+str(overallRecall)
	print "Overall F-Score:\t"+str(overallFscore)
	print ""

	print "Tagwise Accuracy, Precision, Recall and F-Score:\n"
	for pos in uniqueTagsScores.keys():
		print pos+"\tAccuracy: "+str(uniqueTagsScores[pos]['Accuracy'])+"\tPrecision: "+str(uniqueTagsScores[pos]['Precision'])+"\tRecall: "+str(uniqueTagsScores[pos]['Recall'])+"\tF-Score: "+str(uniqueTagsScores[pos]['F-Score'])

if __name__=='__main__':
	t0 = time.time()
	import argparse
	argpar = argparse.ArgumentParser(description='Creates a feature representation for each word in a given file in CoNLL09 format')

	argpar.add_argument('-g','--gold',dest='gold',help='gold',required=True)
	argpar.add_argument('-p','--prediction',dest='prediction',help='prediction',required=True)
	#argpar.add_argument('-o','--output',dest='outputfile',help='output file',required=True)
	args = argpar.parse_args()

	posDict = {}

	#outstream = open(args.outputfile,'w')

	counter=0
	for sid, sentence in enumerate(sentences(codecs.open(args.gold,encoding='utf-8'))):
		for tid,token in enumerate(sentence):
			posDict[counter] = []
			posDict[counter].append(token.pos)
			counter+=1
	counter=0
	for sid, sentence in enumerate(sentences(codecs.open(args.prediction,encoding='utf-8'))):
		for tid,token in enumerate(sentence):
			posDict[counter].append(token.pos)
			counter+=1

	evaluate(posDict)
	t1 = time.time()
	print "\n"
	print "Time: "+ str(t1-t0) + "sec."
	#print >> outstream, " "
	#outstream.close()

<commit_msg>featurevector<commit_after>import codecs
import time

class Token(object):
	def __init__( self, line ):
		entries = line.split('\t')
		self.form = entries[0]
		self.gold_pos = entries[1]
		self.predicted_pos = entries [3]

	def createFeatureVector(self, featvec, currentToken, previousToken, nextToken):
		self.sparseFeatvec = {}
		#print len(featvec)

		self.sparseFeatvec[featvec["current_pos_"+str(currentToken.gold_pos)]] = 1
		if previousToken: self.sparseFeatvec[featvec["prev_pos_"+str(previousToken.gold_pos)]] = 1
		if nextToken: self.sparseFeatvec[featvec["next_pos_"+str(nextToken.gold_pos)]] = 1

		self.sparseFeatvec[featvec["current_form_"+str(currentToken.form)]] = 1
		if previousToken: self.sparseFeatvec[featvec["prev_form_"+str(previousToken.form)]] = 1
		if nextToken: self.sparseFeatvec[featvec["next_form_"+str(nextToken.form)]] = 1
		#for feat in featvec:

def sentences( filestream ):

	sentence = []
	for line in filestream:
		line = line.rstrip()
		if line:
			sentence.append(Token(line))
		elif sentence:
			yield sentence
			sentence = []
	if sentence:
		yield sentence

def evaluate(posDict):
	goldPosSet = set()
	predPosSet = set()
	uniqueTags = {}
	uniqueTagsScores = {}
	correctPredictions = 0

	for key in posDict:

		# creates sets of gold and prediction tags

		goldPosSet.add(posDict[key][0])
		predPosSet.add(posDict[key][1])

	for key in posDict:

		# creates a dictionary with every existing tag (either existing only in gold, only in
		# prediction, or both) the key and a dictionary each as the value containing TP, FN, FP
		# as the key and zero as the value.

		if posDict[key][0] not in uniqueTags.keys():
			uniqueTags[posDict[key][0]] = {'TP':0,'FN':0,'FP':0}
		if posDict[key][1] not in uniqueTags.keys():
			uniqueTags[posDict[key][1]] = {'TP':0,'FN':0,'FP':0}

	for key in posDict:

		# Computes TP, FN and FP for each unique tag.

		if posDict[key][0] == posDict[key][1]:
			correctPredictions+=1
			uniqueTags[posDict[key][0]]['TP']+=1
		else:
			uniqueTags[posDict[key][0]]['FN']+=1
			uniqueTags[posDict[key][1]]['FP']+=1

	for pos in uniqueTags.keys():

		# Computes Precision, Recall, Accuracy and F-Score for each Tag based on TP, FN, FP.

		uniqueTagsScores[pos]={'Precision':0.00, 'Recall':0.00, 'Accuracy':0.00, 'F-Score':0.00}
		if uniqueTags[pos]['TP']+uniqueTags[pos]['FP'] == 0:
			uniqueTagsScores[pos]['Precision'] = 0.00
		else:
			uniqueTagsScores[pos]['Precision']=round((float(uniqueTags[pos]['TP']))/(float(uniqueTags[pos]['TP'])+float(uniqueTags[pos]['FP']))*100.00, 2)

		if uniqueTags[pos]['TP']+uniqueTags[pos]['FN'] == 0:
			uniqueTagsScores[pos]['Recall'] = 0.00
		else: uniqueTagsScores[pos]['Recall']=round((float(uniqueTags[pos]['TP']))/(float(uniqueTags[pos]['TP'])+float(uniqueTags[pos]['FN']))*100.00, 2)

		if uniqueTags[pos]['TP']+uniqueTags[pos]['FP']+uniqueTags[pos]['FN'] == 0:
			uniqueTagsScores[pos]['Accuracy'] = 0.00
		else: uniqueTagsScores[pos]['Accuracy']=round(float(uniqueTags[pos]['TP'])/(float(uniqueTags[pos]['TP'])+float(uniqueTags[pos]['FN'])+float(uniqueTags[pos]['FP']))*100.00, 2)

		if uniqueTagsScores[pos]['Precision']+uniqueTagsScores[pos]['Recall'] == 0.00:
			uniqueTagsScores[pos]['F-Score'] = 0.00
		else: uniqueTagsScores[pos]['F-Score'] = round((2*float(uniqueTagsScores[pos]['Precision'])*float(uniqueTagsScores[pos]['Recall']))/(float(uniqueTagsScores[pos]['Precision'])+float(uniqueTagsScores[pos]['Recall'])), 2)

	precisionSum = 0.0
	recallSum = 0.0
	fscoreSum = 0.0
	predictionCount = len(posDict)

	for pos in uniqueTagsScores.keys():
		precisionSum += uniqueTagsScores[pos]['Precision']
		recallSum += uniqueTagsScores[pos]['Recall']
		fscoreSum += uniqueTagsScores[pos]['F-Score']

	overallPrecision = precisionSum/float(len(uniqueTagsScores))
	overallRecall = recallSum/float(len(uniqueTagsScores))
	overallFscore = fscoreSum/float(len(uniqueTagsScores))

	overallAccuracy = (float(correctPredictions)/float(predictionCount))*100
	falseTags = predictionCount-correctPredictions

	print "Total Predictions:\t"+str(predictionCount)
	print "Correct Predictions:\t"+str(correctPredictions)
	print "False Predictions:\t"+str(falseTags)
	print ""
	print "Overall Accuracy:\t"+str(overallAccuracy)
	print "Overall Precision:\t"+str(overallPrecision)
	print "Overall Recall:\t"+str(overallRecall)
	print "Overall F-Score:\t"+str(overallFscore)
	print ""

	print "Tagwise Accuracy, Precision, Recall and F-Score:\n"
	for pos in uniqueTagsScores.keys():
		print pos+"\tAccuracy: "+str(uniqueTagsScores[pos]['Accuracy'])+"\tPrecision: "+str(uniqueTagsScores[pos]['Precision'])+"\tRecall: "+str(uniqueTagsScores[pos]['Recall'])+"\tF-Score: "+str(uniqueTagsScores[pos]['F-Score'])

def extractFeatures(filein):

	featvec = {}
	for sid, sentence in enumerate(sentences(codecs.open(filein,encoding='utf-8'))):
		for tid,token in enumerate(sentence):

			# pos features

			if not "current_pos_"+str(token.gold_pos) in featvec: featvec["current_pos_"+str(token.gold_pos)] = len(featvec.keys())
			if not "current_pos_"+str(token.predicted_pos) in featvec: featvec["current_pos_"+str(token.predicted_pos)] = len(featvec.keys())
			if not "prev_pos_"+str(token.gold_pos) in featvec: featvec["prev_pos_"+str(token.gold_pos)] = len(featvec.keys())
			if not "prev_pos_"+str(token.predicted_pos) in featvec: featvec["prev_pos_"+str(token.predicted_pos)] = len(featvec.keys())
			if not "next_pos_"+str(token.gold_pos) in featvec: featvec["next_pos_"+str(token.gold_pos)] = len(featvec.keys())
			if not "next_pos_"+str(token.predicted_pos) in featvec: featvec["next_pos_"+str(token.predicted_pos)] = len(featvec.keys())

			# form features

			if not "current_form_"+str(token.form) in featvec: featvec["current_form_"+str(token.form)] = len(featvec.keys())
			if not "prev_form_"+str(token.form) in featvec: featvec["prev_form_"+str(token.form)] = len(featvec.keys())
			if not "next_form_"+str(token.form) in featvec: featvec["next_form_"+str(token.form)] = len(featvec.keys())

	return featvec

if __name__=='__main__':
	t0 = time.time()
	import argparse
	argpar = argparse.ArgumentParser(description='Creates a feature representation for each word in a given file in CoNLL09 format')

	argpar.add_argument('-i','--infile',dest='infile',help='infile',required=True)
	#argpar.add_argument('-g','--gold',dest='gold',help='gold',required=True)
	#argpar.add_argument('-p','--prediction',dest='prediction',help='prediction',required=True)
	#argpar.add_argument('-o','--output',dest='outputfile',help='output file',required=True)
	args = argpar.parse_args()

	posDict = {}

	featvec = extractFeatures(args.infile)
	#outstream = open(args.outputfile,'w')
	counter=0
	for sid, sentence in enumerate(sentences(codecs.open(args.infile,encoding='utf-8'))):
		for tid,token in enumerate(sentence):
			posDict[counter] = [token.gold_pos, token.predicted_pos]
			if tid == 0:
				token.createFeatureVector(featvec, sentence[tid], None, sentence[tid+1])
			elif tid == len(sentence)-1:
				token.createFeatureVector(featvec, sentence[tid], sentence[tid-1], None)
			else:
				token.createFeatureVector(featvec, sentence[tid], sentence[tid-1], sentence[tid+1])
			#print token.sparseFeatvec
			counter+=1
	evaluate(posDict)
	t1 = time.time()
	print "\n"
	print "Time: "+ str(t1-t0) + "sec."
	#print >> outstream, " "
	#outstream.close()

<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Horn Concerto - Evaluation for inference.
Author: Tommaso Soru <tsoru@informatik.uni-leipzig.de>
Version: 0.0.1
Usage:
    Use test endpoint (DBpedia)
    > python evaluation.py <TEST_SET> <INFERRED_TRIPLES>
"""
import sys
from joblib import Parallel, delayed
import numpy as np
import multiprocessing

reload(sys)
sys.setdefaultencoding("utf-8")

VERSION = "0.0.1"

############################### ARGUMENTS ################################
num_cores = multiprocessing.cpu_count()
print "Cores: ", num_cores

TEST_SET = sys.argv[1]
INFERRED = sys.argv[2]

inferred = list()
with open(INFERRED) as f:
    for line in f:
        line = line[:-1].split('\t')
        inferred.append(line)

test = list()

def evaluate(t):
    # corrupted
    t_triple = t.split(' ')
    corr_obj = "{} {}".format(t_triple[0], t_triple[1])
    corr_sub = "{} {}".format(t_triple[1], t_triple[2])
    pos = 1
    for line in inferred:
        tpl = line[1]
        if t == tpl:
            return pos
        triple = tpl.split(' ')
        if corr_obj in tpl:
            # filter out if true
            if not line[1] in test:
                pos += 1
        elif corr_sub in tpl:
            # filter out if true
            if not tpl in test:
                pos += 1
    return None

# index test set
with open(TEST_SET) as f:
    for line in f:
        line = line[:-1].split(' ')
        test.append(line[0] + " " + line[1] + " " + line[2])

def range_test(t):
    pos = evaluate(t)
    # print "{}\n\t{}".format(t, pos)
    if pos is None:
        rr = 1.0 / len(test)
        return rr, 0, 0, 0
    rr = 1.0 / pos
    h1 = 0; h3 = 0; h10 = 0
    if pos <= 10:
        h10 = 1
        if pos <= 3:
            h3 = 1
            if pos <= 1:
                h1 = 1
    return rr, h1, h3, h10

rr, h1, h3, h10 = 0, 0, 0, 0

STEP = 500 * num_cores

for i in range(len(test)):
    if i % STEP == 0:
        start = i / STEP
        result = Parallel(n_jobs=num_cores)(delayed(range_test)(t=t) for t in test[i:i+STEP])
        rr, h1, h3, h10 = np.sum(result, axis=0) + (rr, h1, h3, h10)
        n = i + STEP
        mrr = rr / n
        hitsAt1 = float(h1) / n
        hitsAt3 = float(h3) / n
        hitsAt10 = float(h10) / n
        print "adding range {} to {}".format(i, i+STEP)
        print "|test| = {}".format(n)
        print "MRR = {}".format(mrr)
        print "Hits@1 = {}".format(hitsAt1)
        print "Hits@3 = {}".format(hitsAt3)
        print "Hits@10 = {}".format(hitsAt10)

# add remainder
result = Parallel(n_jobs=num_cores)(delayed(range_test)(t=t) for t in test[STEP*(len(test)/STEP):])
rr, h1, h3, h10 = np.sum(result, axis=0) + (rr, h1, h3, h10)
print "adding range {} to {}".format(STEP*(len(test)/STEP), "END")

mrr = rr / len(test)
hitsAt1 = 100 * float(h1) / len(test)
hitsAt3 = 100 * float(h3) / len(test)
hitsAt10 = 100 * float(h10) / len(test)
print "\nFINAL RESULTS"
print "|test| = {}".format(len(test))
print "MRR = {}".format(mrr)
print "Hits@1 = {}".format(hitsAt1)
print "Hits@3 = {}".format(hitsAt3)
print "Hits@10 = {}".format(hitsAt10)
print "{}\t{}\t{}\t{}".format(mrr, hitsAt1, hitsAt3, hitsAt10)
<commit_msg>evaluation updated to v0.0.6<commit_after>#!/usr/bin/env python
"""
Horn Concerto - Evaluation for inference.
Author: Tommaso Soru <tsoru@informatik.uni-leipzig.de>
Version: 0.0.6
Usage:
    Use test endpoint (DBpedia)
    > python evaluation.py <TEST_SET> <INFERRED_TRIPLES>
"""
import sys
from joblib import Parallel, delayed
import numpy as np
import multiprocessing

reload(sys)
sys.setdefaultencoding("utf-8")

VERSION = "0.0.6"

############################### ARGUMENTS ################################
num_cores = multiprocessing.cpu_count()
print "Cores: ", num_cores

TEST_SET = sys.argv[1]
INFERRED = sys.argv[2]

inferred = list()
with open(INFERRED) as f:
    for line in f:
        line = line[:-1].split('\t')
        inferred.append(line)

test = list()

def evaluate(t):
    # corrupted
    t_triple = t.split(' ')
    corr_obj = "{} {}".format(t_triple[0], t_triple[1])
    corr_sub = "{} {}".format(t_triple[1], t_triple[2])
    pos = 1
    for line in inferred:
        tpl = line[1]
        if t == tpl:
            return pos
        triple = tpl.split(' ')
        if corr_obj in tpl:
            # filter out if true
            if not line[1] in test:
                pos += 1
        elif corr_sub in tpl:
            # filter out if true
            if not tpl in test:
                pos += 1
    return None

# index test set
with open(TEST_SET) as f:
    for line in f:
        line = line[:-1].split(' ')
        test.append(line[0] + " " + line[1] + " " + line[2])

def range_test(t):
    pos = evaluate(t)
    # print "{}\n\t{}".format(t, pos)
    if pos is None:
        rr = 1.0 / len(test)
        return rr, 0, 0, 0
    rr = 1.0 / pos
    h1 = 0; h3 = 0; h10 = 0
    if pos <= 10:
        h10 = 1
        if pos <= 3:
            h3 = 1
            if pos <= 1:
                h1 = 1
    return rr, h1, h3, h10

rr, h1, h3, h10, n = 0, 0, 0, 0, 0
mrr, hitsAt1, hitsAt3, hitsAt10 = 0, 0, 0, 0

STEP = 500 * num_cores

for i in range(len(test)):
    if i % STEP == 0:
        start = i / STEP
        result = Parallel(n_jobs=num_cores)(delayed(range_test)(t=t) for t in test[i:i+STEP])
        print "len=",len(result)
        rr, h1, h3, h10 = np.sum(result, axis=0) + (rr, h1, h3, h10)
        n = n + len(result)
        mrr = rr / n
        hitsAt1 = float(h1) / n
        hitsAt3 = float(h3) / n
        hitsAt10 = float(h10) / n
        print "adding range {} to {}".format(i, i+STEP)
        print "|test| = {}".format(n)
        print "MRR = {}".format(mrr)
        print "Hits@1 = {}".format(hitsAt1)
        print "Hits@3 = {}".format(hitsAt3)
        print "Hits@10 = {}".format(hitsAt10)

print "\nFINAL RESULTS"
print "|test| = {}".format(len(test))
print "MRR = {}".format(mrr)
print "Hits@1 = {}".format(hitsAt1)
print "Hits@3 = {}".format(hitsAt3)
print "Hits@10 = {}".format(hitsAt10)
print "{}\t{}\t{}\t{}".format(mrr, hitsAt1, hitsAt3, hitsAt10)
<|endoftext|>


<commit_before>import json
import requests

from elasticsearch_dsl import Search
from elasticsearch_dsl.result import Response

class ShareSearch(Search):
    BASE_URL = 'http://localhost:8000/api/search/abstractcreativework/_search'
    HEADERS = {'content-type': 'application/json'}
    PARAMS = dict(raw=True)

    def execute(self):
        return Response(
            self._query(self.to_dict()),
            callbacks=self._doc_type_map
        )

    def count(self):
        d = self.to_dict()
        if d.get('aggs'):
            del d['aggs']
        self = ShareSearch.from_dict(d)
        return self._query(self.to_dict(), params=dict(size=0))['hits']['total']

    def scan(self, size=100):
        count = 0
        page = list(self[0:size].execute())
        while(page):
            for hit in page:
                count += 1
                yield hit
            page = list(self[count:count + size].execute())

    def _query(self, data, params=None):
        return requests.post(
            self.BASE_URL,
            headers=self.HEADERS,
            data=json.dumps(self.to_dict()),
            params=params or self.PARAMS
        ).json()

basic_search = ShareSearch()
basic_search.aggs.bucket(
    'sourceAgg',
    'terms',
    field='_type',
    size=0,
    min_doc_count=0
)
<commit_msg>Use new elasticsearch endpoint in sharepa<commit_after>import json
import requests

from elasticsearch_dsl import Search
from elasticsearch_dsl.result import Response

class ShareSearch(Search):
    BASE_URL = 'https://staging-share.osf.io/api/search/abstractcreativework/_search'
    HEADERS = {'content-type': 'application/json'}
    PARAMS = dict(raw=True)

    def execute(self):
        return Response(
            self._query(self.to_dict()),
            callbacks=self._doc_type_map
        )

    def count(self):
        d = self.to_dict()
        if d.get('aggs'):
            del d['aggs']
        self = ShareSearch.from_dict(d)
        return self._query(self.to_dict(), params=dict(size=0))['hits']['total']

    def scan(self, size=100):
        count = 0
        page = list(self[0:size].execute())
        while(page):
            for hit in page:
                count += 1
                yield hit
            page = list(self[count:count + size].execute())

    def _query(self, data, params=None):
        return requests.post(
            self.BASE_URL,
            headers=self.HEADERS,
            data=json.dumps(self.to_dict()),
            params=params or self.PARAMS
        ).json()

basic_search = ShareSearch()
basic_search.aggs.bucket(
    'sourceAgg',
    'terms',
    field='_type',
    size=0,
    min_doc_count=0
)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
    engine.parse
    ~~~~~~~~~~~~

    Implements code syntax parse

    :author:    Feei <wufeifei#wufeifei.com>
    :homepage:  https://github.com/wufeifei/cobra
    :license:   MIT, see LICENSE for more details.
    :copyright: Copyright (c) 2016 Feei. All rights reserved
"""
import os
import sys
import re
import subprocess
import traceback
from utils import log
import logging

log.Log()
logging = logging.getLogger(__name__)

class Parse:
    """
    Parse code syntax
    """

    def __init__(self, rule, file_path, line, code):
        logging.info('---------------------- [1]. Parse code syntax --------------------------------------')
        self.rule = rule
        self.file_path = file_path
        logging.info(file_path)
        self.line = line
        self.code = code

    def functions(self):
        logging.info('---------------------- [-]. Functions --------------------------------------')
        # parse functions
        # `grep` (`ggrep` on Mac)
        grep = '/bin/grep'
        if 'darwin' == sys.platform:
            ggrep = ''
            for root, dir_names, file_names in os.walk('/usr/local/Cellar/grep'):
                for filename in file_names:
                    if 'ggrep' == filename or 'grep' == filename:
                        ggrep = os.path.join(root, filename)
            if ggrep == '':
                logging.critical("brew install ggrep pleases!")
                sys.exit(0)
            else:
                grep = ggrep
        regex = r'(?:function\s+)(\w+)\s*\('
        param = [grep, "-n", "-r", "-P"] + [regex, self.file_path]

        p = subprocess.Popen(param, stdout=subprocess.PIPE)
        result = p.communicate()
        if len(result[0]):
            functions = {}
            # logging.debug(result[0])
            lines = str(result[0]).strip().split("\n")
            prev_function_name = ''
            for index, line in enumerate(lines):
                line = line.strip()
                if line == '':
                    logging.info('Empty')
                    continue
                function = line.split(':')
                if len(function) >= 2 and function[1].strip()[:2] not in ['//', '#', '/*']:
                    function_name = re.findall(regex, function[1].strip())
                    if len(function_name) == 1:
                        function_name = function_name[0]
                        logging.info('F: {0}.{1} - PF: {2}'.format(index, function_name, prev_function_name))
                        if index > 0 and prev_function_name in functions:
                            functions[prev_function_name]['end'] = function[0]
                        prev_function_name = function_name
                        functions[function_name] = {
                            'start': function[0],
                            'end': None  # next function's start
                        }
                    else:
                        logging.info("Can't find function name: {0}".format(line))
                else:
                    logging.info("can't found (:) or this line is comment {0}".format(function[1]))
            end = sum(1 for l in open(self.file_path))
            for name, value in functions.items():
                if value['end'] is None:
                    functions[name]['end'] = end
            return functions
        else:
            return False

    def block_code(self, block_position):
        """
        Get block code
        :param block_position:
                0:up
                1:down
                2:location_line
        :return:
        """
        logging.info('---------------------- [-]. Block code B:{0} --------------------------------------'.format(block_position))
        if block_position == 2:
            line_rule = '{0}p'.format(self.line)
            code = self.get_code(line_rule)
            logging.info("C: {0}".format(code))
            return code
        else:
            functions = self.functions()
            if functions:
                block_start = 0
                block_end = 0
                for function_name, function_value in functions.items():
                    in_this_function = ''
                    if int(function_value['start']) < int(self.line) < int(function_value['end']):
                        in_this_function = '<---- {0}'.format(self.line)
                        if block_position == 0:
                            block_start = function_value['start']
                            block_end = int(self.line) - 1
                        elif block_position == 1:
                            block_start = int(self.line) + 1
                            block_end = function_value['end']
                    logging.info("F: {0} ({1} - {2}) {3}".format(function_name, function_value['start'], function_value['end'], in_this_function))
                # get param block code
                logging.info('C: {0} - {1}p'.format(block_start, block_end))
                line_rule = "{0},{1}p".format(block_start, block_end)
                return self.get_code(line_rule)
            else:
                logging.info("Not found functions")
                return False

    def get_code(self, line_rule):
        param = ['sed', "-n", line_rule, self.file_path]
        p = subprocess.Popen(param, stdout=subprocess.PIPE)
        result = p.communicate()
        if len(result[0]):
            param_block_code = result[0]
            if param_block_code == '':
                param_block_code = False
        else:
            param_block_code = False
        return param_block_code

    def is_controllable_param(self):
        logging.info('---------------------- [2]. Param is controllable --------------------------------------')
        param_name = re.findall(self.rule, self.code)
        if len(param_name) == 1:
            param_name = param_name[0].strip()
            logging.info('P: {0}'.format(param_name))
            # controllable param
            # exclude class const (maybe misuse)
            class_const = re.findall(r'\$this->([_A-Z]*)', param_name)
            if len(class_const) >= 1 and class_const[0] != '':
                logging.info("R: False ($param = CONST || $this->CONST)")
                return False
            if param_name[:1] == '$':
                # get param block code
                param_block_code = self.block_code(0)
                if param_block_code is False:
                    logging.info("R: True (param block code can't match)")
                    return True
                logging.debug(param_block_code)
                # check = "" or = ''
                """
                check string
                $param_name = ""
                $param_name = ''
                """
                logging.info("Check un controllable param {0} = 'something'".format(param_name))
                un_controllable_param_rule = [
                    # ```$param_name = 'http://wufeifei.com'```
                    r'\{0}\s?=\s?\'((\?(?=\\\\\')..|[^\'])*)\''.format(param_name),
                    # ```$param_name = "http://wufeifei.com"```
                    r'\{0}\s?=\s?"((\?(?=\\\\")..|[^"])*)"'.format(param_name),
                    # ```$param_name = CONST_VARIABLE```
                    r'\$path\s?=\s?([A-Z_]*)'.format(param_name)
                ]
                for uc_rule in un_controllable_param_rule:
                    uc_rule_result = re.findall(uc_rule, param_block_code)
                    if len(uc_rule_result) >= 1:
                        logging.info("R: False ($param = '' : {0} = {1})".format(param_name, uc_rule_result[0]))
                        return False

                logging.info("Check controllable param rule")
                controllable_param_rule = [
                    {
                        'rule': r'\\s?=\s?(\$\w+(?:\[(?:[^[\]]|(\?R))*\])*)'.format(param_name),
                        'example': '$param_name = $variable',
                        'test': """
                            $param_name = $_GET
                            $param_name = $_POST
                            $param_name = $_REQUEST
                            $param_name = $_COOKIE
                            $param_name = $var
                            """
                    },
                    {
                        'rule': r'function\s+\w+\s?\(.*(\{0})'.format(param_name),
                        'example': 'function ($param_name)',
                        'test': """
                            function ($param_name)
                            function ($some, $param_name)
                            """
                    }

                ]
                for c_rule in controllable_param_rule:
                    c_rule_result = re.findall(c_rule['rule'], param_block_code)
                    if len(c_rule_result) >= 1:
                        logging.info("R: True (New rule: controllable param: {0}, {1})".format(param_name, c_rule['example']))
                        return True
                return True
            else:
                logging.info("R: False (Not contained $)")
                return False
        else:
            logging.warning("Not Found Param")

    def is_repair(self, repair_rule, block_repair):
        logging.info('---------------------- [3]. Is repair B:{0} --------------------------------------'.format(block_repair))
        code = self.block_code(block_repair)
        if code is False:
            logging.debug("R: Un Repair (repair code not match)")
            return False
        repair_result = re.findall(repair_rule, code)
        logging.debug(code)
        logging.debug(repair_result)
        if len(repair_result) >= 1:
            logging.debug("R: Repaired")
            return True
        else:
            logging.debug("R: Un Repair")
            return False

if __name__ == '__main__':
    try:
        parse = Parse('curl_setopt\s?\(.*,\s?CURLOPT_URL\s?,(.*)\)', '/Volumes/Statics/Project/Company/mogujie/appbeta/classes/crond/trade/chenxitest.php', '60', "curl_setopt($curl, CURLOPT_URL, $file); #output")
        if parse.is_controllable_param():
            parse.is_repair(r'fff', 2)
        else:
            print("UC")
    except Exception as e:
        print(traceback.print_exc())
<commit_msg>add {{PARAM}} for repair rule<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
    engine.parse
    ~~~~~~~~~~~~

    Implements code syntax parse

    :author:    Feei <wufeifei#wufeifei.com>
    :homepage:  https://github.com/wufeifei/cobra
    :license:   MIT, see LICENSE for more details.
    :copyright: Copyright (c) 2016 Feei. All rights reserved
"""
import os
import sys
import re
import subprocess
import traceback
from utils import log
import logging

log.Log()
logging = logging.getLogger(__name__)

class Parse:
    """
    Parse code syntax
    """

    def __init__(self, rule, file_path, line, code):
        logging.info('---------------------- [1]. Parse code syntax --------------------------------------')
        self.rule = rule
        self.file_path = file_path
        logging.info(file_path)
        self.line = line
        self.code = code
        self.param_name = None

    def functions(self):
        logging.info('---------------------- [-]. Functions --------------------------------------')
        # parse functions
        # `grep` (`ggrep` on Mac)
        grep = '/bin/grep'
        if 'darwin' == sys.platform:
            ggrep = ''
            for root, dir_names, file_names in os.walk('/usr/local/Cellar/grep'):
                for filename in file_names:
                    if 'ggrep' == filename or 'grep' == filename:
                        ggrep = os.path.join(root, filename)
            if ggrep == '':
                logging.critical("brew install ggrep pleases!")
                sys.exit(0)
            else:
                grep = ggrep
        regex = r'(?:function\s+)(\w+)\s*\('
        param = [grep, "-n", "-r", "-P"] + [regex, self.file_path]

        p = subprocess.Popen(param, stdout=subprocess.PIPE)
        result = p.communicate()
        if len(result[0]):
            functions = {}
            # logging.debug(result[0])
            lines = str(result[0]).strip().split("\n")
            prev_function_name = ''
            for index, line in enumerate(lines):
                line = line.strip()
                if line == '':
                    logging.info('Empty')
                    continue
                function = line.split(':')
                if len(function) >= 2 and function[1].strip()[:2] not in ['//', '#', '/*']:
                    function_name = re.findall(regex, function[1].strip())
                    if len(function_name) == 1:
                        function_name = function_name[0]
                        logging.info('F: {0}.{1} - PF: {2}'.format(index, function_name, prev_function_name))
                        if index > 0 and prev_function_name in functions:
                            functions[prev_function_name]['end'] = function[0]
                        prev_function_name = function_name
                        functions[function_name] = {
                            'start': function[0],
                            'end': None  # next function's start
                        }
                    else:
                        logging.info("Can't find function name: {0}".format(line))
                else:
                    logging.info("can't found (:) or this line is comment {0}".format(function[1]))
            end = sum(1 for l in open(self.file_path))
            for name, value in functions.items():
                if value['end'] is None:
                    functions[name]['end'] = end
            return functions
        else:
            return False

    def block_code(self, block_position):
        """
        Get block code
        :param block_position:
                0:up
                1:down
                2:location_line
        :return:
        """
        logging.info('---------------------- [-]. Block code B:{0} --------------------------------------'.format(block_position))
        if block_position == 2:
            line_rule = '{0}p'.format(self.line)
            code = self.get_code(line_rule)
            logging.info("C: {0}".format(code))
            return code
        else:
            functions = self.functions()
            if functions:
                block_start = 0
                block_end = 0
                for function_name, function_value in functions.items():
                    in_this_function = ''
                    if int(function_value['start']) < int(self.line) < int(function_value['end']):
                        in_this_function = '<---- {0}'.format(self.line)
                        if block_position == 0:
                            block_start = function_value['start']
                            block_end = int(self.line) - 1
                        elif block_position == 1:
                            block_start = int(self.line) + 1
                            block_end = function_value['end']
                    logging.info("F: {0} ({1} - {2}) {3}".format(function_name, function_value['start'], function_value['end'], in_this_function))
                # get param block code
                logging.info('C: {0} - {1}p'.format(block_start, block_end))
                line_rule = "{0},{1}p".format(block_start, block_end)
                return self.get_code(line_rule)
            else:
                logging.info("Not found functions")
                return False

    def get_code(self, line_rule):
        param = ['sed', "-n", line_rule, self.file_path]
        p = subprocess.Popen(param, stdout=subprocess.PIPE)
        result = p.communicate()
        if len(result[0]):
            param_block_code = result[0]
            if param_block_code == '':
                param_block_code = False
        else:
            param_block_code = False
        return param_block_code

    def is_controllable_param(self):
        logging.info('---------------------- [2]. Param is controllable --------------------------------------')
        param_name = re.findall(self.rule, self.code)
        param_name = param_name[0].strip()
        self.param_name = param_name
        if len(param_name) == 1:
            logging.info('P: {0}'.format(param_name))
            # controllable param
            # exclude class const (maybe misuse)
            class_const = re.findall(r'\$this->([_A-Z]*)', param_name)
            if len(class_const) >= 1 and class_const[0] != '':
                logging.info("R: False ($param = CONST || $this->CONST)")
                return False
            if param_name[:1] == '$':
                # get param block code
                param_block_code = self.block_code(0)
                if param_block_code is False:
                    logging.info("R: True (param block code can't match)")
                    return True
                logging.debug(param_block_code)
                # check = "" or = ''
                """
                check string
                $param_name = ""
                $param_name = ''
                """
                logging.info("Check un controllable param {0} = 'something'".format(param_name))
                un_controllable_param_rule = [
                    # ```$param_name = 'http://wufeifei.com'```
                    r'\{0}\s?=\s?\'((\?(?=\\\\\')..|[^\'])*)\''.format(param_name),
                    # ```$param_name = "http://wufeifei.com"```
                    r'\{0}\s?=\s?"((\?(?=\\\\")..|[^"])*)"'.format(param_name),
                    # ```$param_name = CONST_VARIABLE```
                    r'\$path\s?=\s?([A-Z_]*)'.format(param_name)
                ]
                for uc_rule in un_controllable_param_rule:
                    uc_rule_result = re.findall(uc_rule, param_block_code)
                    if len(uc_rule_result) >= 1:
                        logging.info("R: False ($param = '' : {0} = {1})".format(param_name, uc_rule_result[0]))
                        return False

                logging.info("Check controllable param rule")
                controllable_param_rule = [
                    {
                        'rule': r'\\s?=\s?(\$\w+(?:\[(?:[^[\]]|(\?R))*\])*)'.format(param_name),
                        'example': '$param_name = $variable',
                        'test': """
                            $param_name = $_GET
                            $param_name = $_POST
                            $param_name = $_REQUEST
                            $param_name = $_COOKIE
                            $param_name = $var
                            """
                    },
                    {
                        'rule': r'function\s+\w+\s?\(.*(\{0})'.format(param_name),
                        'example': 'function ($param_name)',
                        'test': """
                            function ($param_name)
                            function ($some, $param_name)
                            """
                    }

                ]
                for c_rule in controllable_param_rule:
                    c_rule_result = re.findall(c_rule['rule'], param_block_code)
                    if len(c_rule_result) >= 1:
                        logging.info("R: True (New rule: controllable param: {0}, {1})".format(param_name, c_rule['example']))
                        return True
                return True
            else:
                logging.info("R: False (Not contained $)")
                return False
        else:
            logging.warning("Not Found Param")

    def is_repair(self, repair_rule, block_repair):
        logging.info('---------------------- [3]. Is repair B:{0} --------------------------------------'.format(block_repair))
        code = self.block_code(block_repair)
        if code is False:
            logging.debug("R: Un Repair (repair code not match)")
            return False
        # replace repair {{PARAM}} const
        if '{{PARAM}' in repair_rule:
            repair_rule = repair_rule.replace('{{PARAM}', self.param_name)
        repair_result = re.findall(repair_rule, code)
        logging.debug(code)
        logging.debug(repair_result)
        if len(repair_result) >= 1:
            logging.debug("R: Repaired")
            return True
        else:
            logging.debug("R: Un Repair")
            return False

if __name__ == '__main__':
    try:
        parse = Parse('curl_setopt\s?\(.*,\s?CURLOPT_URL\s?,(.*)\)', '/Volumes/Statics/Project/Company/mogujie/appbeta/classes/crond/trade/chenxitest.php', '60', "curl_setopt($curl, CURLOPT_URL, $file); #output")
        if parse.is_controllable_param():
            parse.is_repair(r'fff', 2)
        else:
            print("UC")
    except Exception as e:
        print(traceback.print_exc())
<|endoftext|>