<commit_before>__all__ = [
    'LibSvmRegressionDataSet',
    'LibSvmClassificationDataSet',
    'LibSvmOneClassDataSet'
    ]

import numpy as N

import libsvm

def svm_node_dot(x, y):
    # associate node indexes with array indexes
    xidx = dict(zip(x['index'][:-1],range(0,len(x))))
    yidx = dict(zip(y['index'][:-1],range(0,len(y))))
    # indexes in either vector
    indexes = N.unique(N.hstack([x['index'],y['index']]))
    z = 0.
    for j in indexes:
        if j in xidx and j in yidx:
            # dot if index is present in both vectors
            z += x['value'][xidx[j]]*y['value'][yidx[j]]
    return z

class LibSvmDataSet:
    def __init__(self, data, kernel):
        self.data = data
        self.kernel = kernel

    def precompute(self):
        n = 1 + len(data) + 1
        a = N.zeros((n,), dtype=svm_node_dtype)
        a[-1] = -1, 0. # end of record marker
        grammat = [a.copy() for i in range(len(data))]
        for i, a in enumerate(grammat):
            a[0] = 0, i + 1 # id
        for i in range(len(data)):
            for j in range(i, len(data)):
                z = self.kernel(data[i], N.transpose(data[j]))
                grammat[i][j+1]['value'] = z
                grammat[j][i+1]['value'] = z
        return LibSvmPrecomputedDataSet(grammat, self.data, self.kernel)

class LibSvmPrecomputedDataSet:
    def __init__(self, grammat, data, kernel):
        self.grammat = grammat
        self.data = data
        self.kernel = kernel

    def extend(self, data):
        raise NotImplementedError

class LibSvmRegressionDataSet(LibSvmDataSet):
    def __init__(self, data):
        f = lambda x: (x[0], convert_to_svm_node(x[1]))
        LibSvmDataSet.__init__(self, map(f, data))

class LibSvmClassificationDataSet(LibSvmDataSet):
    def __init__(self, data):
        labels = N.array(map(lambda x: x[0]), dtype=N.intc)
        assert N.alltrue(labels >= 0), \
            'labels must be non-negative integers'
        f = lambda x: (x[0],convert_to_svm_node(x[1]))
        LibSvmDataSet.__init__(self, map(f, data))

class LibSvmOneClassDataSet(LibSvmDataSet):
    def __init__(self, data):
        f = map(lambda x: tuple([0,convert_to_svm_node(x)]))
        LibSvmDataSet.__init__(self, map(f, data))

def convert_to_svm_node(x):
    y = N.empty(len(x)+1, dtype=libsvm.svm_node_dtype)
    y[-1] = (-1, 0.)
    if isinstance(x, dict):
        x = x.items()
    if isinstance(x, list):
        x.sort(cmp=lambda x,y: cmp(x[0],y[0]))
        y[:-1] = x
    else:
        y['index'][:-1] = N.arange(1,len(x)+1)
        y['value'][:-1] = x
    assert N.alltrue(y[:-1]['index'] >= 1), \
        'indexes must be positive'
    assert len(x) == len(N.unique(y[:-1]['index'])), \
        'indexes must be unique'
    return y

def svm_node_dot(x, y):
    # associate node indexes with array indexes
    xidx = dict(zip(x['index'][:-1],range(0,len(x))))
    yidx = dict(zip(y['index'][:-1],range(0,len(y))))
    # indexes in either vector
    indexes = N.unique(N.hstack([x['index'],y['index']]))
    z = 0.
    for j in indexes:
        if j in xidx and j in yidx:
            # dot if index is present in both vectors
            z += x['value'][xidx[j]]*y['value'][yidx[j]]
    return z
<commit_msg>minor<commit_after>__all__ = [
    'LibSvmRegressionDataSet',
    'LibSvmClassificationDataSet',
    'LibSvmOneClassDataSet'
    ]

import numpy as N

import libsvm

def svm_node_dot(x, y):
    # associate node indexes with array indexes
    xidx = dict(zip(x['index'][:-1],range(0,len(x))))
    yidx = dict(zip(y['index'][:-1],range(0,len(y))))
    # indexes in either vector
    indexes = N.unique(N.hstack([x['index'],y['index']]))
    z = 0.
    for j in indexes:
        if j in xidx and j in yidx:
            # dot if index is present in both vectors
            z += x['value'][xidx[j]]*y['value'][yidx[j]]
    return z

class LibSvmDataSet:
    def __init__(self, data, kernel):
        self.data = data
        self.kernel = kernel

    def precompute(self):
        n = 1 + len(data) + 1
        a = N.zeros((n,), dtype=svm_node_dtype)
        a[-1] = -1, 0. # end of record marker
        grammat = [a.copy() for i in range(len(data))]
        for i, a in enumerate(grammat):
            a[0] = 0, i + 1 # id
        for i in range(len(data)):
            for j in range(i, len(data)):
                z = self.kernel(data[i], N.transpose(data[j]))
                grammat[i][j+1]['value'] = z
                grammat[j][i+1]['value'] = z
        return LibSvmPrecomputedDataSet(grammat, self.data, self.kernel)

class LibSvmPrecomputedDataSet:
    def __init__(self, grammat, data, kernel):
        self.grammat = grammat
        self.data = data
        self.kernel = kernel

    def extend(self, data):
        raise NotImplementedError

class LibSvmRegressionDataSet(LibSvmDataSet):
    def __init__(self, data):
        f = lambda x: (x[0], convert_to_svm_node(x[1]))
        LibSvmDataSet.__init__(self, map(f, data))

class LibSvmClassificationDataSet(LibSvmDataSet):
    def __init__(self, data):
        labels = N.array(map(lambda x: x[0]), dtype=N.intc)
        assert N.alltrue(labels >= 0), \
            'labels must be non-negative integers'
        f = lambda x: (x[0],convert_to_svm_node(x[1]))
        LibSvmDataSet.__init__(self, map(f, data))

class LibSvmOneClassDataSet(LibSvmDataSet):
    def __init__(self, data):
        f = map(lambda x: tuple([0,convert_to_svm_node(x)]))
        LibSvmDataSet.__init__(self, map(f, data))

def convert_to_svm_node(x):
    y = N.empty(len(x)+1, dtype=libsvm.svm_node_dtype)
    y[-1] = (-1, 0.)
    if isinstance(x, dict):
        x = x.items()
    if isinstance(x, list):
        x.sort(cmp=lambda x,y: cmp(x[0],y[0]))
        y[:-1] = x
    else:
        y['index'][:-1] = N.arange(1,len(x)+1)
        y['value'][:-1] = x
    assert N.alltrue(y[:-1]['index'] >= 1), \
        'indexes must be positive'
    assert len(x) == len(N.unique(y[:-1]['index'])), \
        'indexes must be unique'
    return y
<|endoftext|>


<commit_before>'''
Zen Coding's settings parser
Created on Jun 14, 2009

@author: sergey
'''
from copy import deepcopy

import re
import types
from zen_settings import zen_settings

_original_settings = deepcopy(zen_settings)

TYPE_ABBREVIATION = 'zen-tag',
TYPE_EXPANDO = 'zen-expando',
TYPE_REFERENCE = 'zen-reference';
""" Reference to another abbreviation or tag """

re_tag = r'^<([\w\-]+(?:\:[\w\-]+)?)((?:\s+[\w\-]+(?:\s*=\s*(?:(?:"[^"]*")|(?:\'[^\']*\')|[^>\s]+))?)*)\s*(\/?)>'
"Regular expression for XML tag matching"

re_attrs = r'([\w\-]+)\s*=\s*([\'"])(.*?)\2'
"Regular expression for matching XML attributes"

class Entry:
	"""
	Unified object for parsed data
	"""
	def __init__(self, entry_type, key, value):
		"""
		@type entry_type: str
		@type key: str
		@type value: dict
		"""
		self.type = entry_type
		self.key = key
		self.value = value

def _make_expando(key, value):
	"""
	Make expando from string
	@type key: str
	@type value: str
	@return: Entry
	"""
	return Entry(TYPE_EXPANDO, key, value)

def _make_abbreviation(key, tag_name, attrs, is_empty=False):
	"""
	Make abbreviation from string
	@param key: Abbreviation key
	@type key: str
	@param tag_name: Expanded element's tag name
	@type tag_name: str
	@param attrs: Expanded element's attributes
	@type attrs: str
	@param is_empty: Is expanded element empty or not
	@type is_empty: bool
	@return: dict
	"""
	result = {
		'name': tag_name,
		'is_empty': is_empty
	};

	if attrs:
		result['attributes'] = [];
		for m in re.findall(re_attrs, attrs):
			result['attributes'].append({
				'name': m[0],
				'value': m[2]
			})

	return Entry(TYPE_ABBREVIATION, key, result)

def _parse_abbreviations(obj):
	"""
	Parses all abbreviations inside dictionary
 	@param obj: dict
	"""
	for key, value in obj.items():
		key = key.strip()
		if key[-1] == '+':
#			this is expando, leave 'value' as is
			obj[key] = _make_expando(key, value)
		else:
			m = re.search(re_tag, value)
			if m:
				obj[key] = _make_abbreviation(key, m.group(1), m.group(2), (m.group(3) == '/'))
			else:
#				assume it's reference to another abbreviation
				obj[key] = Entry(TYPE_REFERENCE, key, value)

def parse(settings):
	"""
	Parse user's settings. This function must be called *before* any activity
	in zen coding (for example, expanding abbreviation)
 	@type settings: dict
	"""
	for p, value in settings.items():
		if p == 'abbreviations':
			_parse_abbreviations(value)
		elif p == 'extends':
			settings[p] = [v.strip() for v in value.split(',')]
		elif type(value) == types.DictType:
			parse(value)

def extend(parent, child):
	"""
	Recursevly extends parent dictionary with children's keys. Used for merging
	default settings with user's
	@type parent: dict
	@type child: dict
	"""
	for p, value in child.items():
		if type(value) == types.DictType:
			if p not in parent:
				parent[p] = {}
			extend(parent[p], value)
		else:
			parent[p] = value

def create_maps(obj):
	"""
	Create hash maps on certain string properties of zen settings
	@type obj: dict
	"""
	for p, value in obj.items():
		if p == 'element_types':
			for k, v in value.items():
				if isinstance(v, str):
					value[k] = [el.strip() for el in v.split(',')]
		elif type(value) == types.DictType:
			create_maps(value)

if __name__ == '__main__':
	pass

def get_settings(user_settings=None):
	"""
	Main function that gather all settings and returns parsed dictionary
	@param user_settings: A dictionary of user-defined settings
	"""
	settings = deepcopy(_original_settings)
	create_maps(settings)

	if user_settings:
		user_settings = deepcopy(user_settings)
		create_maps(user_settings)
		extend(settings, user_settings)

	# now we need to parse final set of settings
	parse(settings)

	return settings
	<commit_msg>Removed old settings parser<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright (c) 2009, Roboterclub Aachen e.V.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#     * Neither the name of the Roboterclub Aachen e.V. nor the
#       names of its contributors may be used to endorse or promote products
#       derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY ROBOTERCLUB AACHEN E.V. ''AS IS'' AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL ROBOTERCLUB AACHEN E.V. BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import os
import string
import time
import re
from SCons.Script import *

generationBlockString = """/*
 * WARNING: This file is generated automatically, do not edit!
 * Please modify the corresponding *.in file instead and rebuild this file.
 */
// ----------------------------------------------------------------------------"""

# -----------------------------------------------------------------------------
def template_action(target, source, env):
	if not env.has_key('substitutions'):
		raise SCons.Errors.UserError, "Use 'Template(..., substitutions = ...)'"

	source = source[0].abspath
	target = target[0].abspath

	output = string.Template(open(source, 'r').read()).safe_substitute(env['substitutions'])

	open(target, 'w').write(output)
	return 0

# -----------------------------------------------------------------------------
def jinja2_template_action(target, source, env):
	if not env.has_key('substitutions'):
		raise SCons.Errors.UserError, "Use 'Jinja2Template(..., substitutions = ...)'"

	try:
		import jinja2
	except ImportError:
		env.Error("To use this functionality you need to install the jinja2 template engine")
		Exit(1)

	globals = {
		'time': time.strftime("%d %b %Y, %H:%M:%S", time.localtime()),
		'match': re.match,
		'generation_block': generationBlockString,
	}

	def filter_wordwrap(value, width=79):
		return '\n\n'.join([textwrap.fill(str, width) for str in value.split('\n\n')])

	def filter_indent(value, level=0):
		return ('\n' + '\t' * level).join(value.split('\n'))

	def filter_pad(value, min_width):
		tab_width = 4
		tab_count =  (min_width/tab_width - len(value)/tab_width) + 1
		return value + ('\t' * tab_count)

	def filter_values(lst, key):
		"""
		Goes through the list of dictionaries and
		adds all the values of a certain key
		to a list which is thus returned
		"""
		values = []
		for item in lst:
			if isinstance(item, dict) and key in item:
				if item[key] not in values:
					values.append(item[key])
		return values

	path, filename = os.path.split(source[0].path)
	loader = jinja2.Environment(loader = jinja2.FileSystemLoader(path), extensions=['jinja2.ext.do'])
	loader.filters['xpcc.wordwrap'] = filter_wordwrap
	loader.filters['xpcc.indent'] = filter_indent
	loader.filters['xpcc.pad'] = filter_pad
	loader.filters['xpcc.values'] = filter_values
	if env['XPCC_JINJA2_TEST'] != None:
		loader.tests = env['XPCC_JINJA2_TEST']
	# Jinja2 Line Statements
	loader.line_statement_prefix = '%%'
	loader.line_comment_prefix = '%#'

	template = loader.get_template(filename, globals=globals)

	output = template.render(env['substitutions']).encode('utf-8')
	open(target[0].path, 'w').write(output)

def template_emitter(target, source, env):
	Depends(target, SCons.Node.Python.Value(env['substitutions']))
	return target, source

def template_string(target, source, env):
	return "Template: '%s' to '%s'" % (str(source[0]), str(target[0]))

def template_add_test(env, test_name, test_function, alias='template_add_test'):
	if 'XPCC_JINJA2_TEST' not in env:
		env['XPCC_JINJA2_TEST'] = {}
	env['XPCC_JINJA2_TEST'][test_name] = test_function

# -----------------------------------------------------------------------------
def generate(env, **kw):
	env.Append(
		BUILDERS = {
		'Template': env.Builder(
			action = env.Action(template_action, template_string),
			emitter = template_emitter,
			src_suffix = '.in',
			single_source = True
		),

		'Jinja2Template':  env.Builder(
			action = env.Action(jinja2_template_action, template_string),
			emitter = template_emitter,
			src_suffix = '.in',
			single_source = True
		),
	})
	env.AddMethod(template_add_test, 'AddTemplateJinja2Test')

def exists(env):
	return True
<commit_msg>SCons: Template: make relative includes possible<commit_after>#!/usr/bin/env python
#
# Copyright (c) 2009, Roboterclub Aachen e.V.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#     * Neither the name of the Roboterclub Aachen e.V. nor the
#       names of its contributors may be used to endorse or promote products
#       derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY ROBOTERCLUB AACHEN E.V. ''AS IS'' AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL ROBOTERCLUB AACHEN E.V. BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import os
import string
import time
import re
from SCons.Script import *

generationBlockString = """/*
 * WARNING: This file is generated automatically, do not edit!
 * Please modify the corresponding *.in file instead and rebuild this file.
 */
// ----------------------------------------------------------------------------"""

# -----------------------------------------------------------------------------
def template_action(target, source, env):
	if not env.has_key('substitutions'):
		raise SCons.Errors.UserError, "Use 'Template(..., substitutions = ...)'"

	source = source[0].abspath
	target = target[0].abspath

	output = string.Template(open(source, 'r').read()).safe_substitute(env['substitutions'])

	open(target, 'w').write(output)
	return 0

# -----------------------------------------------------------------------------
def jinja2_template_action(target, source, env):
	if not env.has_key('substitutions'):
		raise SCons.Errors.UserError, "Use 'Jinja2Template(..., substitutions = ...)'"

	try:
		import jinja2
	except ImportError:
		env.Error("To use this functionality you need to install the jinja2 template engine")
		Exit(1)

	globals = {
		'time': time.strftime("%d %b %Y, %H:%M:%S", time.localtime()),
		'match': re.match,
		'generation_block': generationBlockString,
	}

	def filter_wordwrap(value, width=79):
		return '\n\n'.join([textwrap.fill(str, width) for str in value.split('\n\n')])

	def filter_indent(value, level=0):
		return ('\n' + '\t' * level).join(value.split('\n'))

	def filter_pad(value, min_width):
		tab_width = 4
		tab_count =  (min_width/tab_width - len(value)/tab_width) + 1
		return value + ('\t' * tab_count)

	def filter_values(lst, key):
		"""
		Goes through the list of dictionaries and
		adds all the values of a certain key
		to a list which is thus returned
		"""
		values = []
		for item in lst:
			if isinstance(item, dict) and key in item:
				if item[key] not in values:
					values.append(item[key])
		return values

# Overwrite jinja2 Environment in order to enable relative paths
# since this runs locally that should not be a security concern
# Code from:
# http://stackoverflow.com/questions/8512677/how-to-include-a-template-with-relative-path-in-jinja2
	class RelEnvironment(jinja2.Environment):
		"""Override join_path() to enable relative template paths."""
		def join_path(self, template, parent):
			d = os.path.join(os.path.dirname(parent), template)
			return os.path.normpath(d)

	# path, filename = os.path.split(source[0].path)
	path = env['XPCC_LIBRARY_PATH']
	filename = os.path.relpath(source[0].path, path)
	loader = RelEnvironment(loader = jinja2.FileSystemLoader(path), extensions=['jinja2.ext.do'])
	loader.filters['xpcc.wordwrap'] = filter_wordwrap
	loader.filters['xpcc.indent'] = filter_indent
	loader.filters['xpcc.pad'] = filter_pad
	loader.filters['xpcc.values'] = filter_values
	if env['XPCC_JINJA2_TEST'] != None:
		loader.tests = env['XPCC_JINJA2_TEST']
	# Jinja2 Line Statements
	loader.line_statement_prefix = '%%'
	loader.line_comment_prefix = '%#'

	template = loader.get_template(filename, globals=globals)

	output = template.render(env['substitutions']).encode('utf-8')
	open(target[0].path, 'w').write(output)

def template_emitter(target, source, env):
	Depends(target, SCons.Node.Python.Value(env['substitutions']))
	return target, source

def template_string(target, source, env):
	return "Template: '%s' to '%s'" % (str(source[0]), str(target[0]))

def template_add_test(env, test_name, test_function, alias='template_add_test'):
	if 'XPCC_JINJA2_TEST' not in env:
		env['XPCC_JINJA2_TEST'] = {}
	env['XPCC_JINJA2_TEST'][test_name] = test_function

# -----------------------------------------------------------------------------
def generate(env, **kw):
	env.Append(
		BUILDERS = {
		'Template': env.Builder(
			action = env.Action(template_action, template_string),
			emitter = template_emitter,
			src_suffix = '.in',
			single_source = True
		),

		'Jinja2Template':  env.Builder(
			action = env.Action(jinja2_template_action, template_string),
			emitter = template_emitter,
			src_suffix = '.in',
			single_source = True
		),
	})
	env.AddMethod(template_add_test, 'AddTemplateJinja2Test')

def exists(env):
	return True
<|endoftext|>


<commit_before>#!/usr/bin/python
#-*- coding:utf-8 -*-

import sys, os
from subprocess import Popen, PIPE
class Process(object):
    '''memcached rc script'''
    args = {
    'USER': 'memcached',
    'PORT':  11211,
    'MAXCONN': 1024,
    'CACHESIZE': 64,
    'OPTIONS': ''
    }
    def __init__(self, name, program, workdir):
        self.name = name
        self.program = program
        self.workdir = workdir

    # Normal Function
    def _init(self):
        '''/var/tmp/memcached'''
        if not os.path.exists(self.workdir):
        	os.mkdir(self.workdir)
        	os.chdir(self.workdir)

    def _pidFile(self):
    	'''/var/tmp/memcached.pid'''
    	return os.path.join(self.workdir, "%s.pid" % self.name)

    def _writePid(self):
    	if self.pid:
            with open(self._pidFile(), 'w') as fd:
                fd.write(str(self.pid))

    def _readConf(self):
        with open(f) as fd:
            lines = fd.readlines()
            return dict([i.strip().replace('"','').split('=') for i in lines])
    def _parseArgs(self):
        conf = self._readConf('/etc/sysconfig/memcached')
        if 'USER' in conf:
            self.args['USER'] = conf['USER']
        if 'PORT' in conf:
            self.args['PORT'] = conf['PORT']
        if 'MAXCONN' in conf:
            self.args['MAXCONN'] = conf['MAXCONN']
        if 'CACHESIZE' in conf:
            self.args['CACHESIZE'] = conf['CACHESIZE']
        options = [
        '-u', self.args['USER'],
        '-p', self.args['PORT'],
        '-m', self.args['MAXCONN'],
        '-c', self.args['CACHESIZE'],
        ]
        os.system("chown %s %s" % (self.args['USER'], self.workdir))
        return options
    def start(self):
        pid = self._getPid()
        if pid:
            print "%s is running..." % self.name
            sys.exit()
        self._init()
        cmd = [self.program] + self._parseArgs() + ['-d', '-P', self._pidFile()]
        print cmd
        p = Popen(cmd, stdout = PIPE)
        # self.pid = p.pid
        # self._writePid()
        print "%s start Sucessful" % self.name

    def _getPid(self):
        p = Popen(['pidof', self.name], stdout = PIPE)
        pid = p.stdout.read().strip()
        return pid

    def stop(self):
        pid = self._getPid()
        if pid:
            os.kill(int(pid), 15)
            if os.path.exists(self._pidFile()):
                os.remove(self._pidFile())
            print "%s is stopped " % self.name
        pass
    def restart(self):
        self.stop()
        self.start()
    def status(self):
        pid = self._getPid()
        if pid:
            print "%s is already running" % self.name
        else:
            print "%s is not running" % self.name
    def help(self):
        pass

def main():
    name = 'memcached'
    proc = '/usr/bin/memcached'
    wd = '/var/tmp/memcached'
    pm = Process(name = name, program = proc, workdir = wd)
    try:
    	cmd = sys.argv[1]
    except IndexError, e:
    	print "Option Error"
    	sys.exit()
    if cmd == 'start':
    	pm.start()
    elif cmd == 'stop':
    	pm.stop()
    elif cmd == 'restart':
    	pm.restart()
    elif cmd == 'status':
    	pm.status()
    else:
    	pm.help()

if __name__ == '__main__':
    main()
<commit_msg>fix bug<commit_after>#!/usr/bin/python
#-*- coding:utf-8 -*-

import sys, os
from subprocess import Popen, PIPE
class Process(object):
    '''memcached rc script'''
    args = {
    'USER': 'memcached',
    'PORT':  11211,
    'MAXCONN': 1024,
    'CACHESIZE': 64,
    'OPTIONS': ''
    }
    def __init__(self, name, program, workdir):
        self.name = name
        self.program = program
        self.workdir = workdir

    # Normal Function
    def _init(self):
        '''/var/tmp/memcached'''
        if not os.path.exists(self.workdir):
        	os.mkdir(self.workdir)
        	os.chdir(self.workdir)

    def _pidFile(self):
    	'''/var/tmp/memcached.pid'''
    	return os.path.join(self.workdir, "%s.pid" % self.name)

    def _writePid(self):
    	if self.pid:
            with open(self._pidFile(), 'w') as fd:
                fd.write(str(self.pid))

    def _readConf(self, f):
        with open(f) as fd:
            lines = fd.readlines()
            return dict([i.strip().replace('"','').split('=') for i in lines])
    def _parseArgs(self):
        conf = self._readConf('/etc/sysconfig/memcached')
        if 'USER' in conf:
            self.args['USER'] = conf['USER']
        if 'PORT' in conf:
            self.args['PORT'] = conf['PORT']
        if 'MAXCONN' in conf:
            self.args['MAXCONN'] = conf['MAXCONN']
        if 'CACHESIZE' in conf:
            self.args['CACHESIZE'] = conf['CACHESIZE']
        options = [
        '-u', self.args['USER'],
        '-p', self.args['PORT'],
        '-m', self.args['MAXCONN'],
        '-c', self.args['CACHESIZE'],
        ]
        os.system("chown %s %s" % (self.args['USER'], self.workdir))
        return options
    def start(self):
        pid = self._getPid()
        if pid:
            print "%s is running..." % self.name
            sys.exit()
        self._init()
        cmd = [self.program] + self._parseArgs() + ['-d', '-P', self._pidFile()]
        print cmd
        p = Popen(cmd, stdout = PIPE)
        # self.pid = p.pid
        # self._writePid()
        print "%s start Sucessful" % self.name

    def _getPid(self):
        p = Popen(['pidof', self.name], stdout = PIPE)
        pid = p.stdout.read().strip()
        return pid

    def stop(self):
        pid = self._getPid()
        if pid:
            os.kill(int(pid), 15)
            if os.path.exists(self._pidFile()):
                os.remove(self._pidFile())
            print "%s is stopped " % self.name
        pass
    def restart(self):
        self.stop()
        self.start()
    def status(self):
        pid = self._getPid()
        if pid:
            print "%s is already running" % self.name
        else:
            print "%s is not running" % self.name
    def help(self):
        pass

def main():
    name = 'memcached'
    proc = '/usr/bin/memcached'
    wd = '/var/tmp/memcached'
    pm = Process(name = name, program = proc, workdir = wd)
    try:
    	cmd = sys.argv[1]
    except IndexError, e:
    	print "Option Error"
    	sys.exit()
    if cmd == 'start':
    	pm.start()
    elif cmd == 'stop':
    	pm.stop()
    elif cmd == 'restart':
    	pm.restart()
    elif cmd == 'status':
    	pm.status()
    else:
    	pm.help()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from os import uname

class ComposableFunction(object):
    def __init__(self, function):
        self.func = function

    def __call__(self, *args, **kwargs):
        return self.func(*args, **kwargs)

    def __and__(self, f):
        return ComposableFunction(lambda *args, **kwargs: self(*args, **kwargs) and f(*args, **kwargs))

    def __or__(self, f):
        return ComposableFunction(lambda *args, **kwargs: self(*args, **kwargs) or f(*args, **kwargs))

def check_all(*callables):
    def check(**kwargs):
        return all(c(**kwargs) for c in callables)
    return ComposableFunction(check)

def check_any(*callables):
    def check(**kwargs):
        return any(c(**kwargs) for c in callables)
    return ComposableFunction(check)

def is_platform(platform):
    def is_x(**kwargs):
        return uname()[0] == platform
    return ComposableFunction(is_x)

is_linux = is_platform('Linux')
is_darwin = is_platform('Darwin')

def is_arch(xarch):
    def is_x(arch, **kwargs):
        return arch.arch == xarch
    return ComposableFunction(is_x)

def is_api_gt(apiver):
    def is_x(recipe, **kwargs):
        return recipe.ctx.android_api > apiver
    return ComposableFunction(is_x)

def is_api_gte(apiver):
    def is_x(recipe, **kwargs):
        return recipe.ctx.android_api >= apiver
    return ComposableFunction(is_x)

def is_api_lt(apiver):
    def is_x(recipe, **kwargs):
        return recipe.ctx.android_api < apiver
    return ComposableFunction(is_x)

def is_api_lte(apiver):
    def is_x(recipe, **kwargs):
        return recipe.ctx.android_api <= apiver
    return ComposableFunction(is_x)

def is_api(apiver):
    def is_x(recipe, **kwargs):
        return recipe.ctx.android_api == apiver
    return ComposableFunction(is_x)

def will_build(recipe_name):
    def will(recipe, **kwargs):
        return recipe_name in recipe.ctx.recipe_build_order
    return ComposableFunction(will)

def is_ndk(ndk):
    def is_x(recipe, **kwargs):
        return recipe.ctx.ndk == ndk
    return is_x

<commit_msg>Removed ComposableFunction<commit_after>from os import uname

def check_all(*callables):
    def check(**kwargs):
        return all(c(**kwargs) for c in callables)
    return check

def check_any(*callables):
    def check(**kwargs):
        return any(c(**kwargs) for c in callables)
    return check

def is_platform(platform):
    def is_x(**kwargs):
        return uname()[0] == platform
    return is_x

is_linux = is_platform('Linux')
is_darwin = is_platform('Darwin')

def is_arch(xarch):
    def is_x(arch, **kwargs):
        return arch.arch == xarch
    return is_x

def is_api_gt(apiver):
    def is_x(recipe, **kwargs):
        return recipe.ctx.android_api > apiver
    return is_x

def is_api_gte(apiver):
    def is_x(recipe, **kwargs):
        return recipe.ctx.android_api >= apiver
    return is_x

def is_api_lt(apiver):
    def is_x(recipe, **kwargs):
        return recipe.ctx.android_api < apiver
    return is_x

def is_api_lte(apiver):
    def is_x(recipe, **kwargs):
        return recipe.ctx.android_api <= apiver
    return is_x

def is_api(apiver):
    def is_x(recipe, **kwargs):
        return recipe.ctx.android_api == apiver
    return is_x

def will_build(recipe_name):
    def will(recipe, **kwargs):
        return recipe_name in recipe.ctx.recipe_build_order
    return will

def is_ndk(ndk):
    def is_x(recipe, **kwargs):
        return recipe.ctx.ndk == ndk
    return is_x

<|endoftext|>


<commit_before>import random
import json
from django.contrib.auth.decorators import login_required
from django.shortcuts import render, redirect
from django.http import HttpResponse, HttpResponseRedirect
from django.core import serializers
from django.contrib import messages
from django.db.models import Q

from pytutor.views import home

from tutor.templatetags.tutor_extras import syn
from tutor.models import *

@login_required
def study(request, try_again_id=0, study_tag=None):
    """Randomly choose the next question for the user to study.
       If no questions exist, prompt the user to create one."""

    questions = Question.objects.all()
    if not questions:
        messages.add_message(request, messages.INFO, 'There are currently no questions to study.')
        return home(request)

    if try_again_id:
        question = Question.objects.get(pk=try_again_id)
    elif study_tag is not None:
        questions = Question.objects.filter(tags__icontains=study_tag)
        question = random.choice(questions)
    else:
        question = serve_question(request.user)

    response_form = ResponseForm()
    try:
        response = Response.objects.get(user=request.user, question=question)
        attempt = response.attempt + 1
    except:
        attempt = 1

    context = {
        "question": question,
        "response_form" : response_form,
        "questions" : True,
        "attempt" : attempt,
        "tag" : study_tag
    }

    return render(request, 'tutor/respond.html', context)

@login_required
def respond(request):
    """Allow user to write a response to a question."""

    pk = int(request.POST["qpk"])
    user_code = request.POST.get('user_code', False)
    study_tag = request.POST.get('study_tag', False)
    question = Question.objects.get(pk=pk)

    try:
        attempts = Response.objects.all().filter(user=request.user, question=question)
    except:
        attempts = []

    response = Response(attempt=len(attempts) + 1, user=request.user, question=question)
    response.code = user_code

    #evaluate user's code
    response.is_correct = True
    testResults = []
    passed_tests = True
    for t in Test.objects.all().filter(question=pk):
        (test, ex, result) = t.evaluate(user_code)
        testResults.append((test, ex, result))
        if ex is not None:
            passed_tests = False

    context = {"question" : question,
               "response" : response,
               "user_code": syn(user_code),
               "previous_attempt" : response.attempt - 1,
               "tests" : testResults,
               "passed_tests" : passed_tests,
               "study_tag": study_tag }

    response.save()

    return render(request, 'tutor/response_result.html', context)

def tags(request):
    """List the tags in the database"""

    questions = Question.objects.all()
    tags = []
    for question in questions:
        qtags = question.tags.split(",")
        qtags = [q.strip() for q in qtags]
        tags.extend(qtags)

    tags = set(tags)
    tags = sorted([t for t in tags])
    context = {"tags": tags}

    return render(request, 'tutor/tags.html', context)

def serve_question(user):
    """Serves a user the next applicable question."""
    #get history of user's correct and incorrect responses
    # correct_responses = Response.objects.all().filter(user=user, is_correct=True)
    # incorrect_responses = Response.objects.all().filter(user=user, is_correct=False)

    questions = Question.objects.all().filter()
    next_q = random.choice(questions)
    return next_q
<commit_msg>changed message when there are no questions<commit_after>import random
import json
from django.contrib.auth.decorators import login_required
from django.shortcuts import render, redirect
from django.http import HttpResponse, HttpResponseRedirect
from django.core import serializers
from django.contrib import messages
from django.db.models import Q

from pytutor.views import home

from tutor.templatetags.tutor_extras import syn
from tutor.models import *

@login_required
def study(request, try_again_id=0, study_tag=None):
    """Randomly choose the next question for the user to study.
       If no questions exist, prompt the user to create one."""

    questions = Question.objects.all()
    if not questions:
        messages.info(request, 'There are currently no questions to study.')
        return home(request)

    if try_again_id:
        question = Question.objects.get(pk=try_again_id)
    elif study_tag is not None:
        questions = Question.objects.filter(tags__icontains=study_tag)
        question = random.choice(questions)
    else:
        question = serve_question(request.user)

    response_form = ResponseForm()
    try:
        response = Response.objects.get(user=request.user, question=question)
        attempt = response.attempt + 1
    except:
        attempt = 1

    context = {
        "question": question,
        "response_form" : response_form,
        "questions" : True,
        "attempt" : attempt,
        "tag" : study_tag
    }

    return render(request, 'tutor/respond.html', context)

@login_required
def respond(request):
    """Allow user to write a response to a question."""

    pk = int(request.POST["qpk"])
    user_code = request.POST.get('user_code', False)
    study_tag = request.POST.get('study_tag', False)
    question = Question.objects.get(pk=pk)

    try:
        attempts = Response.objects.all().filter(user=request.user, question=question)
    except:
        attempts = []

    response = Response(attempt=len(attempts) + 1, user=request.user, question=question)
    response.code = user_code

    #evaluate user's code
    response.is_correct = True
    testResults = []
    passed_tests = True
    for t in Test.objects.all().filter(question=pk):
        (test, ex, result) = t.evaluate(user_code)
        testResults.append((test, ex, result))
        if ex is not None:
            passed_tests = False

    context = {"question" : question,
               "response" : response,
               "user_code": syn(user_code),
               "previous_attempt" : response.attempt - 1,
               "tests" : testResults,
               "passed_tests" : passed_tests,
               "study_tag": study_tag }

    response.save()

    return render(request, 'tutor/response_result.html', context)

def tags(request):
    """List the tags in the database"""

    questions = Question.objects.all()
    tags = []
    for question in questions:
        qtags = question.tags.split(",")
        qtags = [q.strip() for q in qtags]
        tags.extend(qtags)

    tags = set(tags)
    tags = sorted([t for t in tags])
    context = {"tags": tags}

    return render(request, 'tutor/tags.html', context)

def serve_question(user):
    """Serves a user the next applicable question."""
    #get history of user's correct and incorrect responses
    # correct_responses = Response.objects.all().filter(user=user, is_correct=True)
    # incorrect_responses = Response.objects.all().filter(user=user, is_correct=False)

    questions = Question.objects.all().filter()
    next_q = random.choice(questions)
    return next_q
<|endoftext|>


<commit_before>"""
Harvester for the CaltechAUTHORS repository for the SHARE project

"""

from __future__ import unicode_literals

from scrapi.base import OAIHarvester

class CaltechHarvester(OAIHarvester):
    short_name = 'caltech'
    long_name = 'CaltechAUTHORS'
    url = 'http://authors.library.caltech.edu//'

    base_url = 'http://authors.library.caltech.edu/cgi/oai2'
    property_list = [
        'date', 'resource type', 'format', 'resource identifier', 'relation'
    ]
<commit_msg>Typo fix<commit_after>"""
Harvester for the CaltechAUTHORS repository for the SHARE project

"""

from __future__ import unicode_literals

from scrapi.base import OAIHarvester

class CaltechHarvester(OAIHarvester):
    short_name = 'caltech'
    long_name = 'CaltechAUTHORS'
    url = 'http://authors.library.caltech.edu/'

    base_url = 'http://authors.library.caltech.edu/cgi/oai2'
    property_list = [
        'date', 'resource type', 'format', 'resource identifier', 'relation'
    ]
<|endoftext|>


<commit_before>import logging
from pyuia import PageObject, cacheable
from selenium.common.exceptions import NoSuchElementException, StaleElementReferenceException

__all__ = ['SeleniumPageObject', 'find_by', 'cacheable']
_logger = logging.getLogger(__name__)

class SeleniumPageObject(PageObject):

    def __init__(self, context):
        PageObject.__init__(
            self, context,
            (NoSuchElementException, StaleElementReferenceException))

    @property
    def _driver(self):
        return self._context.driver

    def _is_displayed(self, element):
        try:
            return element.is_displayed()
        except StaleElementReferenceException:
            return False

_strategy_kwargs = ['id_', 'xpath', 'link_text', 'partial_link_text',
                    'name', 'tag_name', 'class_name', 'css_selector']

from pyuia import cacheable as cacheable_decorator # naming conflict between global and parameter names

def find_by(how=None, using=None, multiple=False, cacheable=True, if_exists=False,
            context=None, driver_attr='_driver', max_swipe_times=5, scrollable=None, **kwargs):

    def func(self):

        def swipe(scrollable_view, up=True):
            # Get start point
            horizontal_midpoint_of_scrollable_view = scrollable_view.location.get('x') + scrollable_view.size.get('width')*0.5
            vertical_base_of_scrollable_view = scrollable_view.location.get('y')
            vertical_addition_base = scrollable_view.size.get('height')

            if up:
                # Get start point
                start_point_y = vertical_base_of_scrollable_view + vertical_addition_base*0.9
                start = {'x': horizontal_midpoint_of_scrollable_view, 'y': start_point_y}

                # Get end point
                end_point_y = vertical_base_of_scrollable_view + vertical_addition_base*0.1
                end = {'x': horizontal_midpoint_of_scrollable_view, 'y': end_point_y}

            else:
                # Get start point
                start_point_y = vertical_base_of_scrollable_view + vertical_addition_base*0.1
                start = {'x': horizontal_midpoint_of_scrollable_view, 'y': start_point_y}

                # Get end point
                end_point_y = vertical_base_of_scrollable_view + vertical_addition_base*0.9
                end = {'x': horizontal_midpoint_of_scrollable_view, 'y': end_point_y}

            ctx.swipe(start.get('x'), start.get('y'), end.get('x'), end.get('y'), 1000)

        # context - driver or a certain element
        if callable(context):
            ctx = context(self)
            if not ctx:
                if if_exists:
                    return None
                else:
                    raise NoSuchElementException("The element as the context doesn't exist.")
        else:
            ctx = getattr(self, driver_attr)

        # 'how' AND 'using' take precedence over keyword arguments
        if how and using:
            lookup = ctx.find_elements if multiple else ctx.find_element
            return lookup(how, using)

        if len(kwargs) != 1 or kwargs.keys()[0] not in _strategy_kwargs :
            raise ValueError(
                "If 'how' AND 'using' are not specified, one and only one of the following "
                "valid keyword arguments should be provided: %s." % _strategy_kwargs)

        key = kwargs.keys()[0]; value = kwargs[key]
        suffix = key[:-1] if key.endswith('_') else key # find_element(s)_by_xxx
        prefix = 'find_elements_by' if multiple else 'find_element_by'
        lookup = getattr(ctx, '%s_%s' % (prefix, suffix))

        scrollable_view = scrollable(self) if callable(scrollable) else scrollable

        current_time = 0
        while True:
            try:
                return lookup(value)

            except NoSuchElementException:
                if if_exists: return None
                if current_time ==0:
                    pass
                    # swipe_down(max_swipe_times)
                else:
                    if current_time > max_swipe_times: raise
                    if scrollable_view:
                        swipe(scrollable_view, up=True)

                current_time+=1

    return cacheable_decorator(func, cache_none=not if_exists) if cacheable else func<commit_msg>Support finding element, scrolling up and down<commit_after>import logging
from pyuia import PageObject, cacheable
from selenium.common.exceptions import NoSuchElementException, StaleElementReferenceException

__all__ = ['SeleniumPageObject', 'find_by', 'cacheable']
_logger = logging.getLogger(__name__)

class SeleniumPageObject(PageObject):

    def __init__(self, context):
        PageObject.__init__(
            self, context,
            (NoSuchElementException, StaleElementReferenceException))

    @property
    def _driver(self):
        return self._context.driver

    def _is_displayed(self, element):
        try:
            return element.is_displayed()
        except StaleElementReferenceException:
            return False

_strategy_kwargs = ['id_', 'xpath', 'link_text', 'partial_link_text',
                    'name', 'tag_name', 'class_name', 'css_selector']

from pyuia import cacheable as cacheable_decorator # naming conflict between global and parameter names

def find_by(how=None, using=None, multiple=False, cacheable=True, if_exists=False,
            context=None, driver_attr='_driver', max_swipe_times=5, scrollable=None, **kwargs):

    def func(self):

        def swipe(scrollable_view, up=True, swipe_times=1):
            # Get start point
            horizontal_midpoint_of_scrollable_view = scrollable_view.location.get('x') + scrollable_view.size.get('width')*0.5
            vertical_base_of_scrollable_view = scrollable_view.location.get('y')
            vertical_addition_base = scrollable_view.size.get('height')

            if up:
                # Get start point
                start_point_y = vertical_base_of_scrollable_view + vertical_addition_base*0.9
                start = {'x': horizontal_midpoint_of_scrollable_view, 'y': start_point_y}

                # Get end point
                end_point_y = vertical_base_of_scrollable_view + vertical_addition_base*0.1
                end = {'x': horizontal_midpoint_of_scrollable_view, 'y': end_point_y}

            else:
                # Get start point
                start_point_y = vertical_base_of_scrollable_view + vertical_addition_base*0.1
                start = {'x': horizontal_midpoint_of_scrollable_view, 'y': start_point_y}

                # Get end point
                end_point_y = vertical_base_of_scrollable_view + vertical_addition_base*0.9
                end = {'x': horizontal_midpoint_of_scrollable_view, 'y': end_point_y}

            for x in range(swipe_times):
                ctx.swipe(start.get('x'), start.get('y'), end.get('x'), end.get('y'), 1000)

        # context - driver or a certain element
        if callable(context):
            ctx = context(self)
            if not ctx:
                if if_exists:
                    return None
                else:
                    raise NoSuchElementException("The element as the context doesn't exist.")
        else:
            ctx = getattr(self, driver_attr)

        # 'how' AND 'using' take precedence over keyword arguments
        if how and using:
            lookup = ctx.find_elements if multiple else ctx.find_element
            return lookup(how, using)

        if len(kwargs) != 1 or kwargs.keys()[0] not in _strategy_kwargs :
            raise ValueError(
                "If 'how' AND 'using' are not specified, one and only one of the following "
                "valid keyword arguments should be provided: %s." % _strategy_kwargs)

        key = kwargs.keys()[0]; value = kwargs[key]
        suffix = key[:-1] if key.endswith('_') else key # find_element(s)_by_xxx
        prefix = 'find_elements_by' if multiple else 'find_element_by'
        lookup = getattr(ctx, '%s_%s' % (prefix, suffix))

        scrollable_view = scrollable(self) if callable(scrollable) else scrollable

        current_time = 0
        while True:
            try:
                return lookup(value)

            except NoSuchElementException:
                if if_exists: return None
                if current_time==0 and scrollable_view:
                    swipe(scrollable_view, up=False, swipe_times=max_swipe_times)
                else:
                    if current_time > max_swipe_times: raise
                    if scrollable_view:
                        swipe(scrollable_view, up=True)

                current_time+=1

    return cacheable_decorator(func, cache_none=not if_exists) if cacheable else func<|endoftext|>


<commit_before>import sys
import os
from subprocess import call
#This step constructs the reference index to which the reads are eventually aligned. The input for this script are the core genomes while the output is a Bowtie index called core_genome_index.
cmd ="bowtie2-build core_genomes.fasta core_genome_index"
call(cmd,shell=True)
<commit_msg>Update BIB_prepare_index.py<commit_after>import sys
import os
from subprocess import call
#This step constructs the reference index to which the reads are eventually aligned.
#The input for this script are the core genomes while the output is a Bowtie2 reference index.
#The reference index produced is of the same name as the input core genomes.
argu = sys.argv[1]
fname = argu.split(".")
core_genomes = fname[0]
cmd ="bowtie2-build "+core_genomes+".fasta "+core_genomes
call(cmd,shell=True)
<|endoftext|>


<commit_before># -*- coding: utf-8  -*-
"""
Objects representing Mediawiki log entries
"""
#
# (C) Pywikipedia bot team, 2007-08
#
# Distributed under the terms of the MIT license.
#
__version__ = '$Id: $'

from api import APIError

class LogEntry(object):
    """Generic log entry"""
    def __init__(self, apidata):
        """Initialize object from a logevent dict returned by MW API"""
        raise NotImplementedError(self.__class__)

class BlockEntry(LogEntry):
    pass

class ProtectEntry(LogEntry):
    pass

class RightsEntry(LogEntry):
    pass

class DeleteEntry(LogEntry):
    pass

class UploadEntry(LogEntry):
    pass

class MoveEntry(LogEntry):
    pass

class ImportEntry(LogEntry):
    pass

class PatrolEntry(LogEntry):
    pass

class NewUsersEntry(LogEntry):
    pass

#TODO entries for merge,suppress,makebot,gblblock,renameuser,globalauth,gblrights ?

_logtypes = {
    'block':BlockEntry,
    'protect':ProtectEntry,
    'rights':RightsEntry,
    'delete':DeleteEntry,
    'upload':UploadEntry,
    'move':MoveEntry,
    'import':ImportEntry,
    'patrol':PatrolEntry,
    'newusers':NewUsersEntry
}

def _getEntryClass(logtype):
    """
    Returns the class corresponding to the @logtype string parameter.
    Returns LogEntry if logtype is unknown or not supported
    """
    try:
        return _logtypes[logtype]
    except KeyError:
        return LogEntry

class LogEntryFactory(object):
    """
    LogEntry Factory

    Only available method is create()
    """
    def __init__(self, logtype=None):
        """
        @param logtype: The log type of the log entries, if known in advance.
                        If None, the Factory will fetch the log entry from
                        the data to create each object.
        @type logtype: (letype) str : move/block/patrol/etc...
        """
        if logtype is None:
            self._creator = self._createFromData
        else:
            # Bind a Class object to self._creator:
            # When called, it will initialize a new object of that class
            self._creator = _getEntryClass(logtype)

    def create(self, logdata):
        """
        Instantiates the LogEntry object representing logdata
        @param logdata: <item> returned by the api
        @type logdata: dict

        @return LogEntry object representing logdata
        """
        return self._creator(logdata)

    def _createFromData(self, logdata):
        """
        Checks for logtype from data, and creates the correct LogEntry
        """
        try:
            logtype = logdata['type']
            return _getEntryClass(logtype)(logdata)
        except KeyError:
            raise APIError("Log entry has no 'type' key")
<commit_msg>APIError is not appropriate here '-_-<commit_after># -*- coding: utf-8  -*-
"""
Objects representing Mediawiki log entries
"""
#
# (C) Pywikipedia bot team, 2007-08
#
# Distributed under the terms of the MIT license.
#
__version__ = '$Id: $'

from pywikibot.exceptions import *

class LogEntry(object):
    """Generic log entry"""
    def __init__(self, apidata):
        """Initialize object from a logevent dict returned by MW API"""
        raise NotImplementedError(self.__class__)

class BlockEntry(LogEntry):
    pass

class ProtectEntry(LogEntry):
    pass

class RightsEntry(LogEntry):
    pass

class DeleteEntry(LogEntry):
    pass

class UploadEntry(LogEntry):
    pass

class MoveEntry(LogEntry):
    pass

class ImportEntry(LogEntry):
    pass

class PatrolEntry(LogEntry):
    pass

class NewUsersEntry(LogEntry):
    pass

#TODO entries for merge,suppress,makebot,gblblock,renameuser,globalauth,gblrights ?

_logtypes = {
    'block':BlockEntry,
    'protect':ProtectEntry,
    'rights':RightsEntry,
    'delete':DeleteEntry,
    'upload':UploadEntry,
    'move':MoveEntry,
    'import':ImportEntry,
    'patrol':PatrolEntry,
    'newusers':NewUsersEntry
}

def _getEntryClass(logtype):
    """
    Returns the class corresponding to the @logtype string parameter.
    Returns LogEntry if logtype is unknown or not supported
    """
    try:
        return _logtypes[logtype]
    except KeyError:
        return LogEntry

class LogEntryFactory(object):
    """
    LogEntry Factory

    Only available method is create()
    """
    def __init__(self, logtype=None):
        """
        @param logtype: The log type of the log entries, if known in advance.
                        If None, the Factory will fetch the log entry from
                        the data to create each object.
        @type logtype: (letype) str : move/block/patrol/etc...
        """
        if logtype is None:
            self._creator = self._createFromData
        else:
            # Bind a Class object to self._creator:
            # When called, it will initialize a new object of that class
            self._creator = _getEntryClass(logtype)

    def create(self, logdata):
        """
        Instantiates the LogEntry object representing logdata
        @param logdata: <item> returned by the api
        @type logdata: dict

        @return LogEntry object representing logdata
        """
        return self._creator(logdata)

    def _createFromData(self, logdata):
        """
        Checks for logtype from data, and creates the correct LogEntry
        """
        try:
            logtype = logdata['type']
            return _getEntryClass(logtype)(logdata)
        except KeyError:
            raise Error("Log entry has no 'type' key")
<|endoftext|>


<commit_before>import socket

# Imports for http event forwarder
import requests
import json
import time

import logging

_max_content_bytes = 100000
http_event_collector_SSL_verify = False
http_event_collector_debug = True

log = logging.getLogger(__name__)

hec = None

def returner(ret):
    # Customized to split up the queries and extract the correct sourcetype

    opts = _get_options()
    logging.info("Options: %s" % json.dumps(opts))
    http_event_collector_key = opts['token']
    http_event_collector_host = opts['indexer']
    # Set up the collector
    hec = http_event_collector(http_event_collector_key, http_event_collector_host)
    # st = "salt:hubble:nova"
    data = ret['return']
    minion_id = ret['id']
    jid = ret['jid']
    master = __grains__['master']

    for fai in data['Failure']:
        check_id = fai.keys()[0]
        payload = {}
        event = {}
        event.update({"minion_id": minion_id})
        event.update({"check_result": "Failure"})
        event.update({"check_id": check_id})
        event.update({"job_id": jid})
        event.update({"master": master})
        payload.update({"host": minion_id})
        payload.update({"index": opts['index']})
        payload.update({"sourcetype": opts['sourcetype']})
        payload.update({'event': event})
        hec.batchEvent(payload)

    for suc in data['Success']:
        check_id = suc.keys()[0]
        payload = {}
        event = {}
        event.update({"minion_id": minion_id})
        event.update({"check_result": "Success"})
        event.update({"check_id": check_id})
        event.update({"job_id": jid})
        event.update({"master": master})
        payload.update({"host": minion_id})
        payload.update({"sourcetype": opts['sourcetype']})
        payload.update({"index": opts['index']})
        payload.update({"event": event})
        hec.batchEvent(payload)

    payload = {}
    payload.update({"minion_id": minion_id})
    payload.update({"compliance_percentage": ret['return']['Compliance']})
    hec.batchEvent(payload)
    hec.flushBatch()
    return

def _get_options():
    try:
        token = __salt__['config.get']('splunk_http_forwarder:token')
        indexer = __salt__['config.get']('splunk_http_forwarder:indexer')
        sourcetype = __salt__['config.get']('splunk_http_forwarder:sourcetype')
        index = __salt__['config.get']('splunk_http_forwarder:index')
    except:
        return None
    splunk_opts = {"token": token, "indexer": indexer, "sourcetype": sourcetype, "index": index}
    return splunk_opts

def send_splunk(event, index_override=None, sourcetype_override=None):
    # Get Splunk Options
    # init the payload
    payload = {}

    # Set up the event metadata
    if index_override is None:
        payload.update({"index": opts['index']})
    else:
        payload.update({"index": index_override})

    if sourcetype_override is None:
        payload.update({"sourcetype": opts['sourcetype']})
    else:
        payload.update({"sourcetype": sourcetype_override})

    # Add the event
    payload.update({"event": event})
    logging.info("Payload: %s" % json.dumps(payload))

    # fire it off
    hec.batchEvent(payload)
    return True

# Thanks to George Starcher for the http_event_collector class (https://github.com/georgestarcher/)
# Default batch max size to match splunk's default limits for max byte
# See http_input stanza in limits.conf; note in testing I had to limit to 100,000 to avoid http event collector breaking connection
# Auto flush will occur if next event payload will exceed limit

class http_event_collector:

    def __init__(self, token, http_event_server, host="", http_event_port='8088', http_event_server_ssl=True, max_bytes=_max_content_bytes):
        self.token = token
        self.batchEvents = []
        self.maxByteLength = max_bytes
        self.currentByteLength = 0

        # Set host to specified value or default to localhostname if no value provided
        if host:
            self.host = host
        else:
            self.host = socket.gethostname()

        # Build and set server_uri for http event collector
        # Defaults to SSL if flag not passed
        # Defaults to port 8088 if port not passed

        if http_event_server_ssl:
            buildURI = ['https://']
        else:
            buildURI = ['http://']
        for i in [http_event_server, ':', http_event_port, '/services/collector/event']:
            buildURI.append(i)
        self.server_uri = "".join(buildURI)

        if http_event_collector_debug:
            print self.token
            print self.server_uri

    def sendEvent(self, payload, eventtime=""):
        # Method to immediately send an event to the http event collector

        headers = {'Authorization': 'Splunk ' + self.token}

        # If eventtime in epoch not passed as optional argument use current system time in epoch
        if not eventtime:
            eventtime = str(int(time.time()))

        # Fill in local hostname if not manually populated
        if 'host' not in payload:
            payload.update({"host": self.host})

        # Update time value on payload if need to use system time
        data = {"time": eventtime}
        data.update(payload)

        # send event to http event collector
        r = requests.post(self.server_uri, data=json.dumps(data), headers=headers, verify=http_event_collector_SSL_verify)

        # Print debug info if flag set
        if http_event_collector_debug:
            logger.debug(r.text)
            logger.debug(data)

    def batchEvent(self, payload, eventtime=""):
        # Method to store the event in a batch to flush later

        # Fill in local hostname if not manually populated
        if 'host' not in payload:
            payload.update({"host": self.host})

        payloadLength = len(json.dumps(payload))

        if (self.currentByteLength + payloadLength) > self.maxByteLength:
            self.flushBatch()
            # Print debug info if flag set
            if http_event_collector_debug:
                print "auto flushing"
        else:
            self.currentByteLength = self.currentByteLength + payloadLength

        # If eventtime in epoch not passed as optional argument use current system time in epoch
        if not eventtime:
            eventtime = str(int(time.time()))

        # Update time value on payload if need to use system time
        data = {"time": eventtime}
        data.update(payload)

        self.batchEvents.append(json.dumps(data))

    def flushBatch(self):
        # Method to flush the batch list of events

        if len(self.batchEvents) > 0:
            headers = {'Authorization': 'Splunk ' + self.token}
            r = requests.post(self.server_uri, data=" ".join(self.batchEvents), headers=headers, verify=http_event_collector_SSL_verify)
            self.batchEvents = []
            self.currentByteLength = 0
<commit_msg>Update splunk_nova<commit_after>import socket

# Imports for http event forwarder
import requests
import json
import time

import logging

_max_content_bytes = 100000
http_event_collector_SSL_verify = False
http_event_collector_debug = True

log = logging.getLogger(__name__)

hec = None

def returner(ret):
    # Customized to split up the queries and extract the correct sourcetype

    opts = _get_options()
    logging.info("Options: %s" % json.dumps(opts))
    http_event_collector_key = opts['token']
    http_event_collector_host = opts['indexer']
    # Set up the collector
    hec = http_event_collector(http_event_collector_key, http_event_collector_host)
    # st = "salt:hubble:nova"
    data = ret['return']
    minion_id = ret['id']
    jid = ret['jid']
    master = __grains__['master']

    for fai in data.get('Failure', []):
        check_id = fai.keys()[0]
        payload = {}
        event = {}
        event.update({"minion_id": minion_id})
        event.update({"check_result": "Failure"})
        event.update({"check_id": check_id})
        event.update({"job_id": jid})
        event.update({"master": master})
        payload.update({"host": minion_id})
        payload.update({"index": opts['index']})
        payload.update({"sourcetype": opts['sourcetype']})
        payload.update({'event': event})
        hec.batchEvent(payload)

    for suc in data.get('Success', []):
        check_id = suc.keys()[0]
        payload = {}
        event = {}
        event.update({"minion_id": minion_id})
        event.update({"check_result": "Success"})
        event.update({"check_id": check_id})
        event.update({"job_id": jid})
        event.update({"master": master})
        payload.update({"host": minion_id})
        payload.update({"sourcetype": opts['sourcetype']})
        payload.update({"index": opts['index']})
        payload.update({"event": event})
        hec.batchEvent(payload)

    if data.get('Compliance', None):
        payload = {}
        event = {}
        event.update({"minion_id": minion_id})
        event.update({"job_id": jid})
        event.update({"master": master})
        event.update({"compliance_percentage": data['Compliance']})
        payload.update({"host": minion_id})
        payload.update({"sourcetype": opts['sourcetype']})
        payload.update({"index": opts['index']})
        payload.update({"event": event})
        hec.batchEvent(payload)

    hec.flushBatch()
    return

def _get_options():
    try:
        token = __salt__['config.get']('splunk_http_forwarder:token')
        indexer = __salt__['config.get']('splunk_http_forwarder:indexer')
        sourcetype = __salt__['config.get']('splunk_http_forwarder:sourcetype')
        index = __salt__['config.get']('splunk_http_forwarder:index')
    except:
        return None
    splunk_opts = {"token": token, "indexer": indexer, "sourcetype": sourcetype, "index": index}
    return splunk_opts

def send_splunk(event, index_override=None, sourcetype_override=None):
    # Get Splunk Options
    # init the payload
    payload = {}

    # Set up the event metadata
    if index_override is None:
        payload.update({"index": opts['index']})
    else:
        payload.update({"index": index_override})

    if sourcetype_override is None:
        payload.update({"sourcetype": opts['sourcetype']})
    else:
        payload.update({"sourcetype": sourcetype_override})

    # Add the event
    payload.update({"event": event})
    logging.info("Payload: %s" % json.dumps(payload))

    # fire it off
    hec.batchEvent(payload)
    return True

# Thanks to George Starcher for the http_event_collector class (https://github.com/georgestarcher/)
# Default batch max size to match splunk's default limits for max byte
# See http_input stanza in limits.conf; note in testing I had to limit to 100,000 to avoid http event collector breaking connection
# Auto flush will occur if next event payload will exceed limit

class http_event_collector:

    def __init__(self, token, http_event_server, host="", http_event_port='8088', http_event_server_ssl=True, max_bytes=_max_content_bytes):
        self.token = token
        self.batchEvents = []
        self.maxByteLength = max_bytes
        self.currentByteLength = 0

        # Set host to specified value or default to localhostname if no value provided
        if host:
            self.host = host
        else:
            self.host = socket.gethostname()

        # Build and set server_uri for http event collector
        # Defaults to SSL if flag not passed
        # Defaults to port 8088 if port not passed

        if http_event_server_ssl:
            buildURI = ['https://']
        else:
            buildURI = ['http://']
        for i in [http_event_server, ':', http_event_port, '/services/collector/event']:
            buildURI.append(i)
        self.server_uri = "".join(buildURI)

        if http_event_collector_debug:
            print self.token
            print self.server_uri

    def sendEvent(self, payload, eventtime=""):
        # Method to immediately send an event to the http event collector

        headers = {'Authorization': 'Splunk ' + self.token}

        # If eventtime in epoch not passed as optional argument use current system time in epoch
        if not eventtime:
            eventtime = str(int(time.time()))

        # Fill in local hostname if not manually populated
        if 'host' not in payload:
            payload.update({"host": self.host})

        # Update time value on payload if need to use system time
        data = {"time": eventtime}
        data.update(payload)

        # send event to http event collector
        r = requests.post(self.server_uri, data=json.dumps(data), headers=headers, verify=http_event_collector_SSL_verify)

        # Print debug info if flag set
        if http_event_collector_debug:
            logger.debug(r.text)
            logger.debug(data)

    def batchEvent(self, payload, eventtime=""):
        # Method to store the event in a batch to flush later

        # Fill in local hostname if not manually populated
        if 'host' not in payload:
            payload.update({"host": self.host})

        payloadLength = len(json.dumps(payload))

        if (self.currentByteLength + payloadLength) > self.maxByteLength:
            self.flushBatch()
            # Print debug info if flag set
            if http_event_collector_debug:
                print "auto flushing"
        else:
            self.currentByteLength = self.currentByteLength + payloadLength

        # If eventtime in epoch not passed as optional argument use current system time in epoch
        if not eventtime:
            eventtime = str(int(time.time()))

        # Update time value on payload if need to use system time
        data = {"time": eventtime}
        data.update(payload)

        self.batchEvents.append(json.dumps(data))

    def flushBatch(self):
        # Method to flush the batch list of events

        if len(self.batchEvents) > 0:
            headers = {'Authorization': 'Splunk ' + self.token}
            r = requests.post(self.server_uri, data=" ".join(self.batchEvents), headers=headers, verify=http_event_collector_SSL_verify)
            self.batchEvents = []
            self.currentByteLength = 0
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
#
# Author : @Jason_wbw

"""
Converter MNIST data to np.array

MNIST files are available at http://yann.lecun.com/exdb/mnist/
Before using this program you first need to download files:
train-images.idx3-ubyte.gz : train image data
t10k-images.idx3-ubyte.gz  : test image data
train-labels.idx1-ubyte.gz : train labels data
t10k-labels.idx1-ubyte.gz  : test labels data
and gunzip them.
"""

import numpy as np
import struct
import matplotlib.pyplot as plt

class Converter(object):

	def __init__(self):
		#self.test_read('t10k-images.idx3-ubyte')
		self.train_images = self.read_images('train-images.idx3-ubyte', 60)
		self.test_images = self.read_images('t10k-images.idx3-ubyte', 10)
		self.train_labels = self.read_labels('train-labels.idx1-ubyte', 60)
		self.test_labels = self.read_labels('t10k-labels.idx1-ubyte', 10)
		self.total_train_data = len(self.train_labels)

	def test_read(self, filename):
		images = []

		binfile = open(filename , 'rb')
		buf = binfile.read()

		index = 0
		magic, numImages, numRows, numColumns = struct.unpack_from('>IIII', buf, index) #read 4unsinged int32
		index += struct.calcsize('>IIII')

		im = struct.unpack_from('>784B', buf, index) #read 784unsigned byte
		index += struct.calcsize('>784B')

		im = np.array(im)
		im = im.reshape(28, 28)
		fig = plt.figure()
		plotwindow = fig.add_subplot(111)
		plt.imshow(im , cmap='gray')
		plt.show()

	def read_images(self, filename, read_num):
		images = []

		binfile = open(filename , 'rb')
		buf = binfile.read()

		index = 0
		magic, numImages, numRows, numColumns = struct.unpack_from('>IIII', buf, index) #read 4unsinged int32
		index += struct.calcsize('>IIII')

		for i in xrange(read_num):
			im = struct.unpack_from('>784B', buf, index) #read 784unsigned byte
			index += struct.calcsize('>784B')

			im = np.array(im)
			images.append(im)

		return np.array(images)

	def read_labels(self, filename, read_num):
		labels = []

		binfile = open(filename , 'rb')
		buf = binfile.read()

		index = 0
		magic, numLabels = struct.unpack_from('>II', buf, index) #read 2unsinged int32
		index += struct.calcsize('>II')

		for i in xrange(read_num):
			im = struct.unpack_from('>B' ,buf, index) #read 1 unsigned byte
			index += struct.calcsize('>B')

			im = np.array(im)
			labels.append(im)

		return np.array(labels)
#endclass Converter

if __name__ == '__main__':
	Converter()<commit_msg>add dimension<commit_after>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
#
# Author : @Jason_wbw

"""
Converter MNIST data to np.array

MNIST files are available at http://yann.lecun.com/exdb/mnist/
Before using this program you first need to download files:
train-images.idx3-ubyte.gz : train image data
t10k-images.idx3-ubyte.gz  : test image data
train-labels.idx1-ubyte.gz : train labels data
t10k-labels.idx1-ubyte.gz  : test labels data
and gunzip them.
"""

import numpy as np
import struct
import matplotlib.pyplot as plt

class Converter(object):

	def __init__(self):
		self.test_read('t10k-images.idx3-ubyte')
		self.train_images = self.read_images('train-images.idx3-ubyte', 60)
		self.test_images = self.read_images('t10k-images.idx3-ubyte', 10)
		self.train_labels = self.read_labels('train-labels.idx1-ubyte', 60)
		self.test_labels = self.read_labels('t10k-labels.idx1-ubyte', 10)
		self.total_train_data = len(self.train_labels)
		self.dimensionality = 28 * 28

	def test_read(self, filename):
		images = []

		binfile = open(filename , 'rb')
		buf = binfile.read()

		index = 0
		magic, numImages, numRows, numColumns = struct.unpack_from('>IIII', buf, index) #read 4unsinged int32
		index += struct.calcsize('>IIII')

		im = struct.unpack_from('>%dB' % (28*28), buf, index) #read 784unsigned byte
		index += struct.calcsize('>%dB' % (28*28))

		im = np.array(im)
		im = im.reshape(28, 28)
		fig = plt.figure()
		plotwindow = fig.add_subplot(111)
		plt.imshow(im , cmap='gray')
		plt.show()

	def read_images(self, filename, read_num):
		images = []

		binfile = open(filename , 'rb')
		buf = binfile.read()

		index = 0
		magic, numImages, numRows, numColumns = struct.unpack_from('>IIII', buf, index) #read 4unsinged int32
		index += struct.calcsize('>IIII')

		for i in xrange(read_num):
			im = struct.unpack_from('>%dB' % (28*28), buf, index) #read 784unsigned byte
			index += struct.calcsize('>%dB' % (28*28))

			im = np.array(im)
			images.append(im)

		return np.array(images)

	def read_labels(self, filename, read_num):
		labels = []

		binfile = open(filename , 'rb')
		buf = binfile.read()

		index = 0
		magic, numLabels = struct.unpack_from('>II', buf, index) #read 2unsinged int32
		index += struct.calcsize('>II')

		for i in xrange(read_num):
			im = struct.unpack_from('>B' ,buf, index) #read 1 unsigned byte
			index += struct.calcsize('>B')

			im = np.array(im)
			labels.append(im)

		return np.array(labels)
#endclass Converter

if __name__ == '__main__':
	Converter()<|endoftext|>


<commit_before>import sqlalchemy as alc
from sqlalchemy import create_engine, MetaData
from rdrf.dynamic_data import DynamicDataWrapper
import explorer
from explorer.utils import DatabaseUtils

import logging
logger = logging.getLogger("registry_log")

from django.conf import settings

class ReportingTableGenerator(object):
    DEMOGRAPHIC_FIELDS = [('family_name', 'todo', alc.String),
                          ('given_names', 'todo', alc.String),
                          ('home_phone', 'todo', alc.String),
                          ('email', 'todo', alc.String)]
    TYPE_MAP = {"float": alc.Float,
                "calculated": alc.Float,
                "integer": alc.Integer,
                "int": alc.Integer,
                "string": alc.String,
                }

    def __init__(self, user, registry_model, type_overrides={}):
        self.user = user
        self.counter = 0
        self.col_map = {}
        self.registry_model = registry_model
        self.type_overrides = type_overrides
        self.engine = self._create_engine()
        self.columns = set([])
        self.table = None
        self.reverse_map = {}

    def _create_engine(self):
        report_db_data = settings.DATABASES["reporting"]
        db_user = report_db_data["USER"]
        db_pass = report_db_data["PASSWORD"]
        database = report_db_data["NAME"]
        host = report_db_data["HOST"]
        port = report_db_data["PORT"]
        connection_string = "postgresql://{0}:{1}@{2}:{3}/{4}".format(db_user,
                                                                       db_pass,
                                                                       host,
                                                                       port,
                                                                       database)
        return create_engine(connection_string)

    def create_table(self):
        self.table.create()
        logger.debug("created table based on schema")

    def drop_table(self):
        if self.table is not None:
            self.table.drop(self.engine)

    def _add_reverse_mapping(self, key, value):
        self.reverse_map[key] = value
        logger.debug("reverse map %s --> %s" % (key, value))

    def create_columns(self, sql_metadata, mongo_metadata):
        logger.debug("creating columns from sql and mongo metadata")
        self.columns = set([])
        self.columns.add(self._create_column("context_id", alc.Integer))
        self._add_reverse_mapping("context_id", "context_id") # special case

        for i, column_metadata in enumerate(sql_metadata):
            column_from_sql = self._create_column_from_sql(column_metadata)
            self.columns.add(column_from_sql)
            #to get from tuple
            # map the column's index in tuple to the column name in the dict
            self._add_reverse_mapping(i, column_metadata["name"])

        for form_model, section_model, cde_model in mongo_metadata["column_map"]:
            column_name = mongo_metadata["column_map"][(form_model, section_model, cde_model)]
            column_from_mongo = self._create_column_from_mongo(column_name, form_model, section_model, cde_model)
            self.columns.add(column_from_mongo)

    def _create_column_from_sql(self, column_metadata):
        column_name = column_metadata["name"]
        type_name = column_metadata["type_name"]
        # Not sure if this good idea ...
        # these type_names are coming from introspection on the sql cursor
        if type_name == "varchar":
            datatype = alc.String
        elif type_name == "bool":
            datatype = alc.Boolean
        elif "float" in type_name:
            datatype = alc.FLOAT
        elif "int" in type_name:
            datatype = alc.INTEGER
        elif "date" in type_name:
            datatype = alc.DATE
        else:
            datatype = alc.String

        return self._create_column(column_name, datatype)

    def _create_column_from_mongo(self, column_name, form_model, section_model, cde_model):
        column_data_type = self._get_sql_alchemy_datatype(cde_model)
        self._add_reverse_mapping((form_model, section_model, cde_model), column_name)
        return self._create_column(column_name, column_data_type)

    def run_explorer_query(self, explorer_query):
        self.create_table()
        for result_dict in explorer_query.generate_results(self.reverse_map):
            self.insert_row(result_dict)

    # the methods below dump everything

    def insert_patient_contexts(self, patient_model):
        self.create_table()
        for context_dict in self._get_context_dicts(patient_model):
            self.insert_row(context_dict)

    def insert_row(self, value_dict):
        # each row will be a denormalised section item
        logger.debug("inserting context dict: %s" % value_dict)
        self.engine.execute(self.table.insert().values(**value_dict))

    def _create_column(self, name, datatype=alc.String):
        column = alc.Column(name, datatype, nullable=True)
        self.columns.add(column)
        logger.debug("added column %s type %s" % (name, datatype))
        return column

    def create_schema(self):
        logger.debug("creating table schema")
        logger.debug("There are %s columns" % len(self.columns))
        table_name = self._generate_temporary_table_name()
        logger.debug("table name will be = %s" % table_name)
        self.table = alc.Table(table_name, MetaData(self.engine), *self.columns, schema=None)

    def _generate_temporary_table_name(self):
        return "reporting_table"

    def _get_sql_alchemy_datatype(self, cde_model):
        if cde_model.code in self.type_overrides:
            datatype = self.type_overrides[cde_model.code]
        else:
            datatype = cde_model.datatype

        return self.TYPE_MAP.get(datatype, alc.String)

class MongoFieldSelector(object):
    def __init__(self, user, registry_model, query_model, checkbox_ids=[]):
        if checkbox_ids is not None:
            self.checkbox_ids = checkbox_ids
        else:
            self.checkbox_ids = []

        self.user = user
        self.registry_model = registry_model
        self.query_model = query_model
        if self.query_model:
            self.existing_data = self._get_existing_report_choices(self.query_model)
        else:
            self.existing_data = {}

    def _get_existing_report_choices(self, query_model):
        import json
        try:
            return json.loads(query_model.projection)
        except:
            return []

    @property
    def field_data(self):
        return {"fields": self.fields}

    @property
    def fields(self):
        self.field_info = []
        for form_model in self.registry_model.forms:
            if self.user.can_view(form_model) and not form_model.is_questionnaire:
                for section_model in form_model.section_models:
                    for cde_model in section_model.cde_models:
                        self._get_field_info(form_model, section_model, cde_model)

        return self.field_info

    def _get_saved_value(self, form_model, section_model, cde_model):
        for value_dict in self.existing_data:
            if value_dict["formName"] == form_model.name:
                if value_dict["sectionCode"] == section_model.code:
                    if value_dict["cdeCode"] == cde_model.code:
                        return value_dict["value"]
        return False

    def _get_field_info(self, form_model, section_model, cde_model):
        saved_value = self._get_saved_value(form_model, section_model, cde_model)
        field_id = "cb_%s_%s_%s_%s" % (self.registry_model.code,
                                       form_model.pk,
                                       section_model.pk,
                                       cde_model.pk)
        field_label = cde_model.name
        self.field_info.append({"form": form_model.name,
                                "sectionCode": section_model.code,
                                "sectionName": section_model.display_name,
                                "cdeCode": cde_model.code,
                                "id": field_id,
                                "label": field_label,
                                "savedValue": saved_value})

    @property
    def projections_json(self):
        # this method returns the new projection data back to the client based
        # on the list of checked items passed in
        # the constructed json is independent of db ids ( as these will change
        # is import of registry definition occurs
        import json
        from rdrf.models import Registry, RegistryForm, Section, CommonDataElement
        projected_cdes = []
        for checkbox_id in self.checkbox_ids:
            # <input type="checkbox" name="cb_fh_39_104_CarotidUltrasonography" id="cb_fh_39_104_CarotidUltrasonography">
            # registry code, form pk, section pk, cde_code
            # cb_fh_39_105_EchocardiogramResult
            _, registry_code, form_pk, section_pk, cde_code = checkbox_id.split("_")

            form_model = RegistryForm.objects.get(pk=int(form_pk))
            section_model = Section.objects.get(pk=int(section_pk))
            cde_model = CommonDataElement.objects.get(code=cde_code)

            value_dict = {}
            value_dict["registryCode"] = form_model.registry.code
            value_dict["formName"] = form_model.name
            value_dict["sectionCode"] = section_model.code
            value_dict["cdeCode"] = cde_model.code
            value_dict["value"] = True  # we only need to store the true / checked cdes

            projected_cdes.append(value_dict)

        logger.debug("projected cdes = %s" % projected_cdes)
        return json.dumps(projected_cdes)

<commit_msg>FKRP-161 drop table initially , recreate schema<commit_after>import sqlalchemy as alc
from sqlalchemy import create_engine, MetaData
from rdrf.dynamic_data import DynamicDataWrapper
import explorer
from explorer.utils import DatabaseUtils

import logging
logger = logging.getLogger("registry_log")

from django.conf import settings

class ReportingTableGenerator(object):
    DEMOGRAPHIC_FIELDS = [('family_name', 'todo', alc.String),
                          ('given_names', 'todo', alc.String),
                          ('home_phone', 'todo', alc.String),
                          ('email', 'todo', alc.String)]
    TYPE_MAP = {"float": alc.Float,
                "calculated": alc.Float,
                "integer": alc.Integer,
                "int": alc.Integer,
                "string": alc.String,
                }

    def __init__(self, user, registry_model, type_overrides={}):
        self.user = user
        self.counter = 0
        self.col_map = {}
        self.registry_model = registry_model
        self.type_overrides = type_overrides
        self.engine = self._create_engine()
        self.columns = set([])
        self.table_name = ""
        self.table = None
        self.reverse_map = {}

    def _create_engine(self):
        report_db_data = settings.DATABASES["reporting"]
        db_user = report_db_data["USER"]
        db_pass = report_db_data["PASSWORD"]
        database = report_db_data["NAME"]
        host = report_db_data["HOST"]
        port = report_db_data["PORT"]
        connection_string = "postgresql://{0}:{1}@{2}:{3}/{4}".format(db_user,
                                                                       db_pass,
                                                                       host,
                                                                       port,
                                                                       database)
        return create_engine(connection_string)

    def create_table(self):
        self.table.create()
        logger.debug("created table based on schema")

    def drop_table(self):
        drop_table_sql = "DROP TABLE %s" % self._generate_temporary_table_name()
        conn = self.engine.connect()
        try:
            conn.execute(drop_table_sql)
            logger.debug("dropped temp table")
        except:
            pass
        conn.close()

    def _add_reverse_mapping(self, key, value):
        self.reverse_map[key] = value
        logger.debug("reverse map %s --> %s" % (key, value))

    def create_columns(self, sql_metadata, mongo_metadata):
        logger.debug("creating columns from sql and mongo metadata")
        self.columns = set([])
        self.columns.add(self._create_column("context_id", alc.Integer))
        self._add_reverse_mapping("context_id", "context_id") # special case

        for i, column_metadata in enumerate(sql_metadata):
            column_from_sql = self._create_column_from_sql(column_metadata)
            self.columns.add(column_from_sql)
            #to get from tuple
            # map the column's index in tuple to the column name in the dict
            self._add_reverse_mapping(i, column_metadata["name"])

        for form_model, section_model, cde_model in mongo_metadata["column_map"]:
            column_name = mongo_metadata["column_map"][(form_model, section_model, cde_model)]
            column_from_mongo = self._create_column_from_mongo(column_name, form_model, section_model, cde_model)
            self.columns.add(column_from_mongo)

    def _create_column_from_sql(self, column_metadata):
        column_name = column_metadata["name"]
        type_name = column_metadata["type_name"]
        # Not sure if this good idea ...
        # these type_names are coming from introspection on the sql cursor
        if type_name == "varchar":
            datatype = alc.String
        elif type_name == "bool":
            datatype = alc.Boolean
        elif "float" in type_name:
            datatype = alc.FLOAT
        elif "int" in type_name:
            datatype = alc.INTEGER
        elif "date" in type_name:
            datatype = alc.DATE
        else:
            datatype = alc.String

        return self._create_column(column_name, datatype)

    def _create_column_from_mongo(self, column_name, form_model, section_model, cde_model):
        column_data_type = self._get_sql_alchemy_datatype(cde_model)
        self._add_reverse_mapping((form_model, section_model, cde_model), column_name)
        return self._create_column(column_name, column_data_type)

    def run_explorer_query(self, explorer_query):
        self.create_table()
        for result_dict in explorer_query.generate_results(self.reverse_map):
            self.insert_row(result_dict)

    def insert_row(self, value_dict):
        # each row will be a denormalised section item
        logger.debug("inserting context dict: %s" % value_dict)
        self.engine.execute(self.table.insert().values(**value_dict))

    def _create_column(self, name, datatype=alc.String):
        column = alc.Column(name, datatype, nullable=True)
        self.columns.add(column)
        logger.debug("added column %s type %s" % (name, datatype))
        return column

    def create_schema(self):

        self.drop_table()
        logger.debug("creating table schema")
        logger.debug("There are %s columns" % len(self.columns))
        table_name = self._generate_temporary_table_name()
        logger.debug("table name will be = %s" % table_name)
        self.table = alc.Table(table_name, MetaData(self.engine), *self.columns, schema=None)

    def _generate_temporary_table_name(self):
        return "reporting_table"

    def _get_sql_alchemy_datatype(self, cde_model):
        if cde_model.code in self.type_overrides:
            datatype = self.type_overrides[cde_model.code]
        else:
            datatype = cde_model.datatype

        return self.TYPE_MAP.get(datatype, alc.String)

class MongoFieldSelector(object):
    def __init__(self, user, registry_model, query_model, checkbox_ids=[]):
        if checkbox_ids is not None:
            self.checkbox_ids = checkbox_ids
        else:
            self.checkbox_ids = []

        self.user = user
        self.registry_model = registry_model
        self.query_model = query_model
        if self.query_model:
            self.existing_data = self._get_existing_report_choices(self.query_model)
        else:
            self.existing_data = {}

    def _get_existing_report_choices(self, query_model):
        import json
        try:
            return json.loads(query_model.projection)
        except:
            return []

    @property
    def field_data(self):
        return {"fields": self.fields}

    @property
    def fields(self):
        self.field_info = []
        for form_model in self.registry_model.forms:
            if self.user.can_view(form_model) and not form_model.is_questionnaire:
                for section_model in form_model.section_models:
                    for cde_model in section_model.cde_models:
                        self._get_field_info(form_model, section_model, cde_model)

        return self.field_info

    def _get_saved_value(self, form_model, section_model, cde_model):
        for value_dict in self.existing_data:
            if value_dict["formName"] == form_model.name:
                if value_dict["sectionCode"] == section_model.code:
                    if value_dict["cdeCode"] == cde_model.code:
                        return value_dict["value"]
        return False

    def _get_field_info(self, form_model, section_model, cde_model):
        saved_value = self._get_saved_value(form_model, section_model, cde_model)
        field_id = "cb_%s_%s_%s_%s" % (self.registry_model.code,
                                       form_model.pk,
                                       section_model.pk,
                                       cde_model.pk)
        field_label = cde_model.name
        self.field_info.append({"form": form_model.name,
                                "sectionCode": section_model.code,
                                "sectionName": section_model.display_name,
                                "cdeCode": cde_model.code,
                                "id": field_id,
                                "label": field_label,
                                "savedValue": saved_value})

    @property
    def projections_json(self):
        # this method returns the new projection data back to the client based
        # on the list of checked items passed in
        # the constructed json is independent of db ids ( as these will change
        # is import of registry definition occurs
        import json
        from rdrf.models import Registry, RegistryForm, Section, CommonDataElement
        projected_cdes = []
        for checkbox_id in self.checkbox_ids:
            # <input type="checkbox" name="cb_fh_39_104_CarotidUltrasonography" id="cb_fh_39_104_CarotidUltrasonography">
            # registry code, form pk, section pk, cde_code
            # cb_fh_39_105_EchocardiogramResult
            _, registry_code, form_pk, section_pk, cde_code = checkbox_id.split("_")

            form_model = RegistryForm.objects.get(pk=int(form_pk))
            section_model = Section.objects.get(pk=int(section_pk))
            cde_model = CommonDataElement.objects.get(code=cde_code)

            value_dict = {}
            value_dict["registryCode"] = form_model.registry.code
            value_dict["formName"] = form_model.name
            value_dict["sectionCode"] = section_model.code
            value_dict["cdeCode"] = cde_model.code
            value_dict["value"] = True  # we only need to store the true / checked cdes

            projected_cdes.append(value_dict)

        logger.debug("projected cdes = %s" % projected_cdes)
        return json.dumps(projected_cdes)

    def dump_csv(self):
        import csv
        select_query = alc.sql.select([self.table])
        result = db_connection.execute(select_query)
        fh = open('data.csv', 'wb')
        outcsv = csv.writer(fh)
        outcsv.writerow(result.keys())
        outcsv.writerows(result)
        fh.close()

<|endoftext|>


<commit_before>#!/usr/bin/env python3

import urllib.request  # to download HTML source
import sys  # to access CLI arguments and to use exit codes
from bs4 import BeautifulSoup  # to parse HTML source

version = 0.01

def printUsageInfo():
    helpMsg = """Usage:
    sjp.py <word>
    sjp.py (-h | --help | /?)
    sjp.py (-v | --version)"""
    print(helpMsg)

def printVersionInfo():
    versionMsg = "sjp.py " + str(version)
    print(versionMsg)

def getDefinition(word):
    url = 'http://sjp.pl/' + urllib.parse.quote(word)
    html = urllib.request.urlopen(url).read()

    soup = BeautifulSoup(html)

    # checks if definition is in dictionary:
    if soup.find_all('span', style="color: #e00;"):
        print("[Error] \"" + word + "\" not found")
        sys.exit(1)

    # definition is in dictionary, continue:
    ex = soup.find_all('p',
                       style="margin-top: .5em; "
                             "font-size: medium; "
                             "max-width: 32em; ")
    ex = ex[0]
    return ex.contents[0::2]  # returns a list of lines of definition

def main():
    if len(sys.argv) > 1:
        if sys.argv[1] in ["-h", "--help", "/?"]:
            printUsageInfo()
            sys.exit()
        elif sys.argv[1] in ["-v", "--version"]:
            printVersionInfo()
            sys.exit()
        else:
            print('\n'.join(getDefinition(sys.argv[1])))
    else:
        printUsageInfo()
        sys.exit()

if __name__ == "__main__":
    main()
<commit_msg>Catch exception while connecting (fixes #2)<commit_after>#!/usr/bin/env python3

import urllib.request  # to download HTML source
import sys  # to access CLI arguments and to use exit codes
from bs4 import BeautifulSoup  # to parse HTML source

version = 0.01

def printUsageInfo():
    helpMsg = """Usage:
    sjp.py <word>
    sjp.py (-h | --help | /?)
    sjp.py (-v | --version)"""
    print(helpMsg)

def printVersionInfo():
    versionMsg = "sjp.py " + str(version)
    print(versionMsg)

def getDefinition(word):
    url = 'http://sjp.pl/' + urllib.parse.quote(word)
    try:
        html = urllib.request.urlopen(url).read()
    except urllib.error.URLError:
        print("[Error] Can't connect to the service")
        sys.exit(2)

    soup = BeautifulSoup(html)

    # checks if definition is in dictionary:
    if soup.find_all('span', style="color: #e00;"):
        print("[Error] \"" + word + "\" not found")
        sys.exit(1)

    # definition is in dictionary, continue:
    ex = soup.find_all('p',
                       style="margin-top: .5em; "
                             "font-size: medium; "
                             "max-width: 32em; ")
    ex = ex[0]
    return ex.contents[0::2]  # returns a list of lines of definition

def main():
    if len(sys.argv) > 1:
        if sys.argv[1] in ["-h", "--help", "/?"]:
            printUsageInfo()
            sys.exit()
        elif sys.argv[1] in ["-v", "--version"]:
            printVersionInfo()
            sys.exit()
        else:
            print('\n'.join(getDefinition(sys.argv[1])))
    else:
        printUsageInfo()
        sys.exit()

if __name__ == "__main__":
    main()
<|endoftext|>