<commit_before>#!/usr/bin/env python

import json
import awsiot
import logging
from signal import pause
import RPi.GPIO as GPIO
import time

def pub(topic, value):
    if topic is not None and len(topic) > 0:
        for t in topic:
            publisher.publish(t,
                              json.dumps(
                                  {args.shadow_var: value, awsiot.MESSAGE: "{} {}".format(args.shadow_var, value)}))
    publisher.publish(awsiot.iot_thing_topic(args.thing), awsiot.iot_payload(awsiot.REPORTED, {args.shadow_var: value}))

def get_distance():
    # set Trigger to HIGH
    GPIO.output(GPIO_TRIGGER, True)

    # set Trigger after 0.01ms to LOW
    time.sleep(0.00001)
    GPIO.output(GPIO_TRIGGER, False)

    start_time = time.time()
    stop_time = time.time()

    # save StartTime
    while GPIO.input(GPIO_ECHO) == 0:
        start_time = time.time()

    # save time of arrival
    while GPIO.input(GPIO_ECHO) == 1:
        stop_time = time.time()

    # time difference between start and arrival
    time_lapsed = stop_time - start_time
    # multiply with the sonic speed (34300 cm/s)
    # and divide by 2, because there and back
    distance = (time_lapsed * 34300) / 2

    return distance

if __name__ == "__main__":
    parser = awsiot.iot_arg_parser()
    parser.add_argument("--trigger_pin", help="trigger gpio pin (using BCM numbering)", type=int, required=True)
    parser.add_argument("--echo_pin", help="echo gpio pin (using BCM numbering)", type=int, required=True)
    args = parser.parse_args()

    # GPIO Mode (BOARD / BCM)
    GPIO.setmode(GPIO.BCM)

    # set GPIO Pins
    GPIO_TRIGGER = args.trigger_pin
    GPIO_ECHO = args.echo_pin

    # set GPIO direction (IN / OUT)
    GPIO.setup(GPIO_TRIGGER, GPIO.OUT)
    GPIO.setup(GPIO_ECHO, GPIO.IN)

    logging.basicConfig(filename=awsiot.LOG_FILE, level=args.log_level, format=awsiot.LOG_FORMAT)

    publisher = awsiot.Publisher(args.endpoint, args.rootCA, args.cert, args.key)
    properties = {}
    properties["distance"] = get_distance()
    publisher.publish(awsiot.iot_thing_topic(args.thing), awsiot.iot_payload(awsiot.REPORTED, properties))
    GPIO.cleanup()
<commit_msg>added ultrasonic distance publisher for HC-sr04<commit_after>#!/usr/bin/env python

import json
import awsiot
import logging
import RPi.GPIO as GPIO
import time

def pub(dist):
    if args.topic is not None and len(args.topic) > 0:
        for t in args.topic:
            publisher.publish(t,
                              json.dumps({"distance": dist, awsiot.MESSAGE: "distance: {}".format(dist)}))
    publisher.publish(awsiot.iot_thing_topic(args.thing),
                      awsiot.iot_payload(awsiot.REPORTED, {'distance': dist}))

def get_distance():
    # set Trigger to HIGH
    GPIO.output(GPIO_TRIGGER, True)

    # set Trigger after 0.01ms to LOW
    time.sleep(0.00001)
    GPIO.output(GPIO_TRIGGER, False)

    start_time = time.time()
    stop_time = time.time()

    # save StartTime
    while GPIO.input(GPIO_ECHO) == 0:
        start_time = time.time()

    # save time of arrival
    while GPIO.input(GPIO_ECHO) == 1:
        stop_time = time.time()

    # time difference between start and arrival
    time_lapsed = stop_time - start_time
    # multiply with the sonic speed (34300 cm/s)
    # and divide by 2, because there and back
    distance = (time_lapsed * 34300) / 2

    return distance

if __name__ == "__main__":
    parser = awsiot.iot_arg_parser()
    parser.add_argument("--trigger_pin", help="trigger gpio pin (using BCM numbering)", type=int, required=True)
    parser.add_argument("--echo_pin", help="echo gpio pin (using BCM numbering)", type=int, required=True)
    args = parser.parse_args()

    # GPIO Mode (BOARD / BCM)
    GPIO.setmode(GPIO.BCM)

    # set GPIO Pins
    GPIO_TRIGGER = args.trigger_pin
    GPIO_ECHO = args.echo_pin

    # set GPIO direction (IN / OUT)
    GPIO.setup(GPIO_TRIGGER, GPIO.OUT)
    GPIO.setup(GPIO_ECHO, GPIO.IN)

    logging.basicConfig(filename=awsiot.LOG_FILE, level=args.log_level, format=awsiot.LOG_FORMAT)

    publisher = awsiot.Publisher(args.endpoint, args.rootCA, args.cert, args.key)
    distance = get_distance()
    if 1 < distance < 250:
        pub(distance)
    GPIO.cleanup()
<|endoftext|>


<commit_before>from .models import BlogPost, Comment
from django.forms import ModelForm

class BlogPostForm(ModelForm):
    class Meta:
        model = BlogPost
        exclude = ('user',)

class CommentForm(ModelForm):
    class Meta:
        model = Comment
        exclude = ('post',)
<commit_msg>Exclude fields from CommentForm that will be automatically inserted<commit_after>from .models import BlogPost, Comment
from django.forms import ModelForm

class BlogPostForm(ModelForm):
    class Meta:
        model = BlogPost
        exclude = ('user',)

class CommentForm(ModelForm):
    class Meta:
        model = Comment
        exclude = ('post', 'user', 'date',)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from mock import patch, Mock

from soco import discover
from soco import config

IP_ADDR = '192.168.1.101'
TIMEOUT = 5

@patch('socket.socket', spec=True)
@patch('select.select', spec=True)
@patch('soco.config.SOCO_CLASS')
class TestDiscover:
    def test_discover(self, mock_soco, mock_select, mock_socket):
        socket = mock_socket.return_value
        socket.recvfrom.return_value = ('data', [IP_ADDR])  # (data,
        # address)
        mock_select.return_value = (1, 1, 1)
        # set timeout
        TIMEOUT = 5
        discover(timeout=TIMEOUT)
        # 3 packets should be sent
        assert socket.sendto.call_count == 3
        # select called with the relevant timeout
        mock_select.assert_called_once_with([socket], [], [], TIMEOUT)
        # SoCo should be created with the IP address received
        mock_soco.assert_called_with(IP_ADDR)

        # Now test include_visible parameter. include_invisible=True should
        # result in calling SoCo.all_zones etc
        mock_soco.return_value = Mock(all_zones='ALL', visible_zones='VISIBLE')
        assert discover(include_invisible=True) == 'ALL'
        assert discover(include_invisible=False) == 'VISIBLE'

        # if select does not return within timeout SoCo should not be called
        # at all
        # simulate no data being returned within timeout
        mock_select.return_value = (0, 1, 1)
        discover()
        # Check no SoCo instance created
        mock_soco.assert_not_called
<commit_msg>Fixed unit test for discovery<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from mock import patch, Mock

from soco import discover
from soco import config

IP_ADDR = '192.168.1.101'
TIMEOUT = 5

@patch('socket.socket', spec=True)
@patch('select.select', spec=True)
@patch('soco.config.SOCO_CLASS')
class TestDiscover:
    def test_discover(self, mock_soco, mock_select, mock_socket):
        socket = mock_socket.return_value
        socket.recvfrom.return_value = ('SERVER: Linux UPnP/1.0 Sonos/26.1-76230 (ZPS3)', [IP_ADDR])  # (data,
        # address)
        mock_select.return_value = (1, 1, 1)
        # set timeout
        TIMEOUT = 5
        discover(timeout=TIMEOUT)
        # 3 packets should be sent
        assert socket.sendto.call_count == 3
        # select called with the relevant timeout
        mock_select.assert_called_once_with([socket], [], [], TIMEOUT)
        # SoCo should be created with the IP address received
        mock_soco.assert_called_with(IP_ADDR)

        # Now test include_visible parameter. include_invisible=True should
        # result in calling SoCo.all_zones etc
        mock_soco.return_value = Mock(all_zones='ALL', visible_zones='VISIBLE')
        assert discover(include_invisible=True) == 'ALL'
        assert discover(include_invisible=False) == 'VISIBLE'

        # if select does not return within timeout SoCo should not be called
        # at all
        # simulate no data being returned within timeout
        mock_select.return_value = (0, 1, 1)
        discover()
        # Check no SoCo instance created
        mock_soco.assert_not_called
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import hashlib
import os
import random
import time

from pysnmp.entity.rfc3413.oneliner import cmdgen
from pysnmp.smi import builder, view, error
from pysnmp.proto import rfc1902, rfc1905

from pyasn1.type import univ

# Snmp version constants
V1 = 0
V2 = V2C = 1

# The internal mib builder
__mibBuilder = builder.MibBuilder()
__mibViewController = view.MibViewController(__mibBuilder)

def add_mib_path(path):
	"""Add a directory to the MIB search path"""
	if path not in __mibBuilder.getMibPath() and os.path.isdir(path):
		__mibBuilder.setMibPath(*(__mibBuilder.getMibPath() + (path, )))  # pylint: disable=W0142

def load_mibs(*modules):
	"""Load one or more mibs"""
	for m in modules:
		try:
			__mibBuilder.loadModules(m)
		except error.SmiError as e:
			if 'already exported' in str(e):
				continue
			raise

def get_namedvalues(mibname, objectname):
	return __mibBuilder.mibSymbols[mibname][objectname].syntax.getNamedValues()

def snmp_auth_data(community, version=V2C, snmp_id=None):
	if snmp_id is None:
		sha_256 = hashlib.sha256()  # pylint: disable=E1101
		sha_256.update(community)
		sha_256.update(str(time.time() * 1000))
		sha_256.update(str(random.random()))
		snmp_id = sha_256.hexdigest()[:32]
	return cmdgen.CommunityData(snmp_id, community, version)

def snmp_auth_data_v1(community, version=V1, snmp_id=None):
	return snmp_auth_data(community, version, snmp_id)

def snmp_auth_data_v2c(community, version=V2C, snmp_id=None):
	return snmp_auth_data(community, version, snmp_id)

class SnmpError(Exception):
	def __init__(self, msg, error_indication, error_status, error_index, varbinds):  # pylint: disable=R0913
		self.msg = msg
		self.error_indication = error_indication
		self.error_status = error_status
		self.error_index = error_index
		self.varbinds = varbinds
		Exception.__init__(self, msg)

	def __str__(self):
		return "%s - with error indication %s and error status %s" % (self.msg, str(self.error_indication), str(self.error_status))

def nodeinfo(oid):
	"""Translate dotted-decimal oid to a tuple with symbolic info"""
	if isinstance(oid, str):
		oid = rfc1902.ObjectName(oid)
	return __mibViewController.getNodeLocation(oid), __mibViewController.getNodeName(oid)

def nodeinfos(*oids):
	return [nodeinfo(oid) for oid in oids]

def nodename(oid):
	"""Translate dotted-decimal oid or oid tuple to symbolic name"""
	if isinstance(oid, str):
		oid = rfc1902.ObjectName(oid)
	oid = __mibViewController.getNodeLocation(oid)
	name = '::'.join(oid[:-1])
	noid = '.'.join([str(x) for x in oid[-1]])
	if noid:
		name += '.' + noid
	return name

def nodenames(*oids):
	return [nodename(oid) for oid in oids]

def nodeid(oid):
	"""Translate named oid to dotted-decimal format"""
	if isinstance(oid, tuple):
		return oid
	elif isinstance(oid, rfc1902.ObjectName):
		return oid
	elif isinstance(oid, str):
		ids = oid.split('.')
		try:
			ids_num = [int(x) for x in ids]
		except ValueError:
			symbols = ids[0].split('::')
			ids = tuple([int(x) for x in ids[1:]])
			mibnode, = __mibBuilder.importSymbols(*symbols)  # pylint: disable=W0142
			oid = mibnode.getName() + ids
			return oid
		else:
			return tuple(ids_num)

	else:
		raise AssertionError("Unknown oid format for %r encountered" % oid)

def nodeids(oids):
	oids_list = []
	for oid in oids:
		oids_list.append(nodeid(oid), )
	oids_list.sort()
	return tuple(set(oids_list))

class SnmpClient(object):  # pylint: disable=R0902
	"""Easy access to an snmp deamon on a host"""

	def __init__(self, host, auth, port=161, timeout=2, retries=3):  # pylint: disable=R0913
		"""Set up the client and detect the community to use"""
		self.host = host
		self.port = port
		self.alive = False
		self.sysname = None
		self.sysdescr = None
		self.auth = auth
		self.timeout = timeout
		self.retries = retries
		self.error_indication = self.error_status = self.error_index = self.error_varbinds = None

		(error_indication, error_status, error_index, varbinds) = cmdgen.CommandGenerator().getCmd(auth, cmdgen.UdpTransportTarget((self.host, self.port), timeout=timeout, retries=retries),
																								nodeid('SNMPv2-MIB::sysName.0'),
																								nodeid('SNMPv2-MIB::sysDescr.0'))
		if error_indication or error_status:
			self.__set_error(error_indication, error_status, error_index, varbinds)
		else:
			assert len(varbinds) == 2
			self.sysname = varbinds[0][1]
			self.sysdescr = varbinds[1][1]
			self.alive = True

	def __set_error(self, error_indication, error_status, error_index, varbinds):
		self.error_indication = error_indication
		self.error_status = error_status
		self.error_index = error_index
		self.error_varbinds = varbinds

	def set_auth_data(self, community, version=V2C, community_index=None):
		self.auth = cmdgen.CommunityData(community_index, community, version)

	def get(self, *oids):
		"""Get a specific node in the tree"""
		assert self.alive is True
		# print "oids is", oids
		oids_trans = nodeids(oids)
		# print "oids_trans are", oids_trans
		(error_indication, error_status, error_index, varbinds) = cmdgen.CommandGenerator().getCmd(self.auth, cmdgen.UdpTransportTarget((self.host, self.port), timeout=self.timeout, retries=self.retries), *oids_trans)  # pylint: disable=W0142
		if error_indication or error_status:
			self.__set_error(error_indication, error_status, error_index, varbinds)
			raise SnmpError("SNMP get command on %s of oid %r failed" % (self.host, oids), error_indication, error_status, error_index, varbinds)
		return SnmpVarBinds(varbinds)

	def gettable(self, *oids):
		"""Get a complete subtable"""
		assert self.alive is True
		oids_trans = nodeids(oids)
		(error_indication, error_status, error_index, varbinds) = cmdgen.CommandGenerator().bulkCmd(self.auth, cmdgen.UdpTransportTarget((self.host, self.port), timeout=self.timeout, retries=self.retries), 0, 25, *oids_trans)  # pylint: disable=W0142
		if error_indication or error_status:
			self.__set_error(error_indication, error_status, error_index, varbinds)
			raise SnmpError("SNMP getnext on %s of oid %r failed" % (self.host, oids), error_indication, error_status, error_index, varbinds)
		return SnmpVarBinds(varbinds)

	def set(self, *oidvalues):
		assert self.alive is True
		oidvalues_trans = []
		for oid, value in oidvalues:
			if isinstance(oid, tuple):
				has_str = False
				for entry in oid:
					if isinstance(entry, str):
						has_str = True
						break
				if has_str:  # if oid is a tuple containing strings, assume translation using cmdgen.MibVariable.
					# value must then be a Python type
					assert isinstance(value, int) or isinstance(value, str) or isinstance(value, bool)
					oidvalues_trans.append((cmdgen.MibVariable(*oid), value))  # pylint:disable=W0142
				else:
					# value must be a rfc1902/pyasn1 type
					if not oid[-1] == 0:
						assert isinstance(value, univ.Integer) or isinstance(value, univ.OctetString) or isinstance(value, univ.ObjectIdentifier)
					oidvalues_trans.append((oid, value))
			elif isinstance(oid, str):  # if oid is a string, assume nodeid lookup
				# value must then be a rfc1902/pyasn1 type, if oid is not a scalar
				if not oid.endswith(".0"):
					assert isinstance(value, univ.Integer) or isinstance(value, univ.OctetString) or isinstance(value, univ.ObjectIdentifier)
				oidvalues_trans.append((nodeid(oid), value))

		(error_indication, error_status, error_index, varbinds) = \
			cmdgen.CommandGenerator().setCmd(self.auth, cmdgen.UdpTransportTarget((self.host, self.port), timeout=self.timeout, retries=self.retries), *oidvalues_trans)  # pylint: disable=W0612,W0142
		if error_indication or error_status:
			self.__set_error(error_indication, error_status, error_index, varbinds)
			raise SnmpError("SNMP set command on %s of oid values %r failed" % (self.host, oidvalues_trans), error_indication, error_status, error_index, varbinds)
		return SnmpVarBinds(varbinds)

	def matchtables(self, index, *tables):
		"""Match a list of tables using either a specific index table or the
		common tail of the OIDs in the tables"""
		assert self.alive is True
		oid_to_index = {}
		result = {}
		indexlen = 1
		if index:
			#  Use the index if available
			varbinds = self.gettable(index)
			for entry in varbinds.get_varbinds():
				oid, index = entry[0]
				oid_to_index[oid[-indexlen:]] = index
				result[index] = []
		else:
			# Generate an index from the first table
			baselen = len(nodeid(tables[0]))
			varbinds = self.gettable(tables[0])
			for oid, value in varbinds.get_varbinds():
				indexlen = len(oid) - baselen
				oid_to_index[oid[-indexlen:]] = oid[-indexlen:]
				result[oid[-indexlen:]] = [value]
			tables = tables[1:]
		# Fetch the tables and match indices
		res = self.gettable(*tables)
		for entry in res.get_varbinds():
			for oid, value in entry:
				index = oid_to_index[oid[-indexlen:]]
				result[index].append(value)
		return result

class SnmpVarBinds(object):
	def __init__(self, varbinds):
		self.__varbinds = varbinds
		self.__varbinds_dict = None

	def __str__(self):
		text = str(self.get_json_name())
		return text

	def __repr__(self):
		text = repr(self.get_json_name())
		return text

	def dictify(self):
		if self.__varbinds_dict is None:
			self.__varbinds_dict = {}
			for entry in self.__varbinds:
				if isinstance(entry, list):
					for oid, value in entry:
						# always store internal data using rfc1902.ObjectNames, which are pyasn1 ObjectIdentifiers, which behave like tuples
						if not value.isSameTypeWith(rfc1905.noSuchObject):
							self.__varbinds_dict[rfc1902.ObjectName(oid)] = value
				else:
					oid, value = entry
					if not value.isSameTypeWith(rfc1905.noSuchObject):
						self.__varbinds_dict[rfc1902.ObjectName(oid)] = value

	def get_by_dict(self, oid):
		self.dictify()
		if oid is None and len(self.__varbinds_dict.keys()) == 1:
			oid = self.__varbinds_dict.keys()[0]
		elif oid is None:
			raise RuntimeError("Cannot query oid %r if multiple varBinds keys are present")
		if isinstance(oid, str):
			if self.__varbinds_dict.has_key(oid):
				value = self.__varbinds_dict[oid]
			else:
				value = self.__varbinds_dict[rfc1902.ObjectName(nodeid(oid))]
			if value.isSameTypeWith(rfc1905.noSuchObject):
				return None
			return value
		elif isinstance(oid, tuple) or isinstance(oid, rfc1902.ObjectName):
			value = self.__varbinds_dict[oid]
			if value.isSameTypeWith(rfc1905.noSuchObject):
				return None
			else:
				return value
		else:
			raise RuntimeError("Unknown format of oid %r with type %s" % (oid, oid.__class__.__name__))

	def get_value(self, oid=None):
		return self.get_by_dict(oid)

	def get_named_value(self, oid=None):
		name = nodename(self.get_dict().keys()[0]).split("::")
		mibname = name[0]
		objectname = name[1].split(".0")[0]
		namedvalues = get_namedvalues(mibname, objectname)
		return namedvalues.getName(self.get_by_dict(oid))

	def get_varbinds(self):
		return self.__varbinds

	def get_dict(self):
		self.dictify()
		return self.__varbinds_dict

	def __get_json(self, keytype=str):
		json = {}
		for key, value in self.get_dict().items():
			if isinstance(value, univ.OctetString):
				value = str(value)
			elif isinstance(value, univ.Integer):
				value = int(value)
			elif isinstance(value, univ.ObjectIdentifier):
				value = str(value)
			else:
				raise AssertionError("Unknown type %s encountered for oid %s" % (value.__class__.__name__, key))
			json[keytype(key)] = value
		return json

	def get_json_oid(self):
		return self.__get_json()

	def get_json_name(self):
		return self.__get_json(keytype=nodename)
<commit_msg>python3 compatibility<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import hashlib
import os
import random
import time

from pysnmp.entity.rfc3413.oneliner import cmdgen
from pysnmp.smi import builder, view, error
from pysnmp.proto import rfc1902, rfc1905

from pyasn1.type import univ

# Snmp version constants
V1 = 0
V2 = V2C = 1

# The internal mib builder
__mibBuilder = builder.MibBuilder()
__mibViewController = view.MibViewController(__mibBuilder)

def add_mib_path(path):
	"""Add a directory to the MIB search path"""
	if not os.path.isdir(path):
		return

	for source in __mibBuilder.getMibSources():
		if isinstance(source, builder.DirMibSource):
			if source.fullPath() == path:
				return

	__mibBuilder.setMibSources(*(__mibBuilder.getMibSources() + (builder.DirMibSource(path), )))

def load_mibs(*modules):
	"""Load one or more mibs"""
	for m in modules:
		try:
			__mibBuilder.loadModules(m)
		except error.SmiError as e:
			if 'already exported' in str(e):
				continue
			raise

def get_namedvalues(mibname, objectname):
	return __mibBuilder.mibSymbols[mibname][objectname].syntax.getNamedValues()

def snmp_auth_data(community, version=V2C, snmp_id=None):
	if snmp_id is None:
		sha_256 = hashlib.sha256()  # pylint: disable=E1101
		sha_256.update(community.encode('ascii'))
		sha_256.update(str(time.time() * 1000).encode('ascii'))
		sha_256.update(str(random.random()).encode('ascii'))
		snmp_id = sha_256.hexdigest()[:32]
	return cmdgen.CommunityData(snmp_id, community, version)

def snmp_auth_data_v1(community, version=V1, snmp_id=None):
	return snmp_auth_data(community, version, snmp_id)

def snmp_auth_data_v2c(community, version=V2C, snmp_id=None):
	return snmp_auth_data(community, version, snmp_id)

class SnmpError(Exception):
	def __init__(self, msg, error_indication, error_status, error_index, varbinds):  # pylint: disable=R0913
		self.msg = msg
		self.error_indication = error_indication
		self.error_status = error_status
		self.error_index = error_index
		self.varbinds = varbinds
		Exception.__init__(self, msg)

	def __str__(self):
		return "%s - with error indication %s and error status %s" % (self.msg, str(self.error_indication), str(self.error_status))

def nodeinfo(oid):
	"""Translate dotted-decimal oid to a tuple with symbolic info"""
	if isinstance(oid, str):
		oid = rfc1902.ObjectName(oid)
	return __mibViewController.getNodeLocation(oid), __mibViewController.getNodeName(oid)

def nodeinfos(*oids):
	return [nodeinfo(oid) for oid in oids]

def nodename(oid):
	"""Translate dotted-decimal oid or oid tuple to symbolic name"""
	if isinstance(oid, str):
		oid = rfc1902.ObjectName(oid)
	oid = __mibViewController.getNodeLocation(oid)  # pylint:disable=R0204
	name = '::'.join(oid[:-1])
	noid = '.'.join([str(x) for x in oid[-1]])
	if noid:
		name += '.' + noid
	return name

def nodenames(*oids):
	return [nodename(oid) for oid in oids]

def nodeid(oid):
	"""Translate named oid to dotted-decimal format"""
	if isinstance(oid, tuple):
		return oid
	elif isinstance(oid, rfc1902.ObjectName):
		return oid
	elif isinstance(oid, str):
		ids = oid.split('.')
		try:
			ids_num = [int(x) for x in ids]
		except ValueError:
			symbols = ids[0].split('::')
			ids = tuple([int(x) for x in ids[1:]])
			mibnode, = __mibBuilder.importSymbols(*symbols)
			oid = mibnode.getName() + ids
			return oid
		else:
			return tuple(ids_num)

	else:
		raise AssertionError("Unknown oid format for %r encountered" % oid)

def nodeids(oids):
	oids_list = []
	for oid in oids:
		oids_list.append(nodeid(oid), )
	oids_list.sort()
	return tuple(set(oids_list))

class SnmpClient(object):  # pylint: disable=R0902
	"""Easy access to an snmp deamon on a host"""

	def __init__(self, host, auth, port=161, timeout=2, retries=3):  # pylint: disable=R0913
		"""Set up the client and detect the community to use"""
		self.host = host
		self.port = port
		self.alive = False
		self.sysname = None
		self.sysdescr = None
		self.auth = auth
		self.timeout = timeout
		self.retries = retries
		self.error_indication = self.error_status = self.error_index = self.error_varbinds = None

		(error_indication, error_status, error_index, varbinds) = cmdgen.CommandGenerator().getCmd(
			auth,
			cmdgen.UdpTransportTarget(
				(self.host, self.port),
				timeout=timeout,
				retries=retries
			),
			nodeid('SNMPv2-MIB::sysName.0'),
			nodeid('SNMPv2-MIB::sysDescr.0')
		)
		if error_indication or error_status:
			self.__set_error(error_indication, error_status, error_index, varbinds)
		else:
			assert len(varbinds) == 2
			self.sysname = varbinds[0][1]
			self.sysdescr = varbinds[1][1]
			self.alive = True

	def __set_error(self, error_indication, error_status, error_index, varbinds):
		self.error_indication = error_indication
		self.error_status = error_status
		self.error_index = error_index
		self.error_varbinds = varbinds

	def set_auth_data(self, community, version=V2C, community_index=None):
		self.auth = cmdgen.CommunityData(community_index, community, version)

	def get(self, *oids):
		"""Get a specific node in the tree"""
		assert self.alive is True
		# print "oids is", oids
		oids_trans = nodeids(oids)
		# print "oids_trans are", oids_trans
		(error_indication, error_status, error_index, varbinds) = cmdgen.CommandGenerator().getCmd(self.auth, cmdgen.UdpTransportTarget((self.host, self.port), timeout=self.timeout, retries=self.retries), *oids_trans)
		if error_indication or error_status:
			self.__set_error(error_indication, error_status, error_index, varbinds)
			raise SnmpError("SNMP get command on %s of oid %r failed" % (self.host, oids), error_indication, error_status, error_index, varbinds)
		return SnmpVarBinds(varbinds)

	def gettable(self, *oids):
		"""Get a complete subtable"""
		assert self.alive is True
		oids_trans = nodeids(oids)
		(error_indication, error_status, error_index, varbinds) = cmdgen.CommandGenerator().bulkCmd(self.auth, cmdgen.UdpTransportTarget((self.host, self.port), timeout=self.timeout, retries=self.retries), 0, 25, *oids_trans)
		if error_indication or error_status:
			self.__set_error(error_indication, error_status, error_index, varbinds)
			raise SnmpError("SNMP getnext on %s of oid %r failed" % (self.host, oids), error_indication, error_status, error_index, varbinds)
		return SnmpVarBinds(varbinds)

	def set(self, *oidvalues):
		assert self.alive is True
		oidvalues_trans = []
		for oid, value in oidvalues:
			if isinstance(oid, tuple):
				has_str = False
				for entry in oid:
					if isinstance(entry, str):
						has_str = True
						break
				if has_str:  # if oid is a tuple containing strings, assume translation using cmdgen.MibVariable.
					# value must then be a Python type
					assert isinstance(value, int) or isinstance(value, str) or isinstance(value, bool)
					oidvalues_trans.append((cmdgen.MibVariable(*oid), value))
				else:
					# value must be a rfc1902/pyasn1 type
					if not oid[-1] == 0:
						assert isinstance(value, univ.Integer) or isinstance(value, univ.OctetString) or isinstance(value, univ.ObjectIdentifier)
					oidvalues_trans.append((oid, value))
			elif isinstance(oid, str):  # if oid is a string, assume nodeid lookup
				# value must then be a rfc1902/pyasn1 type, if oid is not a scalar
				if not oid.endswith(".0"):
					assert isinstance(value, univ.Integer) or isinstance(value, univ.OctetString) or isinstance(value, univ.ObjectIdentifier)
				oidvalues_trans.append((nodeid(oid), value))

		(error_indication, error_status, error_index, varbinds) = \
			cmdgen.CommandGenerator().setCmd(self.auth, cmdgen.UdpTransportTarget((self.host, self.port), timeout=self.timeout, retries=self.retries), *oidvalues_trans)  # pylint: disable=W0612
		if error_indication or error_status:
			self.__set_error(error_indication, error_status, error_index, varbinds)
			raise SnmpError("SNMP set command on %s of oid values %r failed" % (self.host, oidvalues_trans), error_indication, error_status, error_index, varbinds)
		return SnmpVarBinds(varbinds)

	def matchtables(self, index, *tables):
		"""Match a list of tables using either a specific index table or the
		common tail of the OIDs in the tables"""
		assert self.alive is True
		oid_to_index = {}
		result = {}
		indexlen = 1
		if index:
			#  Use the index if available
			varbinds = self.gettable(index)
			for entry in varbinds.get_varbinds():
				oid, index = entry[0]
				oid_to_index[oid[-indexlen:]] = index
				result[index] = []
		else:
			# Generate an index from the first table
			baselen = len(nodeid(tables[0]))
			varbinds = self.gettable(tables[0])
			for oid, value in varbinds.get_varbinds():
				indexlen = len(oid) - baselen
				oid_to_index[oid[-indexlen:]] = oid[-indexlen:]
				result[oid[-indexlen:]] = [value]
			tables = tables[1:]
		# Fetch the tables and match indices
		res = self.gettable(*tables)
		for entry in res.get_varbinds():
			for oid, value in entry:
				index = oid_to_index[oid[-indexlen:]]
				result[index].append(value)
		return result

class SnmpVarBinds(object):
	def __init__(self, varbinds):
		self.__varbinds = varbinds
		self.__varbinds_dict = None

	def __str__(self):
		text = str(self.get_json_name())
		return text

	def __repr__(self):
		text = repr(self.get_json_name())
		return text

	def dictify(self):
		if self.__varbinds_dict is None:
			self.__varbinds_dict = {}
			for entry in self.__varbinds:
				if isinstance(entry, list):
					for oid, value in entry:
						# always store internal data using rfc1902.ObjectNames, which are pyasn1 ObjectIdentifiers, which behave like tuples
						if not value.isSameTypeWith(rfc1905.noSuchObject):
							self.__varbinds_dict[rfc1902.ObjectName(oid)] = value
				else:
					oid, value = entry
					if not value.isSameTypeWith(rfc1905.noSuchObject):
						self.__varbinds_dict[rfc1902.ObjectName(oid)] = value

	def get_by_dict(self, oid):
		self.dictify()
		if oid is None and len(list(self.__varbinds_dict.keys())) == 1:
			oid = list(self.__varbinds_dict.keys())[0]
		elif oid is None:
			raise RuntimeError("Cannot query oid %r if multiple varBinds keys are present")
		if isinstance(oid, str):
			if oid in self.__varbinds_dict:
				value = self.__varbinds_dict[oid]
			else:
				value = self.__varbinds_dict[rfc1902.ObjectName(nodeid(oid))]
			if value.isSameTypeWith(rfc1905.noSuchObject):
				return None
			return value
		elif isinstance(oid, tuple) or isinstance(oid, rfc1902.ObjectName):
			value = self.__varbinds_dict[oid]
			if value.isSameTypeWith(rfc1905.noSuchObject):
				return None
			else:
				return value
		else:
			raise RuntimeError("Unknown format of oid %r with type %s" % (oid, oid.__class__.__name__))

	def get_value(self, oid=None):
		return self.get_by_dict(oid)

	def get_named_value(self, oid=None):
		name = nodename(list(self.get_dict().keys())[0]).split("::")
		mibname = name[0]
		objectname = name[1].split(".0")[0]
		namedvalues = get_namedvalues(mibname, objectname)
		return namedvalues.getName(self.get_by_dict(oid))

	def get_varbinds(self):
		return self.__varbinds

	def get_dict(self):
		self.dictify()
		return self.__varbinds_dict

	def __get_json(self, keytype=str):
		json = {}
		for key, value in list(self.get_dict().items()):
			if isinstance(value, univ.OctetString):
				value = str(value)
			elif isinstance(value, univ.Integer):
				value = int(value)  # pylint:disable=R0204
			elif isinstance(value, univ.ObjectIdentifier):
				value = str(value)
			else:
				raise AssertionError("Unknown type %s encountered for oid %s" % (value.__class__.__name__, key))
			json[keytype(key)] = value
		return json

	def get_json_oid(self):
		return self.__get_json()

	def get_json_name(self):
		return self.__get_json(keytype=nodename)
<|endoftext|>


<commit_before>from url_hacks import url

__version__ = "0.1.1"
<commit_msg>version bump<commit_after>from url_hacks import url

__version__ = "0.1.2"
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
    DU task for tagging resolution
    graph after the SW re-engineering by JLM during the 2019 summer.

    As of June 5th, 2015, this is the exemplary code

    Copyright NAVER(C)  2019 Hervé Déjean

    Developed  for the EU project READ. The READ project has received funding
    from the European Union's Horizon 2020 research and innovation programme
    under grant agreement No 674943.

"""

import sys, os

# try: #to ease the use without proper Python installation
#     import TranskribusDU_version
# except ImportError:
#     sys.path.append( os.path.dirname(os.path.dirname( os.path.abspath(sys.argv[0]) )))
#     import TranskribusDU_version
# TranskribusDU_version

from common.trace import traceln

from graph.NodeType_PageXml     import defaultBBoxDeltaFun
from graph.NodeType_PageXml     import NodeType_PageXml_type
from graph.NodeType_PageXml     import NodeType_PageXml
from graph.NodeType_PageXml     import NodeType_PageXml_type_woText
from tasks.DU_Task_Factory      import DU_Task_Factory
from tasks.DU_Task_Features     import Features_June19_Simple
from tasks.DU_Task_Features     import Features_June19_Simple_Separator
from tasks.DU_Task_Features     import Features_June19_Simple_Shift
from tasks.DU_Task_Features     import Features_June19_Simple_Separator_Shift
from tasks.DU_Task_Features     import Features_June19_Full
from tasks.DU_Task_Features     import Features_June19_Full_Separator
from tasks.DU_Task_Features     import Features_June19_Full_Shift
from tasks.DU_Task_Features     import Features_June19_Full_Separator_Shift
from tasks.DU_Task_Features     import FeatureDefinition
from tasks.DU_Task_Features     import *

from graph.Graph_Multi_SinglePageXml import Graph_MultiSinglePageXml
from xml_formats.PageXml import PageXml

# ----------------------------------------------------------------------------

class Features_GHENT_Full(FeatureDefinition):
    """
    All features we had historically (some specific to CRF):
        NODE: geometry, neighbor count, text
        EDGE: type, constant 1, geometry, text of source and target nodes
        The features of the edges are shifted by class, apart the 1-hot ones.
    """

    n_QUANTILES = 16

    bShiftEdgeByClass = False
    bSeparator        = False

    def __init__(self):
        FeatureDefinition.__init__(self)

        # NODES
        self.lNodeFeature = [                               \
              ("geometry"           , Node_Geometry())         # one can set nQuantile=...
#             , ("neighbor_count"     , Node_Neighbour_Count())  # one can set nQuantile=...
#             , ("text"               , Node_Text_NGram( 'char'    # character n-grams
#                                                        , 50     # number of N-grams
#                                                        , (1,2)    # N
#                                                        , False    # lowercase?))
#                                                        ))
                            ]
        node_transformer = FeatureUnion(self.lNodeFeature)

        # EDGES
        # which types of edge can we get??
        # It depends on the type of graph!!
        lEdgeClass = [HorizontalEdge, VerticalEdge]
        # standard set of features, including a constant 1 for CRF
        self.lEdgeFeature = [                                            \
                  ('1hot'   , Edge_Type_1Hot(lEdgeClass=lEdgeClass)) # Edge class 1 hot encoded (PUT IT FIRST)
                , ('1'      , Edge_1())                              # optional constant 1 for CRF
                , ('geom'   , Edge_Geometry())                       # one can set nQuantile=...
#                 , ('src_txt', Edge_Source_Text_NGram( 'char'    # character n-grams
#                                                , 50     # number of N-grams
#                                                , (1,2)    # N
#                                                , False    # lowercase?))
#                                                ))
#                 , ('tgt_txt', Edge_Target_Text_NGram( 'char'    # character n-grams
#                                                , 50     # number of N-grams
#                                                , (1,2)    # N
#                                                , False    # lowercase?))
#                                                ))
                            ]
        if self.bSeparator:
            self.lEdgeFeature = self.lEdgeFeature + [
                  ('sprtr_bool', Separator_boolean())
                , ('sprtr_num' , Separator_num())
                            ]
        fu =  FeatureUnion(self.lEdgeFeature)

        # you can use directly this union of features!
        edge_transformer = fu

        # OPTIONNALLY, you can have one range of features per type of edge.
        # the 1-hot encoding must be the first part of the union and it will determine
        #   by how much the rest of the feature are shifted.
        #
        # IMPORTANT: 1hot is first of union   AND   the correct number of edge classes
        if self.bShiftEdgeByClass:
            ppl = Pipeline([
                      ('fu', fu)
                    , ('shifter', EdgeClassShifter(len(lEdgeClass)))
                    ])
            edge_transformer = ppl

        self.setTransformers(node_transformer, edge_transformer)

class My_NodeTypeGHENT(NodeType_PageXml_type):
    """
    We need this to extract properly the label from the label attribute of the (parent) TableCell element.
    """
    def __init__(self, sNodeTypeName, lsLabel, lsIgnoredLabel=None, bOther=True, BBoxDeltaFun=defaultBBoxDeltaFun):
        super(NodeType_PageXml_type, self).__init__(sNodeTypeName, lsLabel, lsIgnoredLabel, bOther, BBoxDeltaFun)

    def parseDocNodeLabel(self, graph_node, defaultCls=None):
        """
        Parse and set the graph node label and return its class index
        We rely on the standard self.sLabelAttr
        raise a ValueError if the label is missing while bOther was not True
         , or if the label is neither a valid one nor an ignored one
        """

        ndParent = graph_node.node.getparent()
        try:
            sLabel = "%s_%s" % ( self.sLabelAttr,
                            PageXml.getCustomAttr(ndParent, 'structure','type')
                            #ndParent.get(self.sLabelAttr)   # e.g. "row" or "col"
                            )
        except :
            sLabel='type_None'
#         if sLabel == "TR_OTHER": sLabel='type_None'
        return sLabel

# ----------------------------------------------------------------------------
def main(sys_argv_0, sLabelAttribute, cNodeType=NodeType_PageXml_type_woText):

    def getConfiguredGraphClass(_doer):
        """
        In this class method, we must return a configured graph class
        """

        DU_GRAPH = Graph_MultiSinglePageXml

        ntClass = cNodeType
        print (cNodeType)
        lLabels = ['heading','paragraph','paragraph_left','paragraph_right','None']
        #lLabels = ['heading','paragraph','footnote','None']

        #lLabels.append('IGNORE')

        nt = ntClass(sLabelAttribute         #some short prefix because labels below are prefixed with it
                      , lLabels                   # in conjugate, we accept all labels, andNone becomes "none"
                      , []
                      , False                # unused
                      , BBoxDeltaFun=lambda v: max(v * 0.066, min(5, v/3))  #we reduce overlap in this way
                      )

        nt.setLabelAttribute(sLabelAttribute)
        nt.setXpathExpr( (".//pc:TextLine"        #how to find the nodes
                          #, "./pc:TextEquiv")       #how to get their text
                          , ".//pc:Unicode")       #how to get their text
                       )
        DU_GRAPH.addNodeType(nt)

        return DU_GRAPH

    # standard command line options for CRF- ECN- GAT-based methods
    usage, parser = DU_Task_Factory.getStandardOptionsParser(sys_argv_0)
    parser.add_option("--separator", dest='bSeparator', action="store_true"
                      , default=False, help="Use the graphical spearators, if any, as edge features.")
    parser.add_option("--text"       , dest='bText'     , action="store_true"
                      , default=False, help="Use textual information if any, as node and edge features.")
    parser.add_option("--edge_vh", "--edge_hv"    , dest='bShift'    , action="store_true"
                      , default=False, help="Shift edge feature by range depending on edge type.")
    traceln("VERSION: %s" % DU_Task_Factory.getVersion())

    # ---
    #parse the command line
    (options, args) = parser.parse_args()

    try:
        sModelDir, sModelName = args
    except Exception as e:
        traceln("Specify a model folder and a model name!")
        DU_Task_Factory.exit(usage, 1, e)
    if options.bText     : traceln(" - using textual data, if any")
    if options.bSeparator: traceln(" - using graphical separators, if any")
    if options.bShift    : traceln(" - shift edge features by edge type")
    cFeatureDefinition = Features_GHENT_Full
#     if options.bText:
#         if options.bSeparator:
#             if options.bShift:
#                 cFeatureDefinition = Features_June19_Full_Separator_Shift
#             else:
#                 cFeatureDefinition = Features_June19_Full_Separator
#         else:
#             if options.bShift:
#                 cFeatureDefinition = Features_June19_Full_Shift
#             else:
# #                 cFeatureDefinition = Features_June19_Full
#                 cFeatureDefinition = Features_BAR_Full
#
#     else:
#         if options.bSeparator:
#             if options.bShift:
#                 cFeatureDefinition = Features_June19_Simple_Separator_Shift
#             else:
#                 cFeatureDefinition = Features_June19_Simple_Separator
#         else:
#             if options.bShift:
#                 cFeatureDefinition = Features_June19_Simple_Shift
#             else:
#                 cFeatureDefinition = Features_June19_Simple

    # === SETTING the graph type (and its node type) a,d the feature extraction pipe
    doer = DU_Task_Factory.getDoer(sModelDir, sModelName
                                   , options                    = options
                                   , fun_getConfiguredGraphClass= getConfiguredGraphClass
                                   , cFeatureDefinition         = cFeatureDefinition
                                   )

    # == LEARNER CONFIGURATION ===
    # setting the learner configuration, in a standard way
    # (from command line options, or from a JSON configuration file)
    dLearnerConfig = doer.getStandardLearnerConfig(options)

#     # force a balanced weighting
#     print("Forcing balanced weights")
    dLearnerConfig['balanced'] = True

    # of course, you can put yours here instead.
    doer.setLearnerConfiguration(dLearnerConfig)

    # === CONJUGATE MODE ===
    #doer.setConjugateMode()

    # === GO!! ===
    # act as per specified in the command line (--trn , --fold-run, ...)
    doer.standardDo(options)

    del doer

# ----------------------------------------------------------------------------
if __name__ == "__main__":
    #     import better_exceptions
    #     better_exceptions.MAX_LENGTH = None

    main(sys.argv[0], "type", My_NodeTypeGHENT)
<commit_msg>add  doer.setAdditionalDataProvider(getDataToPickle_for_table)<commit_after># -*- coding: utf-8 -*-

"""
    DU task for tagging resolution
    graph after the SW re-engineering by JLM during the 2019 summer.

    As of June 5th, 2015, this is the exemplary code

    Copyright NAVER(C)  2019 Hervé Déjean

    Developed  for the EU project READ. The READ project has received funding
    from the European Union's Horizon 2020 research and innovation programme
    under grant agreement No 674943.

"""

import sys, os

# try: #to ease the use without proper Python installation
#     import TranskribusDU_version
# except ImportError:
#     sys.path.append( os.path.dirname(os.path.dirname( os.path.abspath(sys.argv[0]) )))
#     import TranskribusDU_version
# TranskribusDU_version

from common.trace import traceln

from graph.NodeType_PageXml     import defaultBBoxDeltaFun
from graph.NodeType_PageXml     import NodeType_PageXml_type
from graph.NodeType_PageXml     import NodeType_PageXml
from graph.NodeType_PageXml     import NodeType_PageXml_type_woText
from tasks.DU_Task_Factory      import DU_Task_Factory
from tasks.DU_Task_Features     import Features_June19_Simple
from tasks.DU_Task_Features     import Features_June19_Simple_Separator
from tasks.DU_Task_Features     import Features_June19_Simple_Shift
from tasks.DU_Task_Features     import Features_June19_Simple_Separator_Shift
from tasks.DU_Task_Features     import Features_June19_Full
from tasks.DU_Task_Features     import Features_June19_Full_Separator
from tasks.DU_Task_Features     import Features_June19_Full_Shift
from tasks.DU_Task_Features     import Features_June19_Full_Separator_Shift
from tasks.DU_Task_Features     import FeatureDefinition
from tasks.DU_Task_Features     import *

from graph.Graph_Multi_SinglePageXml import Graph_MultiSinglePageXml
from xml_formats.PageXml import PageXml
def getDataToPickle_for_table(doer, mdl, lGraph):
    """
    data that is specific to this task, which we want to pickle when --pkl is used
    for each node of each graph, we want to store the node text + row and column numbers + rowSpan and colSpan

    ( (text, (x1, y1, x2, y2), (row, col, rowSpan, colSpan) )
    ...
    """
    def attr_to_int(domnode, sAttr, default_value=None):
        s = domnode.get(sAttr)
        try:
            return int(s)
        except (ValueError, TypeError):
            return default_value

    lDataByGraph = []
    for g in lGraph:
        lNodeData = []
        for nd in g.lNode:
            ndCell = nd.node.getparent()
            data = (nd.text
                      , (nd.x1, nd.y1, nd.x2, nd.y2)
                      , (attr_to_int(ndCell, "row")
                         , attr_to_int(ndCell, "col")
                         , attr_to_int(ndCell, "rowSpan")
                         , attr_to_int(ndCell, "colSpan"))
                      )
            lNodeData.append(data)
        lDataByGraph.append(lNodeData)
    return lDataByGraph
# ----------------------------------------------------------------------------

class Features_GHENT_Full(FeatureDefinition):
    """
    All features we had historically (some specific to CRF):
        NODE: geometry, neighbor count, text
        EDGE: type, constant 1, geometry, text of source and target nodes
        The features of the edges are shifted by class, apart the 1-hot ones.
    """

    n_QUANTILES = 16

    bShiftEdgeByClass = False
    bSeparator        = False

    def __init__(self):
        FeatureDefinition.__init__(self)

        # NODES
        self.lNodeFeature = [                               \
              ("geometry"           , Node_Geometry())         # one can set nQuantile=...
#             , ("neighbor_count"     , Node_Neighbour_Count())  # one can set nQuantile=...
#             , ("text"               , Node_Text_NGram( 'char'    # character n-grams
#                                                        , 50     # number of N-grams
#                                                        , (1,2)    # N
#                                                        , False    # lowercase?))
#                                                        ))
                            ]
        node_transformer = FeatureUnion(self.lNodeFeature)

        # EDGES
        # which types of edge can we get??
        # It depends on the type of graph!!
        lEdgeClass = [HorizontalEdge, VerticalEdge]
        # standard set of features, including a constant 1 for CRF
        self.lEdgeFeature = [                                            \
                  ('1hot'   , Edge_Type_1Hot(lEdgeClass=lEdgeClass)) # Edge class 1 hot encoded (PUT IT FIRST)
                , ('1'      , Edge_1())                              # optional constant 1 for CRF
                , ('geom'   , Edge_Geometry())                       # one can set nQuantile=...
#                 , ('src_txt', Edge_Source_Text_NGram( 'char'    # character n-grams
#                                                , 50     # number of N-grams
#                                                , (1,2)    # N
#                                                , False    # lowercase?))
#                                                ))
#                 , ('tgt_txt', Edge_Target_Text_NGram( 'char'    # character n-grams
#                                                , 50     # number of N-grams
#                                                , (1,2)    # N
#                                                , False    # lowercase?))
#                                                ))
                            ]
        if self.bSeparator:
            self.lEdgeFeature = self.lEdgeFeature + [
                  ('sprtr_bool', Separator_boolean())
                , ('sprtr_num' , Separator_num())
                            ]
        fu =  FeatureUnion(self.lEdgeFeature)

        # you can use directly this union of features!
        edge_transformer = fu

        # OPTIONNALLY, you can have one range of features per type of edge.
        # the 1-hot encoding must be the first part of the union and it will determine
        #   by how much the rest of the feature are shifted.
        #
        # IMPORTANT: 1hot is first of union   AND   the correct number of edge classes
        if self.bShiftEdgeByClass:
            ppl = Pipeline([
                      ('fu', fu)
                    , ('shifter', EdgeClassShifter(len(lEdgeClass)))
                    ])
            edge_transformer = ppl

        self.setTransformers(node_transformer, edge_transformer)

class My_NodeTypeGHENT(NodeType_PageXml_type):
    """
    We need this to extract properly the label from the label attribute of the (parent) TableCell element.
    """
    def __init__(self, sNodeTypeName, lsLabel, lsIgnoredLabel=None, bOther=True, BBoxDeltaFun=defaultBBoxDeltaFun):
        super(NodeType_PageXml_type, self).__init__(sNodeTypeName, lsLabel, lsIgnoredLabel, bOther, BBoxDeltaFun)

    def parseDocNodeLabel(self, graph_node, defaultCls=None):
        """
        Parse and set the graph node label and return its class index
        We rely on the standard self.sLabelAttr
        raise a ValueError if the label is missing while bOther was not True
         , or if the label is neither a valid one nor an ignored one
        """

        ndParent = graph_node.node.getparent()
        try:
            sLabel = "%s_%s" % ( self.sLabelAttr,
                            PageXml.getCustomAttr(ndParent, 'structure','type')
                            #ndParent.get(self.sLabelAttr)   # e.g. "row" or "col"
                            )
        except :
            sLabel='type_None'
#         if sLabel == "TR_OTHER": sLabel='type_None'
        return sLabel

# ----------------------------------------------------------------------------
def main(sys_argv_0, sLabelAttribute, cNodeType=NodeType_PageXml_type_woText):

    def getConfiguredGraphClass(_doer):
        """
        In this class method, we must return a configured graph class
        """

        DU_GRAPH = Graph_MultiSinglePageXml

        ntClass = cNodeType
        lLabels = ['heading','paragraph','paragraph_left','paragraph_right','None']
        #lLabels = ['heading','paragraph','footnote','None']

        #lLabels.append('IGNORE')

        nt = ntClass(sLabelAttribute         #some short prefix because labels below are prefixed with it
                      , lLabels                   # in conjugate, we accept all labels, andNone becomes "none"
                      , []
                      , False                # unused
                      , BBoxDeltaFun=lambda v: max(v * 0.066, min(5, v/3))  #we reduce overlap in this way
                      )

        nt.setLabelAttribute(sLabelAttribute)
        nt.setXpathExpr( (".//pc:TextLine"        #how to find the nodes
                          #, "./pc:TextEquiv")       #how to get their text
                          , ".//pc:Unicode")       #how to get their text
                       )
        DU_GRAPH.addNodeType(nt)

        return DU_GRAPH

    # standard command line options for CRF- ECN- GAT-based methods
    usage, parser = DU_Task_Factory.getStandardOptionsParser(sys_argv_0)
    parser.add_option("--separator", dest='bSeparator', action="store_true"
                      , default=False, help="Use the graphical spearators, if any, as edge features.")
    parser.add_option("--text"       , dest='bText'     , action="store_true"
                      , default=False, help="Use textual information if any, as node and edge features.")
    parser.add_option("--edge_vh", "--edge_hv"    , dest='bShift'    , action="store_true"
                      , default=False, help="Shift edge feature by range depending on edge type.")
    traceln("VERSION: %s" % DU_Task_Factory.getVersion())

    # ---
    #parse the command line
    (options, args) = parser.parse_args()

    try:
        sModelDir, sModelName = args
    except Exception as e:
        traceln("Specify a model folder and a model name!")
        DU_Task_Factory.exit(usage, 1, e)
    if options.bText     : traceln(" - using textual data, if any")
    if options.bSeparator: traceln(" - using graphical separators, if any")
    if options.bShift    : traceln(" - shift edge features by edge type")
    cFeatureDefinition = Features_GHENT_Full
#     if options.bText:
#         if options.bSeparator:
#             if options.bShift:
#                 cFeatureDefinition = Features_June19_Full_Separator_Shift
#             else:
#                 cFeatureDefinition = Features_June19_Full_Separator
#         else:
#             if options.bShift:
#                 cFeatureDefinition = Features_June19_Full_Shift
#             else:
# #                 cFeatureDefinition = Features_June19_Full
#                 cFeatureDefinition = Features_BAR_Full
#
#     else:
#         if options.bSeparator:
#             if options.bShift:
#                 cFeatureDefinition = Features_June19_Simple_Separator_Shift
#             else:
#                 cFeatureDefinition = Features_June19_Simple_Separator
#         else:
#             if options.bShift:
#                 cFeatureDefinition = Features_June19_Simple_Shift
#             else:
#                 cFeatureDefinition = Features_June19_Simple

    # === SETTING the graph type (and its node type) a,d the feature extraction pipe
    doer = DU_Task_Factory.getDoer(sModelDir, sModelName
                                   , options                    = options
                                   , fun_getConfiguredGraphClass= getConfiguredGraphClass
                                   , cFeatureDefinition         = cFeatureDefinition
                                   )

    # == LEARNER CONFIGURATION ===
    # setting the learner configuration, in a standard way
    # (from command line options, or from a JSON configuration file)
    dLearnerConfig = doer.getStandardLearnerConfig(options)

#     # force a balanced weighting
#     print("Forcing balanced weights")
    dLearnerConfig['balanced'] = True

    # of course, you can put yours here instead.
    doer.setLearnerConfiguration(dLearnerConfig)
    doer.setAdditionalDataProvider(getDataToPickle_for_table)

    # === CONJUGATE MODE ===
    #doer.setConjugateMode()

    # === GO!! ===
    # act as per specified in the command line (--trn , --fold-run, ...)
    doer.standardDo(options)

    del doer

# ----------------------------------------------------------------------------
if __name__ == "__main__":
    #     import better_exceptions
    #     better_exceptions.MAX_LENGTH = None

    main(sys.argv[0], "type", My_NodeTypeGHENT)
<|endoftext|>


<commit_before>"""
Metric roll-up classes.
"""

from collections import defaultdict
import random
import time

class Metric(object):
    """
    A base metric class that accepts points, slices them into time intervals
    and performs roll-ups within those intervals.
    """

    def add_point(self, value):
        """ Add a point to the given metric. """
        raise NotImplementedError()

    def flush(self, timestamp):
        """ Flush all metrics up to the given timestamp. """
        raise NotImplementedError()

class Gauge(Metric):
    """ A gauge metric. """

    def __init__(self, name, tags):
        self.name = name
        self.tags = tags
        self.value = None

    def add_point(self, value):
        self.value = value

    def flush(self, timestamp):
        return [(timestamp, self.value, self.name, self.tags)]

class Counter(Metric):
    """ A counter metric. """

    def __init__(self, name, tags):
        self.name = name
        self.tags = tags
        self.count = 0

    def add_point(self, value):
        self.count += value

    def flush(self, timestamp):
        return [(timestamp, self.count, self.name, self.tags)]

class Histogram(Metric):
    """ A histogram metric. """

    def __init__(self, name, tags):
        self.name = name
        self.tags = tags
        self.max = float("-inf")
        self.min = float("inf")
        self.sum = 0
        self.count = 0
        self.sample_size = 1000
        self.samples = []
        self.percentiles = [0.75, 0.85, 0.95, 0.99]

    def add_point(self, value):
        self.max = self.max if self.max > value else value
        self.min = self.min if self.min < value else value
        self.sum += value
        if self.count < self.sample_size:
            self.samples.append(value)
        else:
            self.samples[random.randrange(0, self.sample_size)] = value
        self.count += 1

    def flush(self, timestamp):
        if not self.count:
            return []
        metrics = [
            (timestamp, self.min,       '%s.min'   % self.name, self.tags),
            (timestamp, self.max,       '%s.max'   % self.name, self.tags),
            (timestamp, self.count,     '%s.count' % self.name, self.tags),
            (timestamp, self.average(), '%s.avg'   % self.name, self.tags)
        ]
        length = len(self.samples)
        self.samples.sort()
        for p in self.percentiles:
            val = self.samples[int(round(p * length - 1))]
            name = '%s.%spercentile' % (self.name, int(p * 100))
            metrics.append((timestamp, val, name, self.tags))
        return metrics

    def average(self):
        return float(self.sum) / self.count

class MetricsAggregator(object):
    """
    A small class to handle the roll-ups of multiple metrics at once.
    """

    def __init__(self, roll_up_interval=10):
        self._metrics = {}
        self._roll_up_interval = roll_up_interval

    def add_point(self, metric, tags, timestamp, value, metric_class):
        interval = timestamp - timestamp % self._roll_up_interval
        if interval not in self._metrics:
            self._metrics[interval] = {}
        key = (metric, tuple(sorted(tags or [])))
        if key not in self._metrics[interval]:
            self._metrics[interval][key] = metric_class(metric, tags)
        self._metrics[interval][key].add_point(value)

    def flush(self, timestamp):
        """ Flush all metrics up to the given timestamp. """
        interval = timestamp - timestamp % self._roll_up_interval
        past_intervals = sorted((i for i in self._metrics if i < interval))
        metrics = []
        for i in past_intervals:
            for m in self._metrics.pop(i).values():
                metrics += m.flush(i)
        return metrics

<commit_msg>Using default dict for a small perf boost.<commit_after>"""
Metric roll-up classes.
"""

from collections import defaultdict
import random
import time

class Metric(object):
    """
    A base metric class that accepts points, slices them into time intervals
    and performs roll-ups within those intervals.
    """

    def add_point(self, value):
        """ Add a point to the given metric. """
        raise NotImplementedError()

    def flush(self, timestamp):
        """ Flush all metrics up to the given timestamp. """
        raise NotImplementedError()

class Gauge(Metric):
    """ A gauge metric. """

    def __init__(self, name, tags):
        self.name = name
        self.tags = tags
        self.value = None

    def add_point(self, value):
        self.value = value

    def flush(self, timestamp):
        return [(timestamp, self.value, self.name, self.tags)]

class Counter(Metric):
    """ A counter metric. """

    def __init__(self, name, tags):
        self.name = name
        self.tags = tags
        self.count = 0

    def add_point(self, value):
        self.count += value

    def flush(self, timestamp):
        return [(timestamp, self.count, self.name, self.tags)]

class Histogram(Metric):
    """ A histogram metric. """

    def __init__(self, name, tags):
        self.name = name
        self.tags = tags
        self.max = float("-inf")
        self.min = float("inf")
        self.sum = 0
        self.count = 0
        self.sample_size = 1000
        self.samples = []
        self.percentiles = [0.75, 0.85, 0.95, 0.99]

    def add_point(self, value):
        self.max = self.max if self.max > value else value
        self.min = self.min if self.min < value else value
        self.sum += value
        if self.count < self.sample_size:
            self.samples.append(value)
        else:
            self.samples[random.randrange(0, self.sample_size)] = value
        self.count += 1

    def flush(self, timestamp):
        if not self.count:
            return []
        metrics = [
            (timestamp, self.min,       '%s.min'   % self.name, self.tags),
            (timestamp, self.max,       '%s.max'   % self.name, self.tags),
            (timestamp, self.count,     '%s.count' % self.name, self.tags),
            (timestamp, self.average(), '%s.avg'   % self.name, self.tags)
        ]
        length = len(self.samples)
        self.samples.sort()
        for p in self.percentiles:
            val = self.samples[int(round(p * length - 1))]
            name = '%s.%spercentile' % (self.name, int(p * 100))
            metrics.append((timestamp, val, name, self.tags))
        return metrics

    def average(self):
        return float(self.sum) / self.count

class MetricsAggregator(object):
    """
    A small class to handle the roll-ups of multiple metrics at once.
    """

    def __init__(self, roll_up_interval=10):
        self._metrics = defaultdict(lambda: {})
        self._roll_up_interval = roll_up_interval

    def add_point(self, metric, tags, timestamp, value, metric_class):
        interval = timestamp - timestamp % self._roll_up_interval
        key = (metric, tuple(sorted(tags or [])))
        if key not in self._metrics[interval]:
            self._metrics[interval][key] = metric_class(metric, tags)
        self._metrics[interval][key].add_point(value)

    def flush(self, timestamp):
        """ Flush all metrics up to the given timestamp. """
        interval = timestamp - timestamp % self._roll_up_interval
        past_intervals = sorted((i for i in self._metrics if i < interval))
        metrics = []
        for i in past_intervals:
            for m in self._metrics.pop(i).values():
                metrics += m.flush(i)
        return metrics

<|endoftext|>


<commit_before>import numpy as np

def hpfilter( x, lam=1600 ):
    T = x.shape[-1]
    Au = np.array([
        [1, -2],
        [-2, 5]
    ])
    Ad = np.array([
        [5, -2],
        [-2, 1]
    ])
    a = np.diag( np.ones( T )*6 )
    b = np.diag( np.ones( T-1 )*(-4), 1 )
    c = np.diag( np.ones( T-2 ), 2 )
    d = np.diag( np.ones( T-1 )*(-4), -1 )
    e = np.diag( np.ones( T-2 ), -2 )
    M = a + b + c + d + e
    M[0:2,0:2] = Au
    M[-2:,-2:] = Ad
    M *= lam
    M += np.eye(T)

    if x.ndim == 1:
        return np.linalg.solve(M,x)
    elif x.ndim > 3:
        raise Exception('HP filter is not defined for dimension >= 3.')
    else:
        return np.linalg.solve(M,x.T).T<commit_msg>Updated filters.py to incorporate sparse methods and added bandpass filter.<commit_after>import numpy as np
from numpy import sin
import scipy as sp
import scipy.sparse.linalg as spla
from scipy.signal import fftconvolve

def hp_filter(data, lamb=1600):
    """
    This function will apply a Hodrick-Psomething filter to a dataset.
    The return value is the filtered data-set found according to:
        min sum((X[t] - T[t])**2 + lamb*((T[t+1] - T[t]) - (T[t] - T[t-1]))**2)
          T   t

    T = Lambda**-1 * Y

    Parameters
    ----------
        data: array, dtype=float
            The data set for which you want to apply the HP_filter.
            This mustbe a numpy array.
        lamb: array, dtype=float
            This is the value for lambda as used in the equation.

    Returns
    -------
        T: array, dtype=float
            The solution to the minimization equation above (the trend).
        Cycle: array, dtype=float
            This is the 'stationary data' found by Y - T.

    Notes
    -----
        This function implements sparse methods to be efficient enough to handle
        very large data sets.
    """
    Y = np.asarray(data)
    if Y.ndim > 1:
        Y = Y.squeeze()
    lil_t = len(Y)
    big_Lambda = sp.sparse.eye(lil_t, lil_t)
    big_Lambda = sp.sparse.lil_matrix(big_Lambda)

    # Use FOC's to build rows by group. The first and last rows are similar.
    # As are the second-second to last. Then all the ones in the middle...
    first_last = np.array([1 + lamb, -2 * lamb, lamb])
    second = np.array([-2 * lamb, (1 + 5 * lamb), -4 * lamb, lamb])
    middle_stuff = np.array([lamb, -4. * lamb, 1 + 6 * lamb, -4 * lamb, lamb])

    #--------------------------- Putting it together --------------------------#

    # First two rows
    big_Lambda[0, 0:3] = first_last
    big_Lambda[1, 0:4] = second

    # Last two rows. Second to last first
    big_Lambda[lil_t - 2, -4:] = second
    big_Lambda[lil_t - 1, -3:] = first_last

    # Middle rows
    for i in range(2, lil_t - 2):
        big_Lambda[i, i - 2:i + 3] = middle_stuff

    # spla.spsolve requires csr or csc matrix. I choose csr for fun.
    big_Lambda = sp.sparse.csr_matrix(big_Lambda)

    T = spla.spsolve(big_Lambda, Y)

    Cycle = Y - T

    return T, Cycle

def bandpass_filter(data, k, w1, w2):
    """
    This funciton will apply a bandpass filter to data. It will be kth
    order and will select the band bewtween w1 and w2.

    Parameters
    ----------
        data: array, dtype=float
            The data you wish to filter
        k: number, int
            The order of approximation for the filter. A max value for
            this isdata.size/2
        w1: number, float
            This is the lower bound for which frecuecies will pass
            through.
        w2: number, float
            This is the upper bound for which frecuecies will pass
            through.

    Returns
    -------
        y: array, dtype=float
            The filtered data.
    """
    data = np.asarray(data)
    low_w = np.pi * 2 / w2
    high_w = np.pi * 2 / w1
    bweights = np.zeros(2 * k + 1)
    bweights[k] = (high_w - low_w) / np.pi
    j = np.arange(1, int(k) + 1)
    weights = 1 / (np.pi * j) * (sin(high_w * j) - sin(low_w * j))
    bweights[k + j] = weights
    bweights[:k] = weights[::-1]

    bweights -= bweights.mean()

    return fftconvolve(bweights, data, mode='valid')
<|endoftext|>


<commit_before>import numpy as np
from numpy import sin
import scipy as sp
import scipy.sparse.linalg as spla
from scipy.signal import fftconvolve

def hp_filter(data, lamb=1600):
    """
    This function will apply a Hodrick-Psomething filter to a dataset.
    The return value is the filtered data-set found according to:
        min sum((X[t] - T[t])**2 + lamb*((T[t+1] - T[t]) - (T[t] - T[t-1]))**2)
          T   t

    T = Lambda**-1 * Y

    Parameters
    ----------
        data: array, dtype=float
            The data set for which you want to apply the HP_filter.
            This mustbe a numpy array.
        lamb: array, dtype=float
            This is the value for lambda as used in the equation.

    Returns
    -------
        T: array, dtype=float
            The solution to the minimization equation above (the trend).
        Cycle: array, dtype=float
            This is the 'stationary data' found by Y - T.

    Notes
    -----
        This function implements sparse methods to be efficient enough to handle
        very large data sets.
    """
    Y = np.asarray(data)
    if Y.ndim > 1:
        Y = Y.squeeze()
    lil_t = len(Y)
    big_Lambda = sp.sparse.eye(lil_t, lil_t)
    big_Lambda = sp.sparse.lil_matrix(big_Lambda)

    # Use FOC's to build rows by group. The first and last rows are similar.
    # As are the second-second to last. Then all the ones in the middle...
    first_last = np.array([1 + lamb, -2 * lamb, lamb])
    second = np.array([-2 * lamb, (1 + 5 * lamb), -4 * lamb, lamb])
    middle_stuff = np.array([lamb, -4. * lamb, 1 + 6 * lamb, -4 * lamb, lamb])

    #--------------------------- Putting it together --------------------------#

    # First two rows
    big_Lambda[0, 0:3] = first_last
    big_Lambda[1, 0:4] = second

    # Last two rows. Second to last first
    big_Lambda[lil_t - 2, -4:] = second
    big_Lambda[lil_t - 1, -3:] = first_last

    # Middle rows
    for i in range(2, lil_t - 2):
        big_Lambda[i, i - 2:i + 3] = middle_stuff

    # spla.spsolve requires csr or csc matrix. I choose csr for fun.
    big_Lambda = sp.sparse.csr_matrix(big_Lambda)

    T = spla.spsolve(big_Lambda, Y)

    Cycle = Y - T

    return T, Cycle

def bandpass_filter(data, k, w1, w2):
    """
    This funciton will apply a bandpass filter to data. It will be kth
    order and will select the band bewtween w1 and w2.

    Parameters
    ----------
        data: array, dtype=float
            The data you wish to filter
        k: number, int
            The order of approximation for the filter. A max value for
            this isdata.size/2
        w1: number, float
            This is the lower bound for which frecuecies will pass
            through.
        w2: number, float
            This is the upper bound for which frecuecies will pass
            through.

    Returns
    -------
        y: array, dtype=float
            The filtered data.
    """
    data = np.asarray(data)
    low_w = np.pi * 2 / w2
    high_w = np.pi * 2 / w1
    bweights = np.zeros(2 * k + 1)
    bweights[k] = (high_w - low_w) / np.pi
    j = np.arange(1, int(k) + 1)
    weights = 1 / (np.pi * j) * (sin(high_w * j) - sin(low_w * j))
    bweights[k + j] = weights
    bweights[:k] = weights[::-1]

    bweights -= bweights.mean()

    return fftconvolve(bweights, data, mode='valid')
<commit_msg>Returned hpfilter to original version.<commit_after>import numpy as np
from numpy import sin
from scipy.signal import fftconvolve

def hpfilter( x, lam=1600 ):
    T = x.shape[-1]
    Au = np.array([
        [1, -2],
        [-2, 5]
    ])
    Ad = np.array([
        [5, -2],
        [-2, 1]
    ])
    a = np.diag( np.ones( T )*6 )
    b = np.diag( np.ones( T-1 )*(-4), 1 )
    c = np.diag( np.ones( T-2 ), 2 )
    d = np.diag( np.ones( T-1 )*(-4), -1 )
    e = np.diag( np.ones( T-2 ), -2 )
    M = a + b + c + d + e
    M[0:2,0:2] = Au
    M[-2:,-2:] = Ad
    M *= lam
    M += np.eye(T)

    if x.ndim == 1:
        return np.linalg.solve(M,x)
    elif x.ndim > 3:
        raise Exception('HP filter is not defined for dimension >= 3.')
    else:
        return np.linalg.solve(M,x.T).T

def bandpass_filter(data, k, w1, w2):
    """
    This funciton will apply a bandpass filter to data. It will be kth
    order and will select the band bewtween w1 and w2.

    Parameters
    ----------
        data: array, dtype=float
            The data you wish to filter
        k: number, int
            The order of approximation for the filter. A max value for
            this isdata.size/2
        w1: number, float
            This is the lower bound for which frecuecies will pass
            through.
        w2: number, float
            This is the upper bound for which frecuecies will pass
            through.

    Returns
    -------
        y: array, dtype=float
            The filtered data.
    """
    data = np.asarray(data)
    low_w = np.pi * 2 / w2
    high_w = np.pi * 2 / w1
    bweights = np.zeros(2 * k + 1)
    bweights[k] = (high_w - low_w) / np.pi
    j = np.arange(1, int(k) + 1)
    weights = 1 / (np.pi * j) * (sin(high_w * j) - sin(low_w * j))
    bweights[k + j] = weights
    bweights[:k] = weights[::-1]

    bweights -= bweights.mean()

    return fftconvolve(bweights, data, mode='valid')
<|endoftext|>


<commit_before>from django import forms
from os import walk
import random
import re
from django.contrib.auth.forms import UserCreationForm
from django.utils.translation import ugettext_lazy as _
from django.utils.html import format_html
from .contributor import Contributor

AUDIO_DIR = 'userregistration/static'

class CustomUserCreationForm(UserCreationForm):
    first_name = forms.CharField(
            max_length=30,
            label=_('First Name')
            )
    last_name = forms.CharField(
            max_length=30,
            label=_('Last Name')
            )

    username = forms.EmailField(
        label=_("e-mail address"),
        strip=True,
        widget=forms.EmailInput,
    )

    def save(self, commit=True):
        user = super(CustomUserCreationForm, self).save(commit=False)
        user.first_name = self.cleaned_data["first_name"]
        user.last_name = self.cleaned_data["last_name"]
        user.username = self.cleaned_data["username"]
        if commit:
            user.save()
        return user

class ContributorCreationForm(forms.ModelForm):

    class Meta:
        model = Contributor
        fields = ['sex', 'accepted_licence']

    sex = forms.ChoiceField(
        choices=Contributor.SEX_CHOISES + ((None, _('I am...')),)
    )

    accepted_licence = forms.BooleanField(
        label=_(format_html(
                "I accept that my contributions are released under the \
                  \
                 Creative Commons CC BY-SA 4.0 license.")),
        required=True
    )

    def save(self, commit=True):
        contributor = super(ContributorCreationForm, self).save(commit=False)
        contributor.sex = self.cleaned_data["sex"]
        contributor.syllables_list = create_base_syllables_list()
        if commit:
            contributor.save()
        return contributor

# PINYIN_LIST = ['ai1','ai2','ai3','ai4','ai5',
#         'ma1','ma2','ma3','ma4','ma5',
#         'ma1','ma2','ma3','ma4','ma5',
#         'ma1','ma2','ma3','ma4','ma5',
#         'ma1','ma2','ma3','ma4','ma5',
#         'wo1','wo2','wo3','wo4','wo5',
#         ]
PINYIN_LIST = ['ai1']

def read_file():
    """
    return a random filename from the audio directory
    """
    _, _, filenames = next(walk(AUDIO_DIR))
    return random.choice(filenames)

class AudioCaptchaForm(forms.Form):

    def __init__(self, *args, **kwargs):
        super(AudioCaptchaForm, self).__init__(*args, **kwargs)
        self.audio_file = read_file()

    pinyin = forms.CharField()

    def is_valid(self):
        is_valid = super(AudioCaptchaForm, self).is_valid()
        if not is_valid:
            return False
        else:
            pinyin = self.cleaned_data['pinyin']\
                .replace('0', '5')\
                .replace(' ', '')
            if pinyin == PINYIN_LIST[int(self.audio_file.split('.')[0])]:
                return True
            else:
                return False

def create_base_syllables_list():
    with open('syllables_without_tones_short_list', 'r') as text_file:
        syllables = [line.strip('\n') for line in text_file.readlines()]
        selection1 = random.sample(syllables, 4)
        selection2 = [random.choice(syllables) for i in range(4)]
        sound_list = ''
        for i in range(4):
            for j in range(1, 6):
                sound_list += selection1[i] + str(j) + ';'
        for i in range(2):
            for j in range(1, 5):
                for k in range(1, 6):
                    sound_list += selection2[i] + \
                        str(j) + "_" + selection2[i + 1] + str(k) + ';'
        return sound_list

def get_random_syllables():
    with open('syllables_with_tones', 'r') as text_file:
        syllables = [line.strip('\n') for line in text_file.readlines()]
        length = random.randint(1, 4)
        while True:
            selection = random.choice(syllables)
            if selection[-1] != '5':
                break
        for i in range(length - 1):
            selection += '_' + random.choice(syllables)
        return selection

def pretty_pinyin(string):
    replacements = {
        'a1': 'ā', 'a2': 'á', 'a3': 'ǎ', 'a4': 'à', 'a5': 'a', 'e1': 'ē',
        'e2': 'é', 'e3': 'ě', 'e4': 'è', 'e5': 'e', 'i1': 'ī', 'i2': 'í',
        'i3': 'ǐ', 'i4': 'ì', 'i5': 'i', 'o1': 'ō', 'o2': 'ó', 'o3': 'ǒ',
        'o4': 'ò', 'o5': 'o', 'u1': 'ū', 'u2': 'ú', 'u3': 'ǔ', 'u4': 'ù',
        'u5': 'u', 'v1': 'ǖ', 'v2': 'ǘ', 'v3': 'ǚ', 'v4': 'ǜ', 'v5': 'ü',
        'ai1': 'āi', 'ai2': 'ái', 'ai3': 'ǎi', 'ai4': 'ài', 'ai5': 'ai',
        'ei1': 'ēi', 'ei2': 'éi', 'ei3': 'ěi', 'ei4': 'èi', 'ei5': 'ei',
        'ao1': 'āo', 'ao2': 'áo', 'ao3': 'ǎo', 'ao4': 'ào', 'ao5': 'ao',
        'ou1': 'ōu', 'ou2': 'óu', 'ou3': 'ǒu', 'ou4': 'òu', 'ou5': 'ou5',
        'an1': 'ān', 'an2': 'án', 'an3': 'ǎn', 'an4': 'àn', 'an5': 'an',
        'en1': 'ēn', 'en2': 'én', 'en3': 'ěn', 'en4': 'èn', 'en5': 'en',
        'ang1': 'āng', 'ang2': 'áng', 'ang3': 'ǎng', 'ang4': 'àng',
        'ang5': 'ang', 'eng1': 'ēng', 'eng2': 'éng', 'eng3': 'ěng',
        'eng4': 'èng', 'eng5': 'eng', 'ong1': 'ōng', 'ong2': 'óng',
        'ong3': 'ǒng', 'ong4': 'òng', 'ong5': 'ong', 'ia1': 'iā', 'ia2': 'iá',
        'ia3': 'iǎ', 'ia4': 'ià', 'ia5': 'ia', 'iao1': 'iāo', 'iao2': 'iáo',
        'iao3': 'iǎo', 'iao4': 'iào', 'iao5': 'iao', 'ie1': 'iē', 'ie2': 'ié',
        'ie3': 'iě', 'ie4': 'iè', 'ie5': 'ie', 'iu1': 'iū', 'iu2': 'iú',
        'iu3': 'iǔ', 'iu4': 'iù', 'iu5': 'iu', 'ian1': 'iān', 'ian2': 'ián',
        'ian3': 'iǎn', 'ian4': 'iàn', 'ian5': 'ian', 'in1': 'īn', 'in2': 'ín',
        'in3': 'ǐn', 'in4': 'ìn', 'in5': 'in', 'iang1': 'iāng',
        'iang2': 'iáng', 'iang3': 'iǎng', 'iang4': 'iàng', 'iang5': 'iang',
        'ing1': 'īng', 'ing2': 'íng', 'ing3': 'ǐng', 'ing4': 'ìng',
        'ing5': 'ing', 'ua1': 'uā', 'ua2': 'uá', 'ua3': 'uǎ', 'ua4': 'uà',
        'ua5': 'ua', 'uo1': 'uō', 'uo2': 'uó', 'uo3': 'uǒ', 'uo4': 'uò',
        'uo5': 'uo', 'uai1': 'uāi', 'uai2': 'uái', 'uai3': 'uǎi',
        'uai4': 'uài', 'uai5': 'uai', 'ui1': 'uī', 'ui2': 'uí', 'ui3': 'uǐ',
        'ui4': 'uì', 'ui5': 'ui', 'uan1': 'uān', 'uan2': 'uán', 'uan3': 'uǎn',
        'uan4': 'uàn', 'uan5': 'uan', 'un1': 'ūn', 'un2': 'ún', 'un3': 'ǔn',
        'un4': 'ùn', 'un5': 'un', 'uang1': 'uāng', 'uang2': 'uáng',
        'uang3': 'uǎng', 'uang4': 'uàng', 'uang5': 'uang', 'ue1': 'uē',
        'ue2': 'ué', 'ue3': 'uě', 'ue4': 'uè', 'ue5': 'ue', 've1': 'üē',
        've2': 'üé', 've3': 'üě', 've4': 'üè', 've5': 'üe', 'uan1': 'uān',
        'uan2': 'uán', 'uan3': 'uǎn', 'uan4': 'uàn', 'uan5': 'uan',
        'un1': 'ūn', 'un2': 'ún', 'un3': 'ǔn', 'un4': 'ùn', 'un5': 'un', }

    result = None
    for substring in string.split('_'):
        for src, target in replacements.items():
            p = re.compile('([b-df-hj-np-tv-z]+|^)' + src)
            if p.match(substring):
                if result:
                    result += ' ' + p.sub(r'\1' + target, substring)
                else:
                    result = p.sub(r'\1' + target, substring)
    return result
<commit_msg>ref: Make first- and lastname optional<commit_after>from django import forms
from os import walk
import random
import re
from django.contrib.auth.forms import UserCreationForm
from django.utils.translation import ugettext_lazy as _
from django.utils.html import format_html
from .contributor import Contributor

AUDIO_DIR = 'userregistration/static'

class CustomUserCreationForm(UserCreationForm):
    first_name = forms.CharField(
            max_length=30,
            label=_('First Name'),
            required=False,
            )
    last_name = forms.CharField(
            max_length=30,
            label=_('Last Name'),
            required=False,
            )

    username = forms.EmailField(
        label=_("e-mail address"),
        strip=True,
        widget=forms.EmailInput,
    )

    def save(self, commit=True):
        user = super(CustomUserCreationForm, self).save(commit=False)
        user.first_name = self.cleaned_data["first_name"]
        user.last_name = self.cleaned_data["last_name"]
        user.username = self.cleaned_data["username"]
        if commit:
            user.save()
        return user

class ContributorCreationForm(forms.ModelForm):

    class Meta:
        model = Contributor
        fields = ['sex', 'accepted_licence']

    sex = forms.ChoiceField(
        choices=Contributor.SEX_CHOISES + ((None, _('I am...')),)
    )

    accepted_licence = forms.BooleanField(
        label=_(format_html(
                "I accept that my contributions are released under the \
                  \
                 Creative Commons CC BY-SA 4.0 license.")),
        required=True
    )

    def save(self, commit=True):
        contributor = super(ContributorCreationForm, self).save(commit=False)
        contributor.sex = self.cleaned_data["sex"]
        contributor.syllables_list = create_base_syllables_list()
        if commit:
            contributor.save()
        return contributor

# PINYIN_LIST = ['ai1','ai2','ai3','ai4','ai5',
#         'ma1','ma2','ma3','ma4','ma5',
#         'ma1','ma2','ma3','ma4','ma5',
#         'ma1','ma2','ma3','ma4','ma5',
#         'ma1','ma2','ma3','ma4','ma5',
#         'wo1','wo2','wo3','wo4','wo5',
#         ]
PINYIN_LIST = ['ai1']

def read_file():
    """
    return a random filename from the audio directory
    """
    _, _, filenames = next(walk(AUDIO_DIR))
    return random.choice(filenames)

class AudioCaptchaForm(forms.Form):

    def __init__(self, *args, **kwargs):
        super(AudioCaptchaForm, self).__init__(*args, **kwargs)
        self.audio_file = read_file()

    pinyin = forms.CharField()

    def is_valid(self):
        is_valid = super(AudioCaptchaForm, self).is_valid()
        if not is_valid:
            return False
        else:
            pinyin = self.cleaned_data['pinyin']\
                .replace('0', '5')\
                .replace(' ', '')
            if pinyin == PINYIN_LIST[int(self.audio_file.split('.')[0])]:
                return True
            else:
                return False

def create_base_syllables_list():
    with open('syllables_without_tones_short_list', 'r') as text_file:
        syllables = [line.strip('\n') for line in text_file.readlines()]
        selection1 = random.sample(syllables, 4)
        selection2 = [random.choice(syllables) for i in range(4)]
        sound_list = ''
        for i in range(4):
            for j in range(1, 6):
                sound_list += selection1[i] + str(j) + ';'
        for i in range(2):
            for j in range(1, 5):
                for k in range(1, 6):
                    sound_list += selection2[i] + \
                        str(j) + "_" + selection2[i + 1] + str(k) + ';'
        return sound_list

def get_random_syllables():
    with open('syllables_with_tones', 'r') as text_file:
        syllables = [line.strip('\n') for line in text_file.readlines()]
        length = random.randint(1, 4)
        while True:
            selection = random.choice(syllables)
            if selection[-1] != '5':
                break
        for i in range(length - 1):
            selection += '_' + random.choice(syllables)
        return selection

def pretty_pinyin(string):
    replacements = {
        'a1': 'ā', 'a2': 'á', 'a3': 'ǎ', 'a4': 'à', 'a5': 'a', 'e1': 'ē',
        'e2': 'é', 'e3': 'ě', 'e4': 'è', 'e5': 'e', 'i1': 'ī', 'i2': 'í',
        'i3': 'ǐ', 'i4': 'ì', 'i5': 'i', 'o1': 'ō', 'o2': 'ó', 'o3': 'ǒ',
        'o4': 'ò', 'o5': 'o', 'u1': 'ū', 'u2': 'ú', 'u3': 'ǔ', 'u4': 'ù',
        'u5': 'u', 'v1': 'ǖ', 'v2': 'ǘ', 'v3': 'ǚ', 'v4': 'ǜ', 'v5': 'ü',
        'ai1': 'āi', 'ai2': 'ái', 'ai3': 'ǎi', 'ai4': 'ài', 'ai5': 'ai',
        'ei1': 'ēi', 'ei2': 'éi', 'ei3': 'ěi', 'ei4': 'èi', 'ei5': 'ei',
        'ao1': 'āo', 'ao2': 'áo', 'ao3': 'ǎo', 'ao4': 'ào', 'ao5': 'ao',
        'ou1': 'ōu', 'ou2': 'óu', 'ou3': 'ǒu', 'ou4': 'òu', 'ou5': 'ou5',
        'an1': 'ān', 'an2': 'án', 'an3': 'ǎn', 'an4': 'àn', 'an5': 'an',
        'en1': 'ēn', 'en2': 'én', 'en3': 'ěn', 'en4': 'èn', 'en5': 'en',
        'ang1': 'āng', 'ang2': 'áng', 'ang3': 'ǎng', 'ang4': 'àng',
        'ang5': 'ang', 'eng1': 'ēng', 'eng2': 'éng', 'eng3': 'ěng',
        'eng4': 'èng', 'eng5': 'eng', 'ong1': 'ōng', 'ong2': 'óng',
        'ong3': 'ǒng', 'ong4': 'òng', 'ong5': 'ong', 'ia1': 'iā', 'ia2': 'iá',
        'ia3': 'iǎ', 'ia4': 'ià', 'ia5': 'ia', 'iao1': 'iāo', 'iao2': 'iáo',
        'iao3': 'iǎo', 'iao4': 'iào', 'iao5': 'iao', 'ie1': 'iē', 'ie2': 'ié',
        'ie3': 'iě', 'ie4': 'iè', 'ie5': 'ie', 'iu1': 'iū', 'iu2': 'iú',
        'iu3': 'iǔ', 'iu4': 'iù', 'iu5': 'iu', 'ian1': 'iān', 'ian2': 'ián',
        'ian3': 'iǎn', 'ian4': 'iàn', 'ian5': 'ian', 'in1': 'īn', 'in2': 'ín',
        'in3': 'ǐn', 'in4': 'ìn', 'in5': 'in', 'iang1': 'iāng',
        'iang2': 'iáng', 'iang3': 'iǎng', 'iang4': 'iàng', 'iang5': 'iang',
        'ing1': 'īng', 'ing2': 'íng', 'ing3': 'ǐng', 'ing4': 'ìng',
        'ing5': 'ing', 'ua1': 'uā', 'ua2': 'uá', 'ua3': 'uǎ', 'ua4': 'uà',
        'ua5': 'ua', 'uo1': 'uō', 'uo2': 'uó', 'uo3': 'uǒ', 'uo4': 'uò',
        'uo5': 'uo', 'uai1': 'uāi', 'uai2': 'uái', 'uai3': 'uǎi',
        'uai4': 'uài', 'uai5': 'uai', 'ui1': 'uī', 'ui2': 'uí', 'ui3': 'uǐ',
        'ui4': 'uì', 'ui5': 'ui', 'uan1': 'uān', 'uan2': 'uán', 'uan3': 'uǎn',
        'uan4': 'uàn', 'uan5': 'uan', 'un1': 'ūn', 'un2': 'ún', 'un3': 'ǔn',
        'un4': 'ùn', 'un5': 'un', 'uang1': 'uāng', 'uang2': 'uáng',
        'uang3': 'uǎng', 'uang4': 'uàng', 'uang5': 'uang', 'ue1': 'uē',
        'ue2': 'ué', 'ue3': 'uě', 'ue4': 'uè', 'ue5': 'ue', 've1': 'üē',
        've2': 'üé', 've3': 'üě', 've4': 'üè', 've5': 'üe', 'uan1': 'uān',
        'uan2': 'uán', 'uan3': 'uǎn', 'uan4': 'uàn', 'uan5': 'uan',
        'un1': 'ūn', 'un2': 'ún', 'un3': 'ǔn', 'un4': 'ùn', 'un5': 'un', }

    result = None
    for substring in string.split('_'):
        for src, target in replacements.items():
            p = re.compile('([b-df-hj-np-tv-z]+|^)' + src)
            if p.match(substring):
                if result:
                    result += ' ' + p.sub(r'\1' + target, substring)
                else:
                    result = p.sub(r'\1' + target, substring)
    return result
<|endoftext|>


<commit_before># from wakariserver.. but copy pasted becuase we dont wnat
# wakariserver dependency here.
import shutil
import os
script_paths = [
    'vendor/jquery.1.7.2.min.js',
    'vendor/underscore/underscore.js',
    'vendor/backbone/backbone.js',
    "vendor/jquery/js/jquery.mousewheel.js",
    "vendor/buckets/buckets.js",
    "vendor/d3/d3.v2.js",
    "vendor/bokehjs/js/base.js",
    "vendor/bokehjs/js/view_base.js",
    "vendor/bokehjs/js/model_base.js",
    "vendor/bokehjs/js/ui_view.js",
    "vendor/bokehjs/js/ui_model.js",
    "vendor/bokehjs/js/bokeh_views.js",
    "vendor/bokehjs/js/ticks.js",
    "vendor/bokehjs/js/tools.js",
    "vendor/bokehjs/js/bokeh_model.js",
    "vendor/bokehjs/js/custom.js",
    ]

notebook_script_paths = script_paths

css_paths = [
    'vendor/bokehjs/css/bokeh.css',
    'vendor/bokehjs/css/continuum.css',
    ]

def static_join(*dirs):
    root = os.path.dirname(__file__)
    return os.path.join(root, "server", "static", *dirs)

def css_str(paths):
    retval = ""
    for css_path in paths:
        retval += """
                <link href="%s" rel="stylesheet"/>
                """  % ("./static/" + css_path)
    return retval

def script_str(paths):
    retval = ""
    for script in paths:
        retval += """
           <script type=text/javascript src="%s"></script>
           """ % ("./static/" + script)
    return retval

def concat_scripts(paths):
    output_str = ""
    for script in paths:
        f_name = static_join(script)
        output_str += """
          // BEGIN %s
          """ % f_name

        output_str += open(f_name).read()
        output_str += """
          // END %s
         """ % f_name
    return output_str

def concat_css(paths):
    output_str = ""
    for css_path in paths:
        f_name = static_join(css_path)
        output_str += """
          /* BEGIN %s */
         """ % f_name

        output_str += open(f_name).read().decode("utf-8")
        output_str += """
          /* END %s */
         """ % f_name
    return output_str

def dump_paths(paths, dest):
    for path in paths:
        source = os.path.join("static", path)
        dest = os.path.join(dest, "static", path)
        shutil.copy(source, dest)

def inline_scripts(script_paths):
    js = concat_scripts(script_paths)
    jsstr = """
<script type=text/javascript>
%s
</script>
"""
    jsstr = jsstr % js
    return jsstr

def inline_css(css_paths):
    css = concat_css(css_paths)
    cssstr = """
<style>
%s
</style>
"""
    cssstr = cssstr % css
    return cssstr
<commit_msg>new bokehjs files<commit_after># from wakariserver.. but copy pasted becuase we dont wnat
# wakariserver dependency here.
import shutil
import os
script_paths = [
    'vendor/jquery.1.7.2.min.js',
    'vendor/underscore/underscore.js',
    'vendor/backbone/backbone.js',
    "vendor/jquery/js/jquery.mousewheel.js",
    "vendor/buckets/buckets.js",
    "vendor/d3/d3.v2.js",
    "vendor/bokehjs/js/base.js",
    "vendor/bokehjs/js/datasource.js",
    "vendor/bokehjs/js/ranges.js",
    "vendor/bokehjs/js/guides.js",
    "vendor/bokehjs/js/mapper.js",
    "vendor/bokehjs/js/schema_renderers.js",
    "vendor/bokehjs/js/glyph_renderers.js",
    "vendor/bokehjs/js/table.js",
    "vendor/bokehjs/js/tools.js",
    "vendor/bokehjs/js/overlays.js",
    "vendor/bokehjs/js/ticks.js",
    "vendor/bokehjs/js/container.js",
    "vendor/bokehjs/js/testutils.js",
    "vendor/bokehjs/js/custom.js",
    ]

notebook_script_paths = script_paths

css_paths = [
    'vendor/bokehjs/css/bokeh.css',
    'vendor/bokehjs/css/continuum.css',
    ]

def static_join(*dirs):
    root = os.path.dirname(__file__)
    return os.path.join(root, "server", "static", *dirs)

def css_str(paths):
    retval = ""
    for css_path in paths:
        retval += """
                <link href="%s" rel="stylesheet"/>
                """  % ("./static/" + css_path)
    return retval

def script_str(paths):
    retval = ""
    for script in paths:
        retval += """
           <script type=text/javascript src="%s"></script>
           """ % ("./static/" + script)
    return retval

def concat_scripts(paths):
    output_str = ""
    for script in paths:
        f_name = static_join(script)
        output_str += """
          // BEGIN %s
          """ % f_name

        output_str += open(f_name).read()
        output_str += """
          // END %s
         """ % f_name
    return output_str

def concat_css(paths):
    output_str = ""
    for css_path in paths:
        f_name = static_join(css_path)
        output_str += """
          /* BEGIN %s */
         """ % f_name

        output_str += open(f_name).read().decode("utf-8")
        output_str += """
          /* END %s */
         """ % f_name
    return output_str

def dump_paths(paths, dest):
    for path in paths:
        source = os.path.join("static", path)
        dest = os.path.join(dest, "static", path)
        shutil.copy(source, dest)

def inline_scripts(script_paths):
    js = concat_scripts(script_paths)
    jsstr = """
<script type=text/javascript>
%s
</script>
"""
    jsstr = jsstr % js
    return jsstr

def inline_css(css_paths):
    css = concat_css(css_paths)
    cssstr = """
<style>
%s
</style>
"""
    cssstr = cssstr % css
    return cssstr
<|endoftext|>


<commit_before>#  Copyright 2008-2010 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import sys
import os

from unic import unic

def decode_output(string):
    """Decodes string from console encoding to Unicode."""
    if _output_encoding:
        return unic(string, _output_encoding)
    return string

def encode_output(string, errors='replace'):
    """Encodes string from Unicode to console encoding."""
    return string.encode(_output_encoding, errors)

def _get_output_encoding():
    encoding = sys.__stdout__.encoding or sys.__stdin__.encoding
    if os.sep == '/':
        return encoding or _read_encoding_from_env()
    # Use default DOS encoding if no encoding found (guess)
    # or on buggy Jython 2.5: http://bugs.jython.org/issue1568
    if not encoding or sys.platform.startswith('java'):
        return 'cp437'
    return encoding

def _read_encoding_from_env():
    for name in 'LANG', 'LC_CTYPE', 'LANGUAGE', 'LC_ALL':
        if name in os.environ:
            return os.environ[name].split('.')[-1]
    return None

_output_encoding = _get_output_encoding()
<commit_msg>Fixed user name and email Default encoding provided, issue 614.<commit_after>#  Copyright 2008-2010 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import sys
import os

from unic import unic

def decode_output(string):
    """Decodes string from console encoding to Unicode."""
    if _output_encoding:
        return unic(string, _output_encoding)
    return string

def encode_output(string, errors='replace'):
    """Encodes string from Unicode to console encoding."""
    return string.encode(_output_encoding, errors)

def _get_output_encoding():
    encoding = sys.__stdout__.encoding or sys.__stdin__.encoding
    if os.sep == '/':
        return encoding or _read_encoding_from_env()
    # Use default DOS encoding if no encoding found (guess)
    # or on buggy Jython 2.5: http://bugs.jython.org/issue1568
    if not encoding or sys.platform.startswith('java'):
        return 'cp437'
    return encoding

def _read_encoding_from_env():
    for name in 'LANG', 'LC_CTYPE', 'LANGUAGE', 'LC_ALL':
        if name in os.environ:
            return os.environ[name].split('.')[-1]
    return 'ascii'

_output_encoding = _get_output_encoding()
<|endoftext|>