<commit_before>import sys
import os
import logging

import utilities

from ufal.udpipe import Model,Pipeline,ProcessingError

class UDpipeparser():

    def __init__(self, modeldir):

        self.modeldir = os.path.abspath(modeldir)
        self.model = Model.load(self.modeldir)
        self.error = ProcessingError()

        if not self.model:
            sys.stderr.write("Udpipe language model loading failed:"+self.modeldir)
            sys.exit(1)

        self.pipeline = Pipeline(self.model,"tokenize",Pipeline.DEFAULT,Pipeline.DEFAULT,"conllu")

    def udpipe_parse_sent(self, text):
        logger = logging.getLogger('petr_log')

        processed = self.pipeline.process(text,self.error)

        if self.error.occurred():
            raise ValueError(self.error.message)

        #print(processed)
        parsed = []
        for line in processed.split("\n"):
            if line.startswith("#"):
                continue
            parsed.append(line)

        #print(("\n").join(parsed))
        #input(" ")
        return ("\n").join(parsed)

    def udpipe_parse_events(self, events):
        logger = logging.getLogger('petr_log')

        #modeldir = os.path.abspath(PETRglobals.udpipe_modsel_dir)

        #self.model = Model.load(modeldir)
        #self.pipeline = Pipeline(model,"tokenize",Pipeline.DEFAULT,Pipeline.DEFAULT,"conllu")

        total = len(list(event_dict.keys()))
        logger.info("Starting parse of {} stories.'.format(total)")

        for i, key in enumerate(event_dict.keys()):
            if (i / float(total)) * 100 in [10.0, 25.0, 50, 75.0]:
                print('Parse is {}% complete...'.format((i / float(total)) * 100))
            for sent in event_dict[key]['sents']:
                logger.info('Udpipe parsing {}_{}...'.format(key, sent))
                sent_dict = event_dict[key]['sents'][sent]

                processed = self.pipeline.process(sent_dict['content'],self.error)

                if self.error.occurred():
                    #sys.stderr.write("An error occurred when running run_udpipe: ")
                    #sys.stderr.write(error.message)
                    #sys.stderr.write("\n")
                    raise ValueError(self.error.message)

                parsed = []
                for line in processed.split("\n"):
                    if line.startswith("#"):
                        continue
                    parsed.append(line)

                sent_dict['parsed'] = utilities._format_ud_parsed_str(("\n").join(parsed))
                print(sent_dict['parsed'])
                input("")

        #print('Done with UDpipe parse...\n\n')
        logger.info('Done with UDpipe parse.')
        return event_dict
<commit_msg>remove accents of noun phrases in preprocessing<commit_after>import sys
import os
import logging

import utilities
import unidecode

from ufal.udpipe import Model,Pipeline,ProcessingError

class UDpipeparser():

    def __init__(self, modeldir):

        self.modeldir = os.path.abspath(modeldir)
        self.model = Model.load(self.modeldir)
        self.error = ProcessingError()

        if not self.model:
            sys.stderr.write("Udpipe language model loading failed:"+self.modeldir)
            sys.exit(1)

        self.pipeline = Pipeline(self.model,"tokenize",Pipeline.DEFAULT,Pipeline.DEFAULT,"conllu")

    def udpipe_parse_sent(self, text):
        logger = logging.getLogger('petr_log')

        processed = self.pipeline.process(text,self.error)

        if self.error.occurred():
            raise ValueError(self.error.message)

        #print(processed)
        parsed = []
        for line in processed.split("\n"):
            if line.startswith("#"):
                continue
            if "VERB\t" not in line:
                line = unidecode.unidecode(line)
            parsed.append(line)

        #print(("\n").join(parsed))
        #input(" ")
        return ("\n").join(parsed)

    def udpipe_parse_events(self, events):
        logger = logging.getLogger('petr_log')

        #modeldir = os.path.abspath(PETRglobals.udpipe_modsel_dir)

        #self.model = Model.load(modeldir)
        #self.pipeline = Pipeline(model,"tokenize",Pipeline.DEFAULT,Pipeline.DEFAULT,"conllu")

        total = len(list(event_dict.keys()))
        logger.info("Starting parse of {} stories.'.format(total)")

        for i, key in enumerate(event_dict.keys()):
            if (i / float(total)) * 100 in [10.0, 25.0, 50, 75.0]:
                print('Parse is {}% complete...'.format((i / float(total)) * 100))
            for sent in event_dict[key]['sents']:
                logger.info('Udpipe parsing {}_{}...'.format(key, sent))
                sent_dict = event_dict[key]['sents'][sent]

                processed = self.pipeline.process(sent_dict['content'],self.error)

                if self.error.occurred():
                    #sys.stderr.write("An error occurred when running run_udpipe: ")
                    #sys.stderr.write(error.message)
                    #sys.stderr.write("\n")
                    raise ValueError(self.error.message)

                parsed = []
                for line in processed.split("\n"):
                    if line.startswith("#"):
                        continue
                    parsed.append(line)

                sent_dict['parsed'] = utilities._format_ud_parsed_str(("\n").join(parsed))
                print(sent_dict['parsed'])
                input("")

        #print('Done with UDpipe parse...\n\n')
        logger.info('Done with UDpipe parse.')
        return event_dict
<|endoftext|>


<commit_before># coding: utf-8

import urllib.parse
import functools

import lglass.rpsl

class Database(object):
	""" Database is an abstract class which defines some constants and setter/getter
	for subscripts. You have to extend from it by creating a new class and
	overriding __init__, get, list, find, save and delete to conform to the
	database protocol. """

	def __init__(self):
		raise NotImplementedError("Instances of Database are not permitted")

	def get(self, type, primary_key):
		""" Get specific object addressed by type and primary_key from database. Returns
		lglass.rpsl.Object. This method shall raise a KeyError if the object was not
		found. """
		raise NotImplementedError("get() is not implemented")

	def list(self, filter=None, limit=None):
		""" Return list of matching RPSL object specs, filter can be a callable taking
		two arguments (type, primary_key), and limit can be a int. RPSL object specs
		are tuples consisting of two str instances. """
		raise NotImplementedError("list() is not implemented")

	def find(self, key, types=None, limit=None):
		""" Finds an object by searching the whole database for key. It's possible
		to supply a list of acceptable object types and to provide a limit of objects.
		This method returns a list of lglass.rpsl.Object. """
		raise NotImplementedError("find() is not implemented")

	def save(self, obj):
		""" Save object in database. """
		raise NotImplementedError("save() is not implemented")

	def delete(self, type, primary_key):
		""" Delete object in database. """
		raise NotImplementedError("delete() is not implemented")

	def flush(self):
		""" Flush database cache """
		raise NotImplementedError("flush() is not implemented")

	object_types = {
		"as-block",
		"as-set",
		"aut-num",
		"dns",
		"filter-set",
		"inet6num",
		"inetnum",
		"inet-rtr",
		"irt",
		"key-cert",
		"mntner",
		"organisation",
		"peering-set",
		"person",
		"poem",
		"poetic-form",
		"role",
		"route",
		"route6",
		"route-set",
		"rtr-set",
		"schema"
	}

	def save_all(self, objs):
		for obj in objs:
			self.save(obj)

	def get_all(self):
		return (self.get(*spec) for spec in self.list())

	def schema(self, type):
		""" Return schema for type. Raises a KeyError if schema was not found.  """
		if type == "schema":
			return lglass.rpsl.SchemaObject.SCHEMA_SCHEMA
		name = type.upper() + "-SCHEMA"
		specs = [("schema", name), ("schemas", name), ("schema", type),
			("schemas", type)]
		obj = None
		for spec in specs:
			try:
				obj = self.get(*spec)
			except KeyError:
				continue
			break
		if obj is None:
			raise KeyError("schema({})".format(type))
		return lglass.rpsl.SchemaObject(obj)

	def __len__(self):
		return len(self.list())

	def __iter__(self):
		for type, primary_key in self.list():
			yield self.get(type, primary_key)

	def __contains__(self, key):
		if not isinstance(key, tuple):
			raise TypeError("Expected key to be tuple of length 2, got {}".format(key))
		try:
			self.get(*key)
		except KeyError:
			return False
		return True

	def __getitem__(self, key):
		if not isinstance(key, tuple):
			raise TypeError("Expected key to be tuple of length 2, got {}".format(key))
		return self.get(*key)

	def __delitem__(self, key):
		if not isinstance(key, tuple):
			raise TypeError("Expected key to be tuple of length 2, got {}".format(key))
		self.delete(*key)

url_schemes = {}

def register(scheme_or_cls):
	""" This decorator adds the supplied class to the url_schemes dict with
	the schema specified in `name` """
	def decorator(cls):
		if hasattr(cls, "from_url") and callable(cls.from_url):
			url_schemes[(cls.__module__ + "." + cls.__name__).lower()] = cls
			if name is not None:
				url_schemes[name.lower()] = cls
		return cls

	name = None
	if isinstance(scheme_or_cls, type):
		return decorator(scheme_or_cls)
	else:
		name = scheme_or_cls
		return functools.wraps(scheme_or_cls)(decorator)

def from_url(url):
	""" Create database from URL. This function accepts a str or an URL tuple.
	The URL schema may have multiple formats:

		1. {db_name}
		2. whois+{db_name}
		3. whois+{module}+{db_name}
	"""
	import importlib
	if isinstance(url, str):
		url = urllib.parse.urlparse(url)
	scheme = url.scheme.split("+", 2)
	if len(scheme) == 1:
		scheme = scheme[0]
		if "." in scheme:
			importlib.import_module(scheme.rsplit(".", 1)[0])
	elif len(scheme) == 2:
		assert scheme[0] == "whois"
		scheme = scheme[1]
	elif len(scheme) == 3:
		assert scheme[0] == "whois"
		importlib.import_module(scheme[1])
		scheme = scheme[2]
	return url_schemes[scheme].from_url(url)

def build_chain(urls):
	db = None
	for url in urls:
		new_db = from_url(url)
		if db is not None:
			new_db.database = db
		db = new_db
	return db

@register("dict")
class DictDatabase(Database):
	""" This database backend operates completely in memory by using a Python
	dictionary to organize the information. It uses only builtin Python data types
	like list, tuple, and dict. """

	def __init__(self):
		self.backend = dict()

	def save(self, object):
		self.backend[object.real_spec] = object

	def delete(self, type, primary_key):
		del self.backend[type, primary_key]

	def get(self, type, primary_key):
		return self.backend[type, primary_key]

	def list(self):
		return list(self.backend.keys())

	def find(self, primary_key, types=None):
		objects = []
		for type, pkey in self.backend.keys():
			if pkey != primary_key:
				continue
			if types is not None and type not in types:
				continue
			objects.append((type, pkey))
		return [self.get(*spec) for spec in objects]

	@classmethod
	def from_url(cls, url):
		return cls()

<commit_msg>Also add class name to url scheme lookup dict<commit_after># coding: utf-8

import urllib.parse
import functools

import lglass.rpsl

class Database(object):
	""" Database is an abstract class which defines some constants and setter/getter
	for subscripts. You have to extend from it by creating a new class and
	overriding __init__, get, list, find, save and delete to conform to the
	database protocol. """

	def __init__(self):
		raise NotImplementedError("Instances of Database are not permitted")

	def get(self, type, primary_key):
		""" Get specific object addressed by type and primary_key from database. Returns
		lglass.rpsl.Object. This method shall raise a KeyError if the object was not
		found. """
		raise NotImplementedError("get() is not implemented")

	def list(self, filter=None, limit=None):
		""" Return list of matching RPSL object specs, filter can be a callable taking
		two arguments (type, primary_key), and limit can be a int. RPSL object specs
		are tuples consisting of two str instances. """
		raise NotImplementedError("list() is not implemented")

	def find(self, key, types=None, limit=None):
		""" Finds an object by searching the whole database for key. It's possible
		to supply a list of acceptable object types and to provide a limit of objects.
		This method returns a list of lglass.rpsl.Object. """
		raise NotImplementedError("find() is not implemented")

	def save(self, obj):
		""" Save object in database. """
		raise NotImplementedError("save() is not implemented")

	def delete(self, type, primary_key):
		""" Delete object in database. """
		raise NotImplementedError("delete() is not implemented")

	def flush(self):
		""" Flush database cache """
		raise NotImplementedError("flush() is not implemented")

	object_types = {
		"as-block",
		"as-set",
		"aut-num",
		"dns",
		"filter-set",
		"inet6num",
		"inetnum",
		"inet-rtr",
		"irt",
		"key-cert",
		"mntner",
		"organisation",
		"peering-set",
		"person",
		"poem",
		"poetic-form",
		"role",
		"route",
		"route6",
		"route-set",
		"rtr-set",
		"schema"
	}

	def save_all(self, objs):
		for obj in objs:
			self.save(obj)

	def get_all(self):
		return (self.get(*spec) for spec in self.list())

	def schema(self, type):
		""" Return schema for type. Raises a KeyError if schema was not found.  """
		if type == "schema":
			return lglass.rpsl.SchemaObject.SCHEMA_SCHEMA
		name = type.upper() + "-SCHEMA"
		specs = [("schema", name), ("schemas", name), ("schema", type),
			("schemas", type)]
		obj = None
		for spec in specs:
			try:
				obj = self.get(*spec)
			except KeyError:
				continue
			break
		if obj is None:
			raise KeyError("schema({})".format(type))
		return lglass.rpsl.SchemaObject(obj)

	def __len__(self):
		return len(self.list())

	def __iter__(self):
		for type, primary_key in self.list():
			yield self.get(type, primary_key)

	def __contains__(self, key):
		if not isinstance(key, tuple):
			raise TypeError("Expected key to be tuple of length 2, got {}".format(key))
		try:
			self.get(*key)
		except KeyError:
			return False
		return True

	def __getitem__(self, key):
		if not isinstance(key, tuple):
			raise TypeError("Expected key to be tuple of length 2, got {}".format(key))
		return self.get(*key)

	def __delitem__(self, key):
		if not isinstance(key, tuple):
			raise TypeError("Expected key to be tuple of length 2, got {}".format(key))
		self.delete(*key)

url_schemes = {}

def register(scheme_or_cls):
	""" This decorator adds the supplied class to the url_schemes dict with
	the schema specified in `name` """
	def decorator(cls):
		if hasattr(cls, "from_url") and callable(cls.from_url):
			url_schemes[(cls.__module__ + "." + cls.__name__).lower()] = cls
			url_schemes[(cls.__name__).lower()] = cls
			if name is not None:
				url_schemes[name.lower()] = cls
		return cls

	name = None
	if isinstance(scheme_or_cls, type):
		return decorator(scheme_or_cls)
	else:
		name = scheme_or_cls
		return functools.wraps(scheme_or_cls)(decorator)

def from_url(url):
	""" Create database from URL. This function accepts a str or an URL tuple.
	The URL schema may have multiple formats:

		1. {db_name}
		2. whois+{db_name}
		3. whois+{module}+{db_name}
	"""
	import importlib
	if isinstance(url, str):
		url = urllib.parse.urlparse(url)
	scheme = url.scheme.split("+", 2)
	if len(scheme) == 1:
		scheme = scheme[0]
		if "." in scheme:
			importlib.import_module(scheme.rsplit(".", 1)[0])
	elif len(scheme) == 2:
		assert scheme[0] == "whois"
		scheme = scheme[1]
	elif len(scheme) == 3:
		assert scheme[0] == "whois"
		importlib.import_module(scheme[1])
		scheme = scheme[2]
	return url_schemes[scheme].from_url(url)

def build_chain(urls):
	db = None
	for url in urls:
		new_db = from_url(url)
		if db is not None:
			new_db.database = db
		db = new_db
	return db

@register("dict")
class DictDatabase(Database):
	""" This database backend operates completely in memory by using a Python
	dictionary to organize the information. It uses only builtin Python data types
	like list, tuple, and dict. """

	def __init__(self):
		self.backend = dict()

	def save(self, object):
		self.backend[object.real_spec] = object

	def delete(self, type, primary_key):
		del self.backend[type, primary_key]

	def get(self, type, primary_key):
		return self.backend[type, primary_key]

	def list(self):
		return list(self.backend.keys())

	def find(self, primary_key, types=None):
		objects = []
		for type, pkey in self.backend.keys():
			if pkey != primary_key:
				continue
			if types is not None and type not in types:
				continue
			objects.append((type, pkey))
		return [self.get(*spec) for spec in objects]

	@classmethod
	def from_url(cls, url):
		return cls()

<|endoftext|>


<commit_before>"""
Mock http responses
"""

from httpretty import HTTPretty
from stubdata.solr import example_solr_response

class MockSolrResponse:
    """
    context manager that mocks a Solr response
    """
    def __init__(self, api_endpoint):
        """
        :param api_endpoint: name of the API end point
        """

        self.api_endpoint = api_endpoint

        def request_callback(request, uri, headers):
            """
            :param request: HTTP request
            :param uri: URI/URL to send the request
            :param headers: header of the HTTP request
            :return: httpretty response
            """
            return 200, headers, example_solr_response

        HTTPretty.register_uri(
            HTTPretty.GET,
            self.api_endpoint,
            body=request_callback,
            content_type="application/json"
        )

    def __enter__(self):
        HTTPretty.enable()

    def __exit__(self, etype, value, traceback):
        """
        :param etype: exit type
        :param value: exit value
        :param traceback: the traceback for the exit
        """
        HTTPretty.reset()
        HTTPretty.disable()<commit_msg>tests: Mocks should inheret from base Mock context manager<commit_after>"""
Mock http responses
"""

from httpretty import HTTPretty
from stubdata.solr import example_solr_response

class HTTPrettyMock(object):
    """
    httpretty context manager scaffolding
    """

    def __enter__(self):
        HTTPretty.enable()

    def __exit__(self, etype, value, traceback):
        """
        :param etype: exit type
        :param value: exit value
        :param traceback: the traceback for the exit
        """
        HTTPretty.reset()
        HTTPretty.disable()

class MockSolrResponse(HTTPrettyMock):
    """
    context manager that mocks a Solr response
    """
    def __init__(self, api_endpoint):
        """
        :param api_endpoint: name of the API end point
        """

        self.api_endpoint = api_endpoint

        def request_callback(request, uri, headers):
            """
            :param request: HTTP request
            :param uri: URI/URL to send the request
            :param headers: header of the HTTP request
            :return: httpretty response
            """
            return 200, headers, example_solr_response

        HTTPretty.register_uri(
            HTTPretty.GET,
            self.api_endpoint,
            body=request_callback,
            content_type="application/json"
        )<|endoftext|>


<commit_before>"""
generic workspace analysis module to seek through the undiscovered
country looking for pointers to interesting things.

in a previous life, this analysis code lived inside VivWorkspace.analyze()
"""
import logging
from vivisect.const import RTYPE_BASEPTR, LOC_POINTER

logger = logging.getLogger(__name__)

def analyze(vw):

    logger.info('...analyzing pointers.')

    done = []

    # Let's analyze Relocations we know are pointers
    for rva, rtype in vw.reloc_by_va.items():
        if rtype != RTYPE_BASEPTR:
            continue

        for xfr, xto, xtype, xinfo in vw.getXrefsFrom(rva):
            vw.analyzePointer(xto)
            done.append((xfr, xto))
            logger.info('pointer(1): 0x%x -> 0x%x', xfr, xto)

    # Now, we'll analyze the pointers placed by the file wrapper (ELF, PE, MACHO, etc...)
    for pva, tva, fname, pname in vw.getVaSetRows('PointersFromFile'):
        if vw.getLocation(pva) is None:
            if tva is None:
                logger.info('making pointer 0x%x (%r)', pva, pname)
            else:
                logger.info('making pointer 0x%x -> 0x%x (%r)', pva, tva, pname)
            vw.makePointer(pva, tva, follow=True)
            done.append((pva, tva))
            logger.info('pointer(2): 0x%x -> 0x%x', pva, tva)

    for lva, lsz, lt, li in vw.getLocations(LOC_POINTER):
        tva = vw.readMemoryPtr(lva)
        if not vw.isValidPointer(tva):
            continue

        if vw.getLocation(tva) is not None:
            continue

        logger.info('following previously discovered pointer 0x%x -> 0x%x', lva, tva)
        try:
            vw.followPointer(tva)
            done.append((lva, tva))
            logger.info('pointer(3): 0x%x -> 0x%x', lva, tva)
        except Exception:
            logger.info("followPointer() failed for 0x%.8x (pval: 0x%.8x)" % (lva, tva))

    # Now, lets find likely free-hanging pointers
    for addr, pval in vw.findPointers():
        if vw.isDeadData(pval):
            continue
        try:
            vw.makePointer(addr, follow=True)
            done.append((addr, pval))
            logger.info('pointer(4): 0x%x -> 0x%x', addr, pval)
        except Exception:
            logger.info("followPointer() failed for 0x%.8x (pval: 0x%.8x)" % (addr, pval))

    # Now let's see what these guys should be named (if anything)
    for ptr, tgt in done:
        pname = vw.getName(ptr)
        if pname is not None:
            logger.info('skipping renaming of 0x%x (%r)', ptr, pname)
            continue

        tgtname = vw.getName(tgt)
        if tgtname is not None:
            name = vw._addNamePrefix(tgtname, tgt, 'ptr', '_') + '_%.8x' % ptr
            logger.info('   name(0x%x): %r  (%r)', tgt, tgtname, name)
            vw.makeName(ptr, name)
<commit_msg>logging the exception<commit_after>"""
generic workspace analysis module to seek through the undiscovered
country looking for pointers to interesting things.

in a previous life, this analysis code lived inside VivWorkspace.analyze()
"""
import logging
from vivisect.const import RTYPE_BASEPTR, LOC_POINTER

logger = logging.getLogger(__name__)

def analyze(vw):

    logger.info('...analyzing pointers.')

    done = []

    # Let's analyze Relocations we know are pointers
    for rva, rtype in vw.reloc_by_va.items():
        if rtype != RTYPE_BASEPTR:
            continue

        for xfr, xto, xtype, xinfo in vw.getXrefsFrom(rva):
            vw.analyzePointer(xto)
            done.append((xfr, xto))
            logger.info('pointer(1): 0x%x -> 0x%x', xfr, xto)

    # Now, we'll analyze the pointers placed by the file wrapper (ELF, PE, MACHO, etc...)
    for pva, tva, fname, pname in vw.getVaSetRows('PointersFromFile'):
        if vw.getLocation(pva) is None:
            if tva is None:
                logger.info('making pointer 0x%x (%r)', pva, pname)
            else:
                logger.info('making pointer 0x%x -> 0x%x (%r)', pva, tva, pname)
            vw.makePointer(pva, tva, follow=True)
            done.append((pva, tva))
            logger.info('pointer(2): 0x%x -> 0x%x', pva, tva)

    for lva, lsz, lt, li in vw.getLocations(LOC_POINTER):
        tva = vw.readMemoryPtr(lva)
        if not vw.isValidPointer(tva):
            continue

        if vw.getLocation(tva) is not None:
            continue

        logger.info('following previously discovered pointer 0x%x -> 0x%x', lva, tva)
        try:
            vw.followPointer(tva)
            done.append((lva, tva))
            logger.info('pointer(3): 0x%x -> 0x%x', lva, tva)
        except Exception:
            logger.exception("followPointer() failed for 0x%.8x (pval: 0x%.8x)" % (lva, tva))

    # Now, lets find likely free-hanging pointers
    for addr, pval in vw.findPointers():
        if vw.isDeadData(pval):
            continue
        try:
            vw.makePointer(addr, follow=True)
            done.append((addr, pval))
            logger.info('pointer(4): 0x%x -> 0x%x', addr, pval)
        except Exception:
            logger.exception("followPointer() failed for 0x%.8x (pval: 0x%.8x)" % (addr, pval))

    # Now let's see what these guys should be named (if anything)
    for ptr, tgt in done:
        pname = vw.getName(ptr)
        if pname is not None:
            logger.info('skipping renaming of 0x%x (%r)', ptr, pname)
            continue

        tgtname = vw.getName(tgt)
        if tgtname is not None:
            name = vw._addNamePrefix(tgtname, tgt, 'ptr', '_') + '_%.8x' % ptr
            logger.info('   name(0x%x): %r  (%r)', tgt, tgtname, name)
            vw.makeName(ptr, name)
<|endoftext|>


<commit_before>from scipy.interpolate import interp1d
import matplotlib.pyplot as plt

import scipy.stats
import argparse
import scipy.io
import numpy
import math
import sys
import os

import os.path

sys.path.append("./module")
import basicutils

import matplotlib.pyplot as plt

#################################################################################

"""
start_spread_for_rating = []
for ratclass in range(1,8+1):
    start_spread_for_rating.append([])
    for i in range(Nnaz):
#        if (i != 3):
          for j in range(ratings.shape[1]):
              if ratings[i, j] == ratclass:
                  start_spread_for_rating[ratclass-1].append(spread[i, j])

    s = numpy.std(start_spread_for_rating[ratclass-1])
    m = numpy.mean(start_spread_for_rating[ratclass-1])

    print len(start_spread_for_rating[ratclass-1]), m, s

spread_for_rating = []
for ratclass in range(1,8+1):
    s = numpy.std(start_spread_for_rating[ratclass-1])
    m = numpy.mean(start_spread_for_rating[ratclass-1])

    spread_for_rating.append([])

    #spread_for_rating[ratclass-1] = \
    #        [v for v in start_spread_for_rating[ratclass-1]]

    #spread_for_rating[ratclass-1] = \
    #        [v for v in start_spread_for_rating[ratclass-1] if \
    #        (math.fabs(v-m) <= 3.0*s and v != 0.0) ]

    spread_for_rating[ratclass-1] = \
        [v for v in start_spread_for_rating[ratclass-1] if \
        (v > 0.0) ]

    s = numpy.std(spread_for_rating[ratclass-1])
    m = numpy.mean(spread_for_rating[ratclass-1])

    #print len(spread_for_rating[ratclass-1]), m, s

for ratclass in range(1,8+1):
    #m = numpy.mean(spread_for_rating[ratclass-1])
    #for i in range(len(spread_for_rating[ratclass-1])):
    #    spread_for_rating[ratclass-1][i] -= m
    s = numpy.std(spread_for_rating[ratclass-1])
    m = numpy.mean(spread_for_rating[ratclass-1])

    fp1 = open("spread"+str(ratclass)+".txt", "w")
    fp2 = open("lognorm"+str(ratclass)+".txt", "w")
    fp3 = open("spread_cdf"+str(ratclass)+".txt", "w")
    fp4 = open("lognorm_cdf"+str(ratclass)+".txt", "w")

    #y, x = numpy.histogram(spread_for_rating[ratclass-1], 100, normed=True)
    params = scipy.stats.lognorm.fit(spread_for_rating[ratclass-1])
    params = scipy.stats.lognorm.fit(spread_for_rating[ratclass-1], params[0], loc=params[1], scale=params[2])
    params = scipy.stats.lognorm.fit(spread_for_rating[ratclass-1], params[0], loc=params[1], scale=params[2])

    print scipy.stats.kstest(spread_for_rating[ratclass-1], "lognorm", params,
            alternative="two-sided", mode="asymp")

    data_sorted = numpy.sort(spread_for_rating[ratclass-1])
    p = 1. * numpy.arange(len(spread_for_rating[ratclass-1])) / \
            (len(spread_for_rating[ratclass-1]) - 1)

    for i in range(len(data_sorted)):
        fp3.write(str(data_sorted[i]) + " " + str(p[i]) + "\n")

    #fig = plt.figure()

    #ax1 = fig.add_subplot(121)
    #ax1.plot(p, data_sorted)
    #ax1.set_xlabel('$p$')
    #ax1.set_ylabel('$x$')

    #ax2 = fig.add_subplot(122)
    #ax2.plot(data_sorted, p)
    #ax2.set_xlabel('$x$')
    #ax2.set_ylabel('$p$')

    #plt.show()

    count, bins, ignored = plt.hist(spread_for_rating[ratclass-1], 100, \
            normed=True, align="mid")
    for i in range(len(count)):
        fp1.write(str((bins[i]+bins[i+1])/2.0)  + " " + str(count[i]) + "\n")

    x=numpy.linspace(min(bins),max(bins),1000)
    pdf = scipy.stats.lognorm.pdf(x, params[0], loc=params[1], scale=params[2])
    for i in range(len(x)):
        fp2.write(str( x[i]) + " " + str(pdf[i]) + "\n")

    cdf = scipy.stats.lognorm.cdf(data_sorted, params[0], loc=params[1], scale=params[2])
    for i in range(len(data_sorted)):
        fp4.write(str( data_sorted[i]) + " " + str(cdf[i]) + "\n")

    fp1.close()
    fp2.close()
    fp3.close()
    fp4.close()

for ratclass in range(1,8+1):

    log_spread_for_rating = numpy.log(spread_for_rating[ratclass-1])

    s = numpy.std(log_spread_for_rating)
    m = numpy.mean(log_spread_for_rating)

    value = [((v-m)/s) for v in log_spread_for_rating]

    print scipy.stats.jarque_bera (value)
"""

#######################################################################
# adapted from
# https://stackoverflow.com/questions/33345780/empirical-cdf-in-python-similiar-to-matlabs-one

def ecdf(raw_data):

    data = numpy.asarray(raw_data)
    data = numpy.atleast_1d(data)
    quantiles, counts = numpy.unique(data, return_counts=True)
    cumprob = numpy.cumsum(counts).astype(numpy.double) / data.size

    return quantiles, cumprob

#######################################################################
# adapted from
# https://github.com/stochasticresearch/copula-py

def gaussian_copula_rnd (rho, m):

    n = rho.shape[0]
    mu = numpy.zeros(n)
    y = scipy.stats.multivariate_normal(mu,rho)
    mvndata = y.rvs(size = m)
    u = scipy.stats.norm.cdf(mvndata)

    return u

#######################################################################

if __name__ == "__main__" :

    parser = argparse.ArgumentParser()

    parser.add_argument("-m","--mat-filename", help="Observed transition matrix filename", \
            type=str, required=True, dest="matfilename")
    parser.add_argument("-R", "--name-of-rating-matrix", help="Name of the observed transition matrix ", \
            type=str, required=False, default="ratings", dest="nameofratingmatrix")
    parser.add_argument("-S", "--name-of-spread-matrix", help="Name of the spread matrix ", \
            type=str, required=False, default="spread", dest="nameofspreadmatrix")
    parser.add_argument("-P", "--name-of-prating-matrix", help="Name of the rating matrix ", \
            type=str, required=False, default="p_rating", dest="nameofpratingmatrix")
    parser.add_argument("-v", "--verbose", help="increase output verbosity", \
            default=False, action="store_true")
    parser.add_argument("-d", "--num-of-day", help="Number of days to simulate ", \
            type=int, required=False, default=365*3, dest="time")
    parser.add_argument("-n", "--number-of-MCrun", help="Number of MonteCarlo simulation steps ", \
            type=int, required=False, default=200, dest="numofsim")

    if len(sys.argv) == 1:
        parser.print_help()
        exit(1)

    args = parser.parse_args()

    name_rtmt = args.nameofratingmatrix
    name_spmt = args.nameofspreadmatrix
    name_prmt = args.nameofpratingmatrix
    filename = args.matfilename
    verbose = args.verbose
    d = args.time
    Nsim = args.numofsim

    errmsg = []

    if not (os.path.isfile(filename)):
        print ("File " + filename + " does not exist ")
        exit(1)

    matf = scipy.io.loadmat(filename)

    if not(name_rtmt in matf.keys()):
        print "Cannot find " + name_rtmt + " in " + filename
        print matf.keys()
        exit(1)

    if not(name_spmt in matf.keys()):
        print "Cannot find " + name_spmt + " in " + filename
        print matf.keys()
        exit(1)

    ratings = matf[name_rtmt]
    spread = matf[name_spmt]
    p_rating = matf[name_prmt]
    #print ratings.shape
    #print spmt.shape
    #for i in range(ratings.shape[0]):
    #    for j in range(ratings.shape[1]):
    #        print "%3d"%ratings[i, j]

    if ratings.shape != spread.shape:
        print "Error  in matrix dimension"
        exit(1)

    N = numpy.max(ratings)
    Nnaz = spread.shape[0]
    Dst = max(spread.shape)

    #print "N: ", N, "Nnaz: " , Nnaz, "Dst: ", Dst

    inc_spread = numpy.zeros((Nnaz,Dst-1))

    end = spread.shape[1]
    for i in range(Nnaz):
        a = spread[i,1:end] - spread[i,0:end-1]
        b = spread[i,0:end-1]
        inc_spread[i,:] = numpy.divide(a, b, out=numpy.full_like(a, float("Inf")), where=b!=0)

    rttmp = ratings[:,1:end]
    totdim = rttmp.shape[0]*rttmp.shape[1]

    rttmp = rttmp.reshape(totdim, order='F')
    f_inc_spread = inc_spread.reshape(totdim, order='F')

    #for i in f_inc_spread:
    #    print "%10.5f"%(i)

    X = []
    G = []

    for i in range(N):
        tmp = numpy.where(rttmp == i+1)[0]
        dist_sp = [f_inc_spread[j] for j in tmp]
        dist_sp = filter(lambda a: a != float("Inf"), dist_sp)
        mind = scipy.stats.mstats.mquantiles(dist_sp, 0.05)
        maxd = scipy.stats.mstats.mquantiles(dist_sp, 0.95)

        dist_sp = filter(lambda a: a >= mind and a <= maxd, dist_sp)

        x, y = ecdf(dist_sp)
        X.append(x)
        G.append(y)

        #plt.plot(x, y)
        #plt.show()

        #basicutils.vct_to_stdout(numpy.asarray(G[i]))
        #basicutils.vct_to_stdout(numpy.asarray(dist_sp))
        #basicutils.vct_to_stdout(numpy.asarray(X[i]))
        #for j in range(len(G[i])):
        #    print "%10.5f %10.5f"%(X[i][j], G[i][j])

    rho = numpy.corrcoef(spread) # don't need to transpose

    #for i in range(rho.shape[0]):
    #    for j in range(rho.shape[1]):
    #        print "%10.5f "%(rho[i,j])

    #eigvals, m =  numpy.linalg.eig(rho)
    #print numpy.sort(eigvals)

    # copula gaussian Kendall
    #tau = 2.0*numpy.arcsin(rho)/math.pi
    #print tau

    R_in = ratings[:,-1]

    #print R_in

    spread_synth_tmp = numpy.zeros(Nnaz)
    spread_synth = numpy.zeros((Nsim,d,Nnaz));

    entropy_t = 0.4*numpy.ones((Nsim,d))

    #print entropy_t

    for sim in range(Nsim):
        spread_synth[sim,0,:] = spread[:,-1].transpose()
        #print spread_synth[sim,0,:]
        u = gaussian_copula_rnd (rho, d)

        #print u.shape
        #print u[:,0]
        #plt.hist (u[:,1], bins=10)
        #plt.show()

        for j in range(1,d):
            v = numpy.random.uniform(0.0, 1.0, Nnaz)
            pp = numpy.cumsum(p_rating[R_in-1,:],1)
            jj = numpy.zeros(Nnaz, dtype=int)
            for k in range(Nnaz):
                jj[k] = numpy.where(pp[k,:] >= v[k])[0][0]

                #plt.plot(G[jj[k]][1:])

                func = interp1d(G[jj[k]][1:], X[jj[k]][1:], kind='linear')

                xval = u[j,k]
                xmin = min(G[jj[k]][1:])
                xmax = max(G[jj[k]][1:])
                if u[j,k] < xmin:
                    xval = xmin
                if u[j,k] > xmax:
                    xval = xmax

                spread_synth_tmp[k] = max(func(xval), -0.9)

                #xnew = numpy.linspace(min(G[jj[k]][1:]), max(G[jj[k]][1:]), num=10000, endpoint=True)
                #ynew = func(xnew)
                #plt.plot(G[jj[k]][1:], X[jj[k]][1:], 'o', xnew, ynew, '-')
                #plt.show()

            R_in = jj
            spread_synth[sim,j,:] = numpy.multiply( \
                    numpy.squeeze(spread_synth[sim,j-1,:]), \
                    (1+spread_synth_tmp[:].transpose()))

            summa = numpy.sum(spread_synth[sim,j,:])
            if summa != 0.0:
                summa = 1.0e-10

            P_spread = spread_synth[sim,j,:]/numpy.sum(spread_synth[sim,j,:])

            P_spread = P_spread.clip(min=1.0e-15)

            entropy_t[sim,j] =  numpy.sum(numpy.multiply(P_spread, \
                    numpy.log(float(Nnaz)*P_spread)))

    print numpy.mean(entropy_t[:,1:],0)
    plt.plot(numpy.mean(entropy_t[:,1:],0))
    plt.savefig("mean_entropy.png")
<commit_msg>moved KS test into a quick and dirty module<commit_after>from scipy.interpolate import interp1d
import matplotlib.pyplot as plt

import scipy.stats
import argparse
import scipy.io
import numpy
import math
import sys
import os

import os.path

sys.path.append("./module")

import basicutils
import kstest

import matplotlib.pyplot as plt

#######################################################################
# adapted from
# https://stackoverflow.com/questions/33345780/empirical-cdf-in-python-similiar-to-matlabs-one

def ecdf(raw_data):

    data = numpy.asarray(raw_data)
    data = numpy.atleast_1d(data)
    quantiles, counts = numpy.unique(data, return_counts=True)
    cumprob = numpy.cumsum(counts).astype(numpy.double) / data.size

    return quantiles, cumprob

#######################################################################
# adapted from
# https://github.com/stochasticresearch/copula-py

def gaussian_copula_rnd (rho, m):

    n = rho.shape[0]
    mu = numpy.zeros(n)
    y = scipy.stats.multivariate_normal(mu,rho)
    mvndata = y.rvs(size = m)
    u = scipy.stats.norm.cdf(mvndata)

    return u

#######################################################################

if __name__ == "__main__" :

    parser = argparse.ArgumentParser()

    parser.add_argument("-m","--mat-filename", help="Observed transition matrix filename", \
            type=str, required=True, dest="matfilename")
    parser.add_argument("-R", "--name-of-rating-matrix", help="Name of the observed transition matrix ", \
            type=str, required=False, default="ratings", dest="nameofratingmatrix")
    parser.add_argument("-S", "--name-of-spread-matrix", help="Name of the spread matrix ", \
            type=str, required=False, default="spread", dest="nameofspreadmatrix")
    parser.add_argument("-P", "--name-of-prating-matrix", help="Name of the rating matrix ", \
            type=str, required=False, default="p_rating", dest="nameofpratingmatrix")
    parser.add_argument("-v", "--verbose", help="increase output verbosity", \
            default=False, action="store_true")
    parser.add_argument("-d", "--num-of-day", help="Number of days to simulate ", \
            type=int, required=False, default=365*3, dest="time")
    parser.add_argument("-n", "--number-of-MCrun", help="Number of MonteCarlo simulation steps ", \
            type=int, required=False, default=200, dest="numofsim")

    if len(sys.argv) == 1:
        parser.print_help()
        exit(1)

    args = parser.parse_args()

    name_rtmt = args.nameofratingmatrix
    name_spmt = args.nameofspreadmatrix
    name_prmt = args.nameofpratingmatrix
    filename = args.matfilename
    verbose = args.verbose
    d = args.time
    Nsim = args.numofsim

    errmsg = []

    if not (os.path.isfile(filename)):
        print ("File " + filename + " does not exist ")
        exit(1)

    matf = scipy.io.loadmat(filename)

    if not(name_rtmt in matf.keys()):
        print "Cannot find " + name_rtmt + " in " + filename
        print matf.keys()
        exit(1)

    if not(name_spmt in matf.keys()):
        print "Cannot find " + name_spmt + " in " + filename
        print matf.keys()
        exit(1)

    ratings = matf[name_rtmt]
    spread = matf[name_spmt]
    p_rating = matf[name_prmt]

    kstest.slipshod_kstest (spread, ratings)

    #print ratings.shape
    #print spmt.shape
    #for i in range(ratings.shape[0]):
    #    for j in range(ratings.shape[1]):
    #        print "%3d"%ratings[i, j]

    if ratings.shape != spread.shape:
        print "Error  in matrix dimension"
        exit(1)

    N = numpy.max(ratings)
    Nnaz = spread.shape[0]
    Dst = max(spread.shape)

    #print "N: ", N, "Nnaz: " , Nnaz, "Dst: ", Dst

    inc_spread = numpy.zeros((Nnaz,Dst-1))

    end = spread.shape[1]
    for i in range(Nnaz):
        a = spread[i,1:end] - spread[i,0:end-1]
        b = spread[i,0:end-1]
        inc_spread[i,:] = numpy.divide(a, b, out=numpy.full_like(a, float("Inf")), where=b!=0)

    rttmp = ratings[:,1:end]
    totdim = rttmp.shape[0]*rttmp.shape[1]

    rttmp = rttmp.reshape(totdim, order='F')
    f_inc_spread = inc_spread.reshape(totdim, order='F')

    #for i in f_inc_spread:
    #    print "%10.5f"%(i)

    X = []
    G = []

    for i in range(N):
        tmp = numpy.where(rttmp == i+1)[0]
        dist_sp = [f_inc_spread[j] for j in tmp]
        dist_sp = filter(lambda a: a != float("Inf"), dist_sp)
        mind = scipy.stats.mstats.mquantiles(dist_sp, 0.05)
        maxd = scipy.stats.mstats.mquantiles(dist_sp, 0.95)

        dist_sp = filter(lambda a: a >= mind and a <= maxd, dist_sp)

        x, y = ecdf(dist_sp)
        X.append(x)
        G.append(y)

        #plt.plot(x, y)
        #plt.show()

        #basicutils.vct_to_stdout(numpy.asarray(G[i]))
        #basicutils.vct_to_stdout(numpy.asarray(dist_sp))
        #basicutils.vct_to_stdout(numpy.asarray(X[i]))
        #for j in range(len(G[i])):
        #    print "%10.5f %10.5f"%(X[i][j], G[i][j])

    rho = numpy.corrcoef(spread) # don't need to transpose

    #for i in range(rho.shape[0]):
    #    for j in range(rho.shape[1]):
    #        print "%10.5f "%(rho[i,j])

    #eigvals, m =  numpy.linalg.eig(rho)
    #print numpy.sort(eigvals)

    # copula gaussian Kendall
    #tau = 2.0*numpy.arcsin(rho)/math.pi
    #print tau

    R_in = ratings[:,-1]

    #print R_in

    spread_synth_tmp = numpy.zeros(Nnaz)
    spread_synth = numpy.zeros((Nsim,d,Nnaz));

    entropy_t = 0.4*numpy.ones((Nsim,d))

    #print entropy_t

    for sim in range(Nsim):
        spread_synth[sim,0,:] = spread[:,-1].transpose()
        #print spread_synth[sim,0,:]
        u = gaussian_copula_rnd (rho, d)

        #print u.shape
        #print u[:,0]
        #plt.hist (u[:,1], bins=10)
        #plt.show()

        for j in range(1,d):
            v = numpy.random.uniform(0.0, 1.0, Nnaz)
            pp = numpy.cumsum(p_rating[R_in-1,:],1)
            jj = numpy.zeros(Nnaz, dtype=int)
            for k in range(Nnaz):
                jj[k] = numpy.where(pp[k,:] >= v[k])[0][0]

                #plt.plot(G[jj[k]][1:])

                func = interp1d(G[jj[k]][1:], X[jj[k]][1:], kind='linear')

                xval = u[j,k]
                xmin = min(G[jj[k]][1:])
                xmax = max(G[jj[k]][1:])
                if u[j,k] < xmin:
                    xval = xmin
                if u[j,k] > xmax:
                    xval = xmax

                spread_synth_tmp[k] = max(func(xval), -0.9)

                #xnew = numpy.linspace(min(G[jj[k]][1:]), max(G[jj[k]][1:]), num=10000, endpoint=True)
                #ynew = func(xnew)
                #plt.plot(G[jj[k]][1:], X[jj[k]][1:], 'o', xnew, ynew, '-')
                #plt.show()

            R_in = jj
            spread_synth[sim,j,:] = numpy.multiply( \
                    numpy.squeeze(spread_synth[sim,j-1,:]), \
                    (1+spread_synth_tmp[:].transpose()))

            summa = numpy.sum(spread_synth[sim,j,:])
            if summa != 0.0:
                summa = 1.0e-10

            P_spread = spread_synth[sim,j,:]/numpy.sum(spread_synth[sim,j,:])

            P_spread = P_spread.clip(min=1.0e-15)

            entropy_t[sim,j] =  numpy.sum(numpy.multiply(P_spread, \
                    numpy.log(float(Nnaz)*P_spread)))

    print numpy.mean(entropy_t[:,1:],0)
    plt.plot(numpy.mean(entropy_t[:,1:],0))
    plt.savefig("mean_entropy.png")
<|endoftext|>


<commit_before>
#### IMPORT THE LIBRARIES AND MODULES
from gevent import monkey
monkey.patch_all()

from flask import Flask, render_template, request, redirect, url_for, jsonify, session, request
from server_db import *
import datetime

import smbus
import time

from i2c_threads import create_CommunicationThread

from flask.ext.socketio import SocketIO, emit, disconnect
####

#### INITIALISATIONS
app = Flask(__name__)
app.config['SECRET_KEY'] = 'secret!'
app.debug = True

#turn the flask app into a socketio app
socketio = SocketIO(app)

#dict to store the different threads created
comThreads = {}

#i2c bus init
bus = smbus.SMBus(1)

#database init
s = dbInit()
dbEngine = s[0]
dbMetadata = s[1]
dbConnection = s[2]

#init necessary tables in database and take them from it
dbDevicesTable = dbCreateTables(dbMetadata, dbEngine, 0)
dbDOTable = dbCreateTables(dbMetadata, dbEngine, 1)
dbDITable = dbCreateTables(dbMetadata, dbEngine, 2)
dbAOTable = dbCreateTables(dbMetadata, dbEngine, 3)
dbAITable = dbCreateTables(dbMetadata, dbEngine, 4)
dbTablesDict = {'Devices': dbDevicesTable,
                'DO': dbDOTable,
                'DI': dbDITable,
                'AI': dbAITable,
                'AO': dbAOTable}

tablesNames = ['Digital Output', 'Digital Input',
                       'Analog Input', 'Analog Ouptut']
tablesNamesDict ={'Digital Output':'DO', 'Digital Input':'DI',
                       'Analog Input':'A)', 'Analog Ouptut':'AO'}

error = ""
####

#code to execute when entered '/'
@app.route("/")
def hello():
	return render_template('main.html', templateData=templateData) #Render the template

#method for the devices tab
@app.route('/devices')
def devicesList():
        #creates a dict with all the devices in the i2c_devices table
        dbConnection = dbInit()[2]
        devicesTable = dbSelectTable(dbDevicesTable, dbConnection)
        DITable = dbSelectTable(dbDITable, dbConnection)
        DOTable = dbSelectTable(dbDOTable, dbConnection)
        AITable = dbSelectTable(dbAITable, dbConnection)
        AOTable = dbSelectTable(dbAOTable, dbConnection)
        tablesList = [DOTable, DITable, AITable, AOTable]

        # it passes it to the template before rendering
        return render_template('devices.html', DevicesTable=devicesTable,
                               TablesList=tablesList, TablesNames = tablesNames,
                               counter = 0)

#method for the new devices form. What happens to create a new device
@app.route('/devices/new_device', methods=['POST'])
def new_device():
        # gets the information from the form
        name = request.form['inputName']
        address = request.form['inputAddress']
        new_device_args = [{'Name': name, 'Address': address}]

        # gets all the entries from the Devices table in the database
        dbConnection = dbInit()[2]
        values = dbSelectTable(dbDevicesTable, dbConnection)

        # Checks that the name is not already used
        for device in values:
                if device.Name == name:
                        break
        else:
                # in case it really doesn't exist yet, create new item in i2c_devices table in database
                dbInsert(dbDevicesTable, dbConnection, new_device_args)
        return redirect('/devices') # go back to the web page

# method for creating a new node inside a device
@app.route('/devices/new_node', methods=['POST'])
def new_node():
        # gets the information from the form
        input_name = request.form['inputName']
        input_pin = request.form['inputPin']
        signal = request.form['signal']
        device_address = request.form['deviceAddress']

        # Arrange the data in the way that it want to be passed to the database (to the dbInsert method)
        print "name: "+input_name+", address:"+device_address+"."
        args=[{'Name': input_name, 'Address': int(device_address), 'Pin': int(input_pin)}]

        # Depending on the kind of signal chosen, select the appropriate db table
        if signal == 'Digital Output':
                dbInsert(dbDOTable, dbConnection, args)
        elif signal == 'Digital Input':
                dbInsert(dbDITable, dbConnection, args)
        elif signal == 'Analog Input':
                dbInsert(dbAITable, dbConnection, args)
        elif signal == 'Analog Output':
                dbInsert(dbAOTable, dbConnection, args)
        else:
                return str(0)

        return redirect(url_for('devicesList'))

####
#### methods for the socket
####

#connection of the socket
@socketio.on('connect', namespace='/test')
def test_connect():
        pass

# Start reading: an option from a node is selected (delete, on, off, read, cancel, remove)
@socketio.on('start reading', namespace='/test')
def socket_start_reading(msg):
        # gets the relevant information from the form
        sub_inputs = msg['data'].split()
        table = tablesNamesDict[tablesNames[int(sub_inputs[0])]]
        device_name = sub_inputs[1]
        address = sub_inputs[2]
        pin = sub_inputs[3]
        instr = sub_inputs[4]

        # depending on the action that the user want to take place
        # DELETE
        if instr == 'delete':
                dbConnection = dbInit()[2]
                if dbDeleteDevice(dbTablesDict, address, dbConnection):
                        emit('message', {'data': 'success deleting'})
                        emit('redirect',{'url': url_for('devicesList')})
                else:
                        emit('message', {'data': 'fail deleting'})

        # READ
        elif instr == 'read':
                for key in comThreads:  # checks that the thread is not running already
                        if key == (address + pin):
                                emit('reading',
                                     {'data': 'thread already started',
                                      'pin': pin}, namespace='/test')
                                break
                else:   # if it is not created, then creates it
                        myCom = create_CommunicationThread('thread', int(address), int(pin),
                                                    0, 0, 0, bus, 2, socketio)

                        comThreads[address + pin] = myCom
                        comThreads[address + pin].start()

        # CANCEL
        elif instr == 'cancel':
                if (address + pin) in comThreads:
                        if comThreads[address + pin].getExitFlag():
                                emit('message', {'data': 'thread cancelling'})
                                comThreads[address + pin].setExitFlag()
                                comThreads.pop(address + pin, None)
                        elif not comThreads[address + pin].getExitFlag():
                                comThreads.pop(address + pin, None)
                                emit('message', {'data':str(comThreads[address + pin].getExitFlag())})

                        else:
                               emit('message', {'data': 'thread not cancelled'})
                else:
                        emit('message', {'data': 'thread does not exist'})

                emit('reading',{'data': '',
                                'pin': pin}, namespace='/test')

        # REMOVE
        elif instr == 'remove':
                try:
                        dbConnection = dbInit()[2]
                        dbDelete(dbTablesDict[table], int(address), int(pin), dbConnection)
                        emit('redirect',{'url': url_for('devicesList')})
                except Exception, e:
                        global error
                        error = e

        # ON
        elif instr == 'On':
                myCom = create_CommunicationThread('thread', int(address), int(pin),
                                                   1, 1, 1, bus, 0, socketio)

                comThreads[address + pin] = myCom
                comThreads[address + pin].start()

        # OFF
        elif instr == 'Off':
                myCom = create_CommunicationThread('thread', int(address), int(pin),
                                                   1, 1, 0, bus, 0, socketio)

                comThreads[address + pin] = myCom
                comThreads[address + pin].start()

        # OTHER
        else:
                emit('message', {'data': instr})

#run the server
if __name__ == "__main__":
        socketio.run(app, host='0.0.0.0', port=80)

<commit_msg>Update server_i2c_v2.py<commit_after>
#### IMPORT THE LIBRARIES AND MODULES
from gevent import monkey
monkey.patch_all()

from flask import Flask, render_template, request, redirect, url_for, jsonify, session
from server_db import *
import datetime

import smbus
import time

from i2c_threads import create_CommunicationThread

from flask.ext.socketio import SocketIO, emit, disconnect
####

#### INITIALISATIONS
app = Flask(__name__)
app.config['SECRET_KEY'] = 'secret!'
app.debug = True

#turn the flask app into a socketio app
socketio = SocketIO(app)

#dict to store the different threads created
comThreads = {}

#i2c bus init
bus = smbus.SMBus(1)

#database init
s = dbInit()
dbEngine = s[0]
dbMetadata = s[1]
dbConnection = s[2]

#init necessary tables in database and take them from it
dbDevicesTable = dbCreateTables(dbMetadata, dbEngine, 0)
dbDOTable = dbCreateTables(dbMetadata, dbEngine, 1)
dbDITable = dbCreateTables(dbMetadata, dbEngine, 2)
dbAOTable = dbCreateTables(dbMetadata, dbEngine, 3)
dbAITable = dbCreateTables(dbMetadata, dbEngine, 4)
dbTablesDict = {'Devices': dbDevicesTable,
                'DO': dbDOTable,
                'DI': dbDITable,
                'AI': dbAITable,
                'AO': dbAOTable}

tablesNames = ['Digital Output', 'Digital Input',
                       'Analog Input', 'Analog Ouptut']
tablesNamesDict ={'Digital Output':'DO', 'Digital Input':'DI',
                       'Analog Input':'A)', 'Analog Ouptut':'AO'}

error = ""
####

#code to execute when entered '/'
@app.route("/")
def hello():
	return render_template('main.html', templateData=templateData) #Render the template

#method for the devices tab
@app.route('/devices')
def devicesList():
        #creates a dict with all the devices in the i2c_devices table
        dbConnection = dbInit()[2]
        devicesTable = dbSelectTable(dbDevicesTable, dbConnection)
        DITable = dbSelectTable(dbDITable, dbConnection)
        DOTable = dbSelectTable(dbDOTable, dbConnection)
        AITable = dbSelectTable(dbAITable, dbConnection)
        AOTable = dbSelectTable(dbAOTable, dbConnection)
        tablesList = [DOTable, DITable, AITable, AOTable]

        # it passes it to the template before rendering
        return render_template('devices.html', DevicesTable=devicesTable,
                               TablesList=tablesList, TablesNames = tablesNames,
                               counter = 0)

#method for the new devices form. What happens to create a new device
@app.route('/devices/new_device', methods=['POST'])
def new_device():
        # gets the information from the form
        name = request.form['inputName']
        address = request.form['inputAddress']
        new_device_args = [{'Name': name, 'Address': address}]

        # gets all the entries from the Devices table in the database
        dbConnection = dbInit()[2]
        values = dbSelectTable(dbDevicesTable, dbConnection)

        # Checks that the name is not already used
        for device in values:
                if device.Name == name:
                        break
        else:
                # in case it really doesn't exist yet, create new item in i2c_devices table in database
                dbInsert(dbDevicesTable, dbConnection, new_device_args)
        return redirect('/devices') # go back to the web page

# method for creating a new node inside a device
@app.route('/devices/new_node', methods=['POST'])
def new_node():
        # gets the information from the form
        input_name = request.form['inputName']
        input_pin = request.form['inputPin']
        signal = request.form['signal']
        device_address = request.form['deviceAddress']

        # Arrange the data in the way that it want to be passed to the database (to the dbInsert method)
        print "name: "+input_name+", address:"+device_address+"."
        args=[{'Name': input_name, 'Address': int(device_address), 'Pin': int(input_pin)}]

        # Depending on the kind of signal chosen, select the appropriate db table
        if signal == 'Digital Output':
                dbInsert(dbDOTable, dbConnection, args)
        elif signal == 'Digital Input':
                dbInsert(dbDITable, dbConnection, args)
        elif signal == 'Analog Input':
                dbInsert(dbAITable, dbConnection, args)
        elif signal == 'Analog Output':
                dbInsert(dbAOTable, dbConnection, args)
        else:
                return str(0)

        return redirect(url_for('devicesList'))

####
#### methods for the socket
####

#connection of the socket
@socketio.on('connect', namespace='/test')
def test_connect():
        pass

# Start reading: an option from a node is selected (delete, on, off, read, cancel, remove)
@socketio.on('start reading', namespace='/test')
def socket_start_reading(msg):
        # gets the relevant information from the form
        sub_inputs = msg['data'].split()
        table = tablesNamesDict[tablesNames[int(sub_inputs[0])]]
        device_name = sub_inputs[1]
        address = sub_inputs[2]
        pin = sub_inputs[3]
        instr = sub_inputs[4]

        # depending on the action that the user want to take place
        # DELETE
        if instr == 'delete':
                dbConnection = dbInit()[2]
                if dbDeleteDevice(dbTablesDict, address, dbConnection):
                        emit('message', {'data': 'success deleting'})
                        emit('redirect',{'url': url_for('devicesList')})
                else:
                        emit('message', {'data': 'fail deleting'})

        # READ
        elif instr == 'read':
                for key in comThreads:  # checks that the thread is not running already
                        if key == (address + pin):
                                emit('reading',
                                     {'data': 'thread already started',
                                      'pin': pin}, namespace='/test')
                                break
                else:   # if it is not created, then creates it
                        myCom = create_CommunicationThread('thread', int(address), int(pin),
                                                    0, 0, 0, bus, 2, socketio)

                        comThreads[address + pin] = myCom
                        comThreads[address + pin].start()

        # CANCEL
        elif instr == 'cancel':
                if (address + pin) in comThreads:
                        if comThreads[address + pin].getExitFlag():
                                emit('message', {'data': 'thread cancelling'})
                                comThreads[address + pin].setExitFlag()
                                comThreads.pop(address + pin, None)
                        elif not comThreads[address + pin].getExitFlag():
                                comThreads.pop(address + pin, None)
                                emit('message', {'data':str(comThreads[address + pin].getExitFlag())})

                        else:
                               emit('message', {'data': 'thread not cancelled'})
                else:
                        emit('message', {'data': 'thread does not exist'})

                emit('reading',{'data': '',
                                'pin': pin}, namespace='/test')

        # REMOVE
        elif instr == 'remove':
                try:
                        dbConnection = dbInit()[2]
                        dbDelete(dbTablesDict[table], int(address), int(pin), dbConnection)
                        emit('redirect',{'url': url_for('devicesList')})
                except Exception, e:
                        global error
                        error = e

        # ON
        elif instr == 'On':
                myCom = create_CommunicationThread('thread', int(address), int(pin),
                                                   1, 1, 1, bus, 0, socketio)

                comThreads[address + pin] = myCom
                comThreads[address + pin].start()

        # OFF
        elif instr == 'Off':
                myCom = create_CommunicationThread('thread', int(address), int(pin),
                                                   1, 1, 0, bus, 0, socketio)

                comThreads[address + pin] = myCom
                comThreads[address + pin].start()

        # OTHER
        else:
                emit('message', {'data': instr})

#run the server
if __name__ == "__main__":
        socketio.run(app, host='0.0.0.0', port=80)

<|endoftext|>


<commit_before>"""
    Class Name : Event

    Description:
        Class to handle event observers

    Contributors:
        - Patrick Hennessy

    License:
        Arcbot is free software: you can redistribute it and/or modify it
        under the terms of the GNU General Public License v3; as published
        by the Free Software Foundation
"""

import logging
logger = logging.getLogger(__name__)

class EventManager():
    def __init__(self, core):
        self.core = core
        self.events = {}

    def register(self, event):
        """
            Summary:
                Adds an event to so that the bot knows about it

            Args:
                event (str): Name of the event to register

            Returns:
                None
        """
        if event in self.events:
            logger.warning("Event \"{}\" has already been registered".format(event))
        else:
            self.events[event] = []

    def unregister(self, event):
        """
            Summary:
                Removes an event so that it can no longer be emmited.
                This method exists for completion sake

            Args:
                event (str): Name of the event to register

            Returns:
                None
        """
        if event not in self.events:
            logger.warning("Event \"{}\" does not exist.".format(event))
        else:
            self.events.pop(event, None)

    def notify(self, event, **kwargs):
        """
            Summary:
                Queues up each event notification to the threadPool

            Args:
                event (str): Name of the event to register

            Returns:
                None
        """
        for callback in self.events[event]:
            self.core.workers.queue(callback, **kwargs)

    def subscribe(self, event, callback):
        """
            Summary:
                Adds an callback to be invoked when an event occurs

            Args:
                event (str): Name of the event
                callback (bound method): Reference to the method to be called when event occurs

            Returns:
                None
        """
        if(callback not in self.events[event]):
            self.events[event].append(callback)
        else:
            logger.warning("Callback \"{}\" was already subscribed to {}.".format(callback, event))

    def unsubscribe(self, event, callback):
        """
            Summary:
                Removes a callback from an event

            Args:
                event (str): Name of the event
                callback (bound method): Reference to the method to be called when event occurs

            Returns:
                None
        """
        if callback in self.events[event]:
            self.events[event].remove(callback)
        else:
            logger.warning("Callback \"{}\" was not subscribed to any event.".format(callback))
<commit_msg>Added list method to Event because why not<commit_after>"""
    Class Name : Event

    Description:
        Class to handle event observers

    Contributors:
        - Patrick Hennessy

    License:
        Arcbot is free software: you can redistribute it and/or modify it
        under the terms of the GNU General Public License v3; as published
        by the Free Software Foundation
"""

import logging
logger = logging.getLogger(__name__)

class EventManager():
    def __init__(self, core):
        self.core = core
        self.events = {}

    def list(self):
        """
            Summary:
                Returns a list of events known to the bot

            Args:
                None

            Returns:
                None
        """
        return self.events

    def register(self, event):
        """
            Summary:
                Adds an event to so that the bot knows about it

            Args:
                event (str): Name of the event to register

            Returns:
                None
        """
        if event in self.events:
            logger.warning("Event \"{}\" has already been registered".format(event))
        else:
            self.events[event] = []

    def unregister(self, event):
        """
            Summary:
                Removes an event so that it can no longer be emmited.
                This method exists for completion sake

            Args:
                event (str): Name of the event to register

            Returns:
                None
        """
        if event not in self.events:
            logger.warning("Event \"{}\" does not exist.".format(event))
        else:
            self.events.pop(event, None)

    def notify(self, event, **kwargs):
        """
            Summary:
                Queues up each event notification to the threadPool

            Args:
                event (str): Name of the event to register

            Returns:
                None
        """
        for callback in self.events[event]:
            self.core.workers.queue(callback, **kwargs)

    def subscribe(self, event, callback):
        """
            Summary:
                Adds an callback to be invoked when an event occurs

            Args:
                event (str): Name of the event
                callback (bound method): Reference to the method to be called when event occurs

            Returns:
                None
        """
        if(callback not in self.events[event]):
            self.events[event].append(callback)
        else:
            logger.warning("Callback \"{}\" was already subscribed to {}.".format(callback, event))

    def unsubscribe(self, event, callback):
        """
            Summary:
                Removes a callback from an event

            Args:
                event (str): Name of the event
                callback (bound method): Reference to the method to be called when event occurs

            Returns:
                None
        """
        if callback in self.events[event]:
            self.events[event].remove(callback)
        else:
            logger.warning("Callback \"{}\" was not subscribed to any event.".format(callback))
<|endoftext|>


<commit_before># Copyright 2015 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Rules for cloning external git repositories."""

load("@bazel_tools//tools/build_defs/repo:utils.bzl", "patch", "update_attrs", "workspace_and_buildfile")

def _clone_or_update(ctx):
    if ((not ctx.attr.tag and not ctx.attr.commit and not ctx.attr.branch) or
        (ctx.attr.tag and ctx.attr.commit) or
        (ctx.attr.tag and ctx.attr.branch) or
        (ctx.attr.commit and ctx.attr.branch)):
        fail("Exactly one of commit, tag, or branch must be provided")
    shallow = ""
    if ctx.attr.commit:
        ref = ctx.attr.commit
    elif ctx.attr.tag:
        ref = "tags/" + ctx.attr.tag
        shallow = "--depth=1"
    else:
        ref = ctx.attr.branch
        shallow = "--depth=1"
    directory = str(ctx.path("."))
    if ctx.attr.strip_prefix:
        directory = directory + "-tmp"
    if ctx.attr.shallow_since:
        if ctx.attr.tag:
            fail("shallow_since not allowed if a tag is specified; --depth=1 will be used for tags")
        if ctx.attr.branch:
            fail("shallow_since not allowed if a branch is specified; --depth=1 will be used for branches")
        shallow = "--shallow-since=%s" % ctx.attr.shallow_since

    ctx.report_progress("Cloning %s of %s" % (ref, ctx.attr.remote))
    if (ctx.attr.verbose):
        print("git.bzl: Cloning or updating %s repository %s using strip_prefix of [%s]" %
              (
                  " (%s)" % shallow if shallow else "",
                  ctx.name,
                  ctx.attr.strip_prefix if ctx.attr.strip_prefix else "None",
              ))
    bash_exe = ctx.os.environ["BAZEL_SH"] if "BAZEL_SH" in ctx.os.environ else "bash"
    st = ctx.execute([bash_exe, "-c", """
set -ex
( cd {working_dir} &&
    if ! ( cd '{dir_link}' && [[ "$(git rev-parse --git-dir)" == '.git' ]] ) >/dev/null 2>&1; then
      rm -rf '{directory}' '{dir_link}'
      git clone {shallow} '{remote}' '{directory}' || git clone '{remote}' '{directory}'
    fi
    git -C '{directory}' reset --hard {ref} || \
    ((git -C '{directory}' fetch {shallow} origin {ref}:{ref} || \
      git -C '{directory}' fetch origin {ref}:{ref}) && git -C '{directory}' reset --hard {ref})
      git -C '{directory}' clean -xdf )
  """.format(
        working_dir = ctx.path(".").dirname,
        dir_link = ctx.path("."),
        directory = directory,
        remote = ctx.attr.remote,
        ref = ref,
        shallow = shallow,
    )], environment = ctx.os.environ)

    if st.return_code:
        fail("error cloning %s:\n%s" % (ctx.name, st.stderr))

    if ctx.attr.strip_prefix:
        dest_link = "{}/{}".format(directory, ctx.attr.strip_prefix)
        if not ctx.path(dest_link).exists:
            fail("strip_prefix at {} does not exist in repo".format(ctx.attr.strip_prefix))

        ctx.symlink(dest_link, ctx.path("."))
    if ctx.attr.init_submodules:
        ctx.report_progress("Updating submodules")
        st = ctx.execute([bash_exe, "-c", """
set -ex
(   git -C '{directory}' submodule update --init --checkout --force )
  """.format(
            directory = ctx.path("."),
        )], environment = ctx.os.environ)
    if st.return_code:
        fail("error updating submodules %s:\n%s" % (ctx.name, st.stderr))

    ctx.report_progress("Recording actual commit")

    # After the fact, determine the actual commit and its date
    actual_commit = ctx.execute([
        bash_exe,
        "-c",
        "(git -C '{directory}' log -n 1 --pretty='format:%H')".format(
            directory = ctx.path("."),
        ),
    ]).stdout
    shallow_date = ctx.execute([
        bash_exe,
        "-c",
        "(git -C '{directory}' log -n 1 --pretty='format:%cd' --date=raw)".format(
            directory = ctx.path("."),
        ),
    ]).stdout
    return {"commit": actual_commit, "shallow_since": shallow_date}

def _remove_dot_git(ctx):
    # Remove the .git directory, if present
    bash_exe = ctx.os.environ["BAZEL_SH"] if "BAZEL_SH" in ctx.os.environ else "bash"
    ctx.execute([
        bash_exe,
        "-c",
        "rm -rf '{directory}'".format(directory = ctx.path(".git")),
    ])

def _update_git_attrs(orig, keys, override):
    result = update_attrs(orig, keys, override)

    # if we found the actual commit, remove all other means of specifying it,
    # like tag or branch.
    if "commit" in result:
        result.pop("tag", None)
        result.pop("branch", None)
    return result

_common_attrs = {
    "remote": attr.string(mandatory = True),
    "commit": attr.string(default = ""),
    "shallow_since": attr.string(default = ""),
    "tag": attr.string(default = ""),
    "branch": attr.string(default = ""),
    "init_submodules": attr.bool(default = False),
    "verbose": attr.bool(default = False),
    "strip_prefix": attr.string(default = ""),
    "patches": attr.label_list(default = []),
    "patch_tool": attr.string(default = "patch"),
    "patch_args": attr.string_list(default = ["-p0"]),
    "patch_cmds": attr.string_list(default = []),
}

_new_git_repository_attrs = dict(_common_attrs.items() + {
    "build_file": attr.label(allow_single_file = True),
    "build_file_content": attr.string(),
    "workspace_file": attr.label(),
    "workspace_file_content": attr.string(),
}.items())

def _new_git_repository_implementation(ctx):
    if ((not ctx.attr.build_file and not ctx.attr.build_file_content) or
        (ctx.attr.build_file and ctx.attr.build_file_content)):
        fail("Exactly one of build_file and build_file_content must be provided.")
    update = _clone_or_update(ctx)
    workspace_and_buildfile(ctx)
    patch(ctx)
    _remove_dot_git(ctx)
    return _update_git_attrs(ctx.attr, _new_git_repository_attrs.keys(), update)

def _git_repository_implementation(ctx):
    update = _clone_or_update(ctx)
    patch(ctx)
    _remove_dot_git(ctx)
    return _update_git_attrs(ctx.attr, _common_attrs.keys(), update)

new_git_repository = repository_rule(
    implementation = _new_git_repository_implementation,
    attrs = _new_git_repository_attrs,
)
"""Clone an external git repository.

Clones a Git repository, checks out the specified tag, or commit, and
makes its targets available for binding. Also determine the id of the
commit actually checked out and its date, and return a dict with parameters
that provide a reproducible version of this rule (which a tag not necessarily
is).

Args:
  name: A unique name for this repository.

  build_file: The file to use as the BUILD file for this repository.
    Either build_file or build_file_content must be specified.

    This attribute is an absolute label (use '@//' for the main repo). The file
    does not need to be named BUILD, but can be (something like
    BUILD.new-repo-name may work well for distinguishing it from the
    repository's actual BUILD files.

  build_file_content: The content for the BUILD file for this repository.
    Either build_file or build_file_content must be specified.

  workspace_file: The file to use as the `WORKSPACE` file for this repository.

    Either `workspace_file` or `workspace_file_content` can be specified, or
    neither, but not both.
  workspace_file_content: The content for the WORKSPACE file for this repository.

    Either `workspace_file` or `workspace_file_content` can be specified, or
    neither, but not both.
  branch: branch in the remote repository to checked out

  tag: tag in the remote repository to checked out

  commit: specific commit to be checked out
    Precisely one of branch, tag, or commit must be specified.

  shallow_since: an optional date, not after the specified commit; the
    argument is not allowed if a tag is specified (which allows cloning
    with depth 1). Setting such a date close to the specified commit
    allows for a more shallow clone of the repository, saving bandwidth and
    wall-clock time.

  init_submodules: Whether to clone submodules in the repository.

  remote: The URI of the remote Git repository.

  strip_prefix: A directory prefix to strip from the extracted files.

  patches: A list of files that are to be applied as patches after extracting
    the archive.
  patch_tool: the patch(1) utility to use.
  patch_args: arguments given to the patch tool, defaults to ["-p0"]
  patch_cmds: sequence of commands to be applied after patches are applied.
"""

git_repository = repository_rule(
    implementation = _git_repository_implementation,
    attrs = _common_attrs,
)
"""Clone an external git repository.

Clones a Git repository, checks out the specified tag, or commit, and
makes its targets available for binding. Also determine the id of the
commit actually checked out and its date, and return a dict with parameters
that provide a reproducible version of this rule (which a tag not necessarily
is).

Args:
  name: A unique name for this repository.

  init_submodules: Whether to clone submodules in the repository.

  remote: The URI of the remote Git repository.

  branch: branch in the remote repository to checked out

  tag: tag in the remote repository to checked out

  commit: specific commit to be checked out
    Precisely one of branch, tag, or commit must be specified.

  shallow_since: an optional date in the form YYYY-MM-DD, not after
    the specified commit; the argument is not allowed if a tag is specified
    (which allows cloning with depth 1). Setting such a date close to the
    specified commit allows for a more shallow clone of the repository, saving
    bandwidth and wall-clock time.

  strip_prefix: A directory prefix to strip from the extracted files.

  patches: A list of files that are to be applied as patches after extracting
    the archive.
  patch_tool: the patch(1) utility to use.
  patch_args: arguments given to the patch tool, defaults to ["-p0"]
  patch_cmds: sequence of commands to be applied after patches are applied.
"""
<commit_msg>Avoid deletion of repository directory in repository_ctx<commit_after># Copyright 2015 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Rules for cloning external git repositories."""

load("@bazel_tools//tools/build_defs/repo:utils.bzl", "patch", "update_attrs", "workspace_and_buildfile")

def _clone_or_update(ctx):
    if ((not ctx.attr.tag and not ctx.attr.commit and not ctx.attr.branch) or
        (ctx.attr.tag and ctx.attr.commit) or
        (ctx.attr.tag and ctx.attr.branch) or
        (ctx.attr.commit and ctx.attr.branch)):
        fail("Exactly one of commit, tag, or branch must be provided")
    shallow = ""
    if ctx.attr.commit:
        ref = ctx.attr.commit
    elif ctx.attr.tag:
        ref = "tags/" + ctx.attr.tag
        shallow = "--depth=1"
    else:
        ref = ctx.attr.branch
        shallow = "--depth=1"
    directory = str(ctx.path("."))
    if ctx.attr.strip_prefix:
        directory = directory + "-tmp"
    if ctx.attr.shallow_since:
        if ctx.attr.tag:
            fail("shallow_since not allowed if a tag is specified; --depth=1 will be used for tags")
        if ctx.attr.branch:
            fail("shallow_since not allowed if a branch is specified; --depth=1 will be used for branches")
        shallow = "--shallow-since=%s" % ctx.attr.shallow_since

    ctx.report_progress("Cloning %s of %s" % (ref, ctx.attr.remote))
    if (ctx.attr.verbose):
        print("git.bzl: Cloning or updating %s repository %s using strip_prefix of [%s]" %
              (
                  " (%s)" % shallow if shallow else "",
                  ctx.name,
                  ctx.attr.strip_prefix if ctx.attr.strip_prefix else "None",
              ))
    bash_exe = ctx.os.environ["BAZEL_SH"] if "BAZEL_SH" in ctx.os.environ else "bash"
    st = ctx.execute([bash_exe, "-c", """
cd {working_dir}
set -ex
( cd {working_dir} &&
    if ! ( cd '{dir_link}' && [[ "$(git rev-parse --git-dir)" == '.git' ]] ) >/dev/null 2>&1; then
      rm -rf '{directory}' '{dir_link}'
      git clone {shallow} '{remote}' '{directory}' || git clone '{remote}' '{directory}'
    fi
    git -C '{directory}' reset --hard {ref} || \
    ((git -C '{directory}' fetch {shallow} origin {ref}:{ref} || \
      git -C '{directory}' fetch origin {ref}:{ref}) && git -C '{directory}' reset --hard {ref})
      git -C '{directory}' clean -xdf )
  """.format(
        working_dir = ctx.path(".").dirname,
        dir_link = ctx.path("."),
        directory = directory,
        remote = ctx.attr.remote,
        ref = ref,
        shallow = shallow,
    )], environment = ctx.os.environ)

    if st.return_code:
        fail("error cloning %s:\n%s" % (ctx.name, st.stderr))

    if ctx.attr.strip_prefix:
        dest_link = "{}/{}".format(directory, ctx.attr.strip_prefix)
        if not ctx.path(dest_link).exists:
            fail("strip_prefix at {} does not exist in repo".format(ctx.attr.strip_prefix))

        ctx.symlink(dest_link, ctx.path("."))
    if ctx.attr.init_submodules:
        ctx.report_progress("Updating submodules")
        st = ctx.execute([bash_exe, "-c", """
set -ex
(   git -C '{directory}' submodule update --init --checkout --force )
  """.format(
            directory = ctx.path("."),
        )], environment = ctx.os.environ)
    if st.return_code:
        fail("error updating submodules %s:\n%s" % (ctx.name, st.stderr))

    ctx.report_progress("Recording actual commit")

    # After the fact, determine the actual commit and its date
    actual_commit = ctx.execute([
        bash_exe,
        "-c",
        "(git -C '{directory}' log -n 1 --pretty='format:%H')".format(
            directory = ctx.path("."),
        ),
    ]).stdout
    shallow_date = ctx.execute([
        bash_exe,
        "-c",
        "(git -C '{directory}' log -n 1 --pretty='format:%cd' --date=raw)".format(
            directory = ctx.path("."),
        ),
    ]).stdout
    return {"commit": actual_commit, "shallow_since": shallow_date}

def _remove_dot_git(ctx):
    # Remove the .git directory, if present
    bash_exe = ctx.os.environ["BAZEL_SH"] if "BAZEL_SH" in ctx.os.environ else "bash"
    ctx.execute([
        bash_exe,
        "-c",
        "rm -rf '{directory}'".format(directory = ctx.path(".git")),
    ])

def _update_git_attrs(orig, keys, override):
    result = update_attrs(orig, keys, override)

    # if we found the actual commit, remove all other means of specifying it,
    # like tag or branch.
    if "commit" in result:
        result.pop("tag", None)
        result.pop("branch", None)
    return result

_common_attrs = {
    "remote": attr.string(mandatory = True),
    "commit": attr.string(default = ""),
    "shallow_since": attr.string(default = ""),
    "tag": attr.string(default = ""),
    "branch": attr.string(default = ""),
    "init_submodules": attr.bool(default = False),
    "verbose": attr.bool(default = False),
    "strip_prefix": attr.string(default = ""),
    "patches": attr.label_list(default = []),
    "patch_tool": attr.string(default = "patch"),
    "patch_args": attr.string_list(default = ["-p0"]),
    "patch_cmds": attr.string_list(default = []),
}

_new_git_repository_attrs = dict(_common_attrs.items() + {
    "build_file": attr.label(allow_single_file = True),
    "build_file_content": attr.string(),
    "workspace_file": attr.label(),
    "workspace_file_content": attr.string(),
}.items())

def _new_git_repository_implementation(ctx):
    if ((not ctx.attr.build_file and not ctx.attr.build_file_content) or
        (ctx.attr.build_file and ctx.attr.build_file_content)):
        fail("Exactly one of build_file and build_file_content must be provided.")
    update = _clone_or_update(ctx)
    workspace_and_buildfile(ctx)
    patch(ctx)
    _remove_dot_git(ctx)
    return _update_git_attrs(ctx.attr, _new_git_repository_attrs.keys(), update)

def _git_repository_implementation(ctx):
    update = _clone_or_update(ctx)
    patch(ctx)
    _remove_dot_git(ctx)
    return _update_git_attrs(ctx.attr, _common_attrs.keys(), update)

new_git_repository = repository_rule(
    implementation = _new_git_repository_implementation,
    attrs = _new_git_repository_attrs,
)
"""Clone an external git repository.

Clones a Git repository, checks out the specified tag, or commit, and
makes its targets available for binding. Also determine the id of the
commit actually checked out and its date, and return a dict with parameters
that provide a reproducible version of this rule (which a tag not necessarily
is).

Args:
  name: A unique name for this repository.

  build_file: The file to use as the BUILD file for this repository.
    Either build_file or build_file_content must be specified.

    This attribute is an absolute label (use '@//' for the main repo). The file
    does not need to be named BUILD, but can be (something like
    BUILD.new-repo-name may work well for distinguishing it from the
    repository's actual BUILD files.

  build_file_content: The content for the BUILD file for this repository.
    Either build_file or build_file_content must be specified.

  workspace_file: The file to use as the `WORKSPACE` file for this repository.

    Either `workspace_file` or `workspace_file_content` can be specified, or
    neither, but not both.
  workspace_file_content: The content for the WORKSPACE file for this repository.

    Either `workspace_file` or `workspace_file_content` can be specified, or
    neither, but not both.
  branch: branch in the remote repository to checked out

  tag: tag in the remote repository to checked out

  commit: specific commit to be checked out
    Precisely one of branch, tag, or commit must be specified.

  shallow_since: an optional date, not after the specified commit; the
    argument is not allowed if a tag is specified (which allows cloning
    with depth 1). Setting such a date close to the specified commit
    allows for a more shallow clone of the repository, saving bandwidth and
    wall-clock time.

  init_submodules: Whether to clone submodules in the repository.

  remote: The URI of the remote Git repository.

  strip_prefix: A directory prefix to strip from the extracted files.

  patches: A list of files that are to be applied as patches after extracting
    the archive.
  patch_tool: the patch(1) utility to use.
  patch_args: arguments given to the patch tool, defaults to ["-p0"]
  patch_cmds: sequence of commands to be applied after patches are applied.
"""

git_repository = repository_rule(
    implementation = _git_repository_implementation,
    attrs = _common_attrs,
)
"""Clone an external git repository.

Clones a Git repository, checks out the specified tag, or commit, and
makes its targets available for binding. Also determine the id of the
commit actually checked out and its date, and return a dict with parameters
that provide a reproducible version of this rule (which a tag not necessarily
is).

Args:
  name: A unique name for this repository.

  init_submodules: Whether to clone submodules in the repository.

  remote: The URI of the remote Git repository.

  branch: branch in the remote repository to checked out

  tag: tag in the remote repository to checked out

  commit: specific commit to be checked out
    Precisely one of branch, tag, or commit must be specified.

  shallow_since: an optional date in the form YYYY-MM-DD, not after
    the specified commit; the argument is not allowed if a tag is specified
    (which allows cloning with depth 1). Setting such a date close to the
    specified commit allows for a more shallow clone of the repository, saving
    bandwidth and wall-clock time.

  strip_prefix: A directory prefix to strip from the extracted files.

  patches: A list of files that are to be applied as patches after extracting
    the archive.
  patch_tool: the patch(1) utility to use.
  patch_args: arguments given to the patch tool, defaults to ["-p0"]
  patch_cmds: sequence of commands to be applied after patches are applied.
"""
<|endoftext|>