<commit_before>"""Node (User) Class for Paxos Calendar."""

import os
import sys
import time
import thread
import pickle
import socket
import logging
from Bully import bully_algorithm
from Appointment import Appointment
from Calendar import Calendar
from Proposer import Proposer
from Acceptor import Acceptor

class Node(object):
    """
    Node class.

    node_id:        Unique ID used for Node identification as well as for
                    unique proposal number generation; int.
    calendar:       Calendar object which contains Appointment objects.
    proposer:       Proposer object used in Synod Algorithm; passed node_id so
                    it can create unique proposal numbers.
    acceptor:       Acceptor object used in Synod Algorithm.
    log:            Dictionary of Calendar objects used in Paxos Algorithm;
                    intially empty, Synod Algorithm is used to fill each entry
                    of log where integer keys represents slots and the values
                    being the Calendar agreed upon via conscensus.
    leader:         The current leader elected via the bully algorithm;
                    initially None and updated every ~6 seconds.
    """

    _ip_filename = "./IP_translations.txt"

    def __init__(self, node_id):
        """Construct a Node object."""
        if type(node_id) != int:
            raise TypeError("node_id must be an int")
        if node_id < 0:
            raise ValueError("node id must be a nonnegative integer")

        try:
            Node._ip_table = Node._make_ip_table()
        except IOError:
            raise IOError("Node-to-IP translation file: " + ip_filename + " not found.")

        self._node_id = node_id
        self._calendar = Calendar()
        self._proposer = Proposer(node_id)
        self._acceptor = Acceptor()
        self._log = {}
        self._leader = None
        self._is_Node = True

    def insert(self, appointment):
        """Insert an Appointment into this Node's Calendar."""
        print "IN INSERT"

    def delete(self, appointment):
        """Delete an Appointment in this Node's Calendar."""
        print "IN DELETE"

    def elect_leader(self, poll_time=6, timeout=3):
        """Engage this Node in leader selection."""
        def _do_leader_election(self, poll_time, timeout):
            """Do leader election as new thread."""
            recv_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            recv_socket.bind(("0.0.0.0", self._ip_table[self._node_id][1]))
            #backlog; 1 for each Node besides self
            recv_socket.listen(4)

            while True:
                thread.start_new_thread(bully_algorithm, (self, recv_socket, timeout))
                time.sleep(poll_time)
                logging.debug("NEW LEADER IS: " + str(self._leader))

            recv_socket.close()

        thread.start_new_thread(_do_leader_election, (self, poll_time, timeout))

    @staticmethod
    def save(Node, path="./", filename="state.pkl"):
        """Save this Node's log and Acceptor to stable storage."""
        if not hasattr(Node, "_is_Node"):
            raise TypeError("Node parameter must be a Node object")

        if type(filename) != str or type(path) != str:
            raise TypeError("path and filename must be strings")

        if filename[-4:] != ".pkl":
            raise ValueError("filename must have .pkl extension")

        if not os.path.exists(path):
            raise ValueError("path provided does not exist")

        import pickle
        with open(path + filename, 'w') as f:
            state = (Node._node_id, Node._log, Node._acceptor)
            pickle.dump(state, f)

    @staticmethod
    def load(path="./", filename="state.pkl"):
        """
        Load log and Acceptor from stable storage if path and filename exist.
        """

        def _rebuild_calendar(node, log):
            """Rebuild the calendar of node by reconstructing it from log."""
            #Get the latest entry in the log for most up-to-date Calendar
            node._calendar = log[max(log.keys())]

        if type(filename) != str or type(path) != str:
            raise TypeError("path and filename must be strings")

        if filename[-4:] != ".pkl":
            raise ValueError("filename must have .pkl extension")

        if not os.path.exists(path+filename):
            raise ValueError("path provided does not exist")

        with open(path + filename, 'r') as f:
            state = pickle.load(f)
            node_id, log, acceptor = state
            node = Node(node_id)
            node._log = log
            node._acceptor = acceptor
            _rebuild_calendar(node, log)

            return node

    @staticmethod
    def _make_ip_table():
        """Create the ID-to-IP translation table used for socket connection."""
        table = {}

        import re
        pattern = r"^\d+,\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3},\d{4}$"
        with open(Node._ip_filename, "r") as f:
                for translation in f:
                    match = re.match(pattern, translation.strip())
                    if not match:
                        raise ValueError(
                            "Every line in IP_translations.txt must be of "
                            "form ID,IP")
                    ID, IP, PORT = translation.strip().split(',')
                    table[int(ID)] = [IP, int(PORT)]

        return table

    @staticmethod
    def _parse_command(command, node):
        """Parse command provided, possibly involving provided node."""

        def _do_show(argv, node):
            """Perform show command for debugging/user information."""
            if len(argv) == 1:
                raise ValueError(
                    "Invalid show argument; show needs argument "
                    "{calendar,log,acceptor,proposer,all}")

            #Handle showing the calendar
            if argv[1] == "calendar":
                print node._calendar

            #Handle showing the log
            elif argv[1] == "log":

                print "Log:"
                #copy the log into a list ordered by slot number
                ordered_slots = sorted(node._log.items(), key=lambda x: x[0])

                #if -short flag not thrown, print entire log
                if len(argv) == 2:
                    for slot in ordered_slots:
                        print "Slot " + str(slot[0]) + ' ' + str(slot[1])
                #Short flag is thrown, just print names of Appointments in each
                #Calendar slot
                elif len(argv) == 3:
                    if argv[2] == "-s":
                        for slot in ordered_slots:
                            log_string = "Slot " + str(slot[0]) + " Calendar: \t"
                            log_string += ', '.join(
                                slot[1].get_appointment_names())
                            print log_string
                        print
                    else:
                        raise ValueError(
                            "Invalid show arguments; Only flags \"-s\" "
                            "permitted")
                #Bad number of arguments to show log
                else:
                    raise ValueError(
                        "Invalid show arguments; show log supports only a "
                        "single optional flag argument \"-s\"")
            #Handle showing Node's Acceptor object
            elif argv[1] == "acceptor":
                print str(node._acceptor) + '\n'
            #Handle showing Node's Proposer object
            elif argv[1] == "proposer":
                print str(node._proposer) + '\n'
            #Handle printing entire state of Node
            elif argv[1] == "all":
                print "-" * 100
                print "Node ID: " + str(node._node_id)
                _do_show(['show', 'calendar'], node)
                _do_show(['show', 'log', '-s'], node)
                _do_show(['show', 'acceptor'], node)
                _do_show(['show', 'proposer'], node)
                print "-" * 100
            else:
                raise ValueError(
                    "Invalid show argument; show needs argument "
                    "{calendar,log,acceptor,proposer,all}")

        def _parse_appointment(argv):
            """Try to parse an Appointment object from given argv."""
            generic_error_msg = "Invalid command; Schedule and cancel " + \
                    "commands must be of form: \n" + \
                    "{schedule,cancel} [Appointment name] " + \
                    "(user1,user2,...usern) (start_time,end_time) [day]"

            if len(argv) != 5:
                raise ValueError(generic_error_msg)

            name, participants, times, day = argv[1:]
            participants = participants[1:-1].split(",")
            try:
                participants = [int(user[4:]) for user in participants]
            except ValueError:
                raise ValueError(
                    "Invalid command; participants must be of form "
                    "(user1,user2,...,usern)")
            try:
                start, end = times[1:-1].split(',')
            except ValueError:
                raise ValueError(
                    "Invalid command; times must be of form "
                    "(start_time,end_time)")

            try:
                return Appointment(name, day, start, end, participants)
            except ValueError as excinfo:
                raise ValueError("Invalid command; " + excinfo.message)

        def _do_clear():
            """Perform clear command via ASCI escape code."""
            print(chr(27) + "[2J")

        argv = command.split()

        if not argv:
            return

        #If command is to clear, clear the screen
        if argv[0] == "clear":
            _do_clear()
            return

        #If command was to show something, do show
        if argv[0] == "show":
            try:
                _do_show(argv, node)
            except ValueError as excinfo:
                print excinfo
                print
            finally:
                return

        #If command is to schedule or cancel an Appointment, parse then
        #initiate Synod algorithm
        if argv[0] == "schedule":
            try:
                appointment = _parse_appointment(argv)

                #determine if the Appointment the user is trying to schedule
                #is already in their Calendar or in conflict with some
                #Appointment in their Calendar
                conflict_cond = node._calendar._is_appointment_conflicting(
                                                                appointment)
                in_cond = appointment in node._calendar

                #if it's not already in the Calendar and not in conflict with
                #any Appointment in it, begin Synod
                if not conflict_cond and not in_cond:
                    node.insert(appointment)
                else:
                    print "User scheduled appointment already in their " + \
                            "own Calendar or in conflict with their own " + \
                            "Calendar; ignoring.\n"
            except ValueError as excinfo:
                print excinfo
                print
            finally:
                return

        if argv[0] == "cancel":
            try:
                appointment = _parse_appointment(argv)
                if appointment in node._calendar:
                    node.delete(appointment)
                else:
                    print "User cancelled appointment not in their own " + \
                                                        "Calendar; ignoring.\n"
            except ValueError as excinfo:
                print excinfo
                print
            finally:
                return

        print "Invalid command; supported commands = {clear,show,schedule,cancel}"
        print

def set_verbosity(verbose_level=3):
    """Set the level of verbosity of the Preprocessing."""
    if not type(verbose_level) == int:
        raise TypeError("verbose_level must be an int")

    if verbose_level < 0 or verbose_level > 4:
        raise ValueError("verbose_level must be between 0 and 4")

    verbosity = [
        logging.CRITICAL,
        logging.ERROR,
        logging.WARNING,
        logging.INFO,
        logging.DEBUG]

    logging.basicConfig(
        format='%(asctime)s:\t %(message)s',
        datefmt='%m/%d/%Y %I:%M:%S %p',
        level=verbosity[verbose_level])

def main():
    """Quick tests."""
    "schedule yaboi (user0,user1,user2,user3) (4:00pm,6:00pm) Friday"
    "cancel yaboi (user0,user1,user2,user3) (4:00pm,6:00pm) Friday"
    "schedule xxboi (user1,user4,user5) (1:30am,11:30am) Wednesday"
    "cancel xxboi (user1,user4,user5) (1:30am,11:30am) Wednesday"
    "schedule zo (user1,user2,user3) (12:30pm,1:30pm) Friday"

    a1 = Appointment("zo","Friday","12:30pm","1:30pm", [1, 2, 3])
    a2 = Appointment("xxboi","Wednesday","1:30am","11:30am", [1, 4, 5])
    a3 = Appointment("lol","saturday","11:30am","12:30pm", [1])
    a4 = Appointment("yeee","MondAy","11:30am","12:30pm", [1])

    c1 = Calendar(a1)
    c2 = Calendar(a1, a2)
    c3 = Calendar(a1, a2, a3)
    c4 = Calendar(a1, a2, a3, a4)

    set_verbosity(0)

    N = Node(int(sys.argv[1]))

    N._log[0] = c1
    N._log[1] = c2
    N._log[2] = c3
    N._log[3] = c4
    N._calendar = c4

    #try to load a previous state of this Node
    try:
        N = Node.load()
    except ValueError:
        pass
    except IOError:
        pass

    #N.elect_leader(poll_time=6, timeout=3)

    print("@> Node Started")
    while True:
        message = raw_input('')
        if message == "quit":
            Node.save(N)
            break
        else:
            Node._parse_command(message, N)

if __name__ == "__main__":
    main()<commit_msg>Added basic UDP server function starting line 82<commit_after>"""Node (User) Class for Paxos Calendar."""

import os
import sys
import time
import thread
import pickle
import socket
import logging
from Bully import bully_algorithm
from Appointment import Appointment
from Calendar import Calendar
from Proposer import Proposer
from Acceptor import Acceptor

class Node(object):
    """
    Node class.

    node_id:        Unique ID used for Node identification as well as for
                    unique proposal number generation; int.
    calendar:       Calendar object which contains Appointment objects.
    proposer:       Proposer object used in Synod Algorithm; passed node_id so
                    it can create unique proposal numbers.
    acceptor:       Acceptor object used in Synod Algorithm.
    log:            Dictionary of Calendar objects used in Paxos Algorithm;
                    intially empty, Synod Algorithm is used to fill each entry
                    of log where integer keys represents slots and the values
                    being the Calendar agreed upon via conscensus.
    leader:         The current leader elected via the bully algorithm;
                    initially None and updated every ~6 seconds.
    """

    _ip_filename = "./IP_translations.txt"

    def __init__(self, node_id):
        """Construct a Node object."""
        if type(node_id) != int:
            raise TypeError("node_id must be an int")
        if node_id < 0:
            raise ValueError("node id must be a nonnegative integer")

        try:
            Node._ip_table = Node._make_ip_table()
        except IOError:
            raise IOError("Node-to-IP translation file: " + ip_filename + " not found.")

        self._node_id = node_id
        self._calendar = Calendar()
        self._proposer = Proposer(node_id)
        self._acceptor = Acceptor()
        self._log = {}
        self._leader = None
        self._is_Node = True

    def insert(self, appointment):
        """Insert an Appointment into this Node's Calendar."""
        print "IN INSERT"

    def delete(self, appointment):
        """Delete an Appointment in this Node's Calendar."""
        print "IN DELETE"

    def elect_leader(self, poll_time=6, timeout=3):
        """Engage this Node in leader selection."""
        def _do_leader_election(self, poll_time, timeout):
            """Do leader election as new thread."""
            recv_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            recv_socket.bind(("0.0.0.0", self._ip_table[self._node_id][1]))
            #backlog; 1 for each Node besides self
            recv_socket.listen(4)

            while True:
                thread.start_new_thread(bully_algorithm, (self, recv_socket, timeout))
                time.sleep(poll_time)
                logging.debug("NEW LEADER IS: " + str(self._leader))

            recv_socket.close()

        thread.start_new_thread(_do_leader_election, (self, poll_time, timeout))

    def synod(self):
        """UPD server thread"""
        UDP_IP = '0.0.0.0'
        UPD_PORT = '9010'
        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        sock.bind((UDP_IP, UDP_PORT))
        while True:
            data, addr = sock.recvfrom(1024) # buffer size is 1024 bytes
            print "received message:", data

    @staticmethod
    def save(Node, path="./", filename="state.pkl"):
        """Save this Node's log and Acceptor to stable storage."""
        if not hasattr(Node, "_is_Node"):
            raise TypeError("Node parameter must be a Node object")

        if type(filename) != str or type(path) != str:
            raise TypeError("path and filename must be strings")

        if filename[-4:] != ".pkl":
            raise ValueError("filename must have .pkl extension")

        if not os.path.exists(path):
            raise ValueError("path provided does not exist")

        import pickle
        with open(path + filename, 'w') as f:
            state = (Node._node_id, Node._log, Node._acceptor)
            pickle.dump(state, f)

    @staticmethod
    def load(path="./", filename="state.pkl"):
        """
        Load log and Acceptor from stable storage if path and filename exist.
        """

        def _rebuild_calendar(node, log):
            """Rebuild the calendar of node by reconstructing it from log."""
            #Get the latest entry in the log for most up-to-date Calendar
            node._calendar = log[max(log.keys())]

        if type(filename) != str or type(path) != str:
            raise TypeError("path and filename must be strings")

        if filename[-4:] != ".pkl":
            raise ValueError("filename must have .pkl extension")

        if not os.path.exists(path+filename):
            raise ValueError("path provided does not exist")

        with open(path + filename, 'r') as f:
            state = pickle.load(f)
            node_id, log, acceptor = state
            node = Node(node_id)
            node._log = log
            node._acceptor = acceptor
            _rebuild_calendar(node, log)

            return node

    @staticmethod
    def _make_ip_table():
        """Create the ID-to-IP translation table used for socket connection."""
        table = {}

        import re
        pattern = r"^\d+,\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3},\d{4}$"
        with open(Node._ip_filename, "r") as f:
                for translation in f:
                    match = re.match(pattern, translation.strip())
                    if not match:
                        raise ValueError(
                            "Every line in IP_translations.txt must be of "
                            "form ID,IP")
                    ID, IP, PORT = translation.strip().split(',')
                    table[int(ID)] = [IP, int(PORT)]

        return table

    @staticmethod
    def _parse_command(command, node):
        """Parse command provided, possibly involving provided node."""

        def _do_show(argv, node):
            """Perform show command for debugging/user information."""
            if len(argv) == 1:
                raise ValueError(
                    "Invalid show argument; show needs argument "
                    "{calendar,log,acceptor,proposer,all}")

            #Handle showing the calendar
            if argv[1] == "calendar":
                print node._calendar

            #Handle showing the log
            elif argv[1] == "log":

                print "Log:"
                #copy the log into a list ordered by slot number
                ordered_slots = sorted(node._log.items(), key=lambda x: x[0])

                #if -short flag not thrown, print entire log
                if len(argv) == 2:
                    for slot in ordered_slots:
                        print "Slot " + str(slot[0]) + ' ' + str(slot[1])
                #Short flag is thrown, just print names of Appointments in each
                #Calendar slot
                elif len(argv) == 3:
                    if argv[2] == "-s":
                        for slot in ordered_slots:
                            log_string = "Slot " + str(slot[0]) + " Calendar: \t"
                            log_string += ', '.join(
                                slot[1].get_appointment_names())
                            print log_string
                        print
                    else:
                        raise ValueError(
                            "Invalid show arguments; Only flags \"-s\" "
                            "permitted")
                #Bad number of arguments to show log
                else:
                    raise ValueError(
                        "Invalid show arguments; show log supports only a "
                        "single optional flag argument \"-s\"")
            #Handle showing Node's Acceptor object
            elif argv[1] == "acceptor":
                print str(node._acceptor) + '\n'
            #Handle showing Node's Proposer object
            elif argv[1] == "proposer":
                print str(node._proposer) + '\n'
            #Handle printing entire state of Node
            elif argv[1] == "all":
                print "-" * 100
                print "Node ID: " + str(node._node_id)
                _do_show(['show', 'calendar'], node)
                _do_show(['show', 'log', '-s'], node)
                _do_show(['show', 'acceptor'], node)
                _do_show(['show', 'proposer'], node)
                print "-" * 100
            else:
                raise ValueError(
                    "Invalid show argument; show needs argument "
                    "{calendar,log,acceptor,proposer,all}")

        def _parse_appointment(argv):
            """Try to parse an Appointment object from given argv."""
            generic_error_msg = "Invalid command; Schedule and cancel " + \
                    "commands must be of form: \n" + \
                    "{schedule,cancel} [Appointment name] " + \
                    "(user1,user2,...usern) (start_time,end_time) [day]"

            if len(argv) != 5:
                raise ValueError(generic_error_msg)

            name, participants, times, day = argv[1:]
            participants = participants[1:-1].split(",")
            try:
                participants = [int(user[4:]) for user in participants]
            except ValueError:
                raise ValueError(
                    "Invalid command; participants must be of form "
                    "(user1,user2,...,usern)")
            try:
                start, end = times[1:-1].split(',')
            except ValueError:
                raise ValueError(
                    "Invalid command; times must be of form "
                    "(start_time,end_time)")

            try:
                return Appointment(name, day, start, end, participants)
            except ValueError as excinfo:
                raise ValueError("Invalid command; " + excinfo.message)

        def _do_clear():
            """Perform clear command via ASCI escape code."""
            print(chr(27) + "[2J")

        argv = command.split()

        if not argv:
            return

        #If command is to clear, clear the screen
        if argv[0] == "clear":
            _do_clear()
            return

        #If command was to show something, do show
        if argv[0] == "show":
            try:
                _do_show(argv, node)
            except ValueError as excinfo:
                print excinfo
                print
            finally:
                return

        #If command is to schedule or cancel an Appointment, parse then
        #initiate Synod algorithm
        if argv[0] == "schedule":
            try:
                appointment = _parse_appointment(argv)

                #determine if the Appointment the user is trying to schedule
                #is already in their Calendar or in conflict with some
                #Appointment in their Calendar
                conflict_cond = node._calendar._is_appointment_conflicting(
                                                                appointment)
                in_cond = appointment in node._calendar

                #if it's not already in the Calendar and not in conflict with
                #any Appointment in it, begin Synod
                if not conflict_cond and not in_cond:
                    node.insert(appointment)
                else:
                    print "User scheduled appointment already in their " + \
                            "own Calendar or in conflict with their own " + \
                            "Calendar; ignoring.\n"
            except ValueError as excinfo:
                print excinfo
                print
            finally:
                return

        if argv[0] == "cancel":
            try:
                appointment = _parse_appointment(argv)
                if appointment in node._calendar:
                    node.delete(appointment)
                else:
                    print "User cancelled appointment not in their own " + \
                                                        "Calendar; ignoring.\n"
            except ValueError as excinfo:
                print excinfo
                print
            finally:
                return

        print "Invalid command; supported commands = {clear,show,schedule,cancel}"
        print

def set_verbosity(verbose_level=3):
    """Set the level of verbosity of the Preprocessing."""
    if not type(verbose_level) == int:
        raise TypeError("verbose_level must be an int")

    if verbose_level < 0 or verbose_level > 4:
        raise ValueError("verbose_level must be between 0 and 4")

    verbosity = [
        logging.CRITICAL,
        logging.ERROR,
        logging.WARNING,
        logging.INFO,
        logging.DEBUG]

    logging.basicConfig(
        format='%(asctime)s:\t %(message)s',
        datefmt='%m/%d/%Y %I:%M:%S %p',
        level=verbosity[verbose_level])

def main():
    """Quick tests."""
    "schedule yaboi (user0,user1,user2,user3) (4:00pm,6:00pm) Friday"
    "cancel yaboi (user0,user1,user2,user3) (4:00pm,6:00pm) Friday"
    "schedule xxboi (user1,user4,user5) (1:30am,11:30am) Wednesday"
    "cancel xxboi (user1,user4,user5) (1:30am,11:30am) Wednesday"
    "schedule zo (user1,user2,user3) (12:30pm,1:30pm) Friday"

    a1 = Appointment("zo","Friday","12:30pm","1:30pm", [1, 2, 3])
    a2 = Appointment("xxboi","Wednesday","1:30am","11:30am", [1, 4, 5])
    a3 = Appointment("lol","saturday","11:30am","12:30pm", [1])
    a4 = Appointment("yeee","MondAy","11:30am","12:30pm", [1])

    c1 = Calendar(a1)
    c2 = Calendar(a1, a2)
    c3 = Calendar(a1, a2, a3)
    c4 = Calendar(a1, a2, a3, a4)

    set_verbosity(0)

    N = Node(int(sys.argv[1]))

    N._log[0] = c1
    N._log[1] = c2
    N._log[2] = c3
    N._log[3] = c4
    N._calendar = c4

    #try to load a previous state of this Node
    try:
        N = Node.load()
    except ValueError:
        pass
    except IOError:
        pass

    #N.elect_leader(poll_time=6, timeout=3)

    print("@> Node Started")
    while True:
        message = raw_input('')
        if message == "quit":
            Node.save(N)
            break
        else:
            Node._parse_command(message, N)

if __name__ == "__main__":
    main()<|endoftext|>


<commit_before>try:
    import unittest2 as unittest
except ImportError:
    import unittest

import json
import subprocess
import os
from nose_parameterized import parameterized, param

def get_exec_path():
    return os.path.abspath(os.path.dirname(os.path.realpath(__file__)))

def lookup_dir_path(dirname, dir=get_exec_path()):
    for i in range(0,1):
        dir = os.path.dirname(dir)
        for (dirpath, dirnames, filenames) in os.walk(dir):
            if dirname in dirnames:
                return os.path.join(dirpath, dirname)
    return None

def list_files():
    paths = []
    ''' Generate the tests, based on the json files '''
    data_dir = lookup_dir_path("data")
    for (dirpath, dirnames, filenames) in os.walk(data_dir):
        for file in filenames:
            path = os.path.join(data_dir, file)
            paths.append(path)

    return paths

''' The tests have to be called from the project root, otherwise they do not work '''

class TestJsonMethods(unittest.TestCase):

    BINARY_NAME="simple-processor"

    @classmethod
    def lookup_bin_path(self, binname):
        dir = get_exec_path()
        for i in range(0,1):
            dir = os.path.dirname(dir)
            for (dirpath, dirnames, filenames) in os.walk(dir):
                if binname in filenames:
                    test_bin = os.path.join(dirpath, binname)
                    with open(os.devnull, 'w') as devnull:
                        rc = subprocess.call(test_bin+" --version", shell=True, stdout=devnull, stderr=devnull)
                        if rc == 0:
                            return test_bin
        return None

    @parameterized.expand(list_files())
    def test_insitu(self, filename):
        binary_path = self.lookup_bin_path(self.BINARY_NAME)
        subprocess.call("{} -f {} -o result.json -i 1".format(binary_path, filename), shell=True)
        with open(filename) as reference:
            with open("result.json") as generated:
                refjson = json.load(reference)
                genjson = json.load(generated)
                self.assertEqual(refjson, genjson)

    @parameterized.expand(list_files())
    def test_normal(self, filename):
        binary_path = self.lookup_bin_path(self.BINARY_NAME)
        subprocess.call("{} -f {} -o result.json".format(binary_path, filename), shell=True)
        with open(filename) as reference:
            with open("result.json") as generated:
                refjson = json.load(reference)
                genjson = json.load(generated)
                self.assertEqual(refjson, genjson)

if __name__ == '__main__':
    unittest.main()
<commit_msg>getting closer for appveyor to run ...<commit_after>try:
    import unittest2 as unittest
except ImportError:
    import unittest

import json
import subprocess
import os
from nose_parameterized import parameterized, param

def get_exec_path():
    return os.path.abspath(os.path.dirname(os.path.realpath(__file__)))

def lookup_dir_path(dirname, dir=get_exec_path()):
    for i in range(0,1):
        dir = os.path.dirname(dir)
        for (dirpath, dirnames, filenames) in os.walk(dir):
            if dirname in dirnames:
                return os.path.join(dirpath, dirname)
    return None

def list_files():
    paths = []
    ''' Generate the tests, based on the json files '''
    data_dir = lookup_dir_path("data")
    for (dirpath, dirnames, filenames) in os.walk(data_dir):
        for file in filenames:
            path = os.path.join(data_dir, file)
            paths.append(path)

    return paths

''' The tests have to be called from the project root, otherwise they do not work '''

class TestJsonMethods(unittest.TestCase):

    BINARY_NAME="simple-processor"

    @classmethod
    def lookup_bin_path(self, binname):
        dir = get_exec_path()
        for i in range(0,1):
            dir = os.path.dirname(dir)
            for (dirpath, dirnames, filenames) in os.walk(dir):
                if binname in filenames:
                    test_bin = os.path.join(dirpath, binname)
                    with open(os.devnull, 'w') as devnull:
                        rc = subprocess.call(test_bin+" --version", shell=True, stdout=devnull, stderr=devnull)
                        if rc == 0:
                            return test_bin
        return None

    @parameterized.expand(list_files())
    def test_insitu(self, filename):
        binary_path = self.lookup_bin_path(self.BINARY_NAME)
        subprocess.call("{} --file {} --output result.json --insitu 1".format(binary_path, filename), shell=True)
        with open(filename) as reference:
            with open("result.json") as generated:
                refjson = json.load(reference)
                genjson = json.load(generated)
                self.assertEqual(refjson, genjson)

    @parameterized.expand(list_files())
    def test_normal(self, filename):
        binary_path = self.lookup_bin_path(self.BINARY_NAME)
        subprocess.call("{} --file {} --output result.json".format(binary_path, filename), shell=True)
        with open(filename) as reference:
            with open("result.json") as generated:
                refjson = json.load(reference)
                genjson = json.load(generated)
                self.assertEqual(refjson, genjson)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import numpy

class label_Base:
    """ label structure
    convention for label values: 0 is unlabeled, first label is 1 and so on.
    """
    def __init__(self, size):
        self.size = size
        self.dims = size.__len__()

    def getObjects(self):
        # retruns generator expression of the parameters of label-objects.
        # example:   [ [pos], [pos], [pos], ... ] for pixels
        #            [ [pos,r], [pos,r], [pos,r], ... ] for circles
        #            [ [pos,r1,r2], [pos,r1,r2], [pos,r1,r2], ... ] for ellipses
        pass

    def setLabel(self, pos):
        pass

    def getLabel(self, pos):
        pass

class label_Patch(label_Base):
    def __init__(self, size):
        label_Base.__init__(self, size)
        self.init_storage()

    def init_storage(self):
        self.labelArray = {}   # not sure how this performs. better use special storage like arrays in descendant classes.

    def getPatchNrFromPosition(self, pos):
        pass

    def getPositionFromPatchNr(self, nr):
        pass

    def getPatchCount(self):
        lastpos = []
        for el in self.size:
            lastpos.append(el-1)
        return getPatchNrFromPosition(self, lastpos)

    def setLabel(self, pos, label):
        self.labelArray[ self.getPatchNrFromPosition(pos) ] = label

    def getLabel(self, pos):
        return self.labelArray[ self.getPatchNrFromPosition(pos)]

    def getObjects(self):
        # use generator expressions for memory efficiency
        len = self.labelArray.__len__()
        for i in xrange(len):
            el = self.labelArray[i]
            if el != 0:
                yield el

class label_Grid(label_Patch):
    pass

class label_Pixel(label_Grid):
    def __init__(self, size):
        label_Patch.__init__(self, size)

    def init_storage(self):
        self.labelArray = numpy.ndarray(self.getPatchCount())

    def getPatchNrFromPosition(self, pos):
        nr = 0
        blocksize = 1
        for i in xrange(self.dims):
            #attrnr = ord("x")
            #if hasattr(pos, chr(attrnr)):
            #nr += blocksize * getattr(pos, chr(attrnr))
            nr += blocksize * pos[i]
            blocksize *= self.dims[i]

    def getPositionFromPatchNr(self, nr):
        pass

    def getPatchCount(self):
        cnt = self.dims[0]
        for i in xrange(1,self.dims):
            cnt *= self.dims[i]
        return cnt

            <commit_msg>bresenham algorithm for line drawing.<commit_after>import numpy

class label_Base:
    """ label structure
    convention for label values: 0 is unlabeled, first label is 1 and so on.
    """
    def __init__(self, size):
        self.size = size
        self.dims = size.__len__()

    def getObjects(self):
        # retruns generator expression of the parameters of label-objects.
        # example:   [ [pos], [pos], [pos], ... ] for pixels
        #            [ [pos,r], [pos,r], [pos,r], ... ] for circles
        #            [ [pos,r1,r2], [pos,r1,r2], [pos,r1,r2], ... ] for ellipses
        pass

    def getLastObject(self):
        # returns label-object that has been added with the previous "setLabel" call
        pass

    def setLabel(self, pos, label):
        pass

    def setLabelLine2D(self, pos1, pos2, label):
        pass

    def getLabel(self, pos):
        pass

class label_Patch(label_Base):
    def __init__(self, size):
        label_Base.__init__(self, size)
        self.init_storage()
        self.lastPatchNr = 0

    def init_storage(self):
        self.labelArray = {}   # not sure how this performs. better use special storage like arrays in descendant classes.

    def getPatchNrFromPosition(self, pos):
        pass

    def getPositionFromPatchNr(self, nr):
        pass

    def getPatchCount(self):
        lastpos = []
        for el in self.size:
            lastpos.append(el-1)
        return getPatchNrFromPosition(self, lastpos)

    def setLabel(self, pos, label):
        self.lastPatchNr = self.getPatchNrFromPosition(pos)
        self.labelArray[ self.lastPatchNr ] = label

    def setLabelLine2D(self, pos1, pos2, label):
        (x0, y0) = pos1
        (x1, y1) = pos2
        steep = abs(y1 - y0) > abs(x1 - x0)
        if steep:
            x0, y0 = y0, x0
            x1, y1 = y1, x1
        if x0 > x1:
            x0, x1 = x1, x0
            y0, y1 = y1, y0
        if y0 < y1:
            ystep = 1
        else:
            ystep = -1
        deltax = x1 - x0
        deltay = abs(y1 - y0)
        error = -deltax / 2
        y = y0
        for x in range(x0, x1 + 1): # We add 1 to x1 so that the range includes x1
            if steep:
                self.setLabel((y, x), label)
            else:
                self.setLabel((x, y), label)
            error = error + deltay
            if error > 0:
                y = y + ystep
                error = error - deltax

    def getLabel(self, pos):
        return self.labelArray[ self.getPatchNrFromPosition(pos)]

    def getObjects(self):
        # use generator expressions for memory efficiency
        len = self.labelArray.__len__()
        for i in xrange(len):
            el = self.labelArray[i]
            if el != 0:
                yield el

    def getLastObject(self):
         return self.labelArray[ self.lastPatchNr ]

class label_Grid(label_Patch):
    pass

class label_Pixel(label_Grid):
    def __init__(self, size):
        label_Patch.__init__(self, size)

    def init_storage(self):
        self.labelArray = numpy.ndarray(self.getPatchCount())

    def getPatchNrFromPosition(self, pos):
        nr = 0
        blocksize = 1
        for i in xrange(self.dims):
            #attrnr = ord("x")
            #if hasattr(pos, chr(attrnr)):
            #nr += blocksize * getattr(pos, chr(attrnr))
            nr += blocksize * pos[i]
            blocksize *= self.dims[i]

    def getPositionFromPatchNr(self, nr):
        pass

    def getPatchCount(self):
        cnt = self.dims[0]
        for i in xrange(1,self.dims):
            cnt *= self.dims[i]
        return cnt

            <|endoftext|>


<commit_before>#!/usr/bin/env python
"""
This module has functions that generated random values for django fields.
"""
from random import choice
from random import randint

def generate_integer(bits, negative_allowed=True):
    length = randint(1, bits - 1) - 1
    positive = True
    if negative_allowed:
        positive = bool(randint(0, 1))
    if positive:
        low = (1 << length)
        high = 2 * low - 1
        if low == 1:
            low = 0
        return randint(low, high)
    else:
        high = -(1 << length) - 1
        low = 2 * (high + 1)
        if high == -2:
            high = -1
        return randint(low, high)

def generate_big_integer():
    return generate_integer(64)

def generate_int():
    return generate_integer(32)

def generate_small_integer():
    return generate_integer(16)

def generate_positive_integer():
    return generate_integer(32, False)

def generate_positive_small_integer():
    return generate_integer(16, False)

def generate_boolean(null_allowed=False):
    res = randint(0, 1 + int(null_allowed))
    if res < 2:
        return bool(res)
<commit_msg>Added IP generating function.<commit_after>#!/usr/bin/env python
"""
This module has functions that generated random values for django fields.
"""
from random import choice
from random import randint

def generate_integer(bits, negative_allowed=True):
    length = randint(1, bits - 1) - 1
    positive = True
    if negative_allowed:
        positive = bool(randint(0, 1))
    if positive:
        low = (1 << length)
        high = 2 * low - 1
        if low == 1:
            low = 0
        return randint(low, high)
    else:
        high = -(1 << length) - 1
        low = 2 * (high + 1)
        if high == -2:
            high = -1
        return randint(low, high)

def generate_big_integer():
    return generate_integer(64)

def generate_int():
    return generate_integer(32)

def generate_small_integer():
    return generate_integer(16)

def generate_positive_integer():
    return generate_integer(32, False)

def generate_positive_small_integer():
    return generate_integer(16, False)

def generate_boolean(null_allowed=False):
    res = randint(0, 1 + int(null_allowed))
    if res < 2:
        return bool(res)

def generate_ip():
    return str.join('.', [str(randint(0, 255)) for _ in xrange(4)])
<|endoftext|>


<commit_before>from dlstats.fetchers._skeleton import Skeleton
import io
import zipfile
import urllib.request
import xlrd
import os
import time
from datetime import datetime

class WorldBank(Skeleton):
    def __init__(self):
        super().__init__()
    def upsert_a_series(self):
        response = urllib.request.urlopen(
                   'http://siteresources.worldbank.org/INTPROSPECTS/Resources/' +\
                    'GemDataEXTR.zip')
        zipfile_ = zipfile.ZipFile(io.BytesIO(response.read()))
        excelfile = {name : zipfile_.read(name) for name in zipfile_.namelist()}
        series_ = []
        dataset_ = []
        categories_ = []
        for name_series in excelfile.keys():
            excel_file = xlrd.open_workbook(file_contents = excelfile[name_series])
            for sheet_name in excel_file.sheet_names():
                if sheet_name not in ['Sheet1','Sheet2','Sheet3']:
                    label_row_list = excel_file.sheet_by_name(sheet_name).col(0)
                    label_column_list = excel_file.sheet_by_name(sheet_name).row(0)
                    for column_index in range (
                        excel_file.sheet_by_name(sheet_name).ncols):
                        series = {}
                        dataset = {}
                        categories = {}
                        value = []
                        value_dataset_ = []
                        column = excel_file.sheet_by_name(sheet_name).col(column_index)
                        for info in zipfile_.infolist():
                            if info.filename == name_series:
                                series['released_dates'] = datetime(*info.date_time[0:6])
                        if column[0].value not in ('obs') :
                            if name_series[:-5] in ['Commodity Prices']:
                                series['dimensions'] = {'name':'Commodity Prices',
                                'value':column[0].value}
                            if name_series[:-5] not in ['Commodity Prices']:
                                series['dimensions'] = {'name':'country',
                                'value':column[0].value}
                            column_value = column[1:-1]
                            for cell_value in column_value :
                                value.append(cell_value.value)
                            series['value'] = value
                            series['name'] = name_series[:-5]
                            if sheet_name in ('annual') :
                                series['start_date'] = str(
                                    label_row_list[3].value)[:-2]
                                series['end_date'] = str(
                                    label_row_list[-1].value)[:-2]
                            if sheet_name not in ('annual') :
                                series['start_date'] = str(label_row_list[3].value)
                                series['end_date'] = str(label_row_list[-1].value)

                            if sheet_name == 'annual':
                                frequency = 'a'
                            if sheet_name == 'monthly':
                                frequency = 'm'
                            if sheet_name == 'daily':
                                frequency = 'd'
                            series['frequency'] = frequency
                            series['provider'] = 'WorldBank'
                            series['key'] = name_series[:-5].replace(' ',
                                                '_').replace(',', '')+'.'+column[0].value
                            series_.append(series)

        for series in series_:
            self.db.series.insert(series)

    def upsert_categories(self):
        response = urllib.request.urlopen(
                   'http://siteresources.worldbank.org/INTPROSPECTS/Resources/' +\
                    'GemDataEXTR.zip')
        zipfile_ = zipfile.ZipFile(io.BytesIO(response.read()))
        excelfile = {name : zipfile_.read(name) for name in zipfile_.namelist()}
        series_ = []
        dataset_ = []
        categories_ = []
        for name_series in excelfile.keys():
            excel_file = xlrd.open_workbook(file_contents = excelfile[name_series])
            for sheet_name in excel_file.sheet_names():
                if sheet_name not in ['Sheet1','Sheet2','Sheet3']:
                    label_row_list = excel_file.sheet_by_name(sheet_name).col(0)
                    label_column_list = excel_file.sheet_by_name(sheet_name).row(0)
                    for column_index in range (
                        excel_file.sheet_by_name(sheet_name).ncols):
                        categories = {}
                        value = []
                        column = excel_file.sheet_by_name(sheet_name).col(column_index)
                        if column[0].value not in ('obs') :
                            column_value = column[1:-1]
                            for cell_value in column_value :
                                value.append(cell_value.value)
                            categories['name'] = 'WorldBank'
                            categories['categoryCode'] = name_series[:-5].replace(' ',
                                                '_').replace(',', '')+'.'+column[0].value
                            categories['exposed'] = True
                            categories_.append(categories)
        for categories in categories_:
            self.db.categories.insert(categories)

    def upsert_dataset(self):
        response = urllib.request.urlopen(
                   'http://siteresources.worldbank.org/INTPROSPECTS/Resources/' +\
                    'GemDataEXTR.zip')
        zipfile_ = zipfile.ZipFile(io.BytesIO(response.read()))
        excelfile = {name : zipfile_.read(name) for name in zipfile_.namelist()}
        excel_files_list = []
        series_ = []
        dataset_ = []
        categories_ = []
        for name_series in excelfile.keys():
            excel_file = xlrd.open_workbook(file_contents = excelfile[name_series])
            for sheet_name in excel_file.sheet_names():
                if sheet_name not in ['Sheet1','Sheet2','Sheet3']:
                    label_row_list = excel_file.sheet_by_name(sheet_name).col(0)
                    label_column_list = excel_file.sheet_by_name(sheet_name).row(0)
                    for column_index in range (
                        excel_file.sheet_by_name(sheet_name).ncols):
                        dataset = {}
                        value = []
                        value_dataset_ = []
                        column = excel_file.sheet_by_name(sheet_name).col(column_index)
                        if column[0].value not in ('obs') :
                            column_value = column[1:-1]
                            for cell_value in column_value :
                                value.append(cell_value.value)
                            dataset['provider'] = 'WorldBank'
                            dataset['name'] = name_series[:-5]
                            value_dataset_ = []
                            for value_dataset in label_column_list :
                                value_dataset_.append(value_dataset.value)
                            if name_series[:-5] in ['Commodity Prices']:
                                dataset['dimensions'] = {'name':'Commodity Prices'
                                                         ,'value': value_dataset_[1:-1]}
                            if name_series[:-5] not in ['Commodity Prices']:
                                dataset['dimension_list'] = {'name': 'country' ,
                                                             'value': value_dataset_[1:-1]}
                            dataset['dataset_code'] = 'worldBank_'+name_series[:-5]
                            dataset_.append(dataset)
        for dataset in dataset_:
            self.db.dataset.insert(dataset)
    #def _series_update()

<commit_msg>Modify period_index<commit_after>from dlstats.fetchers._skeleton import Skeleton, Category, Series, Dataset
import io
import zipfile
import urllib.request
import xlrd
import os
import time
import pandas
import logging
from datetime import datetime
import pprint

class WorldBank(Skeleton):
    def __init__(self):
        super().__init__()
        self.lgr = logging.getLogger('WorldBank')
        self.lgr.setLevel(logging.DEBUG)
        self.fh = logging.FileHandler('WorldBank.log')
        self.fh.setLevel(logging.DEBUG)
        self.frmt = logging.Formatter(
            '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        self.fh.setFormatter(self.frmt)
        self.lgr.addHandler(self.fh)
        #self.lgr.info('Retrieving %s', self.configuration['Fetchers']['Eurostat']['url_table_of_contents'])
        self.lgr.debug('debug message')
        self.lgr.info('info message')
        self.lgr.warn('warn message')
        self.lgr.error('error message')
        self.lgr.critical('critical message')
    def update_selected_database(self):
        response = urllib.request.urlopen(
                   'http://siteresources.worldbank.org/INTPROSPECTS/Resources/' +\
                   'GemDataEXTR.zip')
        zipfile_ = zipfile.ZipFile(io.BytesIO(response.read()))
        excelfile = {name : zipfile_.read(name) for name in zipfile_.namelist()}
        series_ = []
        dataset_ = []
        categories_ = []
        for name_series in excelfile.keys():
            excel_file = xlrd.open_workbook(file_contents = excelfile[name_series])
            for sheet_name in excel_file.sheet_names():
                if sheet_name not in ['Sheet1','Sheet2','Sheet3']:
                    label_row_list = excel_file.sheet_by_name(sheet_name).col(0)
                    label_column_list = excel_file.sheet_by_name(sheet_name).row(0)
                    for column_index in range (
                        excel_file.sheet_by_name(sheet_name).ncols):
                        series = {}
                        dataset = {}
                        categories = {}
                        value = []
                        value_dataset_ = []
                        column = excel_file.sheet_by_name(sheet_name).col(column_index)
                        for info in zipfile_.infolist():
                            if info.filename == name_series:
                                released_dates = datetime(*info.date_time[0:6])
                        if column[0].value not in ('obs') :
                            if name_series[:-5] in ['Commodity Prices']:
                                dimensions_int = {'name':'Commodity Prices',
                                'value':column[0].value}
                            if name_series[:-5] not in ['Commodity Prices']:
                                dimensions_int = {'name':'country',
                                'value':column[0].value}
                            column_value = column[1:-1]
                            for cell_value in column_value :
                                value.append(cell_value.value)

                            if sheet_name in ('annual') :
                                start_date_b = str(
                                    label_row_list[3].value)[:-2]
                                end_date_b = str(
                                    label_row_list[-1].value)[:-2]

                            if sheet_name not in ('annual') :
                                start_date = str(label_row_list[3].value)
                                start_date_b = start_date.replace('M','-')
                                end_date = str(label_row_list[-1].value)
                                end_date_b = end_date.replace('M','-')

                            if sheet_name == 'annual':
                                frequency = 'year'
                            if sheet_name == 'quarterly':
                                frequency = 'quarter'
                            if sheet_name == 'monthly':
                                frequency = 'month'
                            if sheet_name == 'daily':
                                frequency = 'day'
                           # series['provider'] = 'WorldBank'
                            Key = name_series[:-5].replace(' ',
                                                '_').replace(',', '')+'.'+column[0].value
                            document = Dataset(provider = 'WorldBank', name = name_series[:-5] , datasetCode = Key, attributeList = '', dimensionList = '', docHerf = '',lastUpdate = released_dates)
                            id = document.update_database()

    def categories_db(self):
        response = urllib.request.urlopen(
                   'http://siteresources.worldbank.org/INTPROSPECTS/Resources/' +\
                   'GemDataEXTR.zip')
        zipfile_ = zipfile.ZipFile(io.BytesIO(response.read()))
        excelfile = {name : zipfile_.read(name) for name in zipfile_.namelist()}
        series_ = []
        dataset_ = []
        categories_ = []
        for name_series in excelfile.keys():
            excel_file = xlrd.open_workbook(file_contents = excelfile[name_series])
            for sheet_name in excel_file.sheet_names():
                if sheet_name not in ['Sheet1','Sheet2','Sheet3']:
                    label_row_list = excel_file.sheet_by_name(sheet_name).col(0)
                    label_column_list = excel_file.sheet_by_name(sheet_name).row(0)
                    for column_index in range (
                        excel_file.sheet_by_name(sheet_name).ncols):
                        series = {}
                        dataset = {}
                        categories = {}
                        value = []
                        value_dataset_ = []
                        column = excel_file.sheet_by_name(sheet_name).col(column_index)
                        for info in zipfile_.infolist():
                            if info.filename == name_series:
                                released_dates = datetime(*info.date_time[0:6])
                        if column[0].value not in ('obs') :
                            if name_series[:-5] in ['Commodity Prices']:
                                dimensions_int = {'name':'Commodity Prices',
                                'value':column[0].value}
                            if name_series[:-5] not in ['Commodity Prices']:
                                dimensions_int = {'name':'country',
                                'value':column[0].value}
                            column_value = column[1:-1]
                            for cell_value in column_value :
                                value.append(cell_value.value)
                            if sheet_name in ('annual') :
                                start_date_b = str(
                                    label_row_list[3].value)[:-2]
                                end_date_b = str(
                                    label_row_list[-1].value)[:-2]

                            if sheet_name not in ('annual') :
                                start_date = str(label_row_list[3].value)
                                start_date_b = start_date.replace('M','-')
                                end_date = str(label_row_list[-1].value)
                                end_date_b = end_date.replace('M','-')

                            if sheet_name == 'annual':
                                frequency = 'year'
                            if sheet_name == 'quarterly':
                                frequency = 'quarter'
                            if sheet_name == 'monthly':
                                frequency = 'month'
                            if sheet_name == 'daily':
                                frequency = 'day'
                            Key = name_series[:-5].replace(' ',
                                                '_').replace(',', '')+'.'+column[0].value

                            document = Category(provider = 'WorldBank', name = name_series[:-5] , datasetCode = Key,lastUpdate = released_dates , docHref ='')
                            _id = document.update_database()

    def update_a_series(self):
        response = urllib.request.urlopen(
                   'http://siteresources.worldbank.org/INTPROSPECTS/Resources/' +\
                   'GemDataEXTR.zip')
        zipfile_ = zipfile.ZipFile(io.BytesIO(response.read()))
        excelfile = {name : zipfile_.read(name) for name in zipfile_.namelist()}
        series_ = []
        dataset_ = []
        categories_ = []
        start_date_b_test= []
        end_date_b_test= []
        fre_fre_fre = []
        test = []
        test1 = []
        for name_series in excelfile.keys():
            excel_file = xlrd.open_workbook(file_contents = excelfile[name_series])
            for sheet_name in excel_file.sheet_names():

                if sheet_name not in ['Sheet1','Sheet2','Sheet3','Sheet4']:
                    label_row_list = excel_file.sheet_by_name(sheet_name).col(0)
                    label_column_list = excel_file.sheet_by_name(sheet_name).row(0)
                    for column_index in range (
                        excel_file.sheet_by_name(sheet_name).ncols):
                        series = {}
                        dataset = {}
                        categories = {}
                        value = []
                        value_dataset_ = []
                        released_dates= []

                        column = excel_file.sheet_by_name(sheet_name).col(column_index)
                        for info in zipfile_.infolist():
                            if info.filename == name_series:
                                released_dates.append(datetime(*info.date_time[0:6]))
                        if column[0].value not in ('obs') :
                            if name_series[:-5] in ['Commodity Prices']:
                                dimensions_int = {'name':'Commodity Prices',
                                'value':column[0].value}
                            if name_series[:-5] not in ['Commodity Prices']:
                                dimensions_int = {'name':'country',
                                'value':column[0].value}
                            column_value = column[1:-1]
                            for cell_value in column_value :
                                value.append(str(cell_value.value))
                            if sheet_name in ('annual') :
                                start_date_b = str(
                                    label_row_list[3].value)[:-2]

                                end_date_b = str(
                                    label_row_list[-1].value)[:-2]

                            if sheet_name not in ('annual') :
                                start_date = str(label_row_list[3].value)
                                start_date_b = start_date.replace('M','-')
                                end_date = str(label_row_list[-1].value)
                                end_date_b = end_date.replace('M','-')

                            if sheet_name == 'annual':
                                frequency = 'year'
                            if sheet_name == 'quarterly':
                                frequency = 'quarter'
                            if sheet_name == 'monthly':
                                frequency = 'month'
                            if sheet_name == 'daily':
                                frequency = 'day'
                            Key = name_series[:-5].replace(' ',
                                                '_').replace(',', '')+'.'+column[0].value

                            #period_index = pandas.period_range(start_date_b, end_date_b ,freq = frequency)
                            document = Series(provider = 'WorldBank', name = name_series[:-5] , key = Key, datasetCode = Key, values = value, period_index = pandas.period_range(start_date_b, end_date_b , freq = frequency), releaseDates = released_dates ,frequency=frequency , dimensions =  dimensions_int)
                            _id = document.update_database(key = Key)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#	OpenERP, Open Source Management Solution
#	Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
#
#	This program is free software: you can redistribute it and/or modify
#	it under the terms of the GNU Affero General Public License as
#	published by the Free Software Foundation, either version 3 of the
#	License, or (at your option) any later version.
#
#	This program is distributed in the hope that it will be useful,
#	but WITHOUT ANY WARRANTY; without even the implied warranty of
#	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#	GNU Affero General Public License for more details.
#
#	You should have received a copy of the GNU Affero General Public License
#	along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import openerp
import re
import time
import pytz
import codecs
from datetime import datetime, timedelta, date
from dateutil import parser
from dateutil import rrule
from dateutil.relativedelta import relativedelta
from openerp.osv import fields, osv
from openerp.tools.translate import _
import logging
import StringIO
import base64
_logger = logging.getLogger(__name__)

tdoc_selection = {
	'11' : 'Registro civil',
	'12' : 'Tarjeta de identidad',
	'13' : 'Cédula de ciudadanía',
	'21' : 'Tarjeta de extranjería',
	'22' : 'Cédula de extranjería',
	'31' : 'NIT',
	'41' : 'Pasaporte',
	'42' : 'Tipo de documento extranjero',
	'43' : 'Para uso definido por la DIAN',
	'NU' : 'Número único de identificación',
	'AS' : 'Adulto sin identificación',
	'MS' : 'Menor sin identificación'
	}

tipo_cliente = {
	'1' : 'CONTRIBUTIVO',
	'2' : 'SUBSIDIADO',
	'3' : 'VINCULADO',
	'4' : 'PARTICULAR',
	'5' : 'OTRO',
}

tipo_archivo_rips = {
	'1'  : 'AC',
	'2'  : 'AD',
	'3'  : 'AF',
	'4'  : 'AH',
	'5'  : 'AM',
	'6'  : 'AP',
	'7'  : 'AT',
	'8'  : 'AU',
	'9'  : 'CT',
	'10' : 'US',
}

class radicacion_cuentas(osv.osv):
	'''
	ABOUT RIPS.
	'''
	_description='Modelo para la radicacion de cuentas de RIPS'
	_name ='rips.radicacioncuentas'
	_rec_name = 'secuencia'

	_columns = {
		'secuencia' : fields.char("Cuenta N°", size=200 ),
		'cliente': fields.many2one('doctor.insurer', 'Cliente', required=True, help='Aseguradora'),
		'f_radicacion' : fields.date('Fecha Radicación', required=True),
		'rangofacturas_desde' : fields.date('Desde', required=True),
		'rangofacturas_hasta' : fields.date('Hasta', required=True),
		'numero_radicado' : fields.char("N° Radicado", size=200 ),
		'cantidad_factura' : fields.integer('Cantidad Facturas'),

		'valor_total' : fields.float('Valor Total'),
		'saldo' : fields.float('Saldo'),
		'plano' : fields.binary(string='Archivo RIP'),
		'plano_nombre' : fields.char('File name', 40, readonly=True),
	 	'tipo_usuario_id': fields.selection((('1','Contributivo'), ('2','Subsidiado'),
										   ('3','Vinculado'), ('4','Particular'),
										   ('5','Otro')), 'Tipo de usuario', required= True),
		#Rips
		'rips_ids': fields.one2many('rips.generados', 'radicacioncuentas_id', string='RIPS', required=True, ondelete='restrict'),
		#Facturas
		'invoices_ids': fields.one2many('account.invoice', 'radicacioncuentas_id', string='Invoices', required=True, ondelete='restrict'),

	}

	_defaults = {
		'f_radicacion' : fields.date.context_today,

	}

	def _date_to_dateuser(self,cr, uid,date,context=None):
		dateuser = datetime.strptime(date, "%Y-%m-%d %H:%M:%S")

		user = self.pool.get('res.users').browse(cr, uid, uid)
		tz = pytz.timezone(user.tz) if user.tz else pytz.utc
		dateuser = pytz.utc.localize(dateuser).astimezone(tz)

		dateuser = datetime.strftime(dateuser, "%d-%m-%Y %H:%M:%S")
		return dateuser

	def get_invoices(self, cr, uid, ids, cliente, rangofacturas_desde, rangofacturas_hasta, tipo_usuario_id, context=None):
		id_insurer = self.pool.get("doctor.insurer").browse(cr, uid, cliente).insurer.id
		id_partner= self.pool.get("doctor.insurer").browse(cr, uid, id_insurer).id
		invoices = self.pool.get('account.invoice').search(cr, uid, [('partner_id', '=', id_partner),
																('date_invoice', '>=', rangofacturas_desde),
																('date_invoice', '<=', rangofacturas_hasta),
																('residual', '<>', 0.0),
																('tipo_usuario_id', '=', tipo_usuario_id ),
																('radicada', '=', False)])

		return {'value': {'invoices_ids': invoices}}

	def getParentid(self, cr, uid, context=None):
		#searching the parent_id // RIPS's directory
		try:
			parent_id = self.pool.get('document.directory').search(cr, uid, [('name', '=', 'Rips')])
		except Exception as e:
			raise osv.except_osv(_('Error!'),
					_('No hay un lugar donde almacenar el archivo RIPS. Habilite la opcion gestion de documentos en el menu Configuracion > Conocimiento o contacte al administrador del sistema.'))

		if not parent_id:
			parent_id = self.pool.get('document.directory').create(cr, uid, {'name' : 'Rips', 'parent_id' : 1, 'type': 'directory'})
		return parent_id

	def getSecuencia(self, cr, uid, context=None):

		return True

	def getNombreArchivo(self, cr, uid, fileType, context=None):
		if fileType:
			rips_generados_obj = self.pool.get('rips.generados')
			todos = rips_generados_obj.search(cr, uid, [])
			if todos:
				ultimo_registro = todos[-1]
				nombre = rips_generados_obj.browse(cr, uid, ultimo_registro).nombre_archivo
			else:
				nombre = 'XX000000.txt'
			get_secuencia = int(nombre[2:8])
			aumentando_secuencia = '{0:06}'.format(get_secuencia + 1)
			nueva_secuencia = fileType + aumentando_secuencia +'.txt'
			return nueva_secuencia
		else:
			return 'known.txt'

	def generar_rips(self, cr, uid, ids, context=None):
		for var in self.browse(cr, uid, ids):
			archivo = StringIO.StringIO()
			for factura in var.invoices_ids:
				tipo_archivo = tipo_archivo_rips.get('3')
				nombre_archivo = self.getNombreArchivo(cr,uid,tipo_archivo)
				parent_id = self.getParentid(cr,uid,ids)[0]

				#***********GET CAMPOS********
				todos = self.pool.get('res.company').search(cr, uid, [])
				invoices = self.pool.get('account.invoice')
				company_id = self.pool.get('res.company').browse(cr, uid, todos[0])
				cod_prestadorservicio = company_id.cod_prestadorservicio
				#company_id
				if cod_prestadorservicio:
					archivo.write( cod_prestadorservicio + ',')
				else:
					raise osv.except_osv(_('Error!'),
							_('El campo codigo prestador de servicio de la compañia está vacío.'))

				#Razon social o apellidos y nombre del prestador de servicios
				nombre_prestadorservicio = company_id.partner_id.name
				archivo.write( nombre_prestadorservicio + ',')
				#Tipo de identificación
				tdoc_prestadorservicio = company_id.partner_id.tdoc
				tdoc_nombre = tdoc_selection.get(tdoc_prestadorservicio)
				archivo.write( tdoc_nombre + ',')
				#Número de identificación
				if company_id.partner_id.ref:
					nro_identificacion = company_id.partner_id.ref
				else:
					raise osv.except_osv(_('Error!'),
							_('El campo N° de Identificacion del tercero que corresponde a la compañia está vacío.'))

				archivo.write( nro_identificacion + ',')
				#Número de la factura
				numero_factura = factura.number
				if numero_factura:
					archivo.write( numero_factura + ',')
				else:
					archivo.write( '0' + ',')
				#Fecha expedición de la factura
				fecha_exp_factura = factura.date_invoice
				archivo.write( fecha_exp_factura + ',')
				#Fecha inicio Radicacion Cuentas
				archivo.write( var.rangofacturas_desde + ',')
				#Fecha fin Radicacion Cuentas
				archivo.write( var.rangofacturas_hasta + ',')
				#codigo aseguradora
				archivo.write( var.cliente.code + ',')
				#nombre aseguradora
				archivo.write( var.cliente.insurer.name + ',')
				#plan de beneficios
				archivo.write( tipo_cliente.get(var.tipo_usuario_id) + ',')
				#valor total del pago copmartido (valor paciente)
				archivo.write( str(factura.amount_patient) + ',')
				#valor neto a pagar por la entidad contratante
				archivo.write( str(factura.amount_total) + ',')
				#salto de linea
				archivo.write('\n')
				# actualizar factura a radicada
				invoices.write(cr, uid, factura.id, {'radicada' : True})

		output = base64.encodestring(archivo.getvalue())
		id_attachment = self.pool.get('ir.attachment').create(cr, uid, {'name': nombre_archivo , 'datas_fname': nombre_archivo, 'type': 'binary', 'datas': output, 'parent_id' : parent_id, 'res_model' : 'rips.radicacioncuentas', 'res_id' : ids[0]}, context= context)

		for actual in self.browse(cr, uid, ids):
			self.pool.get('rips.generados').create(cr, uid, {'radicacioncuentas_id': ids[0],
															  'f_generacion': self._date_to_dateuser(cr,uid, date.today().strftime('%Y-%m-%d %H:%M:%S')),
															 'nombre_archivo': nombre_archivo,
															 'f_inicio_radicacion': actual.rangofacturas_desde,
															 'f_fin_radicacion' : actual.rangofacturas_hasta,
															 'archivo' : output}, context=context)
		return True

	def create(self, cr, uid, vals, context=None):
		return super(radicacion_cuentas, self).create(cr, uid, vals, context)

	def confirmar(self, cr, uid, ids, context=None):
		raise osv.except_osv(_('Aviso!'),
				_('Funcionalidad no implementada.'))

	def validar(self, cr, uid, ids, context=None):
		raise osv.except_osv(_('Aviso!'),
				_('Funcionalidad no implementada.'))

radicacion_cuentas()
<commit_msg>bug fixing<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#	OpenERP, Open Source Management Solution
#	Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
#
#	This program is free software: you can redistribute it and/or modify
#	it under the terms of the GNU Affero General Public License as
#	published by the Free Software Foundation, either version 3 of the
#	License, or (at your option) any later version.
#
#	This program is distributed in the hope that it will be useful,
#	but WITHOUT ANY WARRANTY; without even the implied warranty of
#	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#	GNU Affero General Public License for more details.
#
#	You should have received a copy of the GNU Affero General Public License
#	along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import openerp
import re
import time
import pytz
import codecs
from datetime import datetime, timedelta, date
from dateutil import parser
from dateutil import rrule
from dateutil.relativedelta import relativedelta
from openerp.osv import fields, osv
from openerp.tools.translate import _
import logging
import StringIO
import base64
_logger = logging.getLogger(__name__)

tdoc_selection = {
	'11' : 'Registro civil',
	'12' : 'Tarjeta de identidad',
	'13' : 'Cédula de ciudadanía',
	'21' : 'Tarjeta de extranjería',
	'22' : 'Cédula de extranjería',
	'31' : 'NIT',
	'41' : 'Pasaporte',
	'42' : 'Tipo de documento extranjero',
	'43' : 'Para uso definido por la DIAN',
	'NU' : 'Número único de identificación',
	'AS' : 'Adulto sin identificación',
	'MS' : 'Menor sin identificación'
	}

tipo_cliente = {
	'1' : 'CONTRIBUTIVO',
	'2' : 'SUBSIDIADO',
	'3' : 'VINCULADO',
	'4' : 'PARTICULAR',
	'5' : 'OTRO',
}

tipo_archivo_rips = {
	'1'  : 'AC',
	'2'  : 'AD',
	'3'  : 'AF',
	'4'  : 'AH',
	'5'  : 'AM',
	'6'  : 'AP',
	'7'  : 'AT',
	'8'  : 'AU',
	'9'  : 'CT',
	'10' : 'US',
}

class radicacion_cuentas(osv.osv):
	'''
	ABOUT RIPS.
	'''
	_description='Modelo para la radicacion de cuentas de RIPS'
	_name ='rips.radicacioncuentas'
	_rec_name = 'secuencia'

	_columns = {
		'secuencia' : fields.char("Cuenta N°", size=200 ),
		'cliente': fields.many2one('doctor.insurer', 'Cliente', required=True, help='Aseguradora'),
		'f_radicacion' : fields.date('Fecha Radicación', required=True),
		'rangofacturas_desde' : fields.date('Desde', required=True),
		'rangofacturas_hasta' : fields.date('Hasta', required=True),
		'numero_radicado' : fields.char("N° Radicado", size=200 ),
		'cantidad_factura' : fields.integer('Cantidad Facturas'),

		'valor_total' : fields.float('Valor Total'),
		'saldo' : fields.float('Saldo'),
		'plano' : fields.binary(string='Archivo RIP'),
		'plano_nombre' : fields.char('File name', 40, readonly=True),
	 	'tipo_usuario_id': fields.selection((('1','Contributivo'), ('2','Subsidiado'),
										   ('3','Vinculado'), ('4','Particular'),
										   ('5','Otro')), 'Tipo de usuario', required= True),
		#Rips
		'rips_ids': fields.one2many('rips.generados', 'radicacioncuentas_id', string='RIPS', required=True, ondelete='restrict'),
		#Facturas
		'invoices_ids': fields.one2many('account.invoice', 'radicacioncuentas_id', string='Invoices', required=True, ondelete='restrict'),

	}

	def _date_to_dateuser(self,cr, uid,date,context=None):
		dateuser = datetime.strptime(date, "%Y-%m-%d %H:%M:%S")

		user = self.pool.get('res.users').browse(cr, uid, uid)
		tz = pytz.timezone(user.tz) if user.tz else pytz.utc
		dateuser = pytz.utc.localize(dateuser).astimezone(tz)

		dateuser = datetime.strftime(dateuser, "%d-%m-%Y %H:%M:%S")
		return dateuser

	def get_invoices(self, cr, uid, ids, cliente, rangofacturas_desde, rangofacturas_hasta, tipo_usuario_id, context=None):
		id_insurer = self.pool.get("doctor.insurer").browse(cr, uid, cliente).insurer.id
		id_partner= self.pool.get("doctor.insurer").browse(cr, uid, id_insurer).id
		invoices = self.pool.get('account.invoice').search(cr, uid, [('partner_id', '=', id_partner),
																('date_invoice', '>=', rangofacturas_desde),
																('date_invoice', '<=', rangofacturas_hasta),
																('residual', '<>', 0.0),
																('tipo_usuario_id', '=', tipo_usuario_id ),
																('radicada', '=', False)])

		return {'value': {'invoices_ids': invoices}}

	def getParentid(self, cr, uid, context=None):
		#searching the parent_id // RIPS's directory
		try:
			parent_id = self.pool.get('document.directory').search(cr, uid, [('name', '=', 'Rips')])
		except Exception as e:
			raise osv.except_osv(_('Error!'),
					_('No hay un lugar donde almacenar el archivo RIPS. Habilite la opcion gestion de documentos en el menu Configuracion > Conocimiento o contacte al administrador del sistema.'))

		if not parent_id:
			parent_id = self.pool.get('document.directory').create(cr, uid, {'name' : 'Rips', 'parent_id' : 1, 'type': 'directory'})
		return parent_id

	def getSecuencia(self, cr, uid, context=None):

		return True

	def getNombreArchivo(self, cr, uid, fileType, context=None):
		if fileType:
			rips_generados_obj = self.pool.get('rips.generados')
			todos = rips_generados_obj.search(cr, uid, [])
			if todos:
				ultimo_registro = todos[-1]
				nombre = rips_generados_obj.browse(cr, uid, ultimo_registro).nombre_archivo
			else:
				nombre = 'XX000000.txt'
			get_secuencia = int(nombre[2:8])
			aumentando_secuencia = '{0:06}'.format(get_secuencia + 1)
			nueva_secuencia = fileType + aumentando_secuencia +'.txt'
			return nueva_secuencia
		else:
			return 'known.txt'

	def generar_rips(self, cr, uid, ids, context=None):
		for var in self.browse(cr, uid, ids):
			archivo = StringIO.StringIO()
			for factura in var.invoices_ids:
				tipo_archivo = tipo_archivo_rips.get('3')
				nombre_archivo = self.getNombreArchivo(cr,uid,tipo_archivo)
				parent_id = self.getParentid(cr,uid,ids)[0]

				#***********GET CAMPOS********
				todos = self.pool.get('res.company').search(cr, uid, [])
				invoices = self.pool.get('account.invoice')
				company_id = self.pool.get('res.company').browse(cr, uid, todos[0])
				cod_prestadorservicio = company_id.cod_prestadorservicio
				#company_id
				if cod_prestadorservicio:
					archivo.write( cod_prestadorservicio + ',')
				else:
					raise osv.except_osv(_('Error!'),
							_('El campo codigo prestador de servicio de la compañia está vacío.'))

				#Razon social o apellidos y nombre del prestador de servicios
				nombre_prestadorservicio = company_id.partner_id.name
				archivo.write( nombre_prestadorservicio + ',')
				#Tipo de identificación
				tdoc_prestadorservicio = company_id.partner_id.tdoc
				tdoc_nombre = tdoc_selection.get(tdoc_prestadorservicio)
				archivo.write( tdoc_nombre + ',')
				#Número de identificación
				if company_id.partner_id.ref:
					nro_identificacion = company_id.partner_id.ref
				else:
					raise osv.except_osv(_('Error!'),
							_('El campo N° de Identificacion del tercero que corresponde a la compañia está vacío.'))

				archivo.write( nro_identificacion + ',')
				#Número de la factura
				numero_factura = factura.number
				if numero_factura:
					archivo.write( numero_factura + ',')
				else:
					archivo.write( '0' + ',')
				#Fecha expedición de la factura
				fecha_exp_factura = factura.date_invoice
				archivo.write( fecha_exp_factura + ',')
				#Fecha inicio Radicacion Cuentas
				archivo.write( var.rangofacturas_desde + ',')
				#Fecha fin Radicacion Cuentas
				archivo.write( var.rangofacturas_hasta + ',')
				#codigo aseguradora
				archivo.write( var.cliente.code + ',')
				#nombre aseguradora
				archivo.write( var.cliente.insurer.name + ',')
				#plan de beneficios
				archivo.write( tipo_cliente.get(var.tipo_usuario_id) + ',')
				#valor total del pago copmartido (valor paciente)
				archivo.write( str(factura.amount_patient) + ',')
				#valor neto a pagar por la entidad contratante
				archivo.write( str(factura.amount_total) + ',')
				#salto de linea
				archivo.write('\n')
				# actualizar factura a radicada
				invoices.write(cr, uid, factura.id, {'radicada' : True})

		output = base64.encodestring(archivo.getvalue())
		id_attachment = self.pool.get('ir.attachment').create(cr, uid, {'name': nombre_archivo , 'datas_fname': nombre_archivo, 'type': 'binary', 'datas': output, 'parent_id' : parent_id, 'res_model' : 'rips.radicacioncuentas', 'res_id' : ids[0]}, context= context)

		for actual in self.browse(cr, uid, ids):
			self.pool.get('rips.generados').create(cr, uid, {'radicacioncuentas_id': ids[0],
															  'f_generacion': self._date_to_dateuser(cr,uid, date.today().strftime('%Y-%m-%d %H:%M:%S')),
															 'nombre_archivo': nombre_archivo,
															 'f_inicio_radicacion': actual.rangofacturas_desde,
															 'f_fin_radicacion' : actual.rangofacturas_hasta,
															 'archivo' : output}, context=context)
		return True

	def create(self, cr, uid, vals, context=None):
		return super(radicacion_cuentas, self).create(cr, uid, vals, context)

	def confirmar(self, cr, uid, ids, context=None):
		raise osv.except_osv(_('Aviso!'),
				_('Funcionalidad no implementada.'))

	def validar(self, cr, uid, ids, context=None):
		raise osv.except_osv(_('Aviso!'),
				_('Funcionalidad no implementada.'))

radicacion_cuentas()
<|endoftext|>


<commit_before>import calendar
import json
import time
import unittest
import urllib
from urlparse import urlparse, urlunparse

from django.conf import settings
import requests
import websocket

from cla_frontend.apps.legalaid import addressfinder

def unix_timestamp():
    return calendar.timegm(time.gmtime())

class SmokeTests(unittest.TestCase):

    def test_can_access_backend(self):
        "access the backend"
        response = requests.get(settings.BACKEND_BASE_URI + '/status.json')

    def test_can_access_addressfinder(self):
        "lookup a postcode with AddressFinder"
        response = addressfinder.lookup_postcode('sw1a1aa')
        self.assertNotEqual(
            401,
            response.status_code,
            "Invalid token")
        self.assertEqual(200, response.status_code)

    def test_can_access_socketserver(self):
        "connect to socket server"

        parts = urlparse(settings.SOCKETIO_SERVER_URL)
        host, _, port = parts.netloc.partition(':')
        host = host or settings.SITE_HOSTNAME
        port = ':%s' % port if port else ''
        path = parts.path + '/1/'
        parts = list(parts)
        protocol = 'https' if settings.SESSION_COOKIE_SECURE else 'http'
        headers = {
            'Origin': 'http://localhost:8001',
        }

        response = requests.get('{protocol}://{host}{port}{path}?{params}'.format(
                protocol=protocol,
                host=host,
                port=port,
                path=path,
                params=urllib.urlencode({
                    't': unix_timestamp(),
                    'transport': 'polling',
                    'b64': '1'})),
            timeout=10, headers=headers)

        session_id = json.loads(response.text[4:])['sid']

        parts[0] = 'ws'
        parts[1] = parts[1] or host
        parts[2] = path + 'websocket/'
        parts[4] = urllib.urlencode({
            'sid': session_id,
            'transport': 'polling',
            'timestamp': unix_timestamp()})
        ws_url = urlunparse(parts)
        ws = websocket.create_connection(ws_url)
        ws.send('0:::')
<commit_msg>send correct origin in smoke tests for socket server<commit_after>import calendar
import json
import time
import unittest
import urllib
from urlparse import urlparse, urlunparse

from django.conf import settings
import requests
import websocket

from cla_frontend.apps.legalaid import addressfinder

def unix_timestamp():
    return calendar.timegm(time.gmtime())

class SmokeTests(unittest.TestCase):

    def test_can_access_backend(self):
        "access the backend"
        response = requests.get(settings.BACKEND_BASE_URI + '/status.json')

    def test_can_access_addressfinder(self):
        "lookup a postcode with AddressFinder"
        response = addressfinder.lookup_postcode('sw1a1aa')
        self.assertNotEqual(
            401,
            response.status_code,
            "Invalid token")
        self.assertEqual(200, response.status_code)

    def test_can_access_socketserver(self):
        "connect to socket server"

        parts = urlparse(settings.SOCKETIO_SERVER_URL)
        host, _, port = parts.netloc.partition(':')
        host = host or settings.SITE_HOSTNAME
        port = ':%s' % port if port else ''
        path = parts.path + '/1/'
        parts = list(parts)
        protocol = 'https' if settings.SESSION_COOKIE_SECURE else 'http'
        origin = '{protocol}://{host}{port}'.format(
            protocol=protocol,
            host=host,
            port=port
        )
        headers = {
            'Origin': origin,
        }

        response = requests.get('{origin}{path}?{params}'.format(
                origin=origin,
                path=path,
                params=urllib.urlencode({
                    't': unix_timestamp(),
                    'transport': 'polling',
                    'b64': '1'})
            ),
            timeout=10, headers=headers
        )

        session_id = json.loads(response.text[4:])['sid']

        parts[0] = 'ws'
        parts[1] = parts[1] or host
        parts[2] = path + 'websocket/'
        parts[4] = urllib.urlencode({
            'sid': session_id,
            'transport': 'polling',
            'timestamp': unix_timestamp()})
        ws_url = urlunparse(parts)
        ws = websocket.create_connection(ws_url)
        ws.send('0:::')
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# Interpreter version: python 2.7
#
# Imports =====================================================================
import shutil
import os.path
import tempfile
import subprocess

import sh  # TODO: remove

from string import Template
from multiprocessing import Process

# Variables ===================================================================
SERV = None
TMP_DIR = None

# Functions & classes =========================================================
def data_context_name(fn):
    return os.path.join(os.path.dirname(__file__), "data", fn)

def data_context(fn, mode="r"):
    with open(data_context_name(fn), mode) as f:
        return f.read()

def tmp_context_name(fn):
    return os.path.join(TMP_DIR, fn)

def tmp_context(fn, mode="r"):
    with open(tmp_context_name(fn), mode) as f:
        return f.read()

# Environment generators ======================================================
def generate_environment():
    global TMP_DIR
    TMP_DIR = tempfile.mkdtemp()

    # write ZEO server config to  temp directory
    zeo_conf_path = os.path.join(TMP_DIR, "zeo.conf")
    with open(zeo_conf_path, "w") as f:
        f.write(
            Template(data_context("zeo.conf")).substitute(path=TMP_DIR)
        )

    # write client config to temp directory
    client_config_path = os.path.join(TMP_DIR, "zeo_client.conf")
    with open(client_config_path, "w") as f:
        f.write(data_context("zeo_client.conf"))

    # run the ZEO server
    def run_zeo():
        # subprocess.check_call("runzeo -C " + zeo_conf_path, shell=True)
        sh.runzeo(C=zeo_conf_path)

    global SERV
    SERV = Process(target=run_zeo)
    SERV.start()

def cleanup_environment():
    SERV.terminate()
    shutil.rmtree(TMP_DIR)
<commit_msg>#5: Small improvements in tests.<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# Interpreter version: python 2.7
#
# Imports =====================================================================
import shutil
import os.path
import tempfile

import sh

from string import Template
from multiprocessing import Process

# Variables ===================================================================
SERV = None
TMP_DIR = None

# Functions & classes =========================================================
def data_context_name(fn):
    return os.path.join(os.path.dirname(__file__), "data", fn)

def data_context(fn, mode="r"):
    with open(data_context_name(fn), mode) as f:
        return f.read()

def tmp_context_name(fn):
    return os.path.join(TMP_DIR, fn)

def tmp_context(fn, mode="r"):
    with open(tmp_context_name(fn), mode) as f:
        return f.read()

# Environment generators ======================================================
def generate_environment():
    global TMP_DIR
    TMP_DIR = tempfile.mkdtemp()

    # write ZEO server config to  temp directory
    zeo_conf_path = os.path.join(TMP_DIR, "zeo.conf")
    with open(zeo_conf_path, "w") as f:
        f.write(
            Template(data_context("zeo.conf")).substitute(path=TMP_DIR)
        )

    # write client config to temp directory
    client_config_path = os.path.join(TMP_DIR, "zeo_client.conf")
    shutil.copy(data_context_name("zeo_client.conf"), client_config_path)

    # run the ZEO server
    def run_zeo():
        # sh terminates when .terminate() is called, suprocess and os.system()
        # doesn't
        sh.runzeo(C=zeo_conf_path)

    global SERV
    SERV = Process(target=run_zeo)
    SERV.start()

def cleanup_environment():
    SERV.terminate()
    shutil.rmtree(TMP_DIR)
<|endoftext|>


<commit_before>"""
client_test.py: Client test class

"""

__author__ = "Daniel Mazzer"
__copyright__ = "Copyright 2016, NORS project"
__credits__ = ""
__license__ = "MIT"
__maintainer__ = "Daniel Mazzer"
__email__ = "dmazzer@gmail.com"

import sys
import json
import unittest

sys.path.append('../')
from client.client import Nors_Client
from config.config import Nors_Configuration

# def load_configuration():
#     config = Nors_Configuration('config_client_test.ini')
#     server_ip = config.ReadConfig('server', 'ip')
#     server_port = config.ReadConfig('server', 'port')
#
#     client_auth = config.ReadConfig('client', 'auth')
#     client_id = config.ReadConfig('client', 'id')

class Test_Client(unittest.TestCase):
    def setUp(self):
        self.c = Nors_Client(config)

#     def test_gettoken(self):
#         token = self.c.conn.get_token()
#         self.assertNotEqual(token, None)

    def test_get_protected_resource(self):
        r = self.c.conn.get_resource('/sensors/')
        self.assertNotEqual(r, None)

    def test_post_protected_resource(self):
        data = json.dumps({"name":"p3", "id":1000})
        r = self.c.conn.post_resource('/sensors/', data)
#         r = self.c.conn.get_resource('/sensors/1')
        self.assertNotEqual(r, None)

    def tearDown(self):
        pass

if __name__ == '__main__':
    if len(sys.argv) > 1:
        arg1 = sys.argv.pop()
        arg2 = sys.argv.pop()
    config = Nors_Configuration()
    unittest.main()
<commit_msg>fixed problem with url_for<commit_after>"""
client_test.py: Client test class

"""

__author__ = "Daniel Mazzer"
__copyright__ = "Copyright 2016, NORS project"
__credits__ = ""
__license__ = "MIT"
__maintainer__ = "Daniel Mazzer"
__email__ = "dmazzer@gmail.com"

import sys
import json
import unittest

sys.path.append('../')
from client.client import Nors_Client
from config.config import Nors_Configuration

# def load_configuration():
#     config = Nors_Configuration('config_client_test.ini')
#     server_ip = config.ReadConfig('server', 'ip')
#     server_port = config.ReadConfig('server', 'port')
#
#     client_auth = config.ReadConfig('client', 'auth')
#     client_id = config.ReadConfig('client', 'id')

class Test_Client(unittest.TestCase):
    def setUp(self):
        self.c = Nors_Client(config)

#     def test_gettoken(self):
#         token = self.c.conn.get_token()
#         self.assertNotEqual(token, None)

    def test_get_protected_resource(self):
        r = self.c.conn.get_resource('/sensors/')
        self.assertNotEqual(r, None)

    def test_post_protected_resource(self):
        data = json.dumps({"name":"p3", "idd":"1000"})
        r = self.c.conn.post_resource('/sensors/', data)
#         r = self.c.conn.get_resource('/sensors/1')
        self.assertNotEqual(r, None)

    def tearDown(self):
        pass

if __name__ == '__main__':
    if len(sys.argv) > 1:
        arg1 = sys.argv.pop()
        arg2 = sys.argv.pop()
    config = Nors_Configuration()
    unittest.main()
<|endoftext|>


<commit_before>from flask import g, jsonify, request

from golf_stats import app
from golf_stats.models import Hole, Round, User
from golf_stats.actions import update_round
from .authorize import check_authorization

@app.route('/api/rounds')
@check_authorization
def get_rounds():
    rounds = Round.query.all()
    return jsonify({r.id: r.date for r in rounds})

@app.route('/api/round/<round_id>')
@check_authorization
def get_round(round_id):
    golf_round = Round.query.get(round_id)
    if golf_round:
        if g.user.username == golf_round.user.username:
            return jsonify(golf_round.as_dict())
        else:
            return jsonify(error='round belongs to another user')
    else:
        return jsonify(error='not found')

@app.route('/api/round/hole/<hole_id>')
@check_authorization
def get_hole(hole_id):
    hole = Hole.query.get(hole_id)
    if hole_id:
        if g.user.username == hole.round.user.username:
            return jsonify(hole.as_dict())
        else:
            return jsonify(error='hole belongs to another user')
    else:
        return jsonify(error='not found')

@app.route('/api/add_round', methods=['POST'])
@check_authorization
def add_round():
    try:
        user_id = request.form.get('user_id')
        if user_id:
            if g.user.username == User.query.get(int(user_id)):
                return jsonify(update_round(request.get_json()))
            else:
                return jsonify(error='not permitted')
        else:
            return jsonify(error='user_id is required')
    except ValueError as error:
        return jsonify(error="ValueError: invalid 'user_id'")
    except TypeError as error:
        return jsonify(error='TypeError: %s' % error)
<commit_msg>return more info on errors<commit_after>from flask import g, jsonify, request

from golf_stats import app
from golf_stats.models import Hole, Round, User
from golf_stats.actions import update_round
from .authorize import check_authorization

@app.route('/api/rounds')
@check_authorization
def get_rounds():
    rounds = Round.query.all()
    return jsonify({r.id: r.date for r in rounds})

@app.route('/api/round/<round_id>')
@check_authorization
def get_round(round_id):
    golf_round = Round.query.get(round_id)
    if golf_round:
        if g.user.username == golf_round.user.username:
            return jsonify(golf_round.as_dict())
        else:
            return jsonify(error='round belongs to another user')
    else:
        return jsonify(error='not found')

@app.route('/api/round/hole/<hole_id>')
@check_authorization
def get_hole(hole_id):
    hole = Hole.query.get(hole_id)
    if hole_id:
        if g.user.username == hole.round.user.username:
            return jsonify(hole.as_dict())
        else:
            return jsonify(error='hole belongs to another user')
    else:
        return jsonify(error='not found')

@app.route('/api/add_round', methods=['POST'])
@check_authorization
def add_round():
    try:
        if g.user.username == User.query.get(int(request.form['user_id'])):
            return jsonify(update_round(request.get_json()))
        else:
            return jsonify(error='not permitted')
    except ValueError as error:
        return jsonify(error="ValueError: invalid 'user_id'")
    except TypeError as error:
        return jsonify(error='TypeError: %s' % error)
    except KeyError as error:
        return jsonify(error='KeyError: %s' % error)
<|endoftext|>