<commit_before>import os
import sys
import time
import datetime
import logging
log = logging.getLogger(__file__)

import csv
import re

from ott.utils.config_util import ConfigUtil
from ott.utils import otp_utils
from ott.utils import object_utils

class TestResult:
    FAIL=000
    WARN=333
    PASS=111

class Test(object):
    """ Params for test, along with run capability -- Test object is typically built from a row in an .csv test suite
    """

    def __init__(self, param_dict, line_number, ws_url, map_url, date=None):
        """ {
            OTP parmas:
              'From'
              'To'
              'Max dist'
              'Mode'
              'Optimize'
              'Service' - expects 'Saturday' or 'Sunday' or leave empty
              'Time'

            Test params:
              'Arrive by' - expects 'FALSE' if arrive by test should not be ran or leave empty
              'Depart by' - expects 'FALSE' if depart by test should not be ran or leave empty
              'Expected output'
              'Expected trip duration'
              'Expected trip distance'
              'Expected number of legs'

            Misc text:
              'Description/notes'
            }
        """
        self.config = ConfigUtil(section='otp')

        self.ws_url          = ws_url
        self.map_url         = map_url

        self.csv_line_number = line_number
        self.csv_params      = param_dict
        self.date            = date

        self.itinerary       = None
        self.otp_params      = ''
        self.map_params      = ''
        self.is_valid        = True
        self.error_descript  = None
        self.result          = TestResult.FAIL

        self.coord_from      = self.get_param('From')
        self.coord_to        = self.get_param('To')
        self.distance        = self.get_param('Max dist')
        self.mode            = self.get_param('Mode')
        self.optimize        = self.get_param('Optimize')
        self.service         = self.get_param('Service')
        self.time            = self.get_param('Time', strip_all_spaces=True)
        self.description     = self.get_param('Description/notes')
        self.expect_output   = self.get_param('Expected output')
        self.expect_duration = self.get_param('Expected trip duration')
        self.expect_distance = self.get_param('Expected trip distance')
        self.expect_num_legs = self.get_param('Expected number of legs', warn_not_avail=False)
        self.arrive_by       = self.get_param('Arrive by')
        self.depart_by       = self.get_param('Depart by')

        # post process the load ... make params and urls, etc...
        self.date = self.get_date_param(self.date)
        self.init_url_params()
        self.url_distance()
        self.url_mode()
        self.url_optimize()
        self.url_time()

    def did_test_pass(self):
        ret_val = False
        if self.result is not None and self.result is TestResult.PASS:
            ret_val = True
        return ret_val

    def get_param(self, name, def_val=None, strip_all_spaces=False, warn_not_avail=True):
        return object_utils.get_striped_dict_val(self.csv_params, name, def_val, strip_all_spaces, warn_not_avail)

    def append_note(self, note=None):
        if note:
            self.description = "{}{}".format(self.description, note)

    def test_otp_result(self, strict=True):
        """ regexp test of the itinerary output for certain strings
        """
        if self.itinerary == None:
            self.result = TestResult.FAIL if strict else TestResult.WARN
            self.error_descript = "test_otp_result: itinerary is null"
            log.info(self.error_descript)
        else:
            if len(self.itinerary) < 1000:
                self.result = TestResult.FAIL if strict else TestResult.WARN
                self.error_descript = "test_otp_result: itinerary content looks small at " + str(len(self.itinerary)) + " characters."
                log.warn(self.error_descript)
            else:
                self.error_descript = "test_otp_result: itinerary content size is " + str(len(self.itinerary)) + " characters."
                log.info(self.error_descript)
                warn = False
                warn = self.test_expected_response(self.expect_output, warn, strict)
                if self.expect_duration is not None and len(self.expect_duration) > 0:
                    durations = re.findall('<itinerary>.*?<duration>(.*?)</duration>.*?</itinerary>', self.itinerary)
                    error = 0.2
                    high = float(self.expect_duration) * (1 + error)
                    low = float(self.expect_duration) * (1 - error)
                    for duration in durations:
                        if int(duration) > high or int(duration) < low:
                            self.result = TestResult.FAIL if strict else TestResult.WARN
                            self.error_descript += "test_otp_result: an itinerary duration was different than expected by more than {0}%.".format(error * 100)
                            warn = True
                            break
                if self.expect_num_legs is not None and len(self.expect_num_legs) > 0:
                    try:
                        values = [int(i) for i in self.expect_num_legs.split('|')]
                        if len(values) != 2:
                            raise ValueError
                        min_legs = values[0]
                        max_legs = values[1]
                        all_legs = re.findall('<itinerary>.*?<legs>(.*?)</legs>.*?</itinerary>', self.itinerary)
                        for legs in all_legs:
                            num_legs = len(re.findall('<leg .*?>', legs))
                            if num_legs > max_legs or num_legs < min_legs:
                                self.result = TestResult.FAIL if strict else TestResult.WARN
                                self.error_descript += "test_otp_result: an itinerary returned was not between {0} and {1} legs.".format(min_legs, max_legs)
                                warn = True
                                break
                    except ValueError:
                        self.error_descript += "expected number of legs test not in 'min|max' format."
                        warn = True
                if warn:
                    log.warn(self.error_descript)

        return self.result

    def test_expected_response(self, expected_output, initial_val, strict):
        ret_val = initial_val
        if expected_output is not None and len(expected_output) > 0:
            regres = re.search(expected_output, self.itinerary)
            if regres is None:
                self.result = TestResult.FAIL if strict else TestResult.WARN
                self.error_descript += "test_otp_result: couldn't find " + expected_output + " in otp response."
                ret_val = True
        return ret_val

    def init_url_params(self):
        """
        """
        self.otp_params = 'fromPlace={0}&toPlace={1}'.format(self.coord_from, self.coord_to)
        self.map_params = self.otp_params
        if self.coord_from == None or self.coord_from == '' or self.coord_to == None or self.coord_to == '':
            if self.coord_from != None or self.coord_to != None:
                self.error_descript = "no from and/or to coordinate for the otp url (skipping test) - from:{} to:{}".format(self.coord_from, self.coord_to)
                log.warn(self.error_descript)
            self.is_valid = False

    def url_param(self, name, param, default=None):
        """
        """
        p = param if param else default
        if p:
            self.otp_params += '&{0}={1}'.format(name, p)
            self.map_params += '&{0}={1}'.format(name, p)

    def url_distance(self, dist=None):
        self.url_param('maxWalkDistance', dist, self.distance)

    def url_mode(self, mode=None):
        self.url_param('mode', mode, self.mode)

    def url_optimize(self, opt=None):
        self.url_param('optimize', opt, self.optimize)

    def url_arrive_by(self, opt="true"):
        self.url_param('arriveBy', opt, self.optimize)

    def url_time(self, time=None):
        self.url_param('time', time, self.time)

    def url_time_7am(self):
        self.url_param('time', '7:00am')

    def url_time_12pm(self):
        self.url_param('time', '12:00pm')

    def url_time_5pm(self):
        self.url_param('time', '5:00pm')

    def url_service(self, svc=None):
        """
        """
        pass

    def get_date_param(self, date, fmt="%Y-%m-%d"):
        """ provide a default date (set to today) if no service provided...
        """
        if self.otp_params.find('date') < 0:
            if date is None:
                if self.service is None:
                    date = datetime.datetime.now().strftime(fmt)
                elif self.service == 'Saturday':
                    date = self.url_service_next_saturday()
                elif self.service == 'Sunday':
                    date = self.url_service_next_sunday()
                else:
                    date = datetime.datetime.now().strftime(fmt)
                    log.warn("service param '{0}' not valid, using todays date.".format(self.service))

            self.url_param('date', date)
        return date

    def url_service_next_weekday(self):
        """
        """
        pass

    def url_service_next_month_weekday(self):
        """
        """
        pass

    def url_service_next_saturday(self):
        date = datetime.datetime.now()
        day = date.weekday()
        if day == 6:
            date = date+datetime.timedelta(days=6)
        else:
            date = date+datetime.timedelta(days=5-day)
        date = date.strftime("%Y-%m-%d")
        return date

    def url_service_next_sunday(self):
        date = datetime.datetime.now()
        day = date.weekday()
        date = date+datetime.timedelta(days=6-day)
        date = date.strftime("%Y-%m-%d")
        return date

    def depart_by_check(self):
        if self.depart_by == 'FALSE':
            self.is_valid = False

    def arrive_by_check(self):
        if self.arrive_by == 'FALSE':
            self.is_valid = False

    def call_otp(self, url=None):
        ''' calls the trip web service
        '''
        self.itinerary = None
        start = time.time()
        url = url if url else self.get_ws_url()
        self.itinerary = otp_utils.call_planner_svc(url)
        end = time.time()
        self.response_time = end - start
        log.info("call_otp: response time of {} second for url {}".format(self.response_time, url))
        log.debug(self.itinerary)
        if self.response_time <= 30:
            self.result = TestResult.PASS
        else:
            self.result = TestResult.WARN
            log.info("call_otp: :::NOTE::: response time took *longer than 30 seconds* for url {}".format(url))

    @classmethod
    def make_url(cls, url, separater="?submit&module=planner"):
        ret_val = url
        if "?" not in url:
            ret_val = url + separater
        return ret_val

    def get_ws_url(self):
        return "{0}&{1}".format(self.make_url(self.ws_url), self.otp_params)

    def get_map_url(self):
        purl = self.ws_url.split('/')[-1]
        return "{0}&purl=/{1}&{2}".format(self.make_url(self.map_url), purl, self.map_params)

    def get_ridetrimetorg_url(self):
        return "http://ride.trimet.org?submit&" + self.map_params

class TestSuite(object):
    """ this class corresponds to a single .csv 'test suite'
    """

    def __init__(self, suite_dir, file):
        self.suite_dir = suite_dir
        self.file = file
        self.file_path = os.path.join(suite_dir, file)
        self.name = file
        self.params = []
        self.tests  = []
        self.failures = 0
        self.passes   = 0
        self.read()

    def read(self):
        """ read a .csv file, and save each row as a set of test params
        """
        file = open(self.file_path, 'r')
        reader = csv.DictReader(file)
        fn = reader.fieldnames
        for row in reader:
            self.params.append(row)

    def do_test(self, t, strict=True):
        if t.is_valid:
            t.call_otp()
            time.sleep(1)
            t.test_otp_result(strict)
            self.tests.append(t)
            if t.result is TestResult.PASS:
                self.passes += 1
            elif t.result is TestResult.FAIL:
                log.info("test_suite: this test failed " + t.get_ws_url() + "\n")
                self.failures += 1
            sys.stdout.write(".")

    def get_tests(self):
        return self.tests

    def run(self, ws_url, map_url, date=None):
        """ iterate the list of tests from the .csv files, run the test (call otp), and check the output.
        """
        log.info("test_suite {0}: ******* date - {1} *******\n".format(self.name, datetime.datetime.now()))
        for i, p in enumerate(self.params):
            t = Test(p, i+2, ws_url, map_url, date)
            t.depart_by_check()
            self.do_test(t)

            """ arrive by tests
            """
            t = Test(p, i+2, ws_url, map_url, date)
            t.url_arrive_by()
            t.append_note(" ***NOTE***: arrive by test ")
            t.arrive_by_check()
            self.do_test(t, False)

    def printer(self, ws_url, date=None):
        """ iterate the list of tests from the .csv files and print the URLs
        """
        for i, p in enumerate(self.params):
            t = Test(p, i+2, ws_url, date)  # i+2 is the line number in the .csv file, accounting for the header
            t.depart_by_check()
            url = t.get_ws_url()
            if t.is_valid:
                print url

            """ arrive by tests
            """
            t = Test(p, i+2, ws_url, date)
            t.url_arrive_by()
            t.append_note(" ***NOTE***: arrive by test ")
            t.arrive_by_check()
            url = t.get_ws_url()
            if t.is_valid:
                print url

class ListTestSuites(object):
    """ this class corresponds a list of TestSuites.  Created based on all .csv files in the base directory
    """
    ws_url = None
    map_url = None
    suite_dir = None
    files = None
    date = None
    test_suites = []

    def __init__(self, ws_url, map_url, suite_dir, date=None):
        """ this class corresponds to a single .csv 'test suite'
        """
        self.ws_url = ws_url
        self.map_url = map_url
        self.suite_dir = suite_dir
        self.date = date
        self.files = os.listdir(self.suite_dir)
        for f in self.files:
            if f.lower().endswith('.csv'):
                t = TestSuite(self.suite_dir, f)
                self.test_suites.append(t)

    def has_errors(self):
        ret_val = False
        for t in self.test_suites:
            if t.failures > 0 or t.passes <= 0:
                ret_val = True
                log.info("test_suite {0} has {1} error(s) and {2} passes".format(t, t.failures, t.passes))
        return ret_val

    def run(self):
        #import pdb; pdb.set_trace()
        for ts in self.test_suites:
            ts.run(self.ws_url, self.date)

    def printer(self):
        for ts in self.test_suites:
            ts.printer(self.ws_url, self.date)

    def get_suites(self):
        return self.test_suites

<commit_msg>fix up test run and print routines<commit_after>import os
import sys
import time
import datetime
import logging
log = logging.getLogger(__file__)

import csv
import re

from ott.utils.config_util import ConfigUtil
from ott.utils import otp_utils
from ott.utils import object_utils

class TestResult:
    FAIL=000
    WARN=333
    PASS=111

class Test(object):
    """ Params for test, along with run capability -- Test object is typically built from a row in an .csv test suite
    """

    def __init__(self, param_dict, line_number, ws_url, map_url, date=None):
        """ {
            OTP parmas:
              'From'
              'To'
              'Max dist'
              'Mode'
              'Optimize'
              'Service' - expects 'Saturday' or 'Sunday' or leave empty
              'Time'

            Test params:
              'Arrive by' - expects 'FALSE' if arrive by test should not be ran or leave empty
              'Depart by' - expects 'FALSE' if depart by test should not be ran or leave empty
              'Expected output'
              'Expected trip duration'
              'Expected trip distance'
              'Expected number of legs'

            Misc text:
              'Description/notes'
            }
        """
        self.config = ConfigUtil(section='otp')

        self.ws_url          = ws_url
        self.map_url         = map_url

        self.csv_line_number = line_number
        self.csv_params      = param_dict
        self.date            = date

        self.itinerary       = None
        self.otp_params      = ''
        self.map_params      = ''
        self.is_valid        = True
        self.error_descript  = None
        self.result          = TestResult.FAIL

        self.coord_from      = self.get_param('From')
        self.coord_to        = self.get_param('To')
        self.distance        = self.get_param('Max dist')
        self.mode            = self.get_param('Mode')
        self.optimize        = self.get_param('Optimize')
        self.service         = self.get_param('Service')
        self.time            = self.get_param('Time', strip_all_spaces=True)
        self.description     = self.get_param('Description/notes')
        self.expect_output   = self.get_param('Expected output')
        self.expect_duration = self.get_param('Expected trip duration')
        self.expect_distance = self.get_param('Expected trip distance')
        self.expect_num_legs = self.get_param('Expected number of legs', warn_not_avail=False)
        self.arrive_by       = self.get_param('Arrive by')
        self.depart_by       = self.get_param('Depart by')

        # post process the load ... make params and urls, etc...
        self.date = self.get_date_param(self.date)
        self.init_url_params()
        self.url_distance()
        self.url_mode()
        self.url_optimize()
        self.url_time()

    def did_test_pass(self):
        ret_val = False
        if self.result is not None and self.result is TestResult.PASS:
            ret_val = True
        return ret_val

    def get_param(self, name, def_val=None, strip_all_spaces=False, warn_not_avail=True):
        return object_utils.get_striped_dict_val(self.csv_params, name, def_val, strip_all_spaces, warn_not_avail)

    def append_note(self, note=None):
        if note:
            self.description = "{}{}".format(self.description, note)

    def test_otp_result(self, strict=True):
        """ regexp test of the itinerary output for certain strings
        """
        if self.itinerary == None:
            self.result = TestResult.FAIL if strict else TestResult.WARN
            self.error_descript = "test_otp_result: itinerary is null"
            log.info(self.error_descript)
        else:
            if len(self.itinerary) < 1000:
                self.result = TestResult.FAIL if strict else TestResult.WARN
                self.error_descript = "test_otp_result: itinerary content looks small at " + str(len(self.itinerary)) + " characters."
                log.warn(self.error_descript)
            else:
                self.error_descript = "test_otp_result: itinerary content size is " + str(len(self.itinerary)) + " characters."
                log.info(self.error_descript)
                warn = False
                warn = self.test_expected_response(self.expect_output, warn, strict)
                if self.expect_duration is not None and len(self.expect_duration) > 0:
                    durations = re.findall('<itinerary>.*?<duration>(.*?)</duration>.*?</itinerary>', self.itinerary)
                    error = 0.2
                    high = float(self.expect_duration) * (1 + error)
                    low = float(self.expect_duration) * (1 - error)
                    for duration in durations:
                        if int(duration) > high or int(duration) < low:
                            self.result = TestResult.FAIL if strict else TestResult.WARN
                            self.error_descript += "test_otp_result: an itinerary duration was different than expected by more than {0}%.".format(error * 100)
                            warn = True
                            break
                if self.expect_num_legs is not None and len(self.expect_num_legs) > 0:
                    try:
                        values = [int(i) for i in self.expect_num_legs.split('|')]
                        if len(values) != 2:
                            raise ValueError
                        min_legs = values[0]
                        max_legs = values[1]
                        all_legs = re.findall('<itinerary>.*?<legs>(.*?)</legs>.*?</itinerary>', self.itinerary)
                        for legs in all_legs:
                            num_legs = len(re.findall('<leg .*?>', legs))
                            if num_legs > max_legs or num_legs < min_legs:
                                self.result = TestResult.FAIL if strict else TestResult.WARN
                                self.error_descript += "test_otp_result: an itinerary returned was not between {0} and {1} legs.".format(min_legs, max_legs)
                                warn = True
                                break
                    except ValueError:
                        self.error_descript += "expected number of legs test not in 'min|max' format."
                        warn = True
                if warn:
                    log.warn(self.error_descript)

        return self.result

    def test_expected_response(self, expected_output, initial_val, strict):
        ret_val = initial_val
        if expected_output is not None and len(expected_output) > 0:
            regres = re.search(expected_output, self.itinerary)
            if regres is None:
                self.result = TestResult.FAIL if strict else TestResult.WARN
                self.error_descript += "test_otp_result: couldn't find " + expected_output + " in otp response."
                ret_val = True
        return ret_val

    def init_url_params(self):
        """
        """
        self.otp_params = 'fromPlace={0}&toPlace={1}'.format(self.coord_from, self.coord_to)
        self.map_params = self.otp_params
        if self.coord_from == None or self.coord_from == '' or self.coord_to == None or self.coord_to == '':
            if self.coord_from != None or self.coord_to != None:
                self.error_descript = "no from and/or to coordinate for the otp url (skipping test) - from:{} to:{}".format(self.coord_from, self.coord_to)
                log.warn(self.error_descript)
            self.is_valid = False

    def url_param(self, name, param, default=None):
        """
        """
        p = param if param else default
        if p:
            self.otp_params += '&{0}={1}'.format(name, p)
            self.map_params += '&{0}={1}'.format(name, p)

    def url_distance(self, dist=None):
        self.url_param('maxWalkDistance', dist, self.distance)

    def url_mode(self, mode=None):
        self.url_param('mode', mode, self.mode)

    def url_optimize(self, opt=None):
        self.url_param('optimize', opt, self.optimize)

    def url_arrive_by(self, opt="true"):
        self.url_param('arriveBy', opt, self.optimize)

    def url_time(self, time=None):
        self.url_param('time', time, self.time)

    def url_time_7am(self):
        self.url_param('time', '7:00am')

    def url_time_12pm(self):
        self.url_param('time', '12:00pm')

    def url_time_5pm(self):
        self.url_param('time', '5:00pm')

    def url_service(self, svc=None):
        """
        """
        pass

    def get_date_param(self, date, fmt="%Y-%m-%d"):
        """ provide a default date (set to today) if no service provided...
        """
        if self.otp_params.find('date') < 0:
            if date is None:
                if self.service is None:
                    date = datetime.datetime.now().strftime(fmt)
                elif self.service == 'Saturday':
                    date = self.url_service_next_saturday()
                elif self.service == 'Sunday':
                    date = self.url_service_next_sunday()
                else:
                    date = datetime.datetime.now().strftime(fmt)
                    log.warn("service param '{0}' not valid, using todays date.".format(self.service))

            self.url_param('date', date)
        return date

    def url_service_next_weekday(self):
        """
        """
        pass

    def url_service_next_month_weekday(self):
        """
        """
        pass

    def url_service_next_saturday(self):
        date = datetime.datetime.now()
        day = date.weekday()
        if day == 6:
            date = date+datetime.timedelta(days=6)
        else:
            date = date+datetime.timedelta(days=5-day)
        date = date.strftime("%Y-%m-%d")
        return date

    def url_service_next_sunday(self):
        date = datetime.datetime.now()
        day = date.weekday()
        date = date+datetime.timedelta(days=6-day)
        date = date.strftime("%Y-%m-%d")
        return date

    def depart_by_check(self):
        if self.depart_by == 'FALSE':
            self.is_valid = False

    def arrive_by_check(self):
        if self.arrive_by == 'FALSE':
            self.is_valid = False

    def call_otp(self, url=None):
        ''' calls the trip web service
        '''
        self.itinerary = None
        start = time.time()
        url = url if url else self.get_ws_url()
        self.itinerary = otp_utils.call_planner_svc(url)
        end = time.time()
        self.response_time = end - start
        log.info("call_otp: response time of {} second for url {}".format(self.response_time, url))
        log.debug(self.itinerary)
        if self.response_time <= 30:
            self.result = TestResult.PASS
        else:
            self.result = TestResult.WARN
            log.info("call_otp: :::NOTE::: response time took *longer than 30 seconds* for url {}".format(url))

    @classmethod
    def make_url(cls, url, separater="?submit&module=planner"):
        ret_val = url
        if url is None:
            url = "ERROR ERROR ERROR in test_suite.py line 284 ERROR ERROR ERROR"
        if "?" not in url:
            ret_val = "{}{}".format(url, separater)
        return ret_val

    def get_ws_url(self):
        return "{}&{}".format(self.make_url(self.ws_url), self.otp_params)

    def get_map_url(self):
        return "{}&{}".format(self.make_url(self.map_url), self.map_params)

    def get_ridetrimetorg_url(self):
        return "http://ride.trimet.org?submit&" + self.map_params

class TestSuite(object):
    """ this class corresponds to a single .csv 'test suite'
    """

    def __init__(self, suite_dir, file):
        self.suite_dir = suite_dir
        self.file = file
        self.file_path = os.path.join(suite_dir, file)
        self.name = file
        self.params = []
        self.tests  = []
        self.failures = 0
        self.passes   = 0
        self.read()

    def read(self):
        """ read a .csv file, and save each row as a set of test params
        """
        file = open(self.file_path, 'r')
        reader = csv.DictReader(file)
        fn = reader.fieldnames
        for row in reader:
            self.params.append(row)

    def do_test(self, t, strict=True):
        if t.is_valid:
            t.call_otp()
            time.sleep(1)
            t.test_otp_result(strict)
            self.tests.append(t)
            if t.result is TestResult.PASS:
                self.passes += 1
            elif t.result is TestResult.FAIL:
                log.info("test_suite: this test failed " + t.get_ws_url() + "\n")
                self.failures += 1
            sys.stdout.write(".")

    def get_tests(self):
        return self.tests

    def run(self, ws_url, map_url, date=None, printer=False):
        """ iterate the list of tests from the .csv files, run the test (call otp), and check the output.
        """
        log.info("test_suite {0}: ******* date - {1} *******\n".format(self.name, datetime.datetime.now()))
        for i, p in enumerate(self.params):
            t = Test(p, i+2, ws_url, map_url, date)
            t.depart_by_check()
            if printer:
                print t.get_ws_url()
            else:
                self.do_test(t)

            """ arrive by tests
            """
            t = Test(p, i+2, ws_url, map_url, date)
            t.url_arrive_by()
            t.append_note(" ***NOTE***: arrive by test ")
            t.arrive_by_check()
            if printer:
                print t.get_ws_url()
            else:
                self.do_test(t, False)

    def printer(self, ws_url, map_url, date=None):
        self.run(ws_url, map_url, date, printer=True)

class ListTestSuites(object):
    """ this class corresponds a list of TestSuites.  Created based on all .csv files in the base directory
    """
    ws_url = None
    map_url = None
    suite_dir = None
    files = None
    date = None
    test_suites = []

    def __init__(self, ws_url, map_url, suite_dir, date=None):
        """ this class corresponds to a single .csv 'test suite'
        """
        import pdb; pdb.set_trace()
        self.ws_url = ws_url
        self.map_url = map_url
        self.suite_dir = suite_dir
        self.date = date
        self.files = os.listdir(self.suite_dir)
        for f in self.files:
            if f.lower().endswith('.csv'):
                t = TestSuite(self.suite_dir, f)
                self.test_suites.append(t)

    def has_errors(self):
        ret_val = False
        for t in self.test_suites:
            if t.failures > 0 or t.passes <= 0:
                ret_val = True
                log.info("test_suite {0} has {1} error(s) and {2} passes".format(t, t.failures, t.passes))
        return ret_val

    def run(self):
        for ts in self.test_suites:
            ts.run(self.ws_url, self.map_url, self.date)

    def printer(self):
        for ts in self.test_suites:
            ts.printer(self.ws_url, self.map_url, self.date)

    def get_suites(self):
        return self.test_suites

<|endoftext|>


<commit_before>from osmread import parse_file, Way

from ott.utils import json_utils
from ott.utils import date_utils
from ott.utils import file_utils

import logging
log = logging.getLogger(__file__)

class OsmInfo(object):
    """ Utility for getting stats on an osm file
    """
    def __init__(self):
        """ TODO think about rewriting this to simply read file line by line and find latest stamps and count tags...
            will be much faster than osmread on big files...
        """
        self.way_count = 0
        self.highway_count = 0
        class Last(object):
            timestamp = 0
            changeset = 0
        self.last = Last()

    def __str__(self):
        return json_utils.json_repr(self, pretty_print=True)

    def to_json(self, pretty_print=False):
        return json_utils.obj_to_dict(self)

    def write_json_file(self, file_path, pretty_print=False):
        json_utils.object_to_json_file(file_path, self, pretty_print)

    def calculate_osm_stats(self, osm_path):
        """ reads the .osm file and captures certain stats like last update and number of ways, etc...
        """
        log.info("calculating stats for {}".format(osm_path))
        for entity in parse_file(osm_path):
            if isinstance(entity, Way):
                self.way_count += 1
                if 'highway' in entity.tags:
                    self.highway_count += 1
                if entity.timestamp > self.last.timestamp:
                    self.last.timestamp = entity.timestamp
                    self.last.changeset = entity.changeset

        # clean up data
        self.last.changeset_url = "http://openstreetmap.org/changeset/{}".format(self.last.changeset)
        self.last.edit_date = date_utils.pretty_date_from_ms(self.last.timestamp * 1000, fmt="%B %d, %Y")
        self.last.file_date = file_utils.file_pretty_date(osm_path, fmt="%B %d, %Y")

    @classmethod
    def find_osm_files(cls, dir_path):
        ret_val = []
        osm = file_utils.find_files(dir_path, ".osm")
        if osm:
            ret_val.extend(osm)
        pbf = file_utils.find_files(dir_path, ".pbf")
        if pbf:
            ret_val.extend(pbf)
        return ret_val

    @classmethod
    def get_stats_file_path(cls, osm_file, stats_file=None):
        if stats_file is None:
            stats_file = osm_file + "-stats"
        return stats_file

    @classmethod
    def cache_stats(cls, osm_file, stats_file=None, pretty_print=True):
        stats_file = cls.get_stats_file_path(osm_file, stats_file)
        stats = OsmInfo()
        stats.calculate_osm_stats(osm_file)
        stats.write_json_file(stats_file, pretty_print)
        ret_val = stats.to_json()
        return ret_val

    @classmethod
    def get_stats(cls, osm_file, stats_file=None, pretty_print=True):
        """ will either read a cache'd -stats file into memory, or calculate the stats, cache them and then return
            :return dict representing the json stats object
        """
        #import pdb; pdb.set_trace()
        ret_val = None

        # step 1: validate stats file path
        stats_file = cls.get_stats_file_path(osm_file, stats_file)

        # step 2: if the stats file exists and is newere than the .osm file, try to read it in
        if file_utils.exists(stats_file) and file_utils.is_a_newer_than_b(stats_file, osm_file):
            ret_val = json_utils.get_json(stats_file)

        # step 3: if we don't have stats from a cache'd file, calculate new stats and write them out
        if ret_val is None or len(ret_val) < 2:
            ret_val = cls.cache_stats(osm_file, stats_file, pretty_print)

        # step 4: return the stats as a string
        return ret_val

    @classmethod
    def get_osm_feed_msg(cls, file_path, prefix=" ", suffix="\n"):
        """ get osm feed details msg string for the .vlog file
        """
        file_name = file_utils.get_file_name_from_path(file_path)
        stats = OsmInfo.get_stats(file_path)
        msg = "{}{} : file date: {} -- last OSM update: {}, changeset {}{}"\
            .format(prefix, file_name, stats['last'].get('file_date'), stats['last'].get('edit_date'), stats['last'].get('changeset'), suffix)
        return msg

    @classmethod
    def get_cache_msgs(cls, cache_path, def_msg=""):
        """return message for all OSM feeds in the cache directory """
        osm_msg = def_msg
        try:
            osm_files = OsmInfo.find_osm_files(cache_path)
            for f in osm_files:
                osm_msg += OsmInfo.get_osm_feed_msg(f)
        except Exception, e:
            log.info(e)
        return osm_msg

    @classmethod
    def print_stats(cls):
        """ run the SUM loader routines
        """
        from .osm_cache import OsmCache
        c = OsmCache()
        s = OsmInfo.get_stats(c.osm_path)
        print json_utils.dict_to_json_str(s, pretty_print=True)
        print ""
        print OsmInfo.get_cache_msgs(c.cache_dir)
<commit_msg>read the cached' OSM stats file, rather than re-calculate every time we need to do otp.v<commit_after>from osmread import parse_file, Way

from ott.utils import json_utils
from ott.utils import date_utils
from ott.utils import file_utils

import logging
log = logging.getLogger(__file__)

class OsmInfo(object):
    """ Utility for getting stats on an osm file
    """
    def __init__(self):
        """ TODO think about rewriting this to simply read file line by line and find latest stamps and count tags...
            will be much faster than osmread on big files...
        """
        self.way_count = 0
        self.highway_count = 0
        class Last(object):
            timestamp = 0
            changeset = 0
        self.last = Last()

    def __str__(self):
        return json_utils.json_repr(self, pretty_print=True)

    def to_json(self, pretty_print=False):
        return json_utils.obj_to_dict(self)

    def write_json_file(self, file_path, pretty_print=False):
        json_utils.object_to_json_file(file_path, self, pretty_print)

    def calculate_osm_stats(self, osm_path):
        """ reads the .osm file and captures certain stats like last update and number of ways, etc...
        """
        log.info("calculating stats for {}".format(osm_path))
        for entity in parse_file(osm_path):
            if isinstance(entity, Way):
                self.way_count += 1
                if 'highway' in entity.tags:
                    self.highway_count += 1
                if entity.timestamp > self.last.timestamp:
                    self.last.timestamp = entity.timestamp
                    self.last.changeset = entity.changeset

        # clean up data
        self.last.changeset_url = "http://openstreetmap.org/changeset/{}".format(self.last.changeset)
        self.last.edit_date = date_utils.pretty_date_from_ms(self.last.timestamp * 1000, fmt="%B %d, %Y")
        self.last.file_date = file_utils.file_pretty_date(osm_path, fmt="%B %d, %Y")

    @classmethod
    def find_osm_files(cls, dir_path):
        ret_val = []
        osm = file_utils.find_files(dir_path, ".osm")
        if osm:
            ret_val.extend(osm)
        pbf = file_utils.find_files(dir_path, ".pbf")
        if pbf:
            ret_val.extend(pbf)
        return ret_val

    @classmethod
    def get_stats_file_path(cls, osm_file, stats_file=None):
        if stats_file is None:
            stats_file = osm_file + "-stats"
        return stats_file

    @classmethod
    def cache_stats(cls, osm_path, stats_file=None, pretty_print=True):
        """ return OSM .osm-stats info json
            will either read the .osm-stats file or create a new stats file calculated via the .osm file
        """
        import pdb; pdb.set_trace()
        stats_path = cls.get_stats_file_path(osm_path, stats_file)
        is_stats_good = False

        # step 1: read the .osm-stats cache file (if exists)
        is_stats_fresh = file_utils.is_a_newer_than_b(stats_path, osm_path)
        if is_stats_fresh:
            j = json_utils.get_json(stats_path)
            if j.get('way_count') and j.get('highway_count') and j.get('last'):
                is_stats_good = True
                ret_val = j

        # step 2: if .osm-stats don't exists or are out of date, cache them
        if not is_stats_good:
            stats = OsmInfo()
            stats.calculate_osm_stats(osm_path)
            stats.write_json_file(stats_file, pretty_print)
            ret_val = stats.to_json()

        return ret_val

    @classmethod
    def get_stats(cls, osm_file, stats_file=None, pretty_print=True):
        """ will either read a cache'd -stats file into memory, or calculate the stats, cache them and then return
            :return dict representing the json stats object
        """
        ret_val = None

        # step 1: validate stats file path
        stats_file = cls.get_stats_file_path(osm_file, stats_file)

        # step 2: if the stats file exists and is newere than the .osm file, try to read it in
        if file_utils.exists(stats_file) and file_utils.is_a_newer_than_b(stats_file, osm_file):
            ret_val = json_utils.get_json(stats_file)

        # step 3: if we don't have stats from a cache'd file, calculate new stats and write them out
        if ret_val is None or len(ret_val) < 2:
            ret_val = cls.cache_stats(osm_file, stats_file, pretty_print)

        # step 4: return the stats as a string
        return ret_val

    @classmethod
    def get_osm_feed_msg(cls, file_path, prefix=" ", suffix="\n"):
        """ get osm feed details msg string for the .v log file
        """
        file_name = file_utils.get_file_name_from_path(file_path)
        stats = OsmInfo.get_stats(file_path)
        msg = "{}{} : file date: {} -- last OSM update: {}, changeset {}{}"\
            .format(prefix, file_name, stats['last'].get('file_date'), stats['last'].get('edit_date'), stats['last'].get('changeset'), suffix)
        return msg

    @classmethod
    def get_cache_msgs(cls, cache_path, def_msg=""):
        """return message for all OSM feeds in the cache directory """
        osm_msg = def_msg
        try:
            osm_files = OsmInfo.find_osm_files(cache_path)
            for f in osm_files:
                osm_msg += OsmInfo.get_osm_feed_msg(f)
        except Exception, e:
            log.info(e)
        return osm_msg

    @classmethod
    def print_stats(cls):
        """ run the SUM loader routines
        """
        from .osm_cache import OsmCache
        c = OsmCache()
        s = OsmInfo.get_stats(c.osm_path)
        print json_utils.dict_to_json_str(s, pretty_print=True)
        print ""
        print OsmInfo.get_cache_msgs(c.cache_dir)
<|endoftext|>


<commit_before>'''
Created on Jan 22, 2017

@author: mohame11
'''
from scipy.stats import chisquare
from collections import OrderedDict
from multiprocessing import Process, Queue

import pandas as pd
#import plac
import numpy as np
import math
import os.path
import cProfile
import _eval_outlier
from MyEnums import *
from TestSample import *
from bokeh.colors import gold

import sys
from astropy import log
sys.path.append('/Users/mohame11/anaconda/lib/python2.7/site-packages/')
import kenlm

#testDic, quota, coreId, q, store, true_mem_size, hyper2id, obj2id, Theta_zh, Psi_sz, smoothedProbs
class DetectionTechnique():
    def __init__(self):
        self.true_mem_size = None
        self.model_path = None
        self.useWindow = None
        self.type = None
        self.model = None

    def formOriginalSeq(self, tests):
        origSeq = list(tests[0].actions)
        origGoldMarkers = list(tests[0].goldMarkers)
        for i in range(1,len(tests)):
            a = tests[i].actions[-1]
            g = tests[i].goldMarkers[-1]
            origSeq.append(a)
            origGoldMarkers.append(g)
        return origSeq, origGoldMarkers

    def getProbability(self, userId, newSeq):
        pass

    def getAllPossibleActions(self):
        pass

    def getUserId(self, uid):
        pass

class NgramLM (DetectionTechnique):
    def __init__(self):
        DetectionTechnique.__init__(self)
        self.type = SEQ_PROB.NGRAM
        self.allActions = []

    def loadModel(self):
        self.model = kenlm.Model(self.model_path)

    def getProbability(self, userId, newSeq):
        strSeq = ' '.join(newSeq)
        #prob = self.model.score(strSeq, bos = True, eos = True)
        logProb = self.model.score(strSeq)
        #As is the custom in language modeling, all probabilities are log base 10.
        return (10**logProb)

    def prepareTestSet(self):
        testDic = {}
        print(">>> Preparing testset ...")
        testSetCount = 0
        r = open(self.SEQ_FILE_PATH, 'r')
        for line in r:
            line = line.strip()
            tmp = line.split()
            user = tmp[0]

            seq = tmp[1:self.true_mem_size+2]
            for act in seq:
                if(act not in self.allActions):
                    self.allActions.append(act)
            goldMarkers = tmp[self.true_mem_size+2:]
            if(len(goldMarkers) != len(seq)):
                goldMarkers = ['false']*len(seq)
            t = TestSample()
            t.user = user
            t.actions = list(seq)
            t.goldMarkers = list(goldMarkers)

            testSetCount += 1
            if(user in testDic):
                testDic[user].append(t)
            else:
                testDic[user]=[t]
        r.close()
        if(self.useWindow == USE_WINDOW.FALSE): # we need to use the original sequence instead of overlapping windows
            testSetCount = len(testDic)
            for u in testDic:
                tests = testDic[u]
                originalSeq, originalGoldMarkers = self.formOriginalSeq(tests)
                t = TestSample()
                t.user = u
                t.actions = list(originalSeq)
                t.goldMarkers = list(originalGoldMarkers)
                testDic[u] = [t]
        return testDic, testSetCount

#self, testDic, quota, coreId, q, store, true_mem_size, hyper2id, obj2id, Theta_zh, Psi_sz, smoothedProbs

    def getAllPossibleActions(self):
        return self.allActions

    def getUserId(self, uid):
        return uid

class TribeFlow (DetectionTechnique):
    def __init__(self):
        DetectionTechnique.__init__(self)
        self.type = SEQ_PROB.TRIBEFLOW
        self.hyper2id = None
        self.obj2id = None
        self.Theta_zh = None
        self.Psi_sz = None
        self.smoothedProbs = None
        self.trace_fpath = None
        self.STAT_FILE = None
        self.UNBIAS_CATS_WITH_FREQ = None
        self.store = None

    def evaluate(self, userId, history, targetObjId, Theta_zh, Psi_sz, env):
        mem_factor = 1.0
        candidateProb = 0.0
        for j in xrange(len(history)):#for all B
            #i.e. multiply all psi[objid1,z]*psi[objid2,z]*..psi[objidB,z]
            mem_factor *= Psi_sz[history[j], env] # Psi[objId, env z]
        #mem_factor *= 1.0 / (1 - Psi_sz[history[len(history)-1], env])# 1-Psi_sz[mem[B-1],z] == 1-psi_sz[objIdB,z]
        candidateProb += mem_factor * Psi_sz[targetObjId, env] * Theta_zh[env, userId]
        return candidateProb

    def calculateSequenceProb(self, theSequence, true_mem_size, userId, obj2id, Theta_zh, Psi_sz):
        seqProb = 0.0
        window = min(true_mem_size, len(theSequence))
        for z in xrange(Psi_sz.shape[1]): #for envs
            seqProbZ = 1.0
            for targetObjIdx in range(0,len(theSequence)): #targetObjIdx=0 cannot be predicted we have to skip it
                if(targetObjIdx == 0):
                    d = theSequence[targetObjIdx]
                    #all_in = h in hyper2id and d in obj2id
                    #if(all_in):
                    targetObjId = obj2id[d]
                    prior = Psi_sz[targetObjId, z]
                    seqProbZ *= prior
                else:
                    d = theSequence[targetObjIdx]
                    sources = theSequence[max(0,targetObjIdx-window): targetObjIdx] # look back 'window' actions.
                    #all_in = h in hyper2id and d in obj2id
                    #for s in sources:
                    #    all_in = all_in and s in obj2id
                    #if all_in:
                    targetObjId = obj2id[d]
                    #userId = hyper2id[h]
                    history = [obj2id[s] for s in sources]
                    candProb = self.evaluate(userId, history, targetObjId, Theta_zh, Psi_sz, z) #(int[:, ::1] HOs, double[:, ::1] Theta_zh, double[:, ::1] Psi_sz, int[::1] count_z, int env):
                    #candProb = 0.001
                    seqProbZ *= candProb
                    #seqProb += math.log(predTargetProb)
            seqProb += seqProbZ
            #print(seqProb, math.log(seqProb))
        return seqProb

    def createTestingSeqFile(self, store):
        from_ = store['from_'][0][0]
        to = store['to'][0][0]
        trace_fpath = store['trace_fpath']
        Dts = store['Dts']
        winSize = Dts.shape[1]
        tpath = '/home/zahran/Desktop/tribeFlow/zahranData/pinterest/test_traceFile_win5'
        w = open(tpath,'w')
        r = open(trace_fpath[0][0],'r')

        cnt = 0
        for line in r:
            if(cnt > to):
                p = line.strip().split('\t')
                usr = p[winSize]
                sq = p[winSize+1:]
                w.write(str(usr)+'\t')
                for s in sq:
                    w.write(s+'\t')
                w.write('\n')
            cnt += 1
        w.close()
        r.close()
        return tpath

    def calculatingItemsFreq(self, smoothingParam):
        self.smoothedProbs = {}
        if os.path.isfile(self.STAT_FILE):
            r = open(self.STAT_FILE, 'r')
            for line in r:
                parts = line.strip().split('\t')
                self.smoothedProbs[parts[0]] = float(parts[1])

        freqs = {}
        r = open(self.trace_fpath)
        counts = 0
        for line in r:
            cats = line.strip().split('\t')[self.true_mem_size+1:]
            for c in cats:
                if(c in freqs):
                    freqs[c] += 1
                else:
                    freqs[c] = 1
                counts += 1
        for k in freqs:
            prob = float(freqs[k]+ smoothingParam) / float(counts + (len(freqs) * smoothingParam))
            self.smoothedProbs[k] = prob

        w = open(self.STAT_FILE, 'w')
        for key in self.smoothedProbs:
            w.write(key+'\t'+str(self.smoothedProbs[key])+'\n')
        w.close()
        #return self.smoothedProb

    def getProbability(self, userId, newSeq):
        newSeqIds = [self.obj2id[s] for s in newSeq]
        newSeqIds_np = np.array(newSeqIds, dtype = 'i4').copy()
        seqScore = _eval_outlier.calculateSequenceProb(newSeqIds_np, len(newSeqIds_np), self.true_mem_size, userId, self.Theta_zh, self.Psi_sz)
        #seqScore = calculateSequenceProb(newSeq, true_mem_size, userId, obj2id, Theta_zh, Psi_sz)
        if(self.UNBIAS_CATS_WITH_FREQ):
            unbiasingProb = 1.0
            for ac in newSeq:
                if(ac in self.smoothedProbs):
                    unbiasingProb *= self.smoothedProbs[ac]
            seqScore = float(seqScore)/float(unbiasingProb)
        return seqScore

    def prepareTestSet(self):
        testDic = {}
        print(">>> Preparing testset ...")
        testSetCount = 0
        r = open(self.SEQ_FILE_PATH, 'r')
        for line in r:
            line = line.strip()
            tmp = line.split()
            user = tmp[0]
            if(user not in self.hyper2id):
                #print("User: "+str(user)+" is not found in training set !")
                continue
            seq = tmp[1:self.true_mem_size+2]
            goldMarkers = tmp[self.true_mem_size+2:]
            if(len(goldMarkers) != len(seq)):
                goldMarkers = ['false']*len(seq)
            t = TestSample()
            t.user = user
            t.actions = list(seq)
            t.goldMarkers = list(goldMarkers)

            testSetCount += 1
            if(user in testDic):
                testDic[user].append(t)
            else:
                testDic[user]=[t]
        r.close()
        if(self.useWindow == USE_WINDOW.FALSE): # we need to use the original sequence instead of overlapping windows
            testSetCount = len(testDic)
            for u in testDic:
                tests = testDic[u]
                originalSeq, originalGoldMarkers = self.formOriginalSeq(tests)
                t = TestSample()
                t.user = u
                t.actions = list(originalSeq)
                t.goldMarkers = list(originalGoldMarkers)
                testDic[u] = [t]
        return testDic, testSetCount

    def getAllPossibleActions(self):
        return self.obj2id.keys()

    def getUserId(self, uid):
        return self.hyper2id[uid]
        <commit_msg>checking if users are provided in data<commit_after>'''
Created on Jan 22, 2017

@author: mohame11
'''
from scipy.stats import chisquare
from collections import OrderedDict
from multiprocessing import Process, Queue

import pandas as pd
#import plac
import numpy as np
import math
import os.path
import cProfile
import _eval_outlier
from MyEnums import *
from TestSample import *
from bokeh.colors import gold

import sys
from astropy import log
sys.path.append('/Users/mohame11/anaconda/lib/python2.7/site-packages/')
import kenlm

#testDic, quota, coreId, q, store, true_mem_size, hyper2id, obj2id, Theta_zh, Psi_sz, smoothedProbs
class DetectionTechnique():
    def __init__(self):
        self.true_mem_size = None
        self.model_path = None
        self.useWindow = None
        self.type = None
        self.model = None

    def formOriginalSeq(self, tests):
        origSeq = list(tests[0].actions)
        origGoldMarkers = list(tests[0].goldMarkers)
        for i in range(1,len(tests)):
            a = tests[i].actions[-1]
            g = tests[i].goldMarkers[-1]
            origSeq.append(a)
            origGoldMarkers.append(g)
        return origSeq, origGoldMarkers

    def getProbability(self, userId, newSeq):
        pass

    def getAllPossibleActions(self):
        pass

    def getUserId(self, uid):
        pass

class NgramLM (DetectionTechnique):
    def __init__(self):
        DetectionTechnique.__init__(self)
        self.type = SEQ_PROB.NGRAM
        self.allActions = []

    def loadModel(self):
        self.model = kenlm.Model(self.model_path)

    def getProbability(self, userId, newSeq):
        strSeq = ' '.join(newSeq)
        #prob = self.model.score(strSeq, bos = True, eos = True)
        logProb = self.model.score(strSeq)
        #As is the custom in language modeling, all probabilities are log base 10.
        return (10**logProb)

    def prepareTestSet(self):
        testDic = {}
        print(">>> Preparing testset ...")
        testSetCount = 0
        r = open(self.SEQ_FILE_PATH, 'r')
        for line in r:
            line = line.strip()
            tmp = line.split()
            actionStartIndex = 0
            if (self.DATA_HAS_USER_INFO == True):
                user = tmp[0]
                actionStartIndex = 1

            seq = tmp[actionStartIndex:self.true_mem_size+2]
            for act in seq:
                if(act not in self.allActions):
                    self.allActions.append(act)
            goldMarkers = tmp[self.true_mem_size+2:]
            if(len(goldMarkers) != len(seq)):
                goldMarkers = ['false']*len(seq)
            t = TestSample()
            t.user = user
            t.actions = list(seq)
            t.goldMarkers = list(goldMarkers)

            testSetCount += 1
            if(user in testDic):
                testDic[user].append(t)
            else:
                testDic[user]=[t]
        r.close()
        if(self.useWindow == USE_WINDOW.FALSE): # we need to use the original sequence instead of overlapping windows
            testSetCount = len(testDic)
            for u in testDic:
                tests = testDic[u]
                originalSeq, originalGoldMarkers = self.formOriginalSeq(tests)
                t = TestSample()
                t.user = u
                t.actions = list(originalSeq)
                t.goldMarkers = list(originalGoldMarkers)
                testDic[u] = [t]
        return testDic, testSetCount

#self, testDic, quota, coreId, q, store, true_mem_size, hyper2id, obj2id, Theta_zh, Psi_sz, smoothedProbs

    def getAllPossibleActions(self):
        return self.allActions

    def getUserId(self, uid):
        return uid

class TribeFlow (DetectionTechnique):
    def __init__(self):
        DetectionTechnique.__init__(self)
        self.type = SEQ_PROB.TRIBEFLOW
        self.hyper2id = None
        self.obj2id = None
        self.Theta_zh = None
        self.Psi_sz = None
        self.smoothedProbs = None
        self.trace_fpath = None
        self.STAT_FILE = None
        self.UNBIAS_CATS_WITH_FREQ = None
        self.store = None

    def evaluate(self, userId, history, targetObjId, Theta_zh, Psi_sz, env):
        mem_factor = 1.0
        candidateProb = 0.0
        for j in xrange(len(history)):#for all B
            #i.e. multiply all psi[objid1,z]*psi[objid2,z]*..psi[objidB,z]
            mem_factor *= Psi_sz[history[j], env] # Psi[objId, env z]
        #mem_factor *= 1.0 / (1 - Psi_sz[history[len(history)-1], env])# 1-Psi_sz[mem[B-1],z] == 1-psi_sz[objIdB,z]
        candidateProb += mem_factor * Psi_sz[targetObjId, env] * Theta_zh[env, userId]
        return candidateProb

    def calculateSequenceProb(self, theSequence, true_mem_size, userId, obj2id, Theta_zh, Psi_sz):
        seqProb = 0.0
        window = min(true_mem_size, len(theSequence))
        for z in xrange(Psi_sz.shape[1]): #for envs
            seqProbZ = 1.0
            for targetObjIdx in range(0,len(theSequence)): #targetObjIdx=0 cannot be predicted we have to skip it
                if(targetObjIdx == 0):
                    d = theSequence[targetObjIdx]
                    #all_in = h in hyper2id and d in obj2id
                    #if(all_in):
                    targetObjId = obj2id[d]
                    prior = Psi_sz[targetObjId, z]
                    seqProbZ *= prior
                else:
                    d = theSequence[targetObjIdx]
                    sources = theSequence[max(0,targetObjIdx-window): targetObjIdx] # look back 'window' actions.
                    #all_in = h in hyper2id and d in obj2id
                    #for s in sources:
                    #    all_in = all_in and s in obj2id
                    #if all_in:
                    targetObjId = obj2id[d]
                    #userId = hyper2id[h]
                    history = [obj2id[s] for s in sources]
                    candProb = self.evaluate(userId, history, targetObjId, Theta_zh, Psi_sz, z) #(int[:, ::1] HOs, double[:, ::1] Theta_zh, double[:, ::1] Psi_sz, int[::1] count_z, int env):
                    #candProb = 0.001
                    seqProbZ *= candProb
                    #seqProb += math.log(predTargetProb)
            seqProb += seqProbZ
            #print(seqProb, math.log(seqProb))
        return seqProb

    def createTestingSeqFile(self, store):
        from_ = store['from_'][0][0]
        to = store['to'][0][0]
        trace_fpath = store['trace_fpath']
        Dts = store['Dts']
        winSize = Dts.shape[1]
        tpath = '/home/zahran/Desktop/tribeFlow/zahranData/pinterest/test_traceFile_win5'
        w = open(tpath,'w')
        r = open(trace_fpath[0][0],'r')

        cnt = 0
        for line in r:
            if(cnt > to):
                p = line.strip().split('\t')
                usr = p[winSize]
                sq = p[winSize+1:]
                w.write(str(usr)+'\t')
                for s in sq:
                    w.write(s+'\t')
                w.write('\n')
            cnt += 1
        w.close()
        r.close()
        return tpath

    def calculatingItemsFreq(self, smoothingParam):
        self.smoothedProbs = {}
        if os.path.isfile(self.STAT_FILE):
            r = open(self.STAT_FILE, 'r')
            for line in r:
                parts = line.strip().split('\t')
                self.smoothedProbs[parts[0]] = float(parts[1])

        freqs = {}
        r = open(self.trace_fpath)
        counts = 0
        for line in r:
            cats = line.strip().split('\t')[self.true_mem_size+1:]
            for c in cats:
                if(c in freqs):
                    freqs[c] += 1
                else:
                    freqs[c] = 1
                counts += 1
        for k in freqs:
            prob = float(freqs[k]+ smoothingParam) / float(counts + (len(freqs) * smoothingParam))
            self.smoothedProbs[k] = prob

        w = open(self.STAT_FILE, 'w')
        for key in self.smoothedProbs:
            w.write(key+'\t'+str(self.smoothedProbs[key])+'\n')
        w.close()
        #return self.smoothedProb

    def getProbability(self, userId, newSeq):
        newSeqIds = [self.obj2id[s] for s in newSeq]
        newSeqIds_np = np.array(newSeqIds, dtype = 'i4').copy()
        seqScore = _eval_outlier.calculateSequenceProb(newSeqIds_np, len(newSeqIds_np), self.true_mem_size, userId, self.Theta_zh, self.Psi_sz)
        #seqScore = calculateSequenceProb(newSeq, true_mem_size, userId, obj2id, Theta_zh, Psi_sz)
        if(self.UNBIAS_CATS_WITH_FREQ):
            unbiasingProb = 1.0
            for ac in newSeq:
                if(ac in self.smoothedProbs):
                    unbiasingProb *= self.smoothedProbs[ac]
            seqScore = float(seqScore)/float(unbiasingProb)
        return seqScore

    def prepareTestSet(self):
        testDic = {}
        print(">>> Preparing testset ...")
        testSetCount = 0
        r = open(self.SEQ_FILE_PATH, 'r')
        for line in r:
            line = line.strip()
            tmp = line.split()
            actionStartIndex = 0
            user = tmp[0]
            if(user not in self.hyper2id):
                #print("User: "+str(user)+" is not found in training set !")
                continue
            seq = tmp[actionStartIndex:self.true_mem_size+2]
            goldMarkers = tmp[self.true_mem_size+2:]
            if(len(goldMarkers) != len(seq)):
                goldMarkers = ['false']*len(seq)
            t = TestSample()
            t.user = user
            t.actions = list(seq)
            t.goldMarkers = list(goldMarkers)

            testSetCount += 1
            if(user in testDic):
                testDic[user].append(t)
            else:
                testDic[user]=[t]
        r.close()
        if(self.useWindow == USE_WINDOW.FALSE): # we need to use the original sequence instead of overlapping windows
            testSetCount = len(testDic)
            for u in testDic:
                tests = testDic[u]
                originalSeq, originalGoldMarkers = self.formOriginalSeq(tests)
                t = TestSample()
                t.user = u
                t.actions = list(originalSeq)
                t.goldMarkers = list(originalGoldMarkers)
                testDic[u] = [t]
        return testDic, testSetCount

    def getAllPossibleActions(self):
        return self.obj2id.keys()

    def getUserId(self, uid):
        return self.hyper2id[uid]
        <|endoftext|>


<commit_before>from osmread import parse_file, Way

from ott.utils import json_utils
from ott.utils import date_utils

import logging
log = logging.getLogger(__file__)

class OsmInfo(object):
    """ Utility for getting stats on an osm file
    """
    def __init__(self):
        """
        """
        self.way_count = 0
        self.highway_count = 0
        class Last(object):
            timestamp = 0
            changeset = 0
        self.last = Last()

    def __str__(self):
        return json_utils.json_repr(self, pretty_print=True)

    def get_osm_stats(self, osm_path):
        """
        """
        log.info("calculating stats for {}".format(osm_path))
        for entity in parse_file(osm_path):
            if isinstance(entity, Way):
                #import pdb; pdb.set_trace()
                self.way_count += 1
                if 'highway' in entity.tags:
                    self.highway_count += 1
                if entity.timestamp > self.last.timestamp:
                    self.last.timestamp = entity.timestamp
                    self.last.changeset = entity.changeset

        # clean up data
        self.last.changeset_url = "http://openstreetmap.org/changeset/{}".format(self.last.changeset)
        self.last.date = date_utils.pretty_date_from_ms(self.last.timestamp * 1000)

    @classmethod
    def print_stats(cls):
        """ run the SUM loader routines
        """
        from .osm_cache import OsmCache
        c = OsmCache()
        stats = OsmInfo()
        stats.get_osm_stats(c.osm_name)
        print stats
<commit_msg>OSM stats ... print override<commit_after>from osmread import parse_file, Way

from ott.utils import json_utils
from ott.utils import date_utils

import logging
log = logging.getLogger(__file__)

class OsmInfo(object):
    """ Utility for getting stats on an osm file
    """
    def __init__(self):
        """
        """
        self.way_count = 0
        self.highway_count = 0
        class Last(object):
            timestamp = 0
            changeset = 0
        self.last = Last()

    def __str__(self):
        return self.to_json(pretty_print=True)

    def to_json(self, pretty_print=False):
        return json_utils.json_repr(self, pretty_print=pretty_print)

    def to_json_file(self, file_path, pretty_print=False):
        json_utils.object_to_json_file(file_path, self, pretty_print)

    def get_osm_stats(self, osm_path):
        """
        """
        log.info("calculating stats for {}".format(osm_path))
        for entity in parse_file(osm_path):
            if isinstance(entity, Way):
                #import pdb; pdb.set_trace()
                self.way_count += 1
                if 'highway' in entity.tags:
                    self.highway_count += 1
                if entity.timestamp > self.last.timestamp:
                    self.last.timestamp = entity.timestamp
                    self.last.changeset = entity.changeset

        # clean up data
        self.last.changeset_url = "http://openstreetmap.org/changeset/{}".format(self.last.changeset)
        self.last.date = date_utils.pretty_date_from_ms(self.last.timestamp * 1000)

    @classmethod
    def print_stats(cls):
        """ run the SUM loader routines
        """
        from .osm_cache import OsmCache
        c = OsmCache()
        stats = OsmInfo()
        stats.get_osm_stats(c.osm_name)
        print stats
        stats.to_json(c.osm_name + "-stats", pretty_print=True)
<|endoftext|>


<commit_before><commit_msg>Write a full test suite for override_settings<commit_after>from django.conf import settings
from django.test import TestCase
from override_settings import (
    override_settings, SETTING_DELETED, with_apps, without_apps)

@override_settings(FOO="abc")
class TestOverrideSettingsDecoratedClass(TestCase):
    """
    Provide a decorated class.
    """
    def test_override_settings_class_decorator(self):
        """
        Settings overwritten at the class level are available in each method.
        """
        self.assertEqual(settings.FOO, "abc")

    @override_settings(FOO="abc123")
    def test_override_settings_method_decorator(self):
        """
        Method level overrides overwrite class level overrides.
        """
        self.assertEqual(settings.FOO, "abc123")

    @override_settings(FOO="321")
    def test_override_settings_context_manager(self):
        """
        Context managers overwrite method and class level decorations.
        """
        with override_settings(FOO="xyz"):
            self.assertEqual(settings.FOO, "xyz")

class TestOverrideSettingsUndecoratedClass(TestCase):
    """
    Provide an undecorated class.
    """
    @override_settings(FOO="123")
    def test_override_settings_on_a_method(self):
        """
        Override settings can be used for an individual method.
        """
        self.assertEqual(settings.FOO, "123")

    def test_override_settings_as_context_manager(self):
        """
        Can use override_settings as a context manager.
        """
        with override_settings(FOO="321"):
            self.assertEqual(settings.FOO, "321")

class TestAppModifiers(TestCase):
    @with_apps('django.contrib.sites')
    def test_with_apps(self):
        """
        The `with_apps` decorator adds apps to INSTALLED_APPS.
        """
        self.assertTrue('django.contrib.sites' in settings.INSTALLED_APPS)

    @without_apps("django.contrib.sites")
    def test_without_apps(self):
        """
        The `without_apps` decorator removes apps from INSTALLED_APPS.
        """
        self.assertFalse('django.contrib.sites' in settings.INSTALLED_APPS)

class TestSettingDeleted(TestCase):
    def test_dummy_option_exists(self):
        """
        Deleted options should return after the context manager is finished.
        """
        self.assertEqual(settings.DUMMY_OPTION, 42)

        with override_settings(DUMMY_OPTION=SETTING_DELETED):
            self.assertRaises(AttributeError, lambda: settings.DUMMY_OPTION)

        self.assertEqual(settings.DUMMY_OPTION, 42)

    @override_settings(DUMMY_OPTION=SETTING_DELETED)
    def test_delete_dummy_option(self):
        """
        Can delete settings at the method level.
        """
        self.assertRaises(AttributeError, lambda: settings.DUMMY_OPTION)

    def test_dummy_option_exists_after_method_level_delete(self):
        """
        Make sure the option returns after deleting it at the method level.
        """
        self.assertEqual(settings.DUMMY_OPTION, 42)
<|endoftext|>


<commit_before># TODO
#
#   Define an interface I am going to implement with this Python module
#
# EcdsaError ::: LanguagePrimitive
# Boolean ::: LanguagePrimitive
# OctetString ::: LanguagePrimitive
# X509Cert ::: OctetString
# PublicKey ::: OctetString
# Signature ::: OctetString
# Message ::: OctetString
#
# extract_publickey_from_certificate
#       :: (X509Cert -> PublicKey) except EcdsaError
#
# compress_publickey
#       :: (PublicKey -> PublicKey) except EcdsaError
#
# verify_signature
#       :: ((PublicKey, Signature, Message) -> Boolean) except EcdsaError
#

from P256 import q
from P256ECC import (
        E, E_from_bytes, E_to_bytes, E_contains, E_take_x_mod_q, E_eq, E_neg,
        E_dbl, E_add, E_mul, E_InputError)

Z = E(0) # the point at infinity (additive identity) of group E
G = E(1) # the base point that generates the cyclic group E

def verify(publickey, signature, message):
    Q    = _decode_a_public_key_point_from_an_octet_string(publickey)
    r, s = _decode_a_signature_from_an_octet_string(signature)
    h    = _hash_encode_an_octet_string_into_an_integer_modulo_q(message)
    return _is_valid_Q_h_r_s_ecdsa_quadruple(Q, h, r, s)

def extract_an_ecdsa_public_key_octets_from_an_x509_certificate(stream):
    if type(stream) is not bytes:
        raise SignatureVerifierError('bad input')
    seqbody, tail = _parse_an_asn1_sequence(stream)
    if len(tail) != 0:
        raise SignatureVerifierError('bad input')
    #
    # TODO: Need improvement later
    #
    # Because I'm too lazy to actually parse the whole X.509 certificate
    # structure, for now I am going to make a shortcut here...
    #
    splitted = seqbody.split(bytes.fromhex(
            '301306072a8648ce3d020106082a8648ce3d030107034200'))
    if len(splitted) == 1:
        raise SignatureVerifierError('bad input')
    elif len(splitted) >= 3:
        raise SignatureVerifierError('case not implemented yet')
    else:
        stream2 = splitted[1]
        if len(stream2) < 1:
            raise SignatureVerifierError('bad input')
        elif stream2[0] in {0x02, 0x03} and len(stream2) >= 33:
            octets = stream2[:33]
            try:
                Q = E_from_bytes(octets)
            except E_InputError:
                raise SignatureVerifierError('bad input')
            else:
                return octets
        elif stream2[0] in {0x04, 0x06, 0x07} and len(stream2) >= 65:
            octets = stream2[:65]
            try:
                Q = E_from_bytes(octets)
            except E_InputError:
                raise SignatureVerifierError('bad input')
            return octets
        else:
            raise SignatureVerifierError('bad input')

class SignatureVerifierError(Exception):
    pass

def _is_valid_Q_h_r_s_ecdsa_quadruple(Q, h, r, s):
    assert E_contains(Q) and not E_eq(Q, Z)
    assert 0 <= h and h <= q-1
    assert 1 <= r and r <= q-1
    assert 1 <= s and s <= q-1
    r2 = E_take_x_mod_q(E_add(E_mul(G, _div_(h, s)),
                              E_mul(Q, _div_(r, s))))
    return (r - r2) % q == 0

def _div_(a, b):
    return (a * pow(b, q - 2, q)) % q

def _decode_a_public_key_point_from_an_octet_string(pk):
    if type(pk) is not bytes:
        raise SignatureVerifierError('a public key MUST be a bytes')
    try:
        Q = E_from_bytes(pk)
    except E_InputError:
        raise SignatureVerifierError('the bytes is not a valid public key')
    if E_eq(Q, Z):
        raise SignatureVerifierError('the bytes is not a valid public key')
    else:
        return Q

def _decode_a_signature_from_an_octet_string(sig):
    seqbody, tail = _parse_an_asn1_sequence(sig)
    if len(tail) != 0:
        raise SignatureVerifierError('bad input')
    r, tail = parse_asn1_integer(sequence_body)
    s, tail = parse_asn1_integer(tail)
    if len(tail) != 0:
        raise SignatureVerifierError('bad input')
    if 0 < r and r < q and 0 < s and s < q:
        return r, s
    else:
        raise SignatureVerifierError('bad input')

def _hash_encode_an_octet_string_into_an_integer_modulo_q(msg):
    return _rfc6979_bits2int(_sha256(msg)) % q

def _parse_an_asn1_sequence(stream):
    if len(stream) < 2:
        raise SignatureVerifierError('bad input')
    T, L = stream[0], stream[1]
    if T != 0x30:
        raise SignatureVerifierError('bad input')
    if L > 127:
        if L == 128:
            raise SignatureVerifierError('bad input')
        length_of_the_L_field = L - 128
        if len(stream) < 1 + 1 + length_of_the_L_field:
            raise SignatureVerifierError('bad input')
        octets_L_field = stream[2:2+length_of_the_L_field]
        L = int.from_bytes(octets_L_field, byteorder='big', signed=False)
        if len(stream) < 1 + 1 + length_of_the_L_field + L:
            raise SignatureVerifierError('bad input')
        V = stream[1+1+length_of_the_L_field:1+1+length_of_the_L_field+L]
        trailing_octets = stream[1+1+length_of_the_L_field+L:]
        return V, trailing_octets
    else:
        if L > len(stream)-2:
            raise SignatureVerifierError('bad input')
        V = stream[2:2+L]
        trailing_octets = stream[2+L:]
        return V, trailing_octets

def _parse_an_asn1_signed_integer(stream):
    if len(stream) < 3:
        raise SignatureVerifierError('bad input')
    T, L = stream[0], stream[1]
    if T != 0x02:
        raise SignatureVerifierError('bad input')
    if L > 127:
        raise SignatureVerifierError('case not implemented yet')
    if L == 0 or L > len(stream)-2:
        raise SignatureVerifierError('bad input')
    V = int.from_bytes(stream[2:2+L], byteorder='big', signed=True)
    trailing_octets = stream[2+L:]
    return V, trailing_octets

def _sha256(msg_to_hash):
    import hashlib
    sha256_digester = hashlib.sha256()
    sha256_digester.update(msg_to_hash)
    return sha256_digester.digest()

def _octets2int(z):
    return int.from_bytes(z, byteorder='big')

def _rfc6979_bits2int(octetstr):
    blen = len(octetstr) * 8
    qlen = 256
    x = _octets2int(octetstr)
    return (x >> (blen-qlen)) if (blen > qlen) else x
<commit_msg>update<commit_after># TODO: Define an interface I am going to implement with this Python module
#
# EcdsaError ::: LanguagePrimitive
# Boolean ::: LanguagePrimitive
# OctetString ::: LanguagePrimitive
# X509Cert ::: OctetString
# PublicKey ::: OctetString
# Signature ::: OctetString
# Message ::: OctetString
#
# extract_publickey_from_certificate
#       :: (X509Cert -> PublicKey) except EcdsaError
#
# compress_publickey
#       :: (PublicKey -> PublicKey) except EcdsaError
#
# verify_signature
#       :: ((PublicKey, Signature, Message) -> Boolean) except EcdsaError
#

from P256 import q
from P256ECC import (
        E, E_from_bytes, E_to_bytes, E_contains, E_take_x_mod_q, E_eq, E_neg,
        E_dbl, E_add, E_mul, E_InputError)

Z = E(0) # the point at infinity (additive identity) of group E
G = E(1) # the base point that generates the cyclic group E

def verify(publickey, signature, message):
    Q    = _decode_a_public_key_point_from_an_octet_string(publickey)
    r, s = _decode_a_signature_from_an_octet_string(signature)
    h    = _hash_encode_an_octet_string_into_an_integer_modulo_q(message)
    return _is_valid_Q_h_r_s_ecdsa_quadruple(Q, h, r, s)

def extract_an_ecdsa_public_key_octets_from_an_x509_certificate(stream):
    if type(stream) is not bytes:
        raise SignatureVerifierError('bad input')
    seqbody, tail = _parse_an_asn1_sequence(stream)
    if len(tail) != 0:
        raise SignatureVerifierError('bad input')
    #
    # TODO: Need improvement later
    #
    # Because I'm too lazy to actually parse the whole X.509 certificate
    # structure, for now I am going to make a shortcut here...
    #
    splitted = seqbody.split(bytes.fromhex(
            '301306072a8648ce3d020106082a8648ce3d030107034200'))
    if len(splitted) == 1:
        raise SignatureVerifierError('bad input')
    elif len(splitted) >= 3:
        raise SignatureVerifierError('case not implemented yet')
    else:
        stream2 = splitted[1]
        if len(stream2) < 1:
            raise SignatureVerifierError('bad input')
        elif stream2[0] in {0x02, 0x03} and len(stream2) >= 33:
            octets = stream2[:33]
            try:
                Q = E_from_bytes(octets)
            except E_InputError:
                raise SignatureVerifierError('bad input')
            else:
                return octets
        elif stream2[0] in {0x04, 0x06, 0x07} and len(stream2) >= 65:
            octets = stream2[:65]
            try:
                Q = E_from_bytes(octets)
            except E_InputError:
                raise SignatureVerifierError('bad input')
            return octets
        else:
            raise SignatureVerifierError('bad input')

class SignatureVerifierError(Exception):
    pass

def _is_valid_Q_h_r_s_ecdsa_quadruple(Q, h, r, s):
    assert E_contains(Q) and not E_eq(Q, Z)
    assert 0 <= h and h <= q-1
    assert 1 <= r and r <= q-1
    assert 1 <= s and s <= q-1
    r2 = E_take_x_mod_q(E_add(E_mul(G, _div_(h, s)),
                              E_mul(Q, _div_(r, s))))
    return (r - r2) % q == 0

def _div_(a, b):
    return (a * pow(b, q - 2, q)) % q

def _decode_a_public_key_point_from_an_octet_string(pk):
    if type(pk) is not bytes:
        raise SignatureVerifierError('a public key MUST be a bytes')
    try:
        Q = E_from_bytes(pk)
    except E_InputError:
        raise SignatureVerifierError('the bytes is not a valid public key')
    if E_eq(Q, Z):
        raise SignatureVerifierError('the bytes is not a valid public key')
    else:
        return Q

def _decode_a_signature_from_an_octet_string(sig):
    seqbody, tail = _parse_an_asn1_sequence(sig)
    if len(tail) != 0:
        raise SignatureVerifierError('bad input')
    r, tail = parse_asn1_integer(sequence_body)
    s, tail = parse_asn1_integer(tail)
    if len(tail) != 0:
        raise SignatureVerifierError('bad input')
    if 0 < r and r < q and 0 < s and s < q:
        return r, s
    else:
        raise SignatureVerifierError('bad input')

def _hash_encode_an_octet_string_into_an_integer_modulo_q(msg):
    return _rfc6979_bits2int(_sha256(msg)) % q

def _parse_an_asn1_sequence(stream):
    if len(stream) < 2:
        raise SignatureVerifierError('bad input')
    T, L = stream[0], stream[1]
    if T != 0x30:
        raise SignatureVerifierError('bad input')
    if L > 127:
        if L == 128:
            raise SignatureVerifierError('bad input')
        length_of_the_L_field = L - 128
        if len(stream) < 1 + 1 + length_of_the_L_field:
            raise SignatureVerifierError('bad input')
        octets_L_field = stream[2:2+length_of_the_L_field]
        L = int.from_bytes(octets_L_field, byteorder='big', signed=False)
        if len(stream) < 1 + 1 + length_of_the_L_field + L:
            raise SignatureVerifierError('bad input')
        V = stream[1+1+length_of_the_L_field:1+1+length_of_the_L_field+L]
        trailing_octets = stream[1+1+length_of_the_L_field+L:]
        return V, trailing_octets
    else:
        if L > len(stream)-2:
            raise SignatureVerifierError('bad input')
        V = stream[2:2+L]
        trailing_octets = stream[2+L:]
        return V, trailing_octets

def _parse_an_asn1_signed_integer(stream):
    if len(stream) < 3:
        raise SignatureVerifierError('bad input')
    T, L = stream[0], stream[1]
    if T != 0x02:
        raise SignatureVerifierError('bad input')
    if L > 127:
        raise SignatureVerifierError('case not implemented yet')
    if L == 0 or L > len(stream)-2:
        raise SignatureVerifierError('bad input')
    V = int.from_bytes(stream[2:2+L], byteorder='big', signed=True)
    trailing_octets = stream[2+L:]
    return V, trailing_octets

def _sha256(msg_to_hash):
    import hashlib
    sha256_digester = hashlib.sha256()
    sha256_digester.update(msg_to_hash)
    return sha256_digester.digest()

def _octets2int(z):
    return int.from_bytes(z, byteorder='big')

def _rfc6979_bits2int(octetstr):
    blen = len(octetstr) * 8
    qlen = 256
    x = _octets2int(octetstr)
    return (x >> (blen-qlen)) if (blen > qlen) else x
<|endoftext|>


<commit_before>def initKML():
  global ET
  global KMLTag
  from lxml import etree as ET
  KMLTag = ET.Element("kml")
  KMLTag.set("xmlns", "http://www.opengis.net/kml/2.2")

def makeLine(points,color,look=False):
  Document = ET.SubElement(KMLTag,"Document")
  Style = ET.SubElement(Document, "Style", id=color)
  Placemark = ET.SubElement(Document, "Placemark")
  LineStyle = ET.SubElement(Style,"LineStyle")
  colorTag = ET.SubElement(LineStyle,"color")
  colorTag.text = color
  widthTag = ET.SubElement(LineStyle,"width")
  widthTag.text = "2"
  styleUrl = ET.SubElement(Placemark,"styleUrl")
  styleUrl.text = "#"+color
  LineString = ET.SubElement(Placemark, "LineString")
  LineStringTess = ET.SubElement(LineString, "tessellate")
  LineStringTess.text = "1"
  LineStringCoor = ET.SubElement(LineString, "coordinates")
  LineStringCoor.text = ""
  for point in points:
    coor = str(point[0])+","+str(point[1])+","+"0\n"
    LineStringCoor.text = LineStringCoor.text+coor
    print(coor)

def endKML(fileName):
  XML_STR = ET.tostring(KMLTag, encoding = "UTF-8", xml_declaration = True, pretty_print = True)
  with open(fileName+".kml", "w") as File:
      File.write(XML_STR)

initKML()
makeLine([[100, 59], [100, 50.00394857394857]], "7f0000ff")
endKML("TestFile")

<commit_msg>removed .kml from endKML's write<commit_after>def initKML():
  global ET
  global KMLTag
  from lxml import etree as ET
  KMLTag = ET.Element("kml")
  KMLTag.set("xmlns", "http://www.opengis.net/kml/2.2")

def makeLine(points,color,look=False):
  Document = ET.SubElement(KMLTag,"Document")
  Style = ET.SubElement(Document, "Style", id=color)
  Placemark = ET.SubElement(Document, "Placemark")
  LineStyle = ET.SubElement(Style,"LineStyle")
  colorTag = ET.SubElement(LineStyle,"color")
  colorTag.text = color
  widthTag = ET.SubElement(LineStyle,"width")
  widthTag.text = "2"
  styleUrl = ET.SubElement(Placemark,"styleUrl")
  styleUrl.text = "#"+color
  LineString = ET.SubElement(Placemark, "LineString")
  LineStringTess = ET.SubElement(LineString, "tessellate")
  LineStringTess.text = "1"
  LineStringCoor = ET.SubElement(LineString, "coordinates")
  LineStringCoor.text = ""
  for point in points:
    coor = str(point[0])+","+str(point[1])+","+"0\n"
    LineStringCoor.text = LineStringCoor.text+coor
    print(coor)

def endKML(fileName):
  XML_STR = ET.tostring(KMLTag, encoding = "UTF-8", xml_declaration = True, pretty_print = True)
  with open(fileName, "w") as File:
      File.write(XML_STR)

initKML()
makeLine([[100, 59], [100, 50.00394857394857]], "7f0000ff")
endKML("TestFile")

<|endoftext|>


<commit_before>import re
import os
import uuid

from datetime import datetime, timedelta
from pytz import timezone

from django.contrib.auth.models import User
from django.db import models
from django.db.models.signals import post_save
from django.db.models import Sum

from amazonproduct import API

class UserProfile(models.Model):
    user = models.OneToOneField(User)
    piggy_bank = models.FloatField(default=0)
    small_amount = models.FloatField(default=1)
    mid_amount = models.FloatField(default=5)
    large_amount = models.FloatField(default=10)

    def __unicode__(self):
        return "%s's profile" % self.user

def create_user_profile(sender, instance, created, **kwargs):
    if created:
        profile, created = UserProfile.objects.get_or_create(user=instance)

post_save.connect(create_user_profile, sender=User)

class BillableItem(models.Model):
    created = models.DateTimeField(auto_now_add=True)
    updated = models.DateTimeField(auto_now=True)
    user = models.ForeignKey(User)
    title = models.CharField(max_length=255, null=True, blank=True)
    completed = models.BooleanField(default=False)
    amount = models.FloatField()
    is_credit = models.BooleanField(default=False)

class Task(BillableItem):
    doable = models.BooleanField(default=True)
    task_rule = models.ForeignKey('TaskRule', null=True)
    due_date = models.DateTimeField(null=True, blank=True)
    can_be_overdue = models.BooleanField(default=False)

    def __unicode__(self):
        completed = "Incomplete"
        if self.completed:
            completed = "Complete"
        return "[%s] %s (%s)" % (self.user, self.title, completed)

class Wish(BillableItem):
    amazon_link = models.CharField(max_length=255, null=True, blank=True)
    asin = models.CharField(max_length=255, null=True, blank=True)
    image_url = models.CharField(max_length=255, null=True, blank=True)
    featured = models.BooleanField(default=False)
    url = models.CharField(max_length=255, null=True, blank=True)

def create_wish_from_url(user, url):
    AWS_KEY = os.environ['AWS_KEY']
    AWS_SECRET_KEY = os.environ['AWS_SECRET_KEY']
    ASSOCIATE_TAG = os.environ['ASSOCIATE_TAG']
    ASIN_MATCH = 'http://www.amazon.com/([\\w-]+/)?(dp|gp/product)/(\\w+/)?(\\w{10})'

    asin = list(re.match(ASIN_MATCH, url).groups())[-1]

    api = API(locale='us',
              associate_tag=ASSOCIATE_TAG,
              access_key_id=AWS_KEY,
              secret_access_key=AWS_SECRET_KEY)
    result = api.item_lookup(asin, ResponseGroup='ItemAttributes, OfferFull, Images')
    item = result.Items.Item[0]

    title = item.ItemAttributes.Title
    url = item.DetailPageURL

    if item.OfferSummary:
        amount = (item.OfferSummary.LowestNewPrice.Amount / 100)
    elif item.ItemAttributes.ListPrice:
        amount = (item.ItemAttributes.ListPrice.Amount / 100)
    else:
        amount = 0.0

    image_url = item.LargeImage.URL

    wish = Wish(
        user=user,
        asin=asin,
        title=title,
        amount=amount,
        is_credit=False,
        url=url,
        image_url=image_url
    )
    wish.save()

    return wish

FREQUENCY_CHOICES = (
    ('day', 'day'),
    ('week', 'week'),
    ('month', 'month'),
    ('year', 'year')
)

COMPLETABLE_BY = (
    ('Oyster', 'Oyster'),
    ('IFTTT', 'IFTTT')
)

class TaskRule(models.Model):
    created = models.DateTimeField(auto_now_add=True)
    updated = models.DateTimeField(auto_now=True)
    user = models.ForeignKey(User)
    uuid = models.CharField(max_length=255, null=True, blank=True)
    title = models.CharField(max_length=255, null=True, blank=True)
    amount = models.FloatField(null=True)
    start = models.DateTimeField(auto_now_add=True)
    frequency = models.IntegerField(null=True)
    scale = models.CharField(max_length=30, choices=FREQUENCY_CHOICES,
                             null=True, blank=True)
    completable_by = models.CharField(max_length=30, choices=COMPLETABLE_BY,
                                      default='Oyster')
    next_scheduled_run = models.DateTimeField(auto_now_add=True)
    must_be_completed_by_time = models.TimeField(null=True)
    must_be_completed_in_x_days = models.IntegerField(null=True)
    regenerate_on_completion = models.BooleanField(default=False)
    can_be_overdue = models.BooleanField(default=False)
    cancelled = models.BooleanField(default=False)

    def calculate_next_run(self):
        if self.scale == 'day':
            hours = 24
        elif self.scale == 'week':
            hours = 168
        elif self.scale == 'month':
            hours = 720
        elif self.scale == 'year':
            hours = 8760
        else:
            return
        delta = timedelta(hours=(hours / self.frequency))
        next_run = self.next_scheduled_run + delta
        self.next_scheduled_run = next_run
        self.save()

        return self.next_scheduled_run

    def infer_due_date(self):
        time = self.must_be_completed_by_time
        days = self.must_be_completed_in_x_days

        if not time and not days:
            return None

        now_utc = datetime.now(timezone('UTC'))
        now_pacific = now_utc.astimezone(timezone('US/Pacific'))

        due_date = now_pacific

        if time:
            due_date = due_date.replace(hour=time.hour, minute=time.minute)
            if due_date < now_pacific and not days:
                delta = timedelta(days=1)
                due_date = due_date + delta

        if days:
            delta = timedelta(days=days)
            due_date = due_date + delta

        return due_date

    def create_new_task(self):
        if self.cancelled:
            return
        doable = True
        if self.completable_by != 'Oyster':
            doable = False
        new_task = Task(
            user=self.user,
            title=self.title,
            amount=self.amount,
            task_rule=self,
            doable=doable,
            due_date=self.infer_due_date(),
            can_be_overdue=self.can_be_overdue
        )
        new_task.save()
        return new_task

    def get_open_tasks(self):
        return Task.objects.filter(task_rule=self, completed=False).order_by('-created')

    def get_first_open_task(self):
        open_tasks = self.get_open_tasks()
        if open_tasks:
            return open_tasks.first()
        return None

def update_piggy_bank(sender, instance, created, **kwargs):
    profile = instance.user.userprofile
    credits = Task.objects.filter(user=instance.user,
                                  completed=True).aggregate(
                                  amount=Sum('amount'))
    deductions = Wish.objects.filter(user=instance.user,
                                     completed=True).aggregate(
                                     amount=Sum('amount'))
    amount = 0
    if credits.get('amount', 0):
        amount = credits.get('amount', 0)
    if deductions.get('amount', 0):
        amount -= deductions.get('amount', 0)
    profile.piggy_bank = amount
    profile.save()

def complete_task(sender, instance, created, **kwargs):
    if instance.completed and instance.task_rule and instance.task_rule.regenerate_on_completion:
        instance.task_rule.create_new_task()

post_save.connect(update_piggy_bank, sender=Task)
post_save.connect(complete_task, sender=Task)
post_save.connect(update_piggy_bank, sender=Wish)
<commit_msg>fix display of checkmark in history<commit_after>import re
import os
import uuid

from datetime import datetime, timedelta
from pytz import timezone

from django.contrib.auth.models import User
from django.db import models
from django.db.models.signals import post_save
from django.db.models import Sum

from amazonproduct import API

class UserProfile(models.Model):
    user = models.OneToOneField(User)
    piggy_bank = models.FloatField(default=0)
    small_amount = models.FloatField(default=1)
    mid_amount = models.FloatField(default=5)
    large_amount = models.FloatField(default=10)

    def __unicode__(self):
        return "%s's profile" % self.user

def create_user_profile(sender, instance, created, **kwargs):
    if created:
        profile, created = UserProfile.objects.get_or_create(user=instance)

post_save.connect(create_user_profile, sender=User)

class BillableItem(models.Model):
    created = models.DateTimeField(auto_now_add=True)
    updated = models.DateTimeField(auto_now=True)
    user = models.ForeignKey(User)
    title = models.CharField(max_length=255, null=True, blank=True)
    completed = models.BooleanField(default=False)
    amount = models.FloatField()
    is_credit = models.BooleanField(default=False)

class Task(BillableItem):
    doable = models.BooleanField(default=True)
    task_rule = models.ForeignKey('TaskRule', null=True)
    due_date = models.DateTimeField(null=True, blank=True)
    can_be_overdue = models.BooleanField(default=False)

    def __unicode__(self):
        completed = "Incomplete"
        if self.completed:
            completed = "Complete"
        return "[%s] %s (%s)" % (self.user, self.title, completed)

class Wish(BillableItem):
    amazon_link = models.CharField(max_length=255, null=True, blank=True)
    asin = models.CharField(max_length=255, null=True, blank=True)
    image_url = models.CharField(max_length=255, null=True, blank=True)
    featured = models.BooleanField(default=False)
    url = models.CharField(max_length=255, null=True, blank=True)

def create_wish_from_url(user, url):
    AWS_KEY = os.environ['AWS_KEY']
    AWS_SECRET_KEY = os.environ['AWS_SECRET_KEY']
    ASSOCIATE_TAG = os.environ['ASSOCIATE_TAG']
    ASIN_MATCH = 'http://www.amazon.com/([\\w-]+/)?(dp|gp/product)/(\\w+/)?(\\w{10})'

    asin = list(re.match(ASIN_MATCH, url).groups())[-1]

    api = API(locale='us',
              associate_tag=ASSOCIATE_TAG,
              access_key_id=AWS_KEY,
              secret_access_key=AWS_SECRET_KEY)
    result = api.item_lookup(asin, ResponseGroup='ItemAttributes, OfferFull, Images')
    item = result.Items.Item[0]

    title = item.ItemAttributes.Title
    url = item.DetailPageURL

    if item.OfferSummary:
        amount = (item.OfferSummary.LowestNewPrice.Amount / 100)
    elif item.ItemAttributes.ListPrice:
        amount = (item.ItemAttributes.ListPrice.Amount / 100)
    else:
        amount = 0.0

    image_url = item.LargeImage.URL

    wish = Wish(
        user=user,
        asin=asin,
        title=title,
        amount=amount,
        is_credit=False,
        url=url,
        image_url=image_url
    )
    wish.save()

    return wish

FREQUENCY_CHOICES = (
    ('day', 'day'),
    ('week', 'week'),
    ('month', 'month'),
    ('year', 'year')
)

COMPLETABLE_BY = (
    ('Oyster', 'Oyster'),
    ('IFTTT', 'IFTTT')
)

class TaskRule(models.Model):
    created = models.DateTimeField(auto_now_add=True)
    updated = models.DateTimeField(auto_now=True)
    user = models.ForeignKey(User)
    uuid = models.CharField(max_length=255, null=True, blank=True)
    title = models.CharField(max_length=255, null=True, blank=True)
    amount = models.FloatField(null=True)
    start = models.DateTimeField(auto_now_add=True)
    frequency = models.IntegerField(null=True)
    scale = models.CharField(max_length=30, choices=FREQUENCY_CHOICES,
                             null=True, blank=True)
    completable_by = models.CharField(max_length=30, choices=COMPLETABLE_BY,
                                      default='Oyster')
    next_scheduled_run = models.DateTimeField(auto_now_add=True)
    must_be_completed_by_time = models.TimeField(null=True)
    must_be_completed_in_x_days = models.IntegerField(null=True)
    regenerate_on_completion = models.BooleanField(default=False)
    can_be_overdue = models.BooleanField(default=False)
    cancelled = models.BooleanField(default=False)

    def calculate_next_run(self):
        if self.scale == 'day':
            hours = 24
        elif self.scale == 'week':
            hours = 168
        elif self.scale == 'month':
            hours = 720
        elif self.scale == 'year':
            hours = 8760
        else:
            return
        delta = timedelta(hours=(hours / self.frequency))
        next_run = self.next_scheduled_run + delta
        self.next_scheduled_run = next_run
        self.save()

        return self.next_scheduled_run

    def infer_due_date(self):
        time = self.must_be_completed_by_time
        days = self.must_be_completed_in_x_days

        if not time and not days:
            return None

        now_utc = datetime.now(timezone('UTC'))
        now_pacific = now_utc.astimezone(timezone('US/Pacific'))

        due_date = now_pacific

        if time:
            due_date = due_date.replace(hour=time.hour, minute=time.minute)
            if due_date < now_pacific and not days:
                delta = timedelta(days=1)
                due_date = due_date + delta

        if days:
            delta = timedelta(days=days)
            due_date = due_date + delta

        return due_date

    def create_new_task(self):
        if self.cancelled:
            return
        doable = True
        if self.completable_by != 'Oyster':
            doable = False
        new_task = Task(
            user=self.user,
            title=self.title,
            amount=self.amount,
            task_rule=self,
            doable=doable,
            due_date=self.infer_due_date(),
            can_be_overdue=self.can_be_overdue,
            is_credit=True
        )
        new_task.save()
        return new_task

    def get_open_tasks(self):
        return Task.objects.filter(task_rule=self, completed=False).order_by('-created')

    def get_first_open_task(self):
        open_tasks = self.get_open_tasks()
        if open_tasks:
            return open_tasks.first()
        return None

def update_piggy_bank(sender, instance, created, **kwargs):
    profile = instance.user.userprofile
    credits = Task.objects.filter(user=instance.user,
                                  completed=True).aggregate(
                                  amount=Sum('amount'))
    deductions = Wish.objects.filter(user=instance.user,
                                     completed=True).aggregate(
                                     amount=Sum('amount'))
    amount = 0
    if credits.get('amount', 0):
        amount = credits.get('amount', 0)
    if deductions.get('amount', 0):
        amount -= deductions.get('amount', 0)
    profile.piggy_bank = amount
    profile.save()

def complete_task(sender, instance, created, **kwargs):
    if instance.completed and instance.task_rule and instance.task_rule.regenerate_on_completion:
        instance.task_rule.create_new_task()

post_save.connect(update_piggy_bank, sender=Task)
post_save.connect(complete_task, sender=Task)
post_save.connect(update_piggy_bank, sender=Wish)
<|endoftext|>