<commit_before># django settings for script
from django.conf import settings

from djequis.core.utils import sendmail
from djzbar.utils.informix import get_engine
from djzbar.utils.informix import do_sql

# Imports for additional modules and functions written as part of this project
from djequis.adp.aarec import fn_archive_address
from djequis.adp.utilities import fn_validate_field, fn_write_log, fn_write_error, \
    fn_format_phone

DEBUG = settings.INFORMIX_DEBUG

# set up command-line options
desc = """
    Upload ADP data to CX
"""

# write out the .sql file
scr = open("apdtocx_output.sql", "a")

#############################################
# Begin Processing
#############################################

def fn_process_idrec(carth_id, file_number, fullname, lastname, firstname,
        middlename, title, addr_line1, addr_line2, addr_line3, city, st,
        zip, ctry, ctry_cod, ss_no, phone, decsd, eff_date, EARL):
    print("Start ID Rec Processing")
    engine = get_engine(EARL)

    v_id = fn_validate_field(carth_id, "id", "id", "id_rec", "integer",
                              EARL)

    if v_id == 0:
            fn_write_log("ID not found in CX database.  ID = " + carth_id
                         + " Name = " + fullname)
            BODY = "ID not found in CX database for ID " + carth_id \
                   + " Name, " + fullname
            SUBJECT = "CX ID not found"
            # sendmail(
            #     settings.ADP_TO_EMAIL, settings.ADP_FROM_EMAIL,
            #     BODY, SUBJECT)
    else:

        try:

            q_update_id_rec = ('''UPDATE id_rec SET fullname = ?, lastname = ?,
                firstname = ?, middlename = ?, ss_no = ?, decsd = 'N',
                upd_date = ?, ofc_add_by = 'HR'
                WHERE id = ?''')

            q_update_id_args = (fullname, lastname, firstname, middlename, ss_no, eff_date,
                           carth_id)
            # print(q_update_id_rec)
            # print(q_update_id_args)
            fn_write_log("Update basic info in id_rec table for " + fullname +
                         ", ID = " + str(carth_id))
            scr.write(q_update_id_rec + '\n' + str(q_update_id_args) + '\n');
            # logger.info("Update id_rec table");
            engine.execute(q_update_id_rec, q_update_id_args)
        except Exception as err:
            # print(err.message)
            return (err.message)
            fn_write_error("Error in id_rec.py updating basic info.  Error = "
                           + err.message)
            # logger.error(err, exc_info=True)

        #########################################################
        # Title is a problem - most blank in ADP
        # To avoid overwriting, will need to validate and do
        # a separate update of the record
        try:

            if title is not None:
                x = title.replace(".", "")
                vTitle  = fn_validate_field(x.upper(), 'title', 'title', 'title_table',
                                         'char', EARL)
                print("Title = " + str(vTitle))
                if vTitle is not None and vTitle != "":
                    q_update_title = ('''UPDATE id_rec SET title = ?
                                WHERE id = ?''')
                    q_update_title_args = (vTitle, carth_id)
                    fn_write_log("Update Title info in id_rec table for " + fullname +
                                 ", ID = " + str(carth_id))
                    scr.write(q_update_title + '\n' + str(q_update_title_args) + '\n');
                    # logger.info("Update id_rec table");
                    engine.execute(q_update_title, q_update_title_args)
            else:
                print("No Title")

        except Exception as err:
            # print(err.message)
            return (err.message)
            fn_write_error("Error in id_rec.py updating title info.  Error = " + err.message)
            # logger.error(err, exc_info=True)

#########################################################

        # print("Country Code = " + str(len(ctry_cod)))

        try:
            # also need to deal with address changes
            # Search for existing address record
            if ctry_cod.strip() != '' and len(ctry_cod) > 0:
                cntry = fn_validate_field(ctry_cod, 'ctry', 'ctry', 'ctry_table', 'char', EARL)
                print("Valid Country Code = " + cntry)

                print(" In Check Address")
                q_check_addr = '''
                            SELECT id, addr_line1, addr_line2, addr_line3, city,
                                st, zip, ctry, phone
                            FROM id_rec
                            WHERE id = {0}
                                '''.format(carth_id)
                addr_result = do_sql(q_check_addr, key=DEBUG, earl=EARL)
                # scr.write(q_check_addr + '\n');
                row = addr_result.fetchone()
                if row is None:
                    fn_write_log("Data missing in idrec.py address function. \
                                                  Employee not in id rec for id "
                                 "number " + str(
                        carth_id))
                    # print("Employee not in id rec")
                elif str(row[0]) == '0' or str(row[0]) == '':  # No person in id rec? Should never happen
                    fn_write_log("Data missing in idrec.py address function. \
                                  Employee not in id rec for id number " + str(carth_id))
                    # print("Employee not in id rec")
                    BODY = "ID not found in CX database id_rec.py address " \
                           "routine for ID " + carth_id  + " Name, " + fullname
                    SUBJECT = "CX ID not found"
                    # sendmail(
                    #     settings.ADP_TO_EMAIL, settings.ADP_FROM_EMAIL,
                    #     BODY, SUBJECT )

                # Update ID Rec and archive aa rec
                elif (row[1] != addr_line1
                    or row[2] != addr_line2
                    or row[3] != addr_line3
                    or row[4] != city
                    or row[5] != st
                    or row[6] != zip
                    or row[7] != ctry_cod):

                    # print("Update: no address match in ID_REC " + str(carth_id))  #

                    q_update_id_rec_addr = ('''UPDATE id_rec SET addr_line1 = ?,
                         addr_line2 = ?, addr_line3 = ?, city = ?, st = ?, zip = ?,
                         ctry = ?, aa = 'PERM', phone = ? WHERE id = ?''')
                    q_update_id_addr_args = (addr_line1, addr_line2, addr_line3, city, st,
                                            zip, cntry, fn_format_phone(phone), carth_id)

                    # print(q_update_id_rec_addr)
                    # print(q_update_id_addr_args)
                    fn_write_log("Update address info in id_rec table for " +
                                 fullname + ", ID = " + str(carth_id) +
                                 " address = " + addr_line1)
                    engine.execute(q_update_id_rec_addr, q_update_id_addr_args)
                    scr.write(q_update_id_rec_addr + '\n' + str(q_update_id_addr_args) + '\n')

                    #########################################################
                    # Routine to deal with aa_rec
                    #########################################################
                    # now check to see if address is a duplicate in aa_rec
                    # find max start date to determine what date to insert
                    # insert or update as needed
                    if row[1] is None:
                        # This should only happen on initial run, just need to
                        #  ignore the archive process if no address to archive
                        fn_write_log("Empty Address 1 in ID Rec - Nothing to "
                                     "archive")
                    elif row is not None:
                        # print("row[1] = " + row[1])
                        fn_archive_address(carth_id, fullname, row[1], row[2],
                                     row[3], row[4], row[5], row[6], row[7], phone,
                                           EARL)
                    else:
                        fn_write_log("Empty Address 1 in ID Rec - Nothing to "
                                     "archive")

                # else:
                #     print("No Change " + row[1])
            elif ctry_cod is None or len(ctry_cod) == 0:
                # print("invalid country code" + ctry_cod)
                fn_write_log("invalid country code" + ctry_cod)

        except Exception as err:
            # print(err.message)
            fn_write_error("Error in idrec.py for id " + carth_id
                           + ".  Error = " + err.message)

# logger.error(err, exc_info=True)
#     finally:
#          logging.shutdown()

<commit_msg>Commented out debug print statements<commit_after># django settings for script
from django.conf import settings

from djequis.core.utils import sendmail
from djzbar.utils.informix import get_engine
from djzbar.utils.informix import do_sql

# Imports for additional modules and functions written as part of this project
from djequis.adp.aarec import fn_archive_address
from djequis.adp.utilities import fn_validate_field, fn_write_log, fn_write_error, \
    fn_format_phone

DEBUG = settings.INFORMIX_DEBUG

# set up command-line options
desc = """
    Upload ADP data to CX
"""

# write out the .sql file
scr = open("apdtocx_output.sql", "a")

#############################################
# Begin Processing
#############################################

def fn_process_idrec(carth_id, file_number, fullname, lastname, firstname,
        middlename, title, addr_line1, addr_line2, addr_line3, city, st,
        zip, ctry, ctry_cod, ss_no, phone, decsd, eff_date, EARL):
    print("Start ID Rec Processing")
    engine = get_engine(EARL)

    v_id = fn_validate_field(carth_id, "id", "id", "id_rec", "integer",
                              EARL)

    if v_id == 0:
            fn_write_log("ID not found in CX database.  ID = " + carth_id
                         + " Name = " + fullname)
            BODY = "ID not found in CX database for ID " + carth_id \
                   + " Name, " + fullname
            SUBJECT = "CX ID not found"
            # sendmail(
            #     settings.ADP_TO_EMAIL, settings.ADP_FROM_EMAIL,
            #     BODY, SUBJECT)
    else:

        try:

            q_update_id_rec = ('''UPDATE id_rec SET fullname = ?, lastname = ?,
                firstname = ?, middlename = ?, ss_no = ?, decsd = 'N',
                upd_date = ?, ofc_add_by = 'HR'
                WHERE id = ?''')

            q_update_id_args = (fullname, lastname, firstname, middlename, ss_no, eff_date,
                           carth_id)
            # print(q_update_id_rec)
            # print(q_update_id_args)
            fn_write_log("Update basic info in id_rec table for " + fullname +
                         ", ID = " + str(carth_id))
            scr.write(q_update_id_rec + '\n' + str(q_update_id_args) + '\n');
            # logger.info("Update id_rec table");
            engine.execute(q_update_id_rec, q_update_id_args)
        except Exception as err:
            # print(err.message)
            return (err.message)
            fn_write_error("Error in id_rec.py updating basic info.  Error = "
                           + err.message)
            # logger.error(err, exc_info=True)

        #########################################################
        # Title is a problem - most blank in ADP
        # To avoid overwriting, will need to validate and do
        # a separate update of the record
        try:

            if title is not None:
                x = title.replace(".", "")
                vTitle  = fn_validate_field(x.upper(), 'title', 'title', 'title_table',
                                         'char', EARL)
                # print("Title = " + str(vTitle))
                if vTitle is not None and vTitle != "":
                    q_update_title = ('''UPDATE id_rec SET title = ?
                                WHERE id = ?''')
                    q_update_title_args = (vTitle, carth_id)
                    fn_write_log("Update Title info in id_rec table for " + fullname +
                                 ", ID = " + str(carth_id))
                    scr.write(q_update_title + '\n' + str(q_update_title_args) + '\n');
                    # logger.info("Update id_rec table");
                    engine.execute(q_update_title, q_update_title_args)
            # else:
            #     print("No Title")

        except Exception as err:
            # print(err.message)
            return (err.message)
            fn_write_error("Error in id_rec.py updating title info.  Error = " + err.message)
            # logger.error(err, exc_info=True)

#########################################################

        # print("Country Code = " + str(len(ctry_cod)))

        try:
            # also need to deal with address changes
            # Search for existing address record
            if ctry_cod.strip() != '' and len(ctry_cod) > 0:
                cntry = fn_validate_field(ctry_cod, 'ctry', 'ctry', 'ctry_table', 'char', EARL)
                # print("Valid Country Code = " + cntry)

                # print(" In Check Address")
                q_check_addr = '''
                            SELECT id, addr_line1, addr_line2, addr_line3, city,
                                st, zip, ctry, phone
                            FROM id_rec
                            WHERE id = {0}
                                '''.format(carth_id)
                addr_result = do_sql(q_check_addr, key=DEBUG, earl=EARL)
                # scr.write(q_check_addr + '\n');
                row = addr_result.fetchone()
                if row is None:
                    fn_write_log("Data missing in idrec.py address function. \
                                                  Employee not in id rec for id "
                                 "number " + str(
                        carth_id))
                    # print("Employee not in id rec")
                elif str(row[0]) == '0' or str(row[0]) == '':  # No person in id rec? Should never happen
                    fn_write_log("Data missing in idrec.py address function. \
                                  Employee not in id rec for id number " + str(carth_id))
                    # print("Employee not in id rec")
                    BODY = "ID not found in CX database id_rec.py address " \
                           "routine for ID " + carth_id  + " Name, " + fullname
                    SUBJECT = "CX ID not found"
                    # sendmail(
                    #     settings.ADP_TO_EMAIL, settings.ADP_FROM_EMAIL,
                    #     BODY, SUBJECT )

                # Update ID Rec and archive aa rec
                elif (row[1] != addr_line1
                    or row[2] != addr_line2
                    or row[3] != addr_line3
                    or row[4] != city
                    or row[5] != st
                    or row[6] != zip
                    or row[7] != ctry_cod):

                    # print("Update: no address match in ID_REC " + str(carth_id))  #

                    q_update_id_rec_addr = ('''UPDATE id_rec SET addr_line1 = ?,
                         addr_line2 = ?, addr_line3 = ?, city = ?, st = ?, zip = ?,
                         ctry = ?, aa = 'PERM', phone = ? WHERE id = ?''')
                    q_update_id_addr_args = (addr_line1, addr_line2, addr_line3, city, st,
                                            zip, cntry, fn_format_phone(phone), carth_id)

                    # print(q_update_id_rec_addr)
                    # print(q_update_id_addr_args)
                    fn_write_log("Update address info in id_rec table for " +
                                 fullname + ", ID = " + str(carth_id) +
                                 " address = " + addr_line1)
                    engine.execute(q_update_id_rec_addr, q_update_id_addr_args)
                    scr.write(q_update_id_rec_addr + '\n' + str(q_update_id_addr_args) + '\n')

                    #########################################################
                    # Routine to deal with aa_rec
                    #########################################################
                    # now check to see if address is a duplicate in aa_rec
                    # find max start date to determine what date to insert
                    # insert or update as needed
                    if row[1] is None:
                        # This should only happen on initial run, just need to
                        #  ignore the archive process if no address to archive
                        fn_write_log("Empty Address 1 in ID Rec - Nothing to "
                                     "archive")
                    elif row is not None:
                        # print("row[1] = " + row[1])
                        fn_archive_address(carth_id, fullname, row[1], row[2],
                                     row[3], row[4], row[5], row[6], row[7], phone,
                                           EARL)
                    else:
                        fn_write_log("Empty Address 1 in ID Rec - Nothing to "
                                     "archive")

                # else:
                #     print("No Change " + row[1])
            elif ctry_cod is None or len(ctry_cod) == 0:
                # print("invalid country code" + ctry_cod)
                fn_write_log("invalid country code" + ctry_cod)

        except Exception as err:
            # print(err.message)
            fn_write_error("Error in idrec.py for id " + carth_id
                           + ".  Error = " + err.message)

# logger.error(err, exc_info=True)
#     finally:
#          logging.shutdown()

<|endoftext|>


<commit_before>'''
whats_present.py

Given a list of file names provides information about which files are
present and which are absent from within a file listing. There's a good
number of tools out there that will do this as well but in my case I didn't
have access to those.
'''
<commit_msg>Arg parsing now in place<commit_after>'''
whats_present.py

Given a list of file names provides information about which files are
present and which are absent from within a file listing. There's a good
number of tools out there that will do this as well but in my case I didn't
have access to those.

Usage:
  whats_present.py [-h] -i FILEDIR -o FILEOUT -t FILETEST

Options:
    -h --help    show this
    -i FILEDIR   specify directory listing file
    -o FILEOUT   specify output file
    -t FILETEST  specify test file

'''
from docopt import docopt

if __name__ == '__main__':
    arguments = docopt(__doc__, version='Whats Present 0.1')
    print(arguments)
<|endoftext|>


<commit_before>"""DNS proxy response behavior module."""

from re import compile as regex_compile, match
from dnslib import RR, A, QTYPE
from dns.exception import DNSException
import dns.resolver
import logging

module_logger = logging.getLogger('dnsproxy.behavior')

IP_KEY = 'ip'
STRATEGY_KEY = 'strategy'
DEFAULT_STRATEGY = 'forward'
ADDRESS_KEY = 'address'
LOGLEVEL_KEY = 'logLevel'
DEFAULT_LOGLEVEL = 'DEBUG'

def getLogLevelNumber(loglevelname):
    """Parses log level name into log level number.

    Returns int value of log level. On failure, DEBUG level value is returned."""
    number = getattr(logging, loglevelname, None)
    if not isinstance(number, int):
        module_logger.debug("failed to parse log level name '{name}'".format(name = loglevelname))
        number = logging.DEBUG
    return number

class Behavior(object):
    """Behavior for given address. Creates response depending on set strategy.

    Call behavior.handles(address) to check if it's handling given address.
    Call behavior.handle(request) to obtain dns response or None if no response should be sent.
    """

    strategies = dict(
        block = lambda self, req: Behavior.block(self, req),
        forward = lambda self, req: Behavior.forward(self, req),
        respond = lambda self, req: Behavior.respond(self, req))

    def __init__(self, address = '', strategy = DEFAULT_STRATEGY, ip = '', loglevel = DEFAULT_LOGLEVEL):
        """Creates new behavior accepting address which is handled and strategy.

        """
        self.logger = logging.getLogger('dnsproxy.behavior.Behavior')
        self.ip = ip
        self.strategy = strategy
        self.address = address
        self.loglevel = loglevel

    def __str__(self):
        return "Behavior(address='{addr}', strategy='{strategy}', ip = '{ip}', log_level = '{loglevel}')".format(
            addr = self.address,
            strategy = self.strategy,
            ip = self.ip,
            loglevel = self.loglevel)

    def parseloglevel(self):
        return getLogLevelNumber(self.loglevel)

    def handles(self, address):
        """Checks whether given address is handled by this behavior

        Returns True if it handles.
        """
        m = match(self.address, address)
        self.logger.debug("handles check: address '{addr}' by {b}, result: {r}".format(b = str(self), addr=address, r = m != None))
        return m != None

    def handle(self, request):
        """Handles provided request according to set strategy.

        Returns response or None if no response should be sent.
        """
        self.logger.debug("handle: {b}, request: {r}", b = str(self), r = request)
        return self.strategies[self.strategy](self, request)

    def block(self, request):
        """Returns None."""
        address = str(request.questions[0].qname)
        self.logger.log(self.parseloglevel(), "{b} - Blocking request for address:'{addr}'".format(addr=address, b=str(self)))
        return None

    def forward(self, request):
        """Returns response received from system resolver."""
        address = str(request.questions[0].qname)
        self.logger.log(self.parseloglevel(), "{b} - Forwarding request for address:'{addr}'".format(addr=address, b=str(self)))
        try:
            answers = dns.resolver.query(address, 'A')
        except DNSException:
            self.logger.exception("{b} - Exception when forwarding request for address:'{addr}'".format(addr=address, b=str(self)))
        response = request.reply()
        for rdata in answers:
            ip = rdata.address
            response.add_answer(RR(address, QTYPE.A, rdata=A(ip)))
        return response

    def respond(self, request):
        """Returns response containing self.ip."""
        address = str(request.questions[0].qname)
        self.logger.log(self.parseloglevel(), "{b} - Responding to request for address:'{addr}'".format(addr=address, b=str(self)))
        response = request.reply()
        response.add_answer(RR(address, QTYPE.A, rdata=A(self.ip)))
        return response

    def from_json(self, json):
        """Sets attributes from provided JSON object/dict.

        Returns self.
        """
        self.address = json[ADDRESS_KEY]
        self.strategy = json[STRATEGY_KEY]
        if IP_KEY in json:
            self.ip = json[IP_KEY]
        else:
            self.ip = ''
        if LOGLEVEL_KEY in json:
            self.loglevel = json[LOGLEVEL_KEY]
        else:
            self.loglevel = DEFAULT_LOGLEVEL
        return self

    def to_json(self):
        """Creates JSON object/dict representing this Behavior.

        Returns created JSON object (dict).
        """
        return {
            IP_KEY : self.ip,
            STRATEGY_KEY : self.strategy,
            ADDRESS_KEY : self.address,
            LOGLEVEL_KEY : self.loglevel }

def first_or_default(behaviors, request):
    """Finds a behavior in list of behaviors,
    which handles given request.

    Returns behavior if any is found, or default behavior otherwise.
    """
    address = str(request.questions[0].qname)
    if behaviors == None or len(behaviors) == 0:
        return Behavior(address)
    for behavior in behaviors:
        if (behavior.handles(address)):
            return behavior
    return Behavior(address)
<commit_msg>fixed behavior logging<commit_after>"""DNS proxy response behavior module."""

from re import compile as regex_compile, match
from dnslib import RR, A, QTYPE
from dns.exception import DNSException
import dns.resolver
import logging

module_logger = logging.getLogger('dnsproxy.behavior')

IP_KEY = 'ip'
STRATEGY_KEY = 'strategy'
DEFAULT_STRATEGY = 'forward'
ADDRESS_KEY = 'address'
LOGLEVEL_KEY = 'logLevel'
DEFAULT_LOGLEVEL = 'DEBUG'

def getLogLevelNumber(loglevelname):
    """Parses log level name into log level number.

    Returns int value of log level. On failure, DEBUG level value is returned."""
    number = getattr(logging, loglevelname, None)
    if not isinstance(number, int):
        module_logger.debug("failed to parse log level name '{name}'".format(name = loglevelname))
        number = logging.DEBUG
    return number

class Behavior(object):
    """Behavior for given address. Creates response depending on set strategy.

    Call behavior.handles(address) to check if it's handling given address.
    Call behavior.handle(request) to obtain dns response or None if no response should be sent.
    """

    strategies = dict(
        block = lambda self, req: Behavior.block(self, req),
        forward = lambda self, req: Behavior.forward(self, req),
        respond = lambda self, req: Behavior.respond(self, req))

    def __init__(self, address = '', strategy = DEFAULT_STRATEGY, ip = '', loglevel = DEFAULT_LOGLEVEL):
        """Creates new behavior accepting address which is handled and strategy.

        """
        self.logger = logging.getLogger('dnsproxy.behavior.Behavior')
        self.ip = ip
        self.strategy = strategy
        self.address = address
        self.loglevel = loglevel

    def __str__(self):
        return "Behavior(address='{addr}', strategy='{strategy}', ip = '{ip}', log_level = '{loglevel}')".format(
            addr = self.address,
            strategy = self.strategy,
            ip = self.ip,
            loglevel = self.loglevel)

    def parseloglevel(self):
        return getLogLevelNumber(self.loglevel)

    def handles(self, address):
        """Checks whether given address is handled by this behavior

        Returns True if it handles.
        """
        m = match(self.address, address)
        self.logger.debug("handles check: address '{addr}' by {b}, result: {r}".format(b = str(self), addr=address, r = m != None))
        return m != None

    def handle(self, request):
        """Handles provided request according to set strategy.

        Returns response or None if no response should be sent.
        """
        self.logger.debug("handle: {b}, request: {r}".format(b = str(self), r = request))
        return self.strategies[self.strategy](self, request)

    def block(self, request):
        """Returns None."""
        address = str(request.questions[0].qname)
        self.logger.log(self.parseloglevel(), "{b} - Blocking request for address:'{addr}'".format(addr=address, b=str(self)))
        return None

    def forward(self, request):
        """Returns response received from system resolver."""
        address = str(request.questions[0].qname)
        self.logger.log(self.parseloglevel(), "{b} - Forwarding request for address:'{addr}'".format(addr=address, b=str(self)))
        try:
            answers = dns.resolver.query(address, 'A')
        except DNSException:
            self.logger.exception("{b} - Exception when forwarding request for address:'{addr}'".format(addr=address, b=str(self)))
        response = request.reply()
        for rdata in answers:
            ip = rdata.address
            response.add_answer(RR(address, QTYPE.A, rdata=A(ip)))
        return response

    def respond(self, request):
        """Returns response containing self.ip."""
        address = str(request.questions[0].qname)
        self.logger.log(self.parseloglevel(), "{b} - Responding to request for address:'{addr}'".format(addr=address, b=str(self)))
        response = request.reply()
        response.add_answer(RR(address, QTYPE.A, rdata=A(self.ip)))
        return response

    def from_json(self, json):
        """Sets attributes from provided JSON object/dict.

        Returns self.
        """
        self.address = json[ADDRESS_KEY]
        self.strategy = json[STRATEGY_KEY]
        if IP_KEY in json:
            self.ip = json[IP_KEY]
        else:
            self.ip = ''
        if LOGLEVEL_KEY in json:
            self.loglevel = json[LOGLEVEL_KEY]
        else:
            self.loglevel = DEFAULT_LOGLEVEL
        return self

    def to_json(self):
        """Creates JSON object/dict representing this Behavior.

        Returns created JSON object (dict).
        """
        return {
            IP_KEY : self.ip,
            STRATEGY_KEY : self.strategy,
            ADDRESS_KEY : self.address,
            LOGLEVEL_KEY : self.loglevel }

def first_or_default(behaviors, request):
    """Finds a behavior in list of behaviors,
    which handles given request.

    Returns behavior if any is found, or default behavior otherwise.
    """
    address = str(request.questions[0].qname)
    if behaviors == None or len(behaviors) == 0:
        return Behavior(address)
    for behavior in behaviors:
        if (behavior.handles(address)):
            return behavior
    return Behavior(address)
<|endoftext|>


<commit_before># MicroPython SSD1306 OLED driver, I2C and SPI interfaces

from micropython import const
import time
import framebuf

# register definitions
SET_CONTRAST        = const(0x81)
SET_ENTIRE_ON       = const(0xa4)
SET_NORM_INV        = const(0xa6)
SET_DISP            = const(0xae)
SET_MEM_ADDR        = const(0x20)
SET_COL_ADDR        = const(0x21)
SET_PAGE_ADDR       = const(0x22)
SET_DISP_START_LINE = const(0x40)
SET_SEG_REMAP       = const(0xa0)
SET_MUX_RATIO       = const(0xa8)
SET_COM_OUT_DIR     = const(0xc0)
SET_DISP_OFFSET     = const(0xd3)
SET_COM_PIN_CFG     = const(0xda)
SET_DISP_CLK_DIV    = const(0xd5)
SET_PRECHARGE       = const(0xd9)
SET_VCOM_DESEL      = const(0xdb)
SET_CHARGE_PUMP     = const(0x8d)

class SSD1306:
    def __init__(self, width, height, external_vcc):
        self.width = width
        self.height = height
        self.external_vcc = external_vcc
        self.pages = self.height // 8
        self.buffer = bytearray(self.pages * self.width)
        fb = framebuf.FrameBuffer(self.buffer, self.width, self.height, framebuf.MONO_VLSB)
        self.framebuf = fb
        # Provide methods for accessing FrameBuffer graphics primitives. This is a
        # workround because inheritance from a native class is currently unsupported.
        # http://docs.micropython.org/en/latest/pyboard/library/framebuf.html
        self.fill = fb.fill
        self.pixel = fb.pixel
        self.hline = fb.hline
        self.vline = fb.vline
        self.line = fb.line
        self.rect = fb.rect
        self.fill_rect = fb.fill_rect
        self.text = fb.text
        self.scroll = fb.scroll
        self.blit = fb.blit
        self.poweron()
        self.init_display()

    def init_display(self):
        for cmd in (
            SET_DISP | 0x00, # off
            # address setting
            SET_MEM_ADDR, 0x00, # horizontal
            # resolution and layout
            SET_DISP_START_LINE | 0x00,
            SET_SEG_REMAP | 0x01, # column addr 127 mapped to SEG0
            SET_MUX_RATIO, self.height - 1,
            SET_COM_OUT_DIR | 0x08, # scan from COM[N] to COM0
            SET_DISP_OFFSET, 0x00,
            SET_COM_PIN_CFG, 0x02 if self.height == 32 else 0x12,
            # timing and driving scheme
            SET_DISP_CLK_DIV, 0x80,
            SET_PRECHARGE, 0x22 if self.external_vcc else 0xf1,
            SET_VCOM_DESEL, 0x30, # 0.83*Vcc
            # display
            SET_CONTRAST, 0xff, # maximum
            SET_ENTIRE_ON, # output follows RAM contents
            SET_NORM_INV, # not inverted
            # charge pump
            SET_CHARGE_PUMP, 0x10 if self.external_vcc else 0x14,
            SET_DISP | 0x01): # on
            self.write_cmd(cmd)
        self.fill(0)
        self.show()

    def poweroff(self):
        self.write_cmd(SET_DISP | 0x00)

    def contrast(self, contrast):
        self.write_cmd(SET_CONTRAST)
        self.write_cmd(contrast)

    def invert(self, invert):
        self.write_cmd(SET_NORM_INV | (invert & 1))

    def show(self):
        x0 = 0
        x1 = self.width - 1
        if self.width == 64:
            # displays with width of 64 pixels are shifted by 32
            x0 += 32
            x1 += 32
        self.write_cmd(SET_COL_ADDR)
        self.write_cmd(x0)
        self.write_cmd(x1)
        self.write_cmd(SET_PAGE_ADDR)
        self.write_cmd(0)
        self.write_cmd(self.pages - 1)
        self.write_data(self.buffer)

class SSD1306_I2C(SSD1306):
    def __init__(self, width, height, i2c, addr=0x3c, external_vcc=False):
        self.i2c = i2c
        self.addr = addr
        self.temp = bytearray(2)
        super().__init__(width, height, external_vcc)

    def write_cmd(self, cmd):
        self.temp[0] = 0x80 # Co=1, D/C#=0
        self.temp[1] = cmd
        self.i2c.writeto(self.addr, self.temp)

    def write_data(self, buf):
        self.temp[0] = self.addr << 1
        self.temp[1] = 0x40 # Co=0, D/C#=1
        self.i2c.start()
        self.i2c.write(self.temp)
        self.i2c.write(buf)
        self.i2c.stop()

    def poweron(self):
        self.write_cmd(SET_DISP | 0x01)

class SSD1306_SPI(SSD1306):
    def __init__(self, width, height, spi, dc, res, cs, external_vcc=False):
        self.rate = 10 * 1024 * 1024
        dc.init(dc.OUT, value=0)
        res.init(res.OUT, value=0)
        cs.init(cs.OUT, value=1)
        self.spi = spi
        self.dc = dc
        self.res = res
        self.cs = cs
        super().__init__(width, height, external_vcc)

    def write_cmd(self, cmd):
        self.spi.init(baudrate=self.rate, polarity=0, phase=0)
        self.cs(1)
        self.dc(0)
        self.cs(0)
        self.spi.write(bytearray([cmd]))
        self.cs(1)

    def write_data(self, buf):
        self.spi.init(baudrate=self.rate, polarity=0, phase=0)
        self.cs(1)
        self.dc(1)
        self.cs(0)
        self.spi.write(buf)
        self.cs(1)

    def poweron(self):
        self.res(1)
        time.sleep_ms(1)
        self.res(0)
        time.sleep_ms(10)
        self.res(1)
<commit_msg>drivers/display/ssd1306: Make poweron() work the same with SSD1306_SPI.<commit_after># MicroPython SSD1306 OLED driver, I2C and SPI interfaces

from micropython import const
import framebuf

# register definitions
SET_CONTRAST        = const(0x81)
SET_ENTIRE_ON       = const(0xa4)
SET_NORM_INV        = const(0xa6)
SET_DISP            = const(0xae)
SET_MEM_ADDR        = const(0x20)
SET_COL_ADDR        = const(0x21)
SET_PAGE_ADDR       = const(0x22)
SET_DISP_START_LINE = const(0x40)
SET_SEG_REMAP       = const(0xa0)
SET_MUX_RATIO       = const(0xa8)
SET_COM_OUT_DIR     = const(0xc0)
SET_DISP_OFFSET     = const(0xd3)
SET_COM_PIN_CFG     = const(0xda)
SET_DISP_CLK_DIV    = const(0xd5)
SET_PRECHARGE       = const(0xd9)
SET_VCOM_DESEL      = const(0xdb)
SET_CHARGE_PUMP     = const(0x8d)

class SSD1306:
    def __init__(self, width, height, external_vcc):
        self.width = width
        self.height = height
        self.external_vcc = external_vcc
        self.pages = self.height // 8
        self.buffer = bytearray(self.pages * self.width)
        fb = framebuf.FrameBuffer(self.buffer, self.width, self.height, framebuf.MONO_VLSB)
        self.framebuf = fb
        # Provide methods for accessing FrameBuffer graphics primitives. This is a
        # workround because inheritance from a native class is currently unsupported.
        # http://docs.micropython.org/en/latest/pyboard/library/framebuf.html
        self.fill = fb.fill
        self.pixel = fb.pixel
        self.hline = fb.hline
        self.vline = fb.vline
        self.line = fb.line
        self.rect = fb.rect
        self.fill_rect = fb.fill_rect
        self.text = fb.text
        self.scroll = fb.scroll
        self.blit = fb.blit
        self.init_display()

    def init_display(self):
        for cmd in (
            SET_DISP | 0x00, # off
            # address setting
            SET_MEM_ADDR, 0x00, # horizontal
            # resolution and layout
            SET_DISP_START_LINE | 0x00,
            SET_SEG_REMAP | 0x01, # column addr 127 mapped to SEG0
            SET_MUX_RATIO, self.height - 1,
            SET_COM_OUT_DIR | 0x08, # scan from COM[N] to COM0
            SET_DISP_OFFSET, 0x00,
            SET_COM_PIN_CFG, 0x02 if self.height == 32 else 0x12,
            # timing and driving scheme
            SET_DISP_CLK_DIV, 0x80,
            SET_PRECHARGE, 0x22 if self.external_vcc else 0xf1,
            SET_VCOM_DESEL, 0x30, # 0.83*Vcc
            # display
            SET_CONTRAST, 0xff, # maximum
            SET_ENTIRE_ON, # output follows RAM contents
            SET_NORM_INV, # not inverted
            # charge pump
            SET_CHARGE_PUMP, 0x10 if self.external_vcc else 0x14,
            SET_DISP | 0x01): # on
            self.write_cmd(cmd)
        self.fill(0)
        self.show()

    def poweroff(self):
        self.write_cmd(SET_DISP | 0x00)

    def poweron(self):
        self.write_cmd(SET_DISP | 0x01)

    def contrast(self, contrast):
        self.write_cmd(SET_CONTRAST)
        self.write_cmd(contrast)

    def invert(self, invert):
        self.write_cmd(SET_NORM_INV | (invert & 1))

    def show(self):
        x0 = 0
        x1 = self.width - 1
        if self.width == 64:
            # displays with width of 64 pixels are shifted by 32
            x0 += 32
            x1 += 32
        self.write_cmd(SET_COL_ADDR)
        self.write_cmd(x0)
        self.write_cmd(x1)
        self.write_cmd(SET_PAGE_ADDR)
        self.write_cmd(0)
        self.write_cmd(self.pages - 1)
        self.write_data(self.buffer)

class SSD1306_I2C(SSD1306):
    def __init__(self, width, height, i2c, addr=0x3c, external_vcc=False):
        self.i2c = i2c
        self.addr = addr
        self.temp = bytearray(2)
        super().__init__(width, height, external_vcc)

    def write_cmd(self, cmd):
        self.temp[0] = 0x80 # Co=1, D/C#=0
        self.temp[1] = cmd
        self.i2c.writeto(self.addr, self.temp)

    def write_data(self, buf):
        self.temp[0] = self.addr << 1
        self.temp[1] = 0x40 # Co=0, D/C#=1
        self.i2c.start()
        self.i2c.write(self.temp)
        self.i2c.write(buf)
        self.i2c.stop()

class SSD1306_SPI(SSD1306):
    def __init__(self, width, height, spi, dc, res, cs, external_vcc=False):
        self.rate = 10 * 1024 * 1024
        dc.init(dc.OUT, value=0)
        res.init(res.OUT, value=0)
        cs.init(cs.OUT, value=1)
        self.spi = spi
        self.dc = dc
        self.res = res
        self.cs = cs
        import time
        self.res(1)
        time.sleep_ms(1)
        self.res(0)
        time.sleep_ms(10)
        self.res(1)
        super().__init__(width, height, external_vcc)

    def write_cmd(self, cmd):
        self.spi.init(baudrate=self.rate, polarity=0, phase=0)
        self.cs(1)
        self.dc(0)
        self.cs(0)
        self.spi.write(bytearray([cmd]))
        self.cs(1)

    def write_data(self, buf):
        self.spi.init(baudrate=self.rate, polarity=0, phase=0)
        self.cs(1)
        self.dc(1)
        self.cs(0)
        self.spi.write(buf)
        self.cs(1)
<|endoftext|>


<commit_before>from config import make_config_from_argparse
from osgeo import gdal
from importlib import import_module
from multiprocessing import Pool, Array
import joerd.download as download
import joerd.tmpdir as tmpdir
import sys
import argparse
import os
import os.path
import logging
import logging.config
import time
import traceback
import json
import boto3
from contextlib2 import ExitStack, contextmanager
import subprocess
import ctypes
import math

def create_command_parser(fn):
    def create_parser_fn(parser):
        parser.add_argument('--config', required=True,
                            help='The path to the joerd config file.')
        parser.set_defaults(func=fn)
        return parser
    return create_parser_fn

def _remaining_disk(path):
    df = subprocess.Popen(['df', '-k', '-P', path], stdout=subprocess.PIPE)
    remaining = df.communicate()[0]
    remaining = remaining.split('\n')[1]
    remaining = remaining.split()[3]
    return int(remaining) * 1024

@contextmanager
def lock_array(a, **opts):
    a.acquire(**opts)
    try:
        yield a
    finally:
        a.release()

def _make_space(tmps, path):
    #if theres nothing to do bail
    with lock_array(_superfluous, block=True) as superfluous:
        if len(superfluous) and not superfluous[len(superfluous) - 1]:
            raise Exception('Need more space but nothing superfluous to delete.')
        #assume unpacking will need at least this much space
        needed = 0
        for t in tmps:
            needed += os.path.getsize(t.name)
        #keep removing stuff until we have enough
        remaining = _remaining_disk(path)
        for i in range(len(superfluous)):
            if remaining >= needed:
                return
            s = superfluous[i]
            superfluous[i] = ''
            if s:
                try:
                    _logger.info('Removing %s to free up space.' % s)
                    gained = os.path.getsize(s)
                    os.remove(s)
                    remaining += gained
                except:
                    pass
    #still not enough
    if remaining < needed:
        raise Exception('Not enough space left on device to continue, need at least %d more bytes.' % (needed - remaining))

def _init_processes(s, l):
    # in this case its global for each separate process
    global _superfluous
    _superfluous = s
    global _logger
    _logger = l

    # make sure process will error if GDAL fails
    gdal.UseExceptions()

def _download(d):
    try:
        options = d.options().copy()
        options['verifier'] = d.verifier()

        with ExitStack() as stack:
            def _get(u):
                return stack.enter_context(download.get(u, options))

            tmps = [_get(url) for url in d.urls()]

            while True:
                try:
                    d.unpack(*tmps)
                    break
                except Exception as e:
                    #TODO: only catch out of space exception
                    _logger.error(repr(e))
                    _make_space(tmps, os.path.dirname(d.output_file()))

        assert os.path.isfile(d.output_file())

    except:
        raise Exception("".join(traceback.format_exception(*sys.exc_info())))

def _render(t, store):
    try:
        with tmpdir.tmpdir() as d:
            t.render(d)
            store.upload_all(d)

    except:
        raise Exception("".join(traceback.format_exception(*sys.exc_info())))

def _renderstar(args):
    _render(*args)

# ProgressLogger - logs progress towards a goal to the given logger.
# This is useful for letting the user know that something is happening, and
# roughly how far along it is. Progress is logged at given percentage
# intervals or time intervals, whichever is crossed first.
class ProgressLogger(object):
    def __init__(self, logger, total, time_interval=10, pct_interval=5):
        self.logger = logger
        self.total = total
        self.progress = 0
        self.time_interval = time_interval
        self.pct_interval = pct_interval

        self.next_time = time.time() + self.time_interval
        self.next_pct = self.pct_interval

    def increment(self, amount):
        self.progress += amount
        pct = (100.0 * self.progress) / self.total
        now = time.time()

        if pct > self.next_pct or now > self.next_time:
            self.next_pct = pct + self.pct_interval
            self.next_time = now + self.time_interval
            self.logger.info("Progress: %3.1f%%" % pct)

class Joerd:

    def __init__(self, cfg):
        self.sources = self._sources(cfg)
        self.outputs = self._outputs(cfg, self.sources)
        self.num_threads = cfg.num_threads
        self.chunksize = cfg.chunksize
        self.store = self._store(cfg)

    def process(self):
        logger = logging.getLogger('process')

        # fetch index for each source, which speeds up subsequent downloads or
        # queries about which source tiles are available.
        for source in self.sources:
            source.get_index()

        # get the list of all tiles to be generated
        tiles = []
        for output in self.outputs:
            tiles.extend(output.generate_tiles())

        logger.info("Will generate %d tiles." % len(tiles))

        # gather the set of all downloads - upstream source tiles - for all the
        # tiles that will be generated.
        downloads = set()
        progress = ProgressLogger(logger, len(tiles))
        for tile in tiles:
            # each tile intersects a set of downloads for each source, perhaps
            # an empty set. to track those, only sources which intersect the
            # tile are tracked.
            tile_sources = []
            for source in self.sources:
                d = source.downloads_for(tile)
                if d:
                    downloads.update(d)
                    tile_sources.append(source)
            tile.set_sources(tile_sources)
            progress.increment(1)

        # grab a list of the files which aren't currently available
        need_to_download = []
        need_on_disk = set()
        for download in downloads:
            need_on_disk.add(download.output_file())
            if not os.path.isfile(download.output_file()):
                need_to_download.append(download)

        logger.info("Need to download %d source files."
                    % len(need_to_download))

        #grab a list of the files which we could delete if we need to
        superfluous = []
        for source in self.sources:
            for existing in source.existing_files():
                if existing not in need_on_disk:
                    superfluous.append(existing)

        logger.info("%d source files are superfluous to this job."
                    % len(superfluous))

        # give each process a handle to the shared mem
        shared = Array(ctypes.c_char_p, superfluous)
        p = Pool(processes=self.num_threads, initializer=_init_processes,
                 initargs=(shared,logger))

        # make sure we've got a store
        p.map(_download, need_to_download,
              chunksize=self._chunksize(len(need_to_download)))

        logger.info("Starting render of %d tiles." % len(tiles))

        # now render the tiles
        p.map(_renderstar, [(t, self.store) for t in tiles],
              chunksize=self._chunksize(len(tiles)))

        # clean up the Pool.
        p.close()
        p.join()

        logger.info("All done!")

    def _chunksize(self, length):
        """
        Try to determine an appropriate chunk size. The bigger the chunk, the
        lower the overheads, but potentially worse load balance between the
        different threads. A compromise is a fixed fraction of the maximum
        chunk size - in this case, an eighth.

        Chunksize can be overridden in the config, in which case this
        heuristic is ignored.
        """
        if self.chunksize is not None:
              return self.chunksize
        return max(1, length / self.num_threads / 8)

    def _sources(self, cfg):
        sources = []
        for source in cfg.sources:
            source_type = source['type']
            module = import_module('joerd.source.%s' % source_type)
            create_fn = getattr(module, 'create')
            sources.append(create_fn(source))
        return sources

    def _outputs(self, cfg, sources):
        outputs = []
        for output in cfg.outputs:
            output_type = output['type']
            module = import_module('joerd.output.%s' % output_type)
            create_fn = getattr(module, 'create')
            outputs.append(create_fn(cfg.regions, sources, output))
        return outputs

    def _store(self, cfg):
        store_type = cfg.store['type']
        module = import_module('joerd.store.%s' % store_type)
        create_fn = getattr(module, 'create')
        return create_fn(cfg.store)

class JoerdArgumentParser(argparse.ArgumentParser):
    def error(self, message):
        sys.stderr.write('error: %s\n' % message)
        self.print_help()
        sys.exit(2)

def joerd_process(cfg):
    j = Joerd(cfg)
    j.process()

def joerd_server(global_cfg):
    assert global_cfg.sqs_queue_name is not None, \
        "Could not find SQS queue name in config, but this must be configured."

    sqs = boto3.resource('sqs')
    queue = sqs.get_queue_by_name(QueueName=global_cfg.sqs_queue_name)

    while True:
        for message in queue.receive_messages():
            region = json.loads(message.body)
            cfg = global_cfg.copy_with_regions([region])
            joerd_process(cfg)

            # remove the message from the queue - this indicates that it has
            # completed successfully and it won't be retried.
            message.delete()

def joerd_enqueuer(cfg):
    """
    Split the regions in the input file into jobs suitable for enqueueing and
    send them to the SQS queue for the server to work on.
    """

    assert cfg.sqs_queue_name is not None, \
        "Could not find SQS queue name in config, but this must be configured."

    logger = logging.getLogger('enqueuer')
    block_size = cfg.block_size
    bboxes = dict()
    global_region_zoom = None

    for r in cfg.regions:
        rbox = r.bbox.bounds
        zrange = r.zoom_range

        # split anything zoom 8 or greater into blocks
        if zrange[1] >= 8:
            lft = int(block_size * math.floor(rbox[0] / block_size))
            bot = int(block_size * math.floor(rbox[1] / block_size))
            rgt = int(block_size * math.ceil(rbox[2] / block_size))
            top = int(block_size * math.ceil(rbox[3] / block_size))

            # just in case, clip to the world
            lft = max(-180, lft)
            bot = max(-90, bot)
            rgt = min(180, rgt)
            top = min(90, top)

            for x in range(lft, rgt, block_size):
                for y in range(bot, top, block_size):
                    # accumulate the max z for each block
                    zmax = bboxes.get((x, y))
                    bboxes[(x, y)] = max(zmax, zrange[1])

        # for anything with a range < 8, we also do a "global" range
        # starting at the min zoom seen.
        if zrange[0] < 8:
            global_region_zoom = min(global_region_zoom or 8, zrange[0])

    num_jobs = len(bboxes)
    if global_region_zoom is not None:
        num_jobs += 1

    logger.info("Sending %d jobs to the queue" % num_jobs)

    sqs = boto3.resource('sqs')
    queue = sqs.get_queue_by_name(QueueName=cfg.sqs_queue_name)

    # if there's a global region, then do the whole world down to that
    # zoom.
    if global_region_zoom is not None:
        region = {
            'zoom_range': [global_region_zoom, 8],
            'bbox': {
                'left': -180.0,
                'bottom': -90.0,
                'right': 180.0,
                'top': 90.0,
            }
        }
        queue.send_message(MessageBody=json.dumps(region))

    # send messages for all the other bboxes that need rendering.
    for (x, y), max_z in bboxes.iteritems():
        region = {
            'zoom_range': [8, max_z],
            'bbox': {
                'left': x,
                'bottom': y,
                'right': x + block_size,
                'top': y + block_size,
            }
        }
        queue.send_message(MessageBody=json.dumps(region))

    logger.info("Done.")

def joerd_main(argv=None):
    if argv is None:
        argv = sys.argv[1:]

    parser = JoerdArgumentParser()
    subparsers = parser.add_subparsers()

    parser_config = (
        ('process', create_command_parser(joerd_process)),
        ('server', create_command_parser(joerd_server)),
        ('enqueuer', create_command_parser(joerd_enqueuer)),
    )

    for name, func in parser_config:
        subparser = subparsers.add_parser(name)
        func(subparser)

    args = parser.parse_args(argv)
    assert os.path.exists(args.config), \
        'Config file %r does not exist.' % args.config
    cfg = make_config_from_argparse(args.config)

    if cfg.logconfig is not None:
        config_dir = os.path.dirname(args.config)
        logconfig_path = os.path.join(config_dir, cfg.logconfig)
        logging.config.fileConfig(logconfig_path)

    # make sure process will error if GDAL fails
    gdal.UseExceptions()

    args.func(cfg)
<commit_msg>Alter enqueued bounding boxes in an attempt to avoid duplication of jobs.<commit_after>from config import make_config_from_argparse
from osgeo import gdal
from importlib import import_module
from multiprocessing import Pool, Array
import joerd.download as download
import joerd.tmpdir as tmpdir
import sys
import argparse
import os
import os.path
import logging
import logging.config
import time
import traceback
import json
import boto3
from contextlib2 import ExitStack, contextmanager
import subprocess
import ctypes
import math

def create_command_parser(fn):
    def create_parser_fn(parser):
        parser.add_argument('--config', required=True,
                            help='The path to the joerd config file.')
        parser.set_defaults(func=fn)
        return parser
    return create_parser_fn

def _remaining_disk(path):
    df = subprocess.Popen(['df', '-k', '-P', path], stdout=subprocess.PIPE)
    remaining = df.communicate()[0]
    remaining = remaining.split('\n')[1]
    remaining = remaining.split()[3]
    return int(remaining) * 1024

@contextmanager
def lock_array(a, **opts):
    a.acquire(**opts)
    try:
        yield a
    finally:
        a.release()

def _make_space(tmps, path):
    #if theres nothing to do bail
    with lock_array(_superfluous, block=True) as superfluous:
        if len(superfluous) and not superfluous[len(superfluous) - 1]:
            raise Exception('Need more space but nothing superfluous to delete.')
        #assume unpacking will need at least this much space
        needed = 0
        for t in tmps:
            needed += os.path.getsize(t.name)
        #keep removing stuff until we have enough
        remaining = _remaining_disk(path)
        for i in range(len(superfluous)):
            if remaining >= needed:
                return
            s = superfluous[i]
            superfluous[i] = ''
            if s:
                try:
                    _logger.info('Removing %s to free up space.' % s)
                    gained = os.path.getsize(s)
                    os.remove(s)
                    remaining += gained
                except:
                    pass
    #still not enough
    if remaining < needed:
        raise Exception('Not enough space left on device to continue, need at least %d more bytes.' % (needed - remaining))

def _init_processes(s, l):
    # in this case its global for each separate process
    global _superfluous
    _superfluous = s
    global _logger
    _logger = l

    # make sure process will error if GDAL fails
    gdal.UseExceptions()

def _download(d):
    try:
        options = d.options().copy()
        options['verifier'] = d.verifier()

        with ExitStack() as stack:
            def _get(u):
                return stack.enter_context(download.get(u, options))

            tmps = [_get(url) for url in d.urls()]

            while True:
                try:
                    d.unpack(*tmps)
                    break
                except Exception as e:
                    #TODO: only catch out of space exception
                    _logger.error(repr(e))
                    _make_space(tmps, os.path.dirname(d.output_file()))

        assert os.path.isfile(d.output_file())

    except:
        raise Exception("".join(traceback.format_exception(*sys.exc_info())))

def _render(t, store):
    try:
        with tmpdir.tmpdir() as d:
            t.render(d)
            store.upload_all(d)

    except:
        raise Exception("".join(traceback.format_exception(*sys.exc_info())))

def _renderstar(args):
    _render(*args)

# ProgressLogger - logs progress towards a goal to the given logger.
# This is useful for letting the user know that something is happening, and
# roughly how far along it is. Progress is logged at given percentage
# intervals or time intervals, whichever is crossed first.
class ProgressLogger(object):
    def __init__(self, logger, total, time_interval=10, pct_interval=5):
        self.logger = logger
        self.total = total
        self.progress = 0
        self.time_interval = time_interval
        self.pct_interval = pct_interval

        self.next_time = time.time() + self.time_interval
        self.next_pct = self.pct_interval

    def increment(self, amount):
        self.progress += amount
        pct = (100.0 * self.progress) / self.total
        now = time.time()

        if pct > self.next_pct or now > self.next_time:
            self.next_pct = pct + self.pct_interval
            self.next_time = now + self.time_interval
            self.logger.info("Progress: %3.1f%%" % pct)

class Joerd:

    def __init__(self, cfg):
        self.sources = self._sources(cfg)
        self.outputs = self._outputs(cfg, self.sources)
        self.num_threads = cfg.num_threads
        self.chunksize = cfg.chunksize
        self.store = self._store(cfg)

    def process(self):
        logger = logging.getLogger('process')

        # fetch index for each source, which speeds up subsequent downloads or
        # queries about which source tiles are available.
        for source in self.sources:
            source.get_index()

        # get the list of all tiles to be generated
        tiles = []
        for output in self.outputs:
            tiles.extend(output.generate_tiles())

        logger.info("Will generate %d tiles." % len(tiles))

        # gather the set of all downloads - upstream source tiles - for all the
        # tiles that will be generated.
        downloads = set()
        progress = ProgressLogger(logger, len(tiles))
        for tile in tiles:
            # each tile intersects a set of downloads for each source, perhaps
            # an empty set. to track those, only sources which intersect the
            # tile are tracked.
            tile_sources = []
            for source in self.sources:
                d = source.downloads_for(tile)
                if d:
                    downloads.update(d)
                    tile_sources.append(source)
            tile.set_sources(tile_sources)
            progress.increment(1)

        # grab a list of the files which aren't currently available
        need_to_download = []
        need_on_disk = set()
        for download in downloads:
            need_on_disk.add(download.output_file())
            if not os.path.isfile(download.output_file()):
                need_to_download.append(download)

        logger.info("Need to download %d source files."
                    % len(need_to_download))

        #grab a list of the files which we could delete if we need to
        superfluous = []
        for source in self.sources:
            for existing in source.existing_files():
                if existing not in need_on_disk:
                    superfluous.append(existing)

        logger.info("%d source files are superfluous to this job."
                    % len(superfluous))

        # give each process a handle to the shared mem
        shared = Array(ctypes.c_char_p, superfluous)
        p = Pool(processes=self.num_threads, initializer=_init_processes,
                 initargs=(shared,logger))

        # make sure we've got a store
        p.map(_download, need_to_download,
              chunksize=self._chunksize(len(need_to_download)))

        logger.info("Starting render of %d tiles." % len(tiles))

        # now render the tiles
        p.map(_renderstar, [(t, self.store) for t in tiles],
              chunksize=self._chunksize(len(tiles)))

        # clean up the Pool.
        p.close()
        p.join()

        logger.info("All done!")

    def _chunksize(self, length):
        """
        Try to determine an appropriate chunk size. The bigger the chunk, the
        lower the overheads, but potentially worse load balance between the
        different threads. A compromise is a fixed fraction of the maximum
        chunk size - in this case, an eighth.

        Chunksize can be overridden in the config, in which case this
        heuristic is ignored.
        """
        if self.chunksize is not None:
              return self.chunksize
        return max(1, length / self.num_threads / 8)

    def _sources(self, cfg):
        sources = []
        for source in cfg.sources:
            source_type = source['type']
            module = import_module('joerd.source.%s' % source_type)
            create_fn = getattr(module, 'create')
            sources.append(create_fn(source))
        return sources

    def _outputs(self, cfg, sources):
        outputs = []
        for output in cfg.outputs:
            output_type = output['type']
            module = import_module('joerd.output.%s' % output_type)
            create_fn = getattr(module, 'create')
            outputs.append(create_fn(cfg.regions, sources, output))
        return outputs

    def _store(self, cfg):
        store_type = cfg.store['type']
        module = import_module('joerd.store.%s' % store_type)
        create_fn = getattr(module, 'create')
        return create_fn(cfg.store)

class JoerdArgumentParser(argparse.ArgumentParser):
    def error(self, message):
        sys.stderr.write('error: %s\n' % message)
        self.print_help()
        sys.exit(2)

def joerd_process(cfg):
    j = Joerd(cfg)
    j.process()

def joerd_server(global_cfg):
    assert global_cfg.sqs_queue_name is not None, \
        "Could not find SQS queue name in config, but this must be configured."

    sqs = boto3.resource('sqs')
    queue = sqs.get_queue_by_name(QueueName=global_cfg.sqs_queue_name)

    while True:
        for message in queue.receive_messages():
            region = json.loads(message.body)
            cfg = global_cfg.copy_with_regions([region])
            joerd_process(cfg)

            # remove the message from the queue - this indicates that it has
            # completed successfully and it won't be retried.
            message.delete()

def joerd_enqueuer(cfg):
    """
    Split the regions in the input file into jobs suitable for enqueueing and
    send them to the SQS queue for the server to work on.
    """

    assert cfg.sqs_queue_name is not None, \
        "Could not find SQS queue name in config, but this must be configured."

    logger = logging.getLogger('enqueuer')
    block_size = cfg.block_size
    bboxes = dict()
    global_region_zoom = None

    for r in cfg.regions:
        rbox = r.bbox.bounds
        zrange = r.zoom_range

        # split anything zoom 8 or greater into blocks
        if zrange[1] >= 8:
            lft = int(block_size * math.floor(rbox[0] / block_size))
            bot = int(block_size * math.floor(rbox[1] / block_size))
            rgt = int(block_size * math.ceil(rbox[2] / block_size))
            top = int(block_size * math.ceil(rbox[3] / block_size))

            # just in case, clip to the world
            lft = max(-180, lft)
            bot = max(-90, bot)
            rgt = min(180, rgt)
            top = min(90, top)

            for x in range(lft, rgt, block_size):
                for y in range(bot, top, block_size):
                    # accumulate the max z for each block
                    zmax = bboxes.get((x, y))
                    bboxes[(x, y)] = max(zmax, zrange[1])

        # for anything with a range < 8, we also do a "global" range
        # starting at the min zoom seen.
        if zrange[0] < 8:
            global_region_zoom = min(global_region_zoom or 8, zrange[0])

    num_jobs = len(bboxes)
    if global_region_zoom is not None:
        num_jobs += 1

    logger.info("Sending %d jobs to the queue" % num_jobs)

    sqs = boto3.resource('sqs')
    queue = sqs.get_queue_by_name(QueueName=cfg.sqs_queue_name)

    # if there's a global region, then do the whole world down to that
    # zoom.
    if global_region_zoom is not None:
        region = {
            'zoom_range': [global_region_zoom, 8],
            'bbox': {
                'left': -180.0,
                'bottom': -90.0,
                'right': 180.0,
                'top': 90.0,
            }
        }
        queue.send_message(MessageBody=json.dumps(region))

    # inset the boxes by an epsilon amount. this is so that they don't
    # intersect neighbouring boxes, causing them to be rendered twice.
    # 0.0001 is about 1/10th of a zoom 18 tile (in x direction), so
    # should be large enough to avoid duplication, but small enough to
    # ensure all the tiles we want are actually done.
    epsilon = 0.0001

    # send messages for all the other bboxes that need rendering.
    for (x, y), max_z in bboxes.iteritems():
        region = {
            'zoom_range': [8, max_z],
            'bbox': {
                'left': x + epsilon,
                'bottom': y + epsilon,
                'right': x + block_size - epsilon,
                'top': y + block_size - epsilon,
            }
        }
        queue.send_message(MessageBody=json.dumps(region))

    logger.info("Done.")

def joerd_main(argv=None):
    if argv is None:
        argv = sys.argv[1:]

    parser = JoerdArgumentParser()
    subparsers = parser.add_subparsers()

    parser_config = (
        ('process', create_command_parser(joerd_process)),
        ('server', create_command_parser(joerd_server)),
        ('enqueuer', create_command_parser(joerd_enqueuer)),
    )

    for name, func in parser_config:
        subparser = subparsers.add_parser(name)
        func(subparser)

    args = parser.parse_args(argv)
    assert os.path.exists(args.config), \
        'Config file %r does not exist.' % args.config
    cfg = make_config_from_argparse(args.config)

    if cfg.logconfig is not None:
        config_dir = os.path.dirname(args.config)
        logconfig_path = os.path.join(config_dir, cfg.logconfig)
        logging.config.fileConfig(logconfig_path)

    # make sure process will error if GDAL fails
    gdal.UseExceptions()

    args.func(cfg)
<|endoftext|>


<commit_before>"""
Compute all pairs shortest paths using Johnson's algorithm.

Given a networkx graph compute and return shortest s-t paths for every
pair s, t of nodes in the graph.

If the graph contains a negative cycle we'll get a NegativeCycleError
exception.

Author:
  Christos Nitsas
  (nitsas)
  (chrisnitsas)

Language:
  Python 3(.4)

Date:
  August, 2014
"""

import collections
# Modules I've written:
import bellman_ford_sssp
import dijkstra_sssp

__all__ = ['solve', 'johnsons_all_pairs_shortest_paths', 'NegativeCycleError']

NegativeCycleError = bellman_ford_sssp.NegativeCycleError

def _add_supersource(graph):
    """
    This will add a new "supersource" node to the graph as well
    as zero cost edges from the supersource to every other node.

    graph -- a networkx graph
    """
    ss = '_temp_supersource'
    if ss in graph:
        import time
        ss = '_temp_supersource_' + time.asctime()
        if ss in graph:
            raise Exception('Failed to add temporary supersource.')
    graph.add_node(ss)
    for node in graph.nodes_iter():
        graph.add_edge(ss, node, cost=0)
    graph.remove_edge(ss, ss)
    return ss

def _compute_node_weights(graph):
    """
    Compute a weight for each node, such that reweighting each edge (u, v)
    using the formula:
    cost'(u, v) = cost(u, v) + weight(u) - weight(v)
    will result in non-negative edge costs.

    graph -- a networkx graph
    """
    # add a supersource
    ss = _add_supersource(graph)
    # compute distances from the supersource;
    # those will become node weights
    node_weight, node_pred = bellman_ford_sssp.solve(graph, ss)
    # remove the supersource
    graph.remove_node(ss)
    del(node_weight[ss])
    return node_weight

def _reweight_edges_using_node_weights(graph, node_weight,
                                     weight_attr_name=None):
    """
    Reweight each edge (u, v) using the formula:
    cost'[(u, v)] = cost[(u, v)] + node_weight[u] - node_weight[v]
    This will preserve shortest paths.

    graph -- a networkx graph
    node_weight -- a dictionary mapping nodes to weights
    weight_attr_name -- the name of the edge attribute we want to
                        use as weights/costs (default: 'cost')

    For suitable node weights, this will get us non-negative edge
    weights.
    """
    # set name of edge weight attribute
    if weight_attr_name is None:
        weight = 'cost'
    else:
        weight = weight_attr_name
    # use the node weights to assign a new (non-negative) cost
    # to each edge which preserves shortest paths
    for u, v, attrs in graph.edges_iter(data=True):
        attrs[weight] = attrs[weight] + node_weight[u] - node_weight[v]

def johnsons_all_pairs_shortest_paths(graph_):
    """
    Compute a shortest path for every pair of nodes in the graph.

    graph_ -- a networkx graph

    This will not mutate the graph; we work on a temporary copy.
    Will throw a NegativeCycleError exception if the graph contains a
    negative cycle.
    """
    # we work on a copy of the graph
    graph = graph_.copy()
    # compute node weights, for the edge reweighting step
    # (this will throw a NegativeCycleError exception if the graph
    # contains a negative cycle)
    node_weight = _compute_node_weights(graph)
    # reweigh all the graph's edges so that there are no negative edges
    # this operarion preserves shortest paths
    _reweight_edges_using_node_weights(graph, node_weight)
    # call dijkstra's single source shortest paths algorithm once
    # for each node in the graph, to compute all pairs shortest paths
    dist, pred = dict(), dict()
    for node in graph.nodes_iter():
        # compute shortest paths using the current node as a source
        dist[node], pred[node] = dijkstra_sssp.solve(graph, node)
    # remember that we have reweighted the edges; the actual shortest
    # paths didn't change but their lengths did, so we now have to
    # undo the effects of reweighting in the shortest path lengths
    for u in graph.nodes_iter():
        for v in graph.nodes_iter():
            dist[u][v] = dist[u][v] - node_weight[u] + node_weight[v]
    # return all pairs shortest paths distances and predecessors
    return dist, pred

solve = johnsons_all_pairs_shortest_paths
<commit_msg>[johnsons_apsp] Wrap return value in a namedtuple<commit_after>"""
Compute all pairs shortest paths using Johnson's algorithm.

Given a networkx graph compute and return shortest s-t paths for every
pair s, t of nodes in the graph.

If the graph contains a negative cycle we'll get a NegativeCycleError
exception.

Author:
  Christos Nitsas
  (nitsas)
  (chrisnitsas)

Language:
  Python 3(.4)

Date:
  August, 2014
"""

import collections
# Modules I've written:
import bellman_ford_sssp
import dijkstra_sssp

__all__ = ['solve', 'johnsons_all_pairs_shortest_paths', 'NegativeCycleError']

DistAndPred = collections.namedtuple('DistAndPred', ['dist', 'pred'])

NegativeCycleError = bellman_ford_sssp.NegativeCycleError

def _add_supersource(graph):
    """
    This will add a new "supersource" node to the graph as well
    as zero cost edges from the supersource to every other node.

    graph -- a networkx graph
    """
    ss = '_temp_supersource'
    if ss in graph:
        import time
        ss = '_temp_supersource_' + time.asctime()
        if ss in graph:
            raise Exception('Failed to add temporary supersource.')
    graph.add_node(ss)
    for node in graph.nodes_iter():
        graph.add_edge(ss, node, cost=0)
    graph.remove_edge(ss, ss)
    return ss

def _compute_node_weights(graph):
    """
    Compute a weight for each node, such that reweighting each edge (u, v)
    using the formula:
    cost'(u, v) = cost(u, v) + weight(u) - weight(v)
    will result in non-negative edge costs.

    graph -- a networkx graph
    """
    # add a supersource
    ss = _add_supersource(graph)
    # compute distances from the supersource;
    # those will become node weights
    node_weight, node_pred = bellman_ford_sssp.solve(graph, ss)
    # remove the supersource
    graph.remove_node(ss)
    del(node_weight[ss])
    return node_weight

def _reweight_edges_using_node_weights(graph, node_weight,
                                     weight_attr_name=None):
    """
    Reweight each edge (u, v) using the formula:
    cost'[(u, v)] = cost[(u, v)] + node_weight[u] - node_weight[v]
    This will preserve shortest paths.

    graph -- a networkx graph
    node_weight -- a dictionary mapping nodes to weights
    weight_attr_name -- the name of the edge attribute we want to
                        use as weights/costs (default: 'cost')

    For suitable node weights, this will get us non-negative edge
    weights.
    """
    # set name of edge weight attribute
    if weight_attr_name is None:
        weight = 'cost'
    else:
        weight = weight_attr_name
    # use the node weights to assign a new (non-negative) cost
    # to each edge which preserves shortest paths
    for u, v, attrs in graph.edges_iter(data=True):
        attrs[weight] = attrs[weight] + node_weight[u] - node_weight[v]

def johnsons_all_pairs_shortest_paths(graph_):
    """
    Compute a shortest path for every pair of nodes in the graph.

    graph_ -- a networkx graph

    Return a namedtuple of two dictionaries (dist, pred). Both dist and pred
    map each node (source) in the graph to another dict, which in turn maps
    nodes (targets) to distances and predecessors respectively.
    - dist[u][v] contains a shortest path distance from u to v
    - pred[u][v] contains the predecessor of v in a shortest path from u

    This will not mutate the graph; we work on a temporary copy.
    Will throw a NegativeCycleError exception if the graph contains a
    negative cycle.
    """
    # we work on a copy of the graph
    graph = graph_.copy()
    # compute node weights, for the edge reweighting step
    # (this will throw a NegativeCycleError exception if the graph
    # contains a negative cycle)
    node_weight = _compute_node_weights(graph)
    # reweigh all the graph's edges so that there are no negative edges
    # this operarion preserves shortest paths
    _reweight_edges_using_node_weights(graph, node_weight)
    # call dijkstra's single source shortest paths algorithm once
    # for each node in the graph, to compute all pairs shortest paths
    dist, pred = dict(), dict()
    for node in graph.nodes_iter():
        # compute shortest paths using the current node as a source
        dist[node], pred[node] = dijkstra_sssp.solve(graph, node)
    # remember that we have reweighted the edges; the actual shortest
    # paths didn't change but their lengths did, so we now have to
    # undo the effects of reweighting in the shortest path lengths
    for u in graph.nodes_iter():
        for v in graph.nodes_iter():
            dist[u][v] = dist[u][v] - node_weight[u] + node_weight[v]
    # return all pairs shortest paths distances and predecessors
    return DistAndPred(dist, pred)

solve = johnsons_all_pairs_shortest_paths
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Tests for the contrib/localflavor/se form fields.

tests = r"""
# Monkey-patch datetime.date
>>> import datetime
>>> class MockDate(datetime.date):
...     def today(cls):
...         return datetime.date(2008, 5, 14)
...     today = classmethod(today)
...
>>> olddate = datetime.date
>>> datetime.date = MockDate
>>> datetime.date.today()
MockDate(2008, 5, 14)

# SECountySelect #####################################################
>>> from django.contrib.localflavor.se.forms import SECountySelect

>>> w = SECountySelect()
>>> w.render('swedish_county', 'E')
u'<select name="swedish_county">\n<option value="AB">Stockholm</option>\n<option value="AC">V\xe4sterbotten</option>\n<option value="BD">Norrbotten</option>\n<option value="C">Uppsala</option>\n<option value="D">S\xf6dermanland</option>\n<option value="E" selected="selected">\xd6sterg\xf6tland</option>\n<option value="F">J\xf6nk\xf6ping</option>\n<option value="G">Kronoberg</option>\n<option value="H">Kalmar</option>\n<option value="I">Gotland</option>\n<option value="K">Blekinge</option>\n<option value="M">Sk\xe5ne</option>\n<option value="N">Halland</option>\n<option value="O">V\xe4stra G\xf6taland</option>\n<option value="S">V\xe4rmland</option>\n<option value="T">\xd6rebro</option>\n<option value="U">V\xe4stmanland</option>\n<option value="W">Dalarna</option>\n<option value="X">G\xe4vleborg</option>\n<option value="Y">V\xe4sternorrland</option>\n<option value="Z">J\xe4mtland</option>\n</select>'

# SEOrganisationNumberField #######################################

>>> from django.contrib.localflavor.se.forms import SEOrganisationNumberField

>>> f = SEOrganisationNumberField()

# Ordinary personal identity numbers for sole proprietors
# The same rules as for SEPersonalIdentityField applies here
>>> f.clean('870512-1989')
u'198705121989'
>>> f.clean('19870512-1989')
u'198705121989'
>>> f.clean('870512-2128')
u'198705122128'
>>> f.clean('081015-6315')
u'190810156315'
>>> f.clean('081015+6315')
u'180810156315'
>>> f.clean('0810156315')
u'190810156315'

>>> f.clean('081015 6315')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish organisation number.']
>>> f.clean('950231-4496')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish organisation number.']
>>> f.clean('6914104499')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish organisation number.']
>>> f.clean('950d314496')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish organisation number.']
>>> f.clean('invalid!!!')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish organisation number.']
>>> f.clean('870514-1111')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish organisation number.']

# Empty values
>>> f.clean('')
Traceback (most recent call last):
...
ValidationError: [u'This field is required.']

>>> f.clean(None)
Traceback (most recent call last):
...
ValidationError: [u'This field is required.']

# Co-ordination number checking
# Co-ordination numbers are not valid organisation numbers
>>> f.clean('870574-1315')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish organisation number.']

>>> f.clean('870573-1311')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish organisation number.']

# Test some different organisation numbers
>>> f.clean('556074-7569') # IKEA Linköping
u'5560747569'

>>> f.clean('556074-3089') # Volvo Personvagnar
u'5560743089'

>>> f.clean('822001-5476') # LJS (organisation)
u'8220015476'

>>> f.clean('8220015476') # LJS (organisation)
u'8220015476'

>>> f.clean('2120000449') # Katedralskolan Linköping (school)
u'2120000449'

# Faux organisation number, which tests that the checksum can be 0
>>> f.clean('232518-5060')
u'2325185060'

>>> f.clean('556074+3089') # Volvo Personvagnar, bad format
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish organisation number.']

# Invalid checksum
>>> f.clean('2120000441')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish organisation number.']

# Valid checksum but invalid organisation type
f.clean('1120000441')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish organisation number.']

# Empty values with required=False
>>> f = SEOrganisationNumberField(required=False)

>>> f.clean(None)
u''

>>> f.clean('')
u''

# SEPersonalIdentityNumberField #######################################

>>> from django.contrib.localflavor.se.forms import SEPersonalIdentityNumberField

>>> f = SEPersonalIdentityNumberField()

# Valid id numbers
>>> f.clean('870512-1989')
u'198705121989'

>>> f.clean('870512-2128')
u'198705122128'

>>> f.clean('19870512-1989')
u'198705121989'

>>> f.clean('198705121989')
u'198705121989'

>>> f.clean('081015-6315')
u'190810156315'

>>> f.clean('0810156315')
u'190810156315'

# This is a "special-case" in the checksum calculation,
# where the sum is divisible by 10 (the checksum digit == 0)
>>> f.clean('8705141060')
u'198705141060'

# + means that the person is older than 100 years
>>> f.clean('081015+6315')
u'180810156315'

# Bogus values
>>> f.clean('081015 6315')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish personal identity number.']

>>> f.clean('950d314496')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish personal identity number.']

>>> f.clean('invalid!!!')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish personal identity number.']

# Invalid dates

# February 31st does not exist
>>> f.clean('950231-4496')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish personal identity number.']

# Month 14 does not exist
>>> f.clean('6914104499')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish personal identity number.']

# There are no Swedish personal id numbers where year < 1800
>>> f.clean('17430309-7135')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish personal identity number.']

# Invalid checksum
>>> f.clean('870514-1111')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish personal identity number.']

# Empty values
>>> f.clean('')
Traceback (most recent call last):
...
ValidationError: [u'This field is required.']

>>> f.clean(None)
Traceback (most recent call last):
...
ValidationError: [u'This field is required.']

# Co-ordination number checking
>>> f.clean('870574-1315')
u'198705741315'

>>> f.clean('870574+1315')
u'188705741315'

>>> f.clean('198705741315')
u'198705741315'

# Co-ordination number with bad checksum
>>> f.clean('870573-1311')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish personal identity number.']

# Check valid co-ordination numbers, that should not be accepted
# because of coordination_number=False
>>> f = SEPersonalIdentityNumberField(coordination_number=False)

>>> f.clean('870574-1315')
Traceback (most recent call last):
...
ValidationError: [u'Co-ordination numbers are not allowed.']

>>> f.clean('870574+1315')
Traceback (most recent call last):
...
ValidationError: [u'Co-ordination numbers are not allowed.']

>>> f.clean('8705741315')
Traceback (most recent call last):
...
ValidationError: [u'Co-ordination numbers are not allowed.']

# Invalid co-ordination numbers should be treated as invalid, and not
# as co-ordination numbers
>>> f.clean('870573-1311')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish personal identity number.']

# Empty values with required=False
>>> f = SEPersonalIdentityNumberField(required=False)

>>> f.clean(None)
u''

>>> f.clean('')
u''

# SEPostalCodeField ###############################################
>>> from django.contrib.localflavor.se.forms import SEPostalCodeField
>>> f = SEPostalCodeField()
>>>
Postal codes can have spaces
>>> f.clean('589 37')
u'58937'

... but the dont have to
>>> f.clean('58937')
u'58937'
>>> f.clean('abcasfassadf')
Traceback (most recent call last):
...
ValidationError: [u'Enter a Swedish postal code in the format XXXXX.']

# Only one space is allowed for separation
>>> f.clean('589  37')
Traceback (most recent call last):
...
ValidationError: [u'Enter a Swedish postal code in the format XXXXX.']

# The postal code must not start with 0
>>> f.clean('01234')
Traceback (most recent call last):
...
ValidationError: [u'Enter a Swedish postal code in the format XXXXX.']

# Empty values
>>> f.clean('')
Traceback (most recent call last):
...
ValidationError: [u'This field is required.']

>>> f.clean(None)
Traceback (most recent call last):
...
ValidationError: [u'This field is required.']

# Empty values, required=False
>>> f = SEPostalCodeField(required=False)
>>> f.clean('')
u''
>>> f.clean(None)
u''

# Revert the monkey patching
>>> datetime.date = olddate

"""
<commit_msg>Fixed #13374 -- Modified a test so that it can pass under PyPy. This is a reimplementation of part of r12998, reverted in r13008. Thanks to Alex Gaynor.<commit_after># -*- coding: utf-8 -*-
# Tests for the contrib/localflavor/se form fields.

tests = r"""
# Monkey-patch datetime.date
>>> import datetime
>>> class MockDate(datetime.date):
...     def today(cls):
...         return datetime.date(2008, 5, 14)
...     today = classmethod(today)
...
>>> olddate = datetime.date
>>> datetime.date = MockDate
>>> datetime.date.today() == olddate(2008, 5, 14)
True

# SECountySelect #####################################################
>>> from django.contrib.localflavor.se.forms import SECountySelect

>>> w = SECountySelect()
>>> w.render('swedish_county', 'E')
u'<select name="swedish_county">\n<option value="AB">Stockholm</option>\n<option value="AC">V\xe4sterbotten</option>\n<option value="BD">Norrbotten</option>\n<option value="C">Uppsala</option>\n<option value="D">S\xf6dermanland</option>\n<option value="E" selected="selected">\xd6sterg\xf6tland</option>\n<option value="F">J\xf6nk\xf6ping</option>\n<option value="G">Kronoberg</option>\n<option value="H">Kalmar</option>\n<option value="I">Gotland</option>\n<option value="K">Blekinge</option>\n<option value="M">Sk\xe5ne</option>\n<option value="N">Halland</option>\n<option value="O">V\xe4stra G\xf6taland</option>\n<option value="S">V\xe4rmland</option>\n<option value="T">\xd6rebro</option>\n<option value="U">V\xe4stmanland</option>\n<option value="W">Dalarna</option>\n<option value="X">G\xe4vleborg</option>\n<option value="Y">V\xe4sternorrland</option>\n<option value="Z">J\xe4mtland</option>\n</select>'

# SEOrganisationNumberField #######################################

>>> from django.contrib.localflavor.se.forms import SEOrganisationNumberField

>>> f = SEOrganisationNumberField()

# Ordinary personal identity numbers for sole proprietors
# The same rules as for SEPersonalIdentityField applies here
>>> f.clean('870512-1989')
u'198705121989'
>>> f.clean('19870512-1989')
u'198705121989'
>>> f.clean('870512-2128')
u'198705122128'
>>> f.clean('081015-6315')
u'190810156315'
>>> f.clean('081015+6315')
u'180810156315'
>>> f.clean('0810156315')
u'190810156315'

>>> f.clean('081015 6315')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish organisation number.']
>>> f.clean('950231-4496')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish organisation number.']
>>> f.clean('6914104499')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish organisation number.']
>>> f.clean('950d314496')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish organisation number.']
>>> f.clean('invalid!!!')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish organisation number.']
>>> f.clean('870514-1111')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish organisation number.']

# Empty values
>>> f.clean('')
Traceback (most recent call last):
...
ValidationError: [u'This field is required.']

>>> f.clean(None)
Traceback (most recent call last):
...
ValidationError: [u'This field is required.']

# Co-ordination number checking
# Co-ordination numbers are not valid organisation numbers
>>> f.clean('870574-1315')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish organisation number.']

>>> f.clean('870573-1311')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish organisation number.']

# Test some different organisation numbers
>>> f.clean('556074-7569') # IKEA Linköping
u'5560747569'

>>> f.clean('556074-3089') # Volvo Personvagnar
u'5560743089'

>>> f.clean('822001-5476') # LJS (organisation)
u'8220015476'

>>> f.clean('8220015476') # LJS (organisation)
u'8220015476'

>>> f.clean('2120000449') # Katedralskolan Linköping (school)
u'2120000449'

# Faux organisation number, which tests that the checksum can be 0
>>> f.clean('232518-5060')
u'2325185060'

>>> f.clean('556074+3089') # Volvo Personvagnar, bad format
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish organisation number.']

# Invalid checksum
>>> f.clean('2120000441')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish organisation number.']

# Valid checksum but invalid organisation type
f.clean('1120000441')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish organisation number.']

# Empty values with required=False
>>> f = SEOrganisationNumberField(required=False)

>>> f.clean(None)
u''

>>> f.clean('')
u''

# SEPersonalIdentityNumberField #######################################

>>> from django.contrib.localflavor.se.forms import SEPersonalIdentityNumberField

>>> f = SEPersonalIdentityNumberField()

# Valid id numbers
>>> f.clean('870512-1989')
u'198705121989'

>>> f.clean('870512-2128')
u'198705122128'

>>> f.clean('19870512-1989')
u'198705121989'

>>> f.clean('198705121989')
u'198705121989'

>>> f.clean('081015-6315')
u'190810156315'

>>> f.clean('0810156315')
u'190810156315'

# This is a "special-case" in the checksum calculation,
# where the sum is divisible by 10 (the checksum digit == 0)
>>> f.clean('8705141060')
u'198705141060'

# + means that the person is older than 100 years
>>> f.clean('081015+6315')
u'180810156315'

# Bogus values
>>> f.clean('081015 6315')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish personal identity number.']

>>> f.clean('950d314496')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish personal identity number.']

>>> f.clean('invalid!!!')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish personal identity number.']

# Invalid dates

# February 31st does not exist
>>> f.clean('950231-4496')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish personal identity number.']

# Month 14 does not exist
>>> f.clean('6914104499')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish personal identity number.']

# There are no Swedish personal id numbers where year < 1800
>>> f.clean('17430309-7135')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish personal identity number.']

# Invalid checksum
>>> f.clean('870514-1111')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish personal identity number.']

# Empty values
>>> f.clean('')
Traceback (most recent call last):
...
ValidationError: [u'This field is required.']

>>> f.clean(None)
Traceback (most recent call last):
...
ValidationError: [u'This field is required.']

# Co-ordination number checking
>>> f.clean('870574-1315')
u'198705741315'

>>> f.clean('870574+1315')
u'188705741315'

>>> f.clean('198705741315')
u'198705741315'

# Co-ordination number with bad checksum
>>> f.clean('870573-1311')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish personal identity number.']

# Check valid co-ordination numbers, that should not be accepted
# because of coordination_number=False
>>> f = SEPersonalIdentityNumberField(coordination_number=False)

>>> f.clean('870574-1315')
Traceback (most recent call last):
...
ValidationError: [u'Co-ordination numbers are not allowed.']

>>> f.clean('870574+1315')
Traceback (most recent call last):
...
ValidationError: [u'Co-ordination numbers are not allowed.']

>>> f.clean('8705741315')
Traceback (most recent call last):
...
ValidationError: [u'Co-ordination numbers are not allowed.']

# Invalid co-ordination numbers should be treated as invalid, and not
# as co-ordination numbers
>>> f.clean('870573-1311')
Traceback (most recent call last):
...
ValidationError: [u'Enter a valid Swedish personal identity number.']

# Empty values with required=False
>>> f = SEPersonalIdentityNumberField(required=False)

>>> f.clean(None)
u''

>>> f.clean('')
u''

# SEPostalCodeField ###############################################
>>> from django.contrib.localflavor.se.forms import SEPostalCodeField
>>> f = SEPostalCodeField()
>>>
Postal codes can have spaces
>>> f.clean('589 37')
u'58937'

... but the dont have to
>>> f.clean('58937')
u'58937'
>>> f.clean('abcasfassadf')
Traceback (most recent call last):
...
ValidationError: [u'Enter a Swedish postal code in the format XXXXX.']

# Only one space is allowed for separation
>>> f.clean('589  37')
Traceback (most recent call last):
...
ValidationError: [u'Enter a Swedish postal code in the format XXXXX.']

# The postal code must not start with 0
>>> f.clean('01234')
Traceback (most recent call last):
...
ValidationError: [u'Enter a Swedish postal code in the format XXXXX.']

# Empty values
>>> f.clean('')
Traceback (most recent call last):
...
ValidationError: [u'This field is required.']

>>> f.clean(None)
Traceback (most recent call last):
...
ValidationError: [u'This field is required.']

# Empty values, required=False
>>> f = SEPostalCodeField(required=False)
>>> f.clean('')
u''
>>> f.clean(None)
u''

# Revert the monkey patching
>>> datetime.date = olddate

"""
<|endoftext|>


<commit_before>from __future__ import absolute_import

from .error_messages import *
from .extra import *
from .fields import FieldsTests
from .forms import *
from .formsets import *
from .input_formats import *
from .media import *
from .models import *
from .regressions import *
from .util import *
from .validators import TestFieldWithValidators
from .widgets import *
<commit_msg>Fix a Python 2.5-ism.<commit_after>from __future__ import absolute_import

from .error_messages import (FormsErrorMessagesTestCase,
    ModelChoiceFieldErrorMessagesTestCase)
from .extra import FormsExtraTestCase, FormsExtraL10NTestCase
from .fields import FieldsTests
from .forms import FormsTestCase
from .formsets import (FormsFormsetTestCase, FormsetAsFooTests,
    TestIsBoundBehavior, TestEmptyFormSet)
from .input_formats import (LocalizedTimeTests, CustomTimeInputFormatsTests,
    SimpleTimeFormatTests, LocalizedDateTests, CustomDateInputFormatsTests,
    SimpleDateFormatTests, LocalizedDateTimeTests,
    CustomDateTimeInputFormatsTests, SimpleDateTimeFormatTests)
from .media import FormsMediaTestCase, StaticFormsMediaTestCase
from .models import (TestTicket12510, ModelFormCallableModelDefault,
    FormsModelTestCase, RelatedModelFormTests)
from .regressions import FormsRegressionsTestCase
from .util import FormsUtilTestCase
from .validators import TestFieldWithValidators
from .widgets import (FormsWidgetTestCase, FormsI18NWidgetsTestCase,
    WidgetTests, ClearableFileInputTests)
<|endoftext|>