<commit_before># Local tests took 1 hour to insert 5 million lines. Estimated 9 hours to insert all data as of 05/2017.
# Suggested to run on a weekend during off hours

import os
import re
import logging
import boto3
import urllib.request
import pandas as pd

from sqlalchemy.exc import IntegrityError

from dataactcore.logging import configure_logging
from dataactcore.config import CONFIG_BROKER
from dataactcore.interfaces.db import GlobalDB
from dataactcore.models.domainModels import Zips, StateCongressional
from dataactvalidator.scripts.loader_utils import clean_data, insert_dataframe

from dataactvalidator.health_check import create_app

logger = logging.getLogger(__name__)
zip4_line_size = 182
citystate_line_size = 129
chunk_size = 1024 * 10

def hot_swap_zip_tables(sess):
    """ Drop the existing zips table, rename the temp_zips table, and rename all the indexes in a transaction.

        Args:
            sess: the database connection
    """
    # Getting indexes before dropping the table
    indexes = Zips.__table__.indexes

    logger.info("Hot swapping temporary zips table to official zips table.")

    # Do everything in a transaction so it doesn't affect anything until it's completely done
    sess.execute("BEGIN;")

    # Make sure the sequence remains
    sess.execute("ALTER SEQUENCE zips_zips_id_seq OWNED BY temp_zips.zips_id")

    # Drop old zips table and rename the temporary one
    sess.execute("DROP TABLE zips;")
    sess.execute("ALTER TABLE temp_zips "
                 "RENAME TO zips;")

    # Rename all the indexes and constraints to match what they were in the original zips table
    sess.execute("ALTER INDEX temp_zips_pkey RENAME TO zips_pkey;")
    sess.execute("ALTER INDEX temp_zips_zip5_zip_last4_key RENAME TO uniq_zip5_zip_last4;")
    # Get all the indexes swapped out
    for index in indexes:
        index_name = index.name.replace('ix_', '')
        sess.execute("ALTER INDEX temp_{}_idx RENAME TO ix_{}".format(index_name, index_name))
    sess.execute("COMMIT;")

def update_state_congr_table_current(sess):
    """ Update contents of state_congressional table based on zips we just inserted

        Args:
            sess: the database connection
    """
    logger.info("Loading zip codes complete, beginning update of state_congressional table")
    # clear old data out
    sess.query(StateCongressional).delete(synchronize_session=False)
    sess.commit()

    # get new data
    distinct_list = sess.query(Zips.state_abbreviation, Zips.congressional_district_no).distinct().\
        order_by(Zips.state_abbreviation, Zips.congressional_district_no)
    sess.bulk_save_objects([StateCongressional(state_code=state_data.state_abbreviation,
                                               congressional_district_no=state_data.congressional_district_no)
                            for state_data in distinct_list])
    sess.commit()

def update_state_congr_table_census(census_file, sess):
    """ Update contents of state_congressional table to include districts from the census

        Args:
            census_file: file path/url to the census file to read
            sess: the database connection
    """
    logger.info("Adding congressional districts from census to the state_congressional table")

    data = pd.read_csv(census_file, dtype=str)
    model = StateCongressional

    data = clean_data(
        data,
        model,
        {"state_code": "state_code",
         "congressional_district_no": "congressional_district_no",
         "census_year": "census_year"},
        {'congressional_district_no': {"pad_to_length": 2}}
    )

    table_name = model.__table__.name
    insert_dataframe(data, table_name, sess.connection())
    sess.commit()

def add_to_table(data, sess):
    """ Add data to the temp_zips table.

        Args:
            data: dictionary of dictionaries containing zip data to process and add to the table
            sess: the database connection
    """
    value_array = []
    for _, item in data.items():
        value_array.append("(NOW(), NOW(), '{}', '{}', '{}', '{}', '{}')".
                           format(item['zip5'], item['zip_last4'], item['county_number'],
                                  item['state_abbreviation'], item['congressional_district_no']))
    try:
        if value_array:
            sess.execute('INSERT INTO temp_zips '
                         '(updated_at, created_at, zip5, zip_last4, county_number, state_abbreviation, '
                         'congressional_district_no) VALUES {}'.format(", ".join(value_array)))
            sess.commit()
    except IntegrityError:
        sess.rollback()
        logger.error("Attempted to insert duplicate zip. Inserting each row in batch individually.")

        i = 0
        for new_zip in value_array:
            # create an insert statement that overrides old values if there's a conflict
            sess.execute('INSERT INTO temp_zips '
                         '(updated_at, created_at, zip5, zip_last4, county_number, state_abbreviation, '
                         'congressional_district_no) VALUES {} '
                         'ON CONFLICT DO NOTHING'.format(new_zip))

            if i % 1000 == 0:
                logger.info("Inserting row %s of current batch", str(i))
            i += 1
        sess.commit()

def parse_zip4_file(f, sess):
    """ Parse file containing full 9-digit zip data

        Args:
            f: opened file containing zip5 and zip_last4 data
            sess: the database connection
    """
    logger.info("Starting file %s", str(f))
    # pull out the copyright data
    f.read(zip4_line_size)

    data_array = {}
    current_zip = ""
    curr_chunk = ""
    while True:
        # grab the next chunk
        next_chunk = f.read(chunk_size)
        # when streaming from S3 it reads in as bytes, we need to decode it as a utf-8 string
        if not type(next_chunk) == str:
            next_chunk = next_chunk.decode("utf-8")

        # add the new chunk of the file to the current chunk we're processing
        curr_chunk += next_chunk

        # if the current chunk is smaller than the line size, we're done
        if len(curr_chunk) < zip4_line_size:
            break

        # while we can still do more processing on the current chunk, process it per line
        while len(curr_chunk) >= zip4_line_size:
            # grab another line and get the data that's always the same
            curr_row = curr_chunk[:zip4_line_size]
            state = curr_row[157:159]

            # ignore state codes AA, AE, and AP because they're just for military routing
            if state not in ['AA', 'AE', 'AP']:
                # files are ordered by zip5: when it changes, that's the last record with that zip5
                # insert batches by zip5 to avoid conflicts
                zip5 = curr_row[1:6]
                if current_zip != zip5:
                    if len(data_array) > 0:
                        logger.info("Inserting {} records for {}".format(len(data_array), current_zip))
                        add_to_table(data_array, sess)
                        data_array.clear()
                    current_zip = zip5

                # zip of 96898 is a special case
                if zip5 == "96898":
                    congressional_district = "99"
                    state = "UM"
                    county = "450"
                else:
                    county = curr_row[159:162]
                    congressional_district = curr_row[162:164]

                # certain states require specific CDs
                if state in ["AK", "DE", "MT", "ND", "SD", "VT", "WY"]:
                    congressional_district = "00"
                elif state in ["AS", "DC", "GU", "MP", "PR", "VI"]:
                    congressional_district = "98"
                elif state in ["FM", "MH", "PW", "UM"]:
                    congressional_district = "99"

                try:
                    zip4_low = int(curr_row[140:144])
                    zip4_high = int(curr_row[144:148])
                    # if the zip4 low and zip4 high are the same, it's just one zip code and we can just add it
                    if zip4_low == zip4_high:
                        zip_string = str(zip4_low).zfill(4)
                        data_array[zip5 + zip_string] = {"zip5": zip5, "zip_last4": zip_string, "county_number": county,
                                                         "state_abbreviation": state,
                                                         "congressional_district_no": congressional_district}
                    # if the zip codes are different, we have to loop through and add each zip4
                    # as a different object/key
                    else:
                        i = zip4_low
                        while i <= zip4_high:
                            zip_string = str(i).zfill(4)
                            data_array[zip5 + zip_string] = {"zip5": zip5, "zip_last4": zip_string,
                                                             "state_abbreviation": state, "county_number": county,
                                                             "congressional_district_no": congressional_district}
                            i += 1
                # catch entries where zip code isn't an int (12ND for example, ND stands for "no delivery")
                except ValueError:
                    logger.error("Error parsing entry: %s", curr_row)

            # cut the current line out of the chunk we're processing
            curr_chunk = curr_chunk[zip4_line_size:]

    # add the final chunk of data to the DB
    if len(data_array) > 0:
        logger.info("Adding last {} records for current file".format(len(data_array)))
        add_to_table(data_array, sess)
        data_array.clear()

def parse_citystate_file(f, sess):
    """ Parse citystate file data to get remaining 5-digit zips that weren't included in the 9-digit file

        Args:
            f: opened file containing citystate data
            sess: the database connection
    """
    logger.info("Starting file %s", str(f))
    # pull out the copyright data
    f.read(citystate_line_size)

    data_array = {}
    curr_chunk = ""
    while True:
        # grab the next chunk
        next_chunk = f.read(chunk_size)
        # when streaming from S3 it reads in as bytes, we need to decode it as a utf-8 string
        if not type(next_chunk) == str:
            next_chunk = next_chunk.decode("utf-8")

        # add the new chunk of the file to the current chunk we're processing
        curr_chunk += next_chunk

        # if the current chunk is smaller than the line size, we're done
        if len(curr_chunk) < citystate_line_size:
            break

        # while we can still do more processing on the current chunk, process it per line
        while len(curr_chunk) >= citystate_line_size:
            # grab another line and get the data if it's a "detail record"
            curr_row = curr_chunk[:citystate_line_size]
            if curr_row[0] == "D":
                state = curr_row[99:101]

                # ignore state codes AA, AE, and AP because they're just for military routing
                if state not in ['AA', 'AE', 'AP']:
                    zip5 = curr_row[1:6]
                    # zip of 96898 is a special case
                    if zip5 == "96898":
                        congressional_district = "99"
                        state = "UM"
                        county = "450"
                    else:
                        congressional_district = None
                        county = curr_row[101:104]

                    # certain states require specific CDs
                    if state in ["AK", "DE", "MT", "ND", "SD", "VT", "WY"]:
                        congressional_district = "00"
                    elif state in ["AS", "DC", "GU", "MP", "PR", "VI"]:
                        congressional_district = "98"
                    elif state in ["FM", "MH", "PW", "UM"]:
                        congressional_district = "99"

                    data_array[zip5] = {"zip5": zip5, "zip_last4": None, "state_abbreviation": state,
                                        "county_number": county, "congressional_district_no": congressional_district}

            # cut the current line out of the chunk we're processing
            curr_chunk = curr_chunk[citystate_line_size:]

    # remove all zip5s that already exist in the table
    distinct_zip5 = sess.execute('SELECT DISTINCT zip5 FROM temp_zips').fetchall()
    for item in distinct_zip5:
        if item.zip5 in data_array:
            del data_array[item.zip5]

    logger.info("Starting insert on zip5 data")
    add_to_table(data_array, sess)
    return f

def read_zips():
    """ Update zip codes in the zips table. """
    with create_app().app_context():
        sess = GlobalDB.db().session

        # Create temporary table to do work in so we don't disrupt the site for too long by altering the actual table
        sess.execute('CREATE TABLE IF NOT EXISTS temp_zips (LIKE zips INCLUDING ALL);')
        # Truncating in case we didn't clear out this table after a failure in the script
        sess.execute('TRUNCATE TABLE temp_zips;')
        sess.commit()

        if CONFIG_BROKER["use_aws"]:
            zip_folder = CONFIG_BROKER["zip_folder"] + "/"
            s3_client = boto3.client('s3', region_name=CONFIG_BROKER['aws_region'])
            response = s3_client.list_objects_v2(Bucket=CONFIG_BROKER['sf_133_bucket'], Prefix=zip_folder)
            for obj in response.get('Contents', []):
                if obj['Key'] != zip_folder:
                    zip_4_file_path = s3_client.generate_presigned_url('get_object',
                                                                       {'Bucket': CONFIG_BROKER['sf_133_bucket'],
                                                                        'Key': obj['Key']}, ExpiresIn=600)
                    parse_zip4_file(urllib.request.urlopen(zip_4_file_path), sess)

            # parse remaining 5 digit zips that weren't in the first file
            citystate_file = s3_client.generate_presigned_url('get_object', {'Bucket': CONFIG_BROKER['sf_133_bucket'],
                                                                             'Key': "ctystate.txt"}, ExpiresIn=600)
            parse_citystate_file(urllib.request.urlopen(citystate_file), sess)

            census_file = s3_client.generate_presigned_url('get_object', {'Bucket': CONFIG_BROKER['sf_133_bucket'],
                                                                          'Key': "census_congressional_districts.csv"},
                                                           ExpiresIn=600)
        else:
            base_path = os.path.join(CONFIG_BROKER["path"], "dataactvalidator", "config", CONFIG_BROKER["zip_folder"])
            # creating the list while ignoring hidden files on mac
            file_list = [f for f in os.listdir(base_path) if not re.match('^\.', f)]
            for file in file_list:
                parse_zip4_file(open(os.path.join(base_path, file)), sess)

            # parse remaining 5 digit zips that weren't in the first file
            citystate_file = os.path.join(CONFIG_BROKER["path"], "dataactvalidator", "config", "ctystate.txt")
            parse_citystate_file(open(citystate_file), sess)

            census_file = os.path.join(base_path, "census_congressional_districts.csv")

        hot_swap_zip_tables(sess)
        update_state_congr_table_current(sess)
        update_state_congr_table_census(census_file, sess)

        logger.info("Zipcode script complete")

if __name__ == '__main__':
    configure_logging()
    read_zips()
<commit_msg>flake<commit_after># Local tests took 1 hour to insert 5 million lines. Estimated 9 hours to insert all data as of 05/2017.
# Suggested to run on a weekend during off hours

import os
import re
import logging
import boto3
import urllib.request
import pandas as pd

from sqlalchemy.exc import IntegrityError

from dataactcore.logging import configure_logging
from dataactcore.config import CONFIG_BROKER
from dataactcore.interfaces.db import GlobalDB
from dataactcore.models.domainModels import Zips, StateCongressional
from dataactvalidator.scripts.loader_utils import clean_data, insert_dataframe

from dataactvalidator.health_check import create_app

logger = logging.getLogger(__name__)
zip4_line_size = 182
citystate_line_size = 129
chunk_size = 1024 * 10

def hot_swap_zip_tables(sess):
    """ Drop the existing zips table, rename the temp_zips table, and rename all the indexes in a transaction.

        Args:
            sess: the database connection
    """
    # Getting indexes before dropping the table
    indexes = Zips.__table__.indexes

    logger.info("Hot swapping temporary zips table to official zips table.")

    # Do everything in a transaction so it doesn't affect anything until it's completely done
    sess.execute("BEGIN;")

    # Make sure the sequence remains
    sess.execute("ALTER SEQUENCE zips_zips_id_seq OWNED BY temp_zips.zips_id")

    # Drop old zips table and rename the temporary one
    sess.execute("DROP TABLE zips;")
    sess.execute("ALTER TABLE temp_zips "
                 "RENAME TO zips;")

    # Rename all the indexes and constraints to match what they were in the original zips table
    sess.execute("ALTER INDEX temp_zips_pkey RENAME TO zips_pkey;")
    sess.execute("ALTER INDEX temp_zips_zip5_zip_last4_key RENAME TO uniq_zip5_zip_last4;")
    # Get all the indexes swapped out
    for index in indexes:
        index_name = index.name.replace('ix_', '')
        sess.execute("ALTER INDEX temp_{}_idx RENAME TO ix_{}".format(index_name, index_name))
    sess.execute("COMMIT;")

def update_state_congr_table_current(sess):
    """ Update contents of state_congressional table based on zips we just inserted

        Args:
            sess: the database connection
    """
    logger.info("Loading zip codes complete, beginning update of state_congressional table")
    # clear old data out
    sess.query(StateCongressional).delete(synchronize_session=False)
    sess.commit()

    # get new data
    distinct_list = sess.query(Zips.state_abbreviation, Zips.congressional_district_no).distinct().\
        order_by(Zips.state_abbreviation, Zips.congressional_district_no)
    sess.bulk_save_objects([StateCongressional(state_code=state_data.state_abbreviation,
                                               congressional_district_no=state_data.congressional_district_no)
                            for state_data in distinct_list])
    sess.commit()

def update_state_congr_table_census(census_file, sess):
    """ Update contents of state_congressional table to include districts from the census

        Args:
            census_file: file path/url to the census file to read
            sess: the database connection
    """
    logger.info("Adding congressional districts from census to the state_congressional table")

    data = pd.read_csv(census_file, dtype=str)
    model = StateCongressional

    data = clean_data(
        data,
        model,
        {"state_code": "state_code",
         "congressional_district_no": "congressional_district_no",
         "census_year": "census_year"},
        {'congressional_district_no': {"pad_to_length": 2}}
    )

    table_name = model.__table__.name
    insert_dataframe(data, table_name, sess.connection())
    sess.commit()

def add_to_table(data, sess):
    """ Add data to the temp_zips table.

        Args:
            data: dictionary of dictionaries containing zip data to process and add to the table
            sess: the database connection
    """
    value_array = []
    for _, item in data.items():
        value_array.append("(NOW(), NOW(), '{}', '{}', '{}', '{}', '{}')".
                           format(item['zip5'], item['zip_last4'], item['county_number'],
                                  item['state_abbreviation'], item['congressional_district_no']))
    try:
        if value_array:
            sess.execute('INSERT INTO temp_zips '
                         '(updated_at, created_at, zip5, zip_last4, county_number, state_abbreviation, '
                         'congressional_district_no) VALUES {}'.format(", ".join(value_array)))
            sess.commit()
    except IntegrityError:
        sess.rollback()
        logger.error("Attempted to insert duplicate zip. Inserting each row in batch individually.")

        i = 0
        for new_zip in value_array:
            # create an insert statement that overrides old values if there's a conflict
            sess.execute('INSERT INTO temp_zips '
                         '(updated_at, created_at, zip5, zip_last4, county_number, state_abbreviation, '
                         'congressional_district_no) VALUES {} '
                         'ON CONFLICT DO NOTHING'.format(new_zip))

            if i % 1000 == 0:
                logger.info("Inserting row %s of current batch", str(i))
            i += 1
        sess.commit()

def parse_zip4_file(f, sess):
    """ Parse file containing full 9-digit zip data

        Args:
            f: opened file containing zip5 and zip_last4 data
            sess: the database connection
    """
    logger.info("Starting file %s", str(f))
    # pull out the copyright data
    f.read(zip4_line_size)

    data_array = {}
    current_zip = ""
    curr_chunk = ""
    while True:
        # grab the next chunk
        next_chunk = f.read(chunk_size)
        # when streaming from S3 it reads in as bytes, we need to decode it as a utf-8 string
        if not type(next_chunk) == str:
            next_chunk = next_chunk.decode("utf-8")

        # add the new chunk of the file to the current chunk we're processing
        curr_chunk += next_chunk

        # if the current chunk is smaller than the line size, we're done
        if len(curr_chunk) < zip4_line_size:
            break

        # while we can still do more processing on the current chunk, process it per line
        while len(curr_chunk) >= zip4_line_size:
            # grab another line and get the data that's always the same
            curr_row = curr_chunk[:zip4_line_size]
            state = curr_row[157:159]

            # ignore state codes AA, AE, and AP because they're just for military routing
            if state not in ['AA', 'AE', 'AP']:
                # files are ordered by zip5: when it changes, that's the last record with that zip5
                # insert batches by zip5 to avoid conflicts
                zip5 = curr_row[1:6]
                if current_zip != zip5:
                    if len(data_array) > 0:
                        logger.info("Inserting {} records for {}".format(len(data_array), current_zip))
                        add_to_table(data_array, sess)
                        data_array.clear()
                    current_zip = zip5

                # zip of 96898 is a special case
                if zip5 == "96898":
                    congressional_district = "99"
                    state = "UM"
                    county = "450"
                else:
                    county = curr_row[159:162]
                    congressional_district = curr_row[162:164]

                # certain states require specific CDs
                if state in ["AK", "DE", "MT", "ND", "SD", "VT", "WY"]:
                    congressional_district = "00"
                elif state in ["AS", "DC", "GU", "MP", "PR", "VI"]:
                    congressional_district = "98"
                elif state in ["FM", "MH", "PW", "UM"]:
                    congressional_district = "99"

                try:
                    zip4_low = int(curr_row[140:144])
                    zip4_high = int(curr_row[144:148])
                    # if the zip4 low and zip4 high are the same, it's just one zip code and we can just add it
                    if zip4_low == zip4_high:
                        zip_string = str(zip4_low).zfill(4)
                        data_array[zip5 + zip_string] = {"zip5": zip5, "zip_last4": zip_string, "county_number": county,
                                                         "state_abbreviation": state,
                                                         "congressional_district_no": congressional_district}
                    # if the zip codes are different, we have to loop through and add each zip4
                    # as a different object/key
                    else:
                        i = zip4_low
                        while i <= zip4_high:
                            zip_string = str(i).zfill(4)
                            data_array[zip5 + zip_string] = {"zip5": zip5, "zip_last4": zip_string,
                                                             "state_abbreviation": state, "county_number": county,
                                                             "congressional_district_no": congressional_district}
                            i += 1
                # catch entries where zip code isn't an int (12ND for example, ND stands for "no delivery")
                except ValueError:
                    logger.error("Error parsing entry: %s", curr_row)

            # cut the current line out of the chunk we're processing
            curr_chunk = curr_chunk[zip4_line_size:]

    # add the final chunk of data to the DB
    if len(data_array) > 0:
        logger.info("Adding last {} records for current file".format(len(data_array)))
        add_to_table(data_array, sess)
        data_array.clear()

def parse_citystate_file(f, sess):
    """ Parse citystate file data to get remaining 5-digit zips that weren't included in the 9-digit file

        Args:
            f: opened file containing citystate data
            sess: the database connection
    """
    logger.info("Starting file %s", str(f))
    # pull out the copyright data
    f.read(citystate_line_size)

    data_array = {}
    curr_chunk = ""
    while True:
        # grab the next chunk
        next_chunk = f.read(chunk_size)
        # when streaming from S3 it reads in as bytes, we need to decode it as a utf-8 string
        if not type(next_chunk) == str:
            next_chunk = next_chunk.decode("utf-8")

        # add the new chunk of the file to the current chunk we're processing
        curr_chunk += next_chunk

        # if the current chunk is smaller than the line size, we're done
        if len(curr_chunk) < citystate_line_size:
            break

        # while we can still do more processing on the current chunk, process it per line
        while len(curr_chunk) >= citystate_line_size:
            # grab another line and get the data if it's a "detail record"
            curr_row = curr_chunk[:citystate_line_size]
            if curr_row[0] == "D":
                state = curr_row[99:101]

                # ignore state codes AA, AE, and AP because they're just for military routing
                if state not in ['AA', 'AE', 'AP']:
                    zip5 = curr_row[1:6]
                    # zip of 96898 is a special case
                    if zip5 == "96898":
                        congressional_district = "99"
                        state = "UM"
                        county = "450"
                    else:
                        congressional_district = None
                        county = curr_row[101:104]

                    # certain states require specific CDs
                    if state in ["AK", "DE", "MT", "ND", "SD", "VT", "WY"]:
                        congressional_district = "00"
                    elif state in ["AS", "DC", "GU", "MP", "PR", "VI"]:
                        congressional_district = "98"
                    elif state in ["FM", "MH", "PW", "UM"]:
                        congressional_district = "99"

                    data_array[zip5] = {"zip5": zip5, "zip_last4": None, "state_abbreviation": state,
                                        "county_number": county, "congressional_district_no": congressional_district}

            # cut the current line out of the chunk we're processing
            curr_chunk = curr_chunk[citystate_line_size:]

    # remove all zip5s that already exist in the table
    distinct_zip5 = sess.execute('SELECT DISTINCT zip5 FROM temp_zips').fetchall()
    for item in distinct_zip5:
        if item.zip5 in data_array:
            del data_array[item.zip5]

    logger.info("Starting insert on zip5 data")
    add_to_table(data_array, sess)
    return f

def read_zips():
    """ Update zip codes in the zips table. """
    with create_app().app_context():
        sess = GlobalDB.db().session

        # Create temporary table to do work in so we don't disrupt the site for too long by altering the actual table
        sess.execute('CREATE TABLE IF NOT EXISTS temp_zips (LIKE zips INCLUDING ALL);')
        # Truncating in case we didn't clear out this table after a failure in the script
        sess.execute('TRUNCATE TABLE temp_zips;')
        sess.commit()

        if CONFIG_BROKER["use_aws"]:
            zip_folder = CONFIG_BROKER["zip_folder"] + "/"
            s3_client = boto3.client('s3', region_name=CONFIG_BROKER['aws_region'])
            response = s3_client.list_objects_v2(Bucket=CONFIG_BROKER['sf_133_bucket'], Prefix=zip_folder)
            for obj in response.get('Contents', []):
                if obj['Key'] != zip_folder:
                    zip_4_file_path = s3_client.generate_presigned_url('get_object',
                                                                       {'Bucket': CONFIG_BROKER['sf_133_bucket'],
                                                                        'Key': obj['Key']}, ExpiresIn=600)
                    parse_zip4_file(urllib.request.urlopen(zip_4_file_path), sess)

            # parse remaining 5 digit zips that weren't in the first file
            citystate_file = s3_client.generate_presigned_url('get_object', {'Bucket': CONFIG_BROKER['sf_133_bucket'],
                                                                             'Key': "ctystate.txt"}, ExpiresIn=600)
            parse_citystate_file(urllib.request.urlopen(citystate_file), sess)

            census_file = s3_client.generate_presigned_url('get_object', {'Bucket': CONFIG_BROKER['sf_133_bucket'],
                                                                          'Key': "census_congressional_districts.csv"},
                                                           ExpiresIn=600)
        else:
            base_path = os.path.join(CONFIG_BROKER["path"], "dataactvalidator", "config", CONFIG_BROKER["zip_folder"])
            # creating the list while ignoring hidden files on mac
            file_list = [f for f in os.listdir(base_path) if not re.match('^\.', f)]
            for file in file_list:
                parse_zip4_file(open(os.path.join(base_path, file)), sess)

            # parse remaining 5 digit zips that weren't in the first file
            citystate_file = os.path.join(CONFIG_BROKER["path"], "dataactvalidator", "config", "ctystate.txt")
            parse_citystate_file(open(citystate_file), sess)

            census_file = os.path.join(base_path, "census_congressional_districts.csv")

        hot_swap_zip_tables(sess)
        update_state_congr_table_current(sess)
        update_state_congr_table_census(census_file, sess)

        logger.info("Zipcode script complete")

if __name__ == '__main__':
    configure_logging()
    read_zips()
<|endoftext|>


<commit_before># Copyright 2012 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Image Magick demo."""

from __future__ import with_statement

__author__ = 'kbrisbin@google.com (Kathryn Hurley)'

import os
import random

import lib_path
import google_cloud.gce as gce
import google_cloud.gce_appengine as gce_appengine
import google_cloud.gcs_appengine as gcs_appengine
import google_cloud.oauth as oauth
import jinja2
import oauth2client.appengine as oauth2client
import user_data
import webapp2

from google.appengine.api import users

DEMO_NAME = 'image-magick'
IMAGE = 'image-magick-demo-image'
IMAGES = ['android', 'appengine', 'apps', 'chrome', 'games', 'gplus',
          'maps', 'wallet', 'youtube']
SEQUENCES = ['5 5 360', '355 -5 0']

jinja_environment = jinja2.Environment(loader=jinja2.FileSystemLoader(''))
oauth_decorator = oauth.decorator
user_data.DEFAULTS[user_data.GCS_BUCKET]['label'] += (' (must have CORS and '
                                                      'public-read ACLs set)')
parameters = [
    user_data.DEFAULTS[user_data.GCE_PROJECT_ID],
    user_data.DEFAULTS[user_data.GCS_PROJECT_ID],
    user_data.DEFAULTS[user_data.GCS_BUCKET],
    user_data.DEFAULTS[user_data.GCS_DIRECTORY],
]
data_handler = user_data.DataHandler(DEMO_NAME, parameters)

class ImageMagick(webapp2.RequestHandler):
  """Show main page for the Image Magick demo."""

  @oauth_decorator.oauth_required
  @data_handler.data_required
  def get(self):
    """Display the main page for the Image Magick demo."""

    if not oauth_decorator.credentials.refresh_token:
      self.redirect(oauth_decorator.authorize_url() + '&approval_prompt=force')

    gcs_bucket = data_handler.stored_user_data[user_data.GCS_BUCKET]
    gcs_directory = data_handler.stored_user_data.get(
        user_data.GCS_DIRECTORY, None)
    variables = {
        'demo_name': DEMO_NAME,
        'bucket': gcs_bucket,
        'directory': gcs_directory,
    }
    template = jinja_environment.get_template(
        'demos/%s/templates/index.html' % DEMO_NAME)
    self.response.out.write(template.render(variables))

class Instance(webapp2.RequestHandler):
  """Start and list instances."""

  @oauth_decorator.oauth_required
  @data_handler.data_required
  def get(self):
    """Get and return the list of instances with names containing the tag."""

    gce_project_id = data_handler.stored_user_data[user_data.GCE_PROJECT_ID]
    gce_project = gce.GceProject(
        oauth_decorator.credentials, project_id=gce_project_id)
    gce_appengine.GceAppEngine().list_demo_instances(
        self, gce_project, DEMO_NAME)

  @data_handler.data_required
  def post(self):
    """Insert instances with a startup script, metadata, and scopes.

    Startup script is randomly chosen to either rotate images left or right.
    Metadata includes the image to rotate, the demo name tag, and the machine
    number. Service account scopes include Compute and storage.
    """

    user = users.get_current_user()
    credentials = oauth2client.StorageByKeyName(
        oauth2client.CredentialsModel, user.user_id(), 'credentials').get()
    gce_project_id = data_handler.stored_user_data[user_data.GCE_PROJECT_ID]
    gce_project = gce.GceProject(credentials, project_id=gce_project_id)

    # Get the bucket info for the instance metadata.
    gcs_bucket = data_handler.stored_user_data[user_data.GCS_BUCKET]
    gcs_directory = data_handler.stored_user_data.get(
        user_data.GCS_DIRECTORY, None)
    gcs_path = None
    if gcs_directory:
      gcs_path = '%s/%s' % (gcs_bucket, gcs_directory)
    else:
      gcs_path = gcs_bucket

    # Figure out the image.  Use custom image if it exists.
    (image_project, image_name) = self._get_image_name(gce_project)

    # Create a list of instances to insert.
    instances = []
    num_instances = int(self.request.get('num_instances'))
    network = gce.Network('default')
    network.gce_project = gce_project
    ext_net = [{'network': network.url,
                'accessConfigs': [{'name': 'External IP access config',
                                   'type': 'ONE_TO_ONE_NAT'
                                  }]
               }]
    for i in range(num_instances):
      startup_script = os.path.join(os.path.dirname(__file__), 'startup.sh')
      instance_name='%s-%d' % (DEMO_NAME, i)
      instances.append(gce.Instance(
          name=instance_name, network_interfaces=ext_net,
          service_accounts=gce_project.settings['cloud_service_account'],
          disk_mounts=[gce.DiskMount(boot=True,
                                     init_disk_name=instance_name,
                                     init_disk_image=image_name,
                                     init_disk_project=image_project,
                                     auto_delete=True)],
          metadata=[
              {'key': 'startup-script', 'value': open(
                  startup_script, 'r').read()},
              {'key': 'image', 'value': random.choice(IMAGES)},
              {'key': 'seq', 'value': random.choice(SEQUENCES)},
              {'key': 'machine-num', 'value': i},
              {'key': 'tag', 'value': DEMO_NAME},
              {'key': 'gcs-path', 'value': gcs_path}]))

    response = gce_appengine.GceAppEngine().run_gce_request(
        self,
        gce_project.bulk_insert,
        'Error inserting instances: ',
        resources=instances)

    if response:
      self.response.headers['Content-Type'] = 'text/plain'
      self.response.out.write('starting cluster')

  def _get_image_name(self, gce_project):
    """Finds the appropriate image to use.

    Args:
      gce_project: An instance of gce.GceProject

    Returns:
      A tuple containing the image project and image name.
    """
    if gce_project.list_images(filter='name eq ^%s$' % IMAGE):
      return (gce_project.project_id, IMAGE)
    return (None, None)

class GceCleanup(webapp2.RequestHandler):
  """Stop instances."""

  @data_handler.data_required
  def post(self):
    """Stop instances with names containing the tag."""

    user = users.get_current_user()
    credentials = oauth2client.StorageByKeyName(
        oauth2client.CredentialsModel, user.user_id(), 'credentials').get()
    gce_project_id = data_handler.stored_user_data[user_data.GCE_PROJECT_ID]
    gce_project = gce.GceProject(credentials, project_id=gce_project_id)
    gce_appengine.GceAppEngine().delete_demo_instances(
        self, gce_project, DEMO_NAME)

class GcsCleanup(webapp2.RequestHandler):
  """Remove Cloud Storage files."""

  @data_handler.data_required
  def post(self):
    """Remove all cloud storage contents from the given bucket and dir."""

    user_id = users.get_current_user().user_id()
    credentials = oauth2client.StorageByKeyName(
        oauth2client.CredentialsModel, user_id, 'credentials').get()
    gcs_project_id = data_handler.stored_user_data[user_data.GCS_PROJECT_ID]
    gcs_bucket = data_handler.stored_user_data[user_data.GCS_BUCKET]
    gcs_directory = data_handler.stored_user_data.get(
        user_data.GCS_DIRECTORY, None)
    gcs_helper = gcs_appengine.GcsAppEngineHelper(credentials, gcs_project_id)
    file_regex = None
    if gcs_directory:
      file_regex = r'^%s/%s.*' % (gcs_directory, DEMO_NAME)
    else:
      file_regex = r'^%s.*' % DEMO_NAME
    gcs_helper.delete_bucket_contents(
        gcs_bucket, gcs_directory, file_regex)
    self.response.headers['Content-Type'] = 'text/plain'
    self.response.out.write('cleaning cloud storage bucket')

app = webapp2.WSGIApplication(
    [
        ('/%s' % DEMO_NAME, ImageMagick),
        ('/%s/instance' % DEMO_NAME, Instance),
        ('/%s/gce-cleanup' % DEMO_NAME, GceCleanup),
        ('/%s/gcs-cleanup' % DEMO_NAME, GcsCleanup),
        (data_handler.url_path, data_handler.data_handler),
    ], debug=True, config={'config': 'imagemagick'})
<commit_msg>simplify pattern matching when searching for image to use<commit_after># Copyright 2012 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Image Magick demo."""

from __future__ import with_statement

__author__ = 'kbrisbin@google.com (Kathryn Hurley)'

import os
import random

import lib_path
import google_cloud.gce as gce
import google_cloud.gce_appengine as gce_appengine
import google_cloud.gcs_appengine as gcs_appengine
import google_cloud.oauth as oauth
import jinja2
import oauth2client.appengine as oauth2client
import user_data
import webapp2

from google.appengine.api import users

DEMO_NAME = 'image-magick'
IMAGE = 'image-magick-demo-image'
IMAGES = ['android', 'appengine', 'apps', 'chrome', 'games', 'gplus',
          'maps', 'wallet', 'youtube']
SEQUENCES = ['5 5 360', '355 -5 0']

jinja_environment = jinja2.Environment(loader=jinja2.FileSystemLoader(''))
oauth_decorator = oauth.decorator
user_data.DEFAULTS[user_data.GCS_BUCKET]['label'] += (' (must have CORS and '
                                                      'public-read ACLs set)')
parameters = [
    user_data.DEFAULTS[user_data.GCE_PROJECT_ID],
    user_data.DEFAULTS[user_data.GCS_PROJECT_ID],
    user_data.DEFAULTS[user_data.GCS_BUCKET],
    user_data.DEFAULTS[user_data.GCS_DIRECTORY],
]
data_handler = user_data.DataHandler(DEMO_NAME, parameters)

class ImageMagick(webapp2.RequestHandler):
  """Show main page for the Image Magick demo."""

  @oauth_decorator.oauth_required
  @data_handler.data_required
  def get(self):
    """Display the main page for the Image Magick demo."""

    if not oauth_decorator.credentials.refresh_token:
      self.redirect(oauth_decorator.authorize_url() + '&approval_prompt=force')

    gcs_bucket = data_handler.stored_user_data[user_data.GCS_BUCKET]
    gcs_directory = data_handler.stored_user_data.get(
        user_data.GCS_DIRECTORY, None)
    variables = {
        'demo_name': DEMO_NAME,
        'bucket': gcs_bucket,
        'directory': gcs_directory,
    }
    template = jinja_environment.get_template(
        'demos/%s/templates/index.html' % DEMO_NAME)
    self.response.out.write(template.render(variables))

class Instance(webapp2.RequestHandler):
  """Start and list instances."""

  @oauth_decorator.oauth_required
  @data_handler.data_required
  def get(self):
    """Get and return the list of instances with names containing the tag."""

    gce_project_id = data_handler.stored_user_data[user_data.GCE_PROJECT_ID]
    gce_project = gce.GceProject(
        oauth_decorator.credentials, project_id=gce_project_id)
    gce_appengine.GceAppEngine().list_demo_instances(
        self, gce_project, DEMO_NAME)

  @data_handler.data_required
  def post(self):
    """Insert instances with a startup script, metadata, and scopes.

    Startup script is randomly chosen to either rotate images left or right.
    Metadata includes the image to rotate, the demo name tag, and the machine
    number. Service account scopes include Compute and storage.
    """

    user = users.get_current_user()
    credentials = oauth2client.StorageByKeyName(
        oauth2client.CredentialsModel, user.user_id(), 'credentials').get()
    gce_project_id = data_handler.stored_user_data[user_data.GCE_PROJECT_ID]
    gce_project = gce.GceProject(credentials, project_id=gce_project_id)

    # Get the bucket info for the instance metadata.
    gcs_bucket = data_handler.stored_user_data[user_data.GCS_BUCKET]
    gcs_directory = data_handler.stored_user_data.get(
        user_data.GCS_DIRECTORY, None)
    gcs_path = None
    if gcs_directory:
      gcs_path = '%s/%s' % (gcs_bucket, gcs_directory)
    else:
      gcs_path = gcs_bucket

    # Figure out the image.  Use custom image if it exists.
    (image_project, image_name) = self._get_image_name(gce_project)

    # Create a list of instances to insert.
    instances = []
    num_instances = int(self.request.get('num_instances'))
    network = gce.Network('default')
    network.gce_project = gce_project
    ext_net = [{'network': network.url,
                'accessConfigs': [{'name': 'External IP access config',
                                   'type': 'ONE_TO_ONE_NAT'
                                  }]
               }]
    for i in range(num_instances):
      startup_script = os.path.join(os.path.dirname(__file__), 'startup.sh')
      instance_name='%s-%d' % (DEMO_NAME, i)
      instances.append(gce.Instance(
          name=instance_name, network_interfaces=ext_net,
          service_accounts=gce_project.settings['cloud_service_account'],
          disk_mounts=[gce.DiskMount(boot=True,
                                     init_disk_name=instance_name,
                                     init_disk_image=image_name,
                                     init_disk_project=image_project,
                                     auto_delete=True)],
          metadata=[
              {'key': 'startup-script', 'value': open(
                  startup_script, 'r').read()},
              {'key': 'image', 'value': random.choice(IMAGES)},
              {'key': 'seq', 'value': random.choice(SEQUENCES)},
              {'key': 'machine-num', 'value': i},
              {'key': 'tag', 'value': DEMO_NAME},
              {'key': 'gcs-path', 'value': gcs_path}]))

    response = gce_appengine.GceAppEngine().run_gce_request(
        self,
        gce_project.bulk_insert,
        'Error inserting instances: ',
        resources=instances)

    if response:
      self.response.headers['Content-Type'] = 'text/plain'
      self.response.out.write('starting cluster')

  def _get_image_name(self, gce_project):
    """Finds the appropriate image to use.

    Args:
      gce_project: An instance of gce.GceProject

    Returns:
      A tuple containing the image project and image name.
    """
    if gce_project.list_images(filter='name eq ' + IMAGE):
      return (gce_project.project_id, IMAGE)
    return (None, None)

class GceCleanup(webapp2.RequestHandler):
  """Stop instances."""

  @data_handler.data_required
  def post(self):
    """Stop instances with names containing the tag."""

    user = users.get_current_user()
    credentials = oauth2client.StorageByKeyName(
        oauth2client.CredentialsModel, user.user_id(), 'credentials').get()
    gce_project_id = data_handler.stored_user_data[user_data.GCE_PROJECT_ID]
    gce_project = gce.GceProject(credentials, project_id=gce_project_id)
    gce_appengine.GceAppEngine().delete_demo_instances(
        self, gce_project, DEMO_NAME)

class GcsCleanup(webapp2.RequestHandler):
  """Remove Cloud Storage files."""

  @data_handler.data_required
  def post(self):
    """Remove all cloud storage contents from the given bucket and dir."""

    user_id = users.get_current_user().user_id()
    credentials = oauth2client.StorageByKeyName(
        oauth2client.CredentialsModel, user_id, 'credentials').get()
    gcs_project_id = data_handler.stored_user_data[user_data.GCS_PROJECT_ID]
    gcs_bucket = data_handler.stored_user_data[user_data.GCS_BUCKET]
    gcs_directory = data_handler.stored_user_data.get(
        user_data.GCS_DIRECTORY, None)
    gcs_helper = gcs_appengine.GcsAppEngineHelper(credentials, gcs_project_id)
    file_regex = None
    if gcs_directory:
      file_regex = r'^%s/%s.*' % (gcs_directory, DEMO_NAME)
    else:
      file_regex = r'^%s.*' % DEMO_NAME
    gcs_helper.delete_bucket_contents(
        gcs_bucket, gcs_directory, file_regex)
    self.response.headers['Content-Type'] = 'text/plain'
    self.response.out.write('cleaning cloud storage bucket')

app = webapp2.WSGIApplication(
    [
        ('/%s' % DEMO_NAME, ImageMagick),
        ('/%s/instance' % DEMO_NAME, Instance),
        ('/%s/gce-cleanup' % DEMO_NAME, GceCleanup),
        ('/%s/gcs-cleanup' % DEMO_NAME, GcsCleanup),
        (data_handler.url_path, data_handler.data_handler),
    ], debug=True, config={'config': 'imagemagick'})
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import unicode_literals

class AppSettings(object):

    def _setting(self, name, default=None):
        from django.conf import settings
        return getattr(settings, name, default)

    @property
    def APP_LABEL(self):
        from django.core.exceptions import ImproperlyConfigured

        result = self._setting('SHOP_APP_LABEL')
        if not result:
            raise ImproperlyConfigured("SHOP_APP_LABEL setting must be set")
        return result

    @property
    def DEFAULT_CURRENCY(self):
        return self._setting('SHOP_DEFAULT_CURRENCY', 'EUR')

    @property
    def MONEY_FORMAT(self):
        """
        When rendering an amount of type Money, use this format.
        Possible placeholders are:
        {symbol}: This can be €, $, £, etc.
        {currency}: This can be EUR, USD, GBP, etc.
        {amount}: The localized amount.
        """
        return self._setting('SHOP_MONEY_FORMAT', '{symbol} {amount}')

    @property
    def USE_TZ(self):
        return True

    @property
    def DECIMAL_PLACES(self):
        """
        Number of decimal places for the internal representation of a price.
        Not visible to the customer.
        """
        return self._setting('SHOP_DECIMAL_PLACES', 2)

    @property
    def CUSTOMER_SERIALIZER(self):
        from django.core.exceptions import ImproperlyConfigured
        from django.utils.module_loading import import_string
        from shop.serializers.bases import BaseCustomerSerializer

        s = self._setting('SHOP_CUSTOMER_SERIALIZER', 'shop.serializers.defaults.CustomerSerializer')
        CustomerSerializer = import_string(s)
        if not issubclass(CustomerSerializer, BaseCustomerSerializer):
            raise ImproperlyConfigured(
                "Serializer class must inherit from 'BaseCustomerSerializer'.")
        return CustomerSerializer

    @property
    def PRODUCT_SUMMARY_SERIALIZER(self):
        from django.core.exceptions import ImproperlyConfigured
        from django.utils.module_loading import import_string
        from shop.serializers.bases import BaseProductSummarySerializer

        s = self._setting('SHOP_PRODUCT_SUMMARY_SERIALIZER')
        if not s:
            raise ImproperlyConfigured("SHOP_PRODUCT_SUMMARY_SERIALIZER setting must be set")
        ProductSummarySerializer = import_string(s)
        if not issubclass(ProductSummarySerializer, BaseProductSummarySerializer):
            raise ImproperlyConfigured(
                "Serializer class must inherit from 'BaseProductSummarySerializer'.")
        return ProductSummarySerializer

    @property
    def PRODUCT_SELECT_SERIALIZER(self):
        from django.utils.module_loading import import_string

        s = self._setting('SHOP_PRODUCT_SELECT_SERIALIZER', 'shop.serializers.defaults.ProductSelectSerializer')
        ProductSelectSerializer = import_string(s)
        return ProductSelectSerializer

    @property
    def ORDER_ITEM_SERIALIZER(self):
        from django.core.exceptions import ImproperlyConfigured
        from django.utils.module_loading import import_string
        from shop.serializers.bases import BaseOrderItemSerializer

        s = self._setting('SHOP_ORDER_ITEM_SERIALIZER', 'shop.serializers.defaults.OrderItemSerializer')
        OrderItemSerializer = import_string(s)
        if not issubclass(OrderItemSerializer, BaseOrderItemSerializer):
            raise ImproperlyConfigured(
                "Serializer class must inherit from 'BaseOrderItemSerializer'.")
        return OrderItemSerializer

    @property
    def CART_MODIFIERS(self):
        from django.utils.module_loading import import_string
        return tuple(
            import_string(mc)
            for mc in self._setting('SHOP_CART_MODIFIERS', ('shop.modifiers.defaults.DefaultCartModifier',))
        )

    @property
    def VALUE_ADDED_TAX(self):
        from decimal import Decimal
        return self._setting('SHOP_VALUE_ADDED_TAX', Decimal('20'))

    @property
    def ORDER_WORKFLOWS(self):
        from django.utils.module_loading import import_string
        return tuple(
            import_string(mc)
            for mc in self._setting('SHOP_ORDER_WORKFLOWS', ())
        )

    @property
    def ADD2CART_NG_MODEL_OPTIONS(self):
        return self._setting('SHOP_ADD2CART_NG_MODEL_OPTIONS', "{updateOn: 'default blur', debounce: {'default': 500, 'blur': 0}}")

    @property
    def EDITCART_NG_MODEL_OPTIONS(self):
        """
        Use ``SHOP_EDITCART_NG_MODEL_OPTIONS`` and ``SHOP_ADD2CART_NG_MODEL_OPTIONS`` to configure the
        update behavior when changing the quantity of a cart item. For more information refer to the
        AngularJS docs: https://code.angularjs.org/1.3.7/docs/api/ng/directive/ngModelOptions
        """
        return self._setting('SHOP_EDITCART_NG_MODEL_OPTIONS', "{updateOn: 'default blur', debounce: {'default': 500, 'blur': 0}}")

    @property
    def GUEST_IS_ACTIVE_USER(self):
        """
        If ``SHOP_GUEST_IS_ACTIVE_USER`` is True, Customers which declared themselves as guests, may request
        a password reset, so that they can log into their account at a later time. The default is False.
        """
        return self._setting('SHOP_GUEST_IS_ACTIVE_USER', False)

    @property
    def CACHE_DURATIONS(self):
        result = self._setting('SHOP_CACHE_DURATIONS') or {}
        result.setdefault('product_html_snippet', 86400)
        return result

# Change the export value of the module, to allow importing with `from shop import app_settings`
# For more details, see http://mail.python.org/pipermail/python-ideas/2012-May/014969.html
import sys  # noqa
app_settings = AppSettings()
app_settings.__name__ = __name__
sys.modules[__name__] = app_settings
<commit_msg>updated docstrings<commit_after># -*- coding: utf-8 -*-

from __future__ import unicode_literals

class AppSettings(object):

    def _setting(self, name, default=None):
        from django.conf import settings
        return getattr(settings, name, default)

    @property
    def APP_LABEL(self):
        from django.core.exceptions import ImproperlyConfigured

        result = self._setting('SHOP_APP_LABEL')
        if not result:
            raise ImproperlyConfigured("SHOP_APP_LABEL setting must be set")
        return result

    @property
    def DEFAULT_CURRENCY(self):
        return self._setting('SHOP_DEFAULT_CURRENCY', 'EUR')

    @property
    def MONEY_FORMAT(self):
        """
        When rendering an amount of type Money, use this format.
        Possible placeholders are:
        {symbol}: This is replaced by €, $, £, etc.
        {currency}: This is replaced by EUR, USD, GBP, etc.
        {amount}: The localized amount.
        """
        return self._setting('SHOP_MONEY_FORMAT', '{symbol} {amount}')

    @property
    def USE_TZ(self):
        return True

    @property
    def DECIMAL_PLACES(self):
        """
        Number of decimal places for the internal representation of a price.
        This is used by the Django admin and is not visible to the customer.
        """
        return self._setting('SHOP_DECIMAL_PLACES', 2)

    @property
    def CUSTOMER_SERIALIZER(self):
        from django.core.exceptions import ImproperlyConfigured
        from django.utils.module_loading import import_string
        from shop.serializers.bases import BaseCustomerSerializer

        s = self._setting('SHOP_CUSTOMER_SERIALIZER', 'shop.serializers.defaults.CustomerSerializer')
        CustomerSerializer = import_string(s)
        if not issubclass(CustomerSerializer, BaseCustomerSerializer):
            raise ImproperlyConfigured(
                "Serializer class must inherit from 'BaseCustomerSerializer'.")
        return CustomerSerializer

    @property
    def PRODUCT_SUMMARY_SERIALIZER(self):
        from django.core.exceptions import ImproperlyConfigured
        from django.utils.module_loading import import_string
        from shop.serializers.bases import BaseProductSummarySerializer

        s = self._setting('SHOP_PRODUCT_SUMMARY_SERIALIZER')
        if not s:
            raise ImproperlyConfigured("SHOP_PRODUCT_SUMMARY_SERIALIZER setting must be set")
        ProductSummarySerializer = import_string(s)
        if not issubclass(ProductSummarySerializer, BaseProductSummarySerializer):
            raise ImproperlyConfigured(
                "Serializer class must inherit from 'BaseProductSummarySerializer'.")
        return ProductSummarySerializer

    @property
    def PRODUCT_SELECT_SERIALIZER(self):
        from django.utils.module_loading import import_string

        s = self._setting('SHOP_PRODUCT_SELECT_SERIALIZER', 'shop.serializers.defaults.ProductSelectSerializer')
        ProductSelectSerializer = import_string(s)
        return ProductSelectSerializer

    @property
    def ORDER_ITEM_SERIALIZER(self):
        from django.core.exceptions import ImproperlyConfigured
        from django.utils.module_loading import import_string
        from shop.serializers.bases import BaseOrderItemSerializer

        s = self._setting('SHOP_ORDER_ITEM_SERIALIZER', 'shop.serializers.defaults.OrderItemSerializer')
        OrderItemSerializer = import_string(s)
        if not issubclass(OrderItemSerializer, BaseOrderItemSerializer):
            raise ImproperlyConfigured(
                "Serializer class must inherit from 'BaseOrderItemSerializer'.")
        return OrderItemSerializer

    @property
    def CART_MODIFIERS(self):
        from django.utils.module_loading import import_string

        return tuple(
            import_string(mc)
            for mc in self._setting('SHOP_CART_MODIFIERS', ('shop.modifiers.defaults.DefaultCartModifier',))
        )

    @property
    def VALUE_ADDED_TAX(self):
        from decimal import Decimal
        return self._setting('SHOP_VALUE_ADDED_TAX', Decimal('20'))

    @property
    def ORDER_WORKFLOWS(self):
        from django.utils.module_loading import import_string

        return tuple(
            import_string(mc)
            for mc in self._setting('SHOP_ORDER_WORKFLOWS', ())
        )

    @property
    def ADD2CART_NG_MODEL_OPTIONS(self):
        """
        Used to configure the update behavior when changing the quantity of a product, when adding
        it to the cart. For more information refer to the AngularJS docs at:
        https://code.angularjs.org/1.3.7/docs/api/ng/directive/ngModelOptions
        """
        return self._setting('SHOP_ADD2CART_NG_MODEL_OPTIONS',
                             "{updateOn: 'default blur', debounce: {'default': 500, 'blur': 0}}")

    @property
    def EDITCART_NG_MODEL_OPTIONS(self):
        """
        Used to configure the update behavior when changing the quantity of a cart item.
        For more information refer to the AngularJS docs at:
        https://code.angularjs.org/1.3.7/docs/api/ng/directive/ngModelOptions
        """
        return self._setting('SHOP_EDITCART_NG_MODEL_OPTIONS',
                             "{updateOn: 'default blur', debounce: {'default': 500, 'blur': 0}}")

    @property
    def GUEST_IS_ACTIVE_USER(self):
        """
        If this directive is True, customers which declared themselves as guests, may request
        a password reset, so that they can log into their account at a later time. The default is False.
        """
        return self._setting('SHOP_GUEST_IS_ACTIVE_USER', False)

    @property
    def CACHE_DURATIONS(self):
        result = self._setting('SHOP_CACHE_DURATIONS') or {}
        result.setdefault('product_html_snippet', 86400)
        return result

# Change the export value of the module, to allow importing with `from shop import app_settings`
# For more details, see http://mail.python.org/pipermail/python-ideas/2012-May/014969.html
import sys  # noqa
app_settings = AppSettings()
app_settings.__name__ = __name__
sys.modules[__name__] = app_settings
<|endoftext|>


<commit_before>from django.conf.urls import include, url
from django.contrib import admin

urlpatterns = [
    #url(r'^admin/', include(admin.site.urls)),
    url(r'^', include('djguicore.urls')),
    url(r'^', include('{{ app_name }}.urls')),
]<commit_msg>removed old urls<commit_after><|endoftext|>


<commit_before><commit_msg>Add ToDo<commit_after><|endoftext|>


<commit_before><commit_msg>path *really* fixed on linux<commit_after><|endoftext|>


<commit_before><commit_msg>Trying to mask out mangroves from annual biomass gain in non-mangrove natural forests.<commit_after><|endoftext|>


<commit_before>import os, sys, time
nimport zmq
import chroma.api as api
api.use_cuda()
import numpy as np
from chroma.sim import Simulation
from chroma.event import Photons
import chroma.event

from uboone import uboone

import ratchromadata_pb2
#import photonHit_pb2

context = zmq.Context().instance()

backend = context.socket(zmq.REP)

backport = '5556' #talks to REQ frontend of server (on backend)

backend.connect( "tcp://localhost:%s"%(backport) )

poll_Server = zmq.Poller()
poll_Server.register(backend, zmq.POLLIN)
#can set polling options to kill and restart on user input or
#set a timeout
det = uboone()

sim = Simulation(det,geant4_processes=0,nthreads_per_block = 1, max_blocks = 1024)

def main():
    while True:
        socks = dict(poll_Server.poll())

        if socks.get(backend) == zmq.POLLIN:
            msg = backend.recv()#_multipart()
            #^decide whether to use multipart or not^
            if not msg:
                break

            chromaData = ratchromadata_pb2.ChromaData()
            chromaData.ParseFromString(msg)
            print chromaData
            nsphotons = 0
            for i,sData in enumerate(chromaData.stepdata):
                pass #use np.random.exponential here
            dphi = np.random.uniform(0,2.0*np.pi, nphotons)
            dcos = np.random.uniform(-1.0, 1.0, nphotons)
            dir = np.array( zip( np.sqrt(1-dcos[:]*dcos[:])*np.cos(dphi[:]), np.sqrt(1-dcos[:]*dcos[:])*np.sin(dphi[:]), dcos[:] ), dtype=np.float32 )
            pos = np.zeros((nphotons,3), dtype=np.float32)
            for i,sData in enumerate(chromaData.stepdata):
                pos[i,0] = random.uniform(sData.step_start_x())
                pos[i,1] = random.uniform(sData.step_start_y())
                pos[i,2] = random.uniform(sData.step_start_z())

            """for cherenkov photons"""
            #nphotons = sum(1 for p in enumerate(chromaData.stepdata))
            #print "NUMPHOTONS: ",nphotons
            # dir = np.zeros((nphotons,3),
            #               dtype = np.float32)
            # for i,cData in enumerate(chromaData.cherenkovdata):
            #   dir[i,0] = cData.dx()
            #     dir[i,1] = cData.dy()
            #     dir[i,2] = cData.dz()
            # pol = np.zeros_like(dir)
            # for i,cData in (chromaData.cherenkovdata):
            #     pol[i,0] = cData.px()
            #     pol[i,1] = cData.py()
            #     pol[i,2] = cData.pz()

            # pos = np.zeros_like(dir)
            # for i,cData in (chromaData.cherenkovdata):
            #     pos[i,0] = cData.x()
            #     pos[i,1] = cData.y()
            #     pos[i,2] = cData.z()

            # t = np.zeros((nphotons),
            #              dtype=np.float32)
            # for i,cData in (chromaData.cherenkovdata):
            #     t[i] = cData.t()

            # wavelengths = np.zeros_like(t)
            # for i,cData in (chromaData.cherenkovdata):
            #     wavelengths[i] = cData.wavelengths()

            #ship to GPU, do some stuff, send data back
            photons = Photons(pos=pos, dir=dir, pol=pol, t=t,
                              wavelengths = wavelengths)

            events = sim.simulate(photons, keep_photons_end=True, max_steps=2000)

            for ev in events:
                detected_photons = ev.photons_end.flags[:] & chroma.event.SURFACE_DETECT
                channelhit = np.zeros(len(detected_photons),dtype=np.int)
                channelhit[:] = det.solid_id_to_channel_index[ det.solid_id[ev.photons_end.last_hit_triangles[:] ] ]
                for n,f in enumerate(detected_photons):
                    if f!=0:
                        print "hit detID:",channelhit[n]," pos=",ev.photons_end.pos[n,:]," time=",ev.photons_end.t[n]-100.0

if __name__ == "__main__":
    main()
<commit_msg>working on generating scint. photons<commit_after>import os, sys, time
nimport zmq
import chroma.api as api
api.use_cuda()
import numpy as np
from chroma.sim import Simulation
from chroma.event import Photons
import chroma.event

from uboone import uboone

import ratchromadata_pb2
#import photonHit_pb2

context = zmq.Context().instance()

backend = context.socket(zmq.REP)

backport = '5556' #talks to REQ frontend of server (on backend)

backend.connect( "tcp://localhost:%s"%(backport) )

poll_Server = zmq.Poller()
poll_Server.register(backend, zmq.POLLIN)
#can set polling options to kill and restart on user input or
#set a timeout
det = uboone()

sim = Simulation(det,geant4_processes=0,nthreads_per_block = 1, max_blocks = 1024)
def GenScintPhotons(protoString):
    nsphotons = 0
    for i,sData in enumerate(protoString.stepdata):
        nsphotons += sData.nphotons()
    pos = np.zeros((nsphotons,3),dtype=np.float32)
    pol = np.zeros_like(pos)
    t = np.zeros(nsphotons, dtype=np.float32)
    wavelengths = np.empty(nsphotons, np.float32)
    wavelengths.fill(128.0)
    dphi = np.random.uniform(0,2.0*np.pi, nphotons)
    dcos = np.random.uniform(-1.0, 1.0, nphotons)
    dir = np.array( zip( np.sqrt(1-dcos[:]*dcos[:])*np.cos(dphi[:]), np.sqrt(1-dcos[:]*dcos[:])*np.sin(dphi[:]), dcos[:] ), dtype=np.float32 )

    stepPhotons = 0
    for i,sData in enumerate(protoString.stepdata):
        for j in xrange(stepPhotons, (stepPhotons+sData.nphotons())):
            pos[j,0] = np.random.uniform(sData.step_start_x(),sData.step_end_x())
            pos[j,1] = np.random.uniform(sData.step_start_y(),sData.step_end_y())
            pos[j,2] = np.random.uniform(sData.step_start_z(),sData.step_end_z())
        for j in xrange(stepPhotons, (stepPhotons+sData.nphotons())):
            pol[j,0] = random.uniform(0,((1/3.0)**.5))
            pol[j,1] = random.uniform(0,((1/3.0)**.5))
            pol[j,2] = ((1 - pol[j,0]**2 - pol[j,1]**2)**.5)
        for j in xrange(stepPhotons, (stepPhotons+sData.nphotons())):
            t[j] = (np.random.exponential(1/45.0) + (sData.step_end_t()-sData.step_start_t()))
        stepPhotons += sData.nphotons()
    return Photons(pos = pos, pol = pol, t = t, dir = dir, wavelengths = wavelengths)
def main():
    while True:
        socks = dict(poll_Server.poll())

        if socks.get(backend) == zmq.POLLIN:
            msg = backend.recv()#_multipart()
            #^decide whether to use multipart or not^
            if not msg:
                break

            chromaData = ratchromadata_pb2.ChromaData()
            chromaData.ParseFromString(msg)
            print chromaData
<<<<<<< HEAD
            photons = GenScintPhotons(chromaData)
=======
            nsphotons = 0
            for i,sData in enumerate(chromaData.stepdata):
                pass #use np.random.exponential here
            dphi = np.random.uniform(0,2.0*np.pi, nphotons)
            dcos = np.random.uniform(-1.0, 1.0, nphotons)
            dir = np.array( zip( np.sqrt(1-dcos[:]*dcos[:])*np.cos(dphi[:]), np.sqrt(1-dcos[:]*dcos[:])*np.sin(dphi[:]), dcos[:] ), dtype=np.float32 )
            pos = np.zeros((nphotons,3), dtype=np.float32)
            for i,sData in enumerate(chromaData.stepdata):
                pos[i,0] = random.uniform(sData.step_start_x())
                pos[i,1] = random.uniform(sData.step_start_y())
                pos[i,2] = random.uniform(sData.step_start_z())

>>>>>>> 3e02b02ca38c3e8e6043f319de759ae737522a31

            """for cherenkov photons"""
            #nphotons = sum(1 for p in enumerate(chromaData.stepdata))
            #print "NUMPHOTONS: ",nphotons
            # dir = np.zeros((nphotons,3),
            #               dtype = np.float32)
            # for i,cData in enumerate(chromaData.cherenkovdata):
            #   dir[i,0] = cData.dx()
            #     dir[i,1] = cData.dy()
            #     dir[i,2] = cData.dz()
            # pol = np.zeros_like(dir)
            # for i,cData in (chromaData.cherenkovdata):
            #     pol[i,0] = cData.px()
            #     pol[i,1] = cData.py()
            #     pol[i,2] = cData.pz()

            # pos = np.zeros_like(dir)
            # for i,cData in (chromaData.cherenkovdata):
            #     pos[i,0] = cData.x()
            #     pos[i,1] = cData.y()
            #     pos[i,2] = cData.z()

            # t = np.zeros((nphotons),
            #              dtype=np.float32)
            # for i,cData in (chromaData.cherenkovdata):
            #     t[i] = cData.t()

            # wavelengths = np.zeros_like(t)
            # for i,cData in (chromaData.cherenkovdata):
            #     wavelengths[i] = cData.wavelengths()

            #ship to GPU, do some stuff, send data back
            #photons = Photons(pos=pos, dir=dir, pol=pol, t=t,
            #                 wavelengths = wavelengths)

            events = sim.simulate(photons, keep_photons_end=True, max_steps=2000)

            for ev in events:
                detected_photons = ev.photons_end.flags[:] & chroma.event.SURFACE_DETECT
                channelhit = np.zeros(len(detected_photons),dtype=np.int)
                channelhit[:] = det.solid_id_to_channel_index[ det.solid_id[ev.photons_end.last_hit_triangles[:] ] ]
                for n,f in enumerate(detected_photons):
                    if f!=0:
                        print "hit detID:",channelhit[n]," pos=",ev.photons_end.pos[n,:]," time=",ev.photons_end.t[n]-100.0

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import re
from dateutil              import parser
from scrapy.spiders        import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from artbot_scraper.items  import EventItem
from pytz                  import timezone

class PompomSpider(CrawlSpider):
    name            = 'Galerie pompom'
    allowed_domains = ['galeriepompom.com']
    start_urls      = ['http://galeriepompom.com/galerie-pompom--exhibitions.html']
    rules           = (Rule(LinkExtractor(deny=('exhibitions-2016', ), allow=('2016', )), callback='parse_exhibition'),)

    def parse_exhibition(self, response):
        item          = EventItem()
        item['url']   = response.url
        item['venue'] = self.name
        item['title'] = response.xpath('//div[contains(@data-muse-type, "txt_frame")]/p[2]/text()').extract_first().strip()
        item['image'] = response.urljoin(response.xpath('//div[contains(@class, "SlideShowWidget")]//img/@data-src').extract_first())

        season = response.xpath('//div[contains(@data-muse-type, "txt_frame")]/p[1]/text()').extract_first().strip()
        match  = re.match('(?P<start>[\d+\s+\w+]*)[\s\-]*(?P<end>\d+\s+\w+\s+\d+)', season)

        if (match):
            tz = timezone('Australia/Sydney')

            if len(match.group('start')) <= 2:
                item['end']   = tz.localize(parser.parse(match.group('end')))
                item['start'] = item['end'].replace(day = int(match.group('start')))
            else:
                item['start'] = tz.localize(parser.parse(match.group('start')))
                item['end']   = tz.localize(parser.parse(match.group('end')))

        yield item
<commit_msg>Fixed Galerie pompom spider post site change.<commit_after># -*- coding: utf-8 -*-
import re
from dateutil              import parser
from scrapy.spiders        import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from artbot_scraper.items  import EventItem
from pytz                  import timezone

class PompomSpider(CrawlSpider):
    name            = 'Galerie pompom'
    allowed_domains = ['galeriepompom.com']
    start_urls      = ['http://galeriepompom.com/galerie-pompom--exhibitions.html']
    rules           = (Rule(LinkExtractor(deny=('exhibitions-2016','exhibitions-2017', ), allow=('2017', )), callback='parse_exhibition'),)

    def parse_exhibition(self, response):
        item          = EventItem()
        item['url']   = response.url
        item['venue'] = self.name
        item['title'] = response.xpath('//div[contains(@data-muse-type, "txt_frame")]/p[2]/text()').extract_first().strip()
        item['image'] = response.urljoin(response.xpath('//div[contains(@class, "SlideShowWidget")]//img/@data-src').extract_first())

        season = response.xpath('//div[contains(@data-muse-type, "txt_frame")]/p[1]/text()').extract_first().strip()
        match  = re.match('(?P<start>[\d+\s+\w+]*)[\s\-]*(?P<end>\d+\s+\w+\s+\d+)', season)

        if (match):
            tz = timezone('Australia/Sydney')

            if len(match.group('start')) <= 2:
                item['end']   = tz.localize(parser.parse(match.group('end')))
                item['start'] = item['end'].replace(day = int(match.group('start')))
            else:
                item['start'] = tz.localize(parser.parse(match.group('start')))
                item['end']   = tz.localize(parser.parse(match.group('end')))

        yield item
<|endoftext|>


<commit_before>import os
import uuid

from integration_test import validate
from notifications_python_client.notifications import NotificationsAPIClient

def send_sms_notification_test_response(python_client):
    mobile_number = os.environ['FUNCTIONAL_TEST_NUMBER']
    template_id = os.environ['SMS_TEMPLATE_ID']
    unique_name = str(uuid.uuid4())
    personalisation = {'name': unique_name}
    response = python_client.send_sms_notification(to=mobile_number,
                                                   template_id=template_id,
                                                   personalisation=personalisation)
    validate(response, 'POST_notification_return_sms.json')
    return response['data']['notification']['id']

def send_email_notification_test_response(python_client):
    email_address = os.environ['FUNCTIONAL_TEST_EMAIL']
    template_id = os.environ['EMAIL_TEMPLATE_ID']
    unique_name = str(uuid.uuid4())
    personalisation = {'name': unique_name}
    response = python_client.send_email_notification(to=email_address,
                                                     template_id=template_id,
                                                     personalisation=personalisation)
    validate(response, 'POST_notification_return_email.json')
    return response['data']['notification']['id']

def get_notification_by_id(python_client, id, notification_type):
    response = python_client.get_notification_by_id(id)

    if notification_type == 'email':
        validate(response, 'GET_notification_return_email.json')
    elif notification_type == 'sms':
        validate(response, 'GET_notification_return_sms.json')
    else:
        raise KeyError("notification type should be email|sms")

def get_all_notifications(client):
    response = client.get_all_notifications()
    validate(response, 'GET_notifications_return.json')

if __name__ == "__main__":
    client = NotificationsAPIClient(
        os.environ['NOTIFY_API_URL'],
        os.environ['SERVICE_ID'],
        os.environ['API_KEY']
    )

    sms_id = send_sms_notification_test_response(client)
    email_id = send_email_notification_test_response(client)
    get_notification_by_id(client, sms_id, 'sms')
    get_notification_by_id(client, email_id, 'email')

    get_all_notifications(client)

    print("notifications-python-client integration tests are successful")
<commit_msg>update client integration tests to use kwargs<commit_after>import os
import uuid

from integration_test import validate
from notifications_python_client.notifications import NotificationsAPIClient

def send_sms_notification_test_response(python_client):
    mobile_number = os.environ['FUNCTIONAL_TEST_NUMBER']
    template_id = os.environ['SMS_TEMPLATE_ID']
    unique_name = str(uuid.uuid4())
    personalisation = {'name': unique_name}
    response = python_client.send_sms_notification(to=mobile_number,
                                                   template_id=template_id,
                                                   personalisation=personalisation)
    validate(response, 'POST_notification_return_sms.json')
    return response['data']['notification']['id']

def send_email_notification_test_response(python_client):
    email_address = os.environ['FUNCTIONAL_TEST_EMAIL']
    template_id = os.environ['EMAIL_TEMPLATE_ID']
    unique_name = str(uuid.uuid4())
    personalisation = {'name': unique_name}
    response = python_client.send_email_notification(to=email_address,
                                                     template_id=template_id,
                                                     personalisation=personalisation)
    validate(response, 'POST_notification_return_email.json')
    return response['data']['notification']['id']

def get_notification_by_id(python_client, id, notification_type):
    response = python_client.get_notification_by_id(id)

    if notification_type == 'email':
        validate(response, 'GET_notification_return_email.json')
    elif notification_type == 'sms':
        validate(response, 'GET_notification_return_sms.json')
    else:
        raise KeyError("notification type should be email|sms")

def get_all_notifications(client):
    response = client.get_all_notifications()
    validate(response, 'GET_notifications_return.json')

if __name__ == "__main__":
    client = NotificationsAPIClient(
        base_url=os.environ['NOTIFY_API_URL'],
        service_id=os.environ['SERVICE_ID'],
        api_key=os.environ['API_KEY']
    )

    sms_id = send_sms_notification_test_response(client)
    email_id = send_email_notification_test_response(client)
    get_notification_by_id(client, sms_id, 'sms')
    get_notification_by_id(client, email_id, 'email')

    get_all_notifications(client)

    print("notifications-python-client integration tests are successful")
<|endoftext|>


<commit_before><commit_msg>adding FtpDataSource class<commit_after><|endoftext|>


<commit_before><commit_msg>revert DEBUG to False<commit_after><|endoftext|>


<commit_before># Django development settings for sikteeri project.
# For production use, override with
#
#   --settings=production_settings
#
# and have production_settings.py
#
#   import * from settings
#

import django.conf.global_settings as DEFAULT_SETTINGS

DEBUG = True
TEMPLATE_DEBUG = True

ADMINS = (
    # ('Your Name', 'your_email@example.com'),
)

MANAGERS = ADMINS

# brew install postgresql && createdb sikteeri
DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.postgresql_psycopg2',
        'NAME': 'sikteeri',
        'USER': '',
        'PASSWORD': '',
        'HOST': '',
        'PORT': '',
    }
}

# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.4/ref/settings/#allowed-hosts
ALLOWED_HOSTS = []

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'Europe/Helsinki'
SHORT_DATE_FORMAT = 'd.m.Y'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'fi-fi'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = False

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/home/media/media.lawrence.com/media/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
MEDIA_URL = ''

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = ''

# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
    'sikteeri/static',
    'membership/static',
)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = 'z$g77fzdhwi%lrn-8zezn995*=m!s_nb*k&7=fo(9ppf6(s%_a'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

TEMPLATE_CONTEXT_PROCESSORS = DEFAULT_SETTINGS.TEMPLATE_CONTEXT_PROCESSORS + (
    "sikteeri.context_processors.is_production",
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'sikteeri.ForceDefaultLanguageMiddleware.ForceDefaultLanguageMiddleware',
    'sikteeri.GitVersionHeaderMiddleware.GitVersionHeaderMiddleware',
    'django.middleware.locale.LocaleMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'sikteeri.urls'

TEMPLATE_DIRS = (
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
    "sikteeri/templates/",
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.admin',
    'django.contrib.humanize',
    'django.contrib.comments',
    'django.contrib.staticfiles',
    'membership',
    'services',
    'south',
    'debug_toolbar',
)

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

LOGIN_URL = 'login'

SOUTH_TESTS_MIGRATE = False

# If set, this string will be displayed and sikteeri is disabled
MAINTENANCE_MESSAGE = None

MESSAGE_STORAGE = 'django.contrib.messages.storage.cookie.CookieStorage'

#USE_X_FORWARDED_HOST = True

# Show 30 items per page in listview
ENTRIES_PER_PAGE=30

# Secure cookies require https
# SESSION_COOKIE_SECURE = False

# Hosts allowed to fetch statistics etc. without authentication
TRUSTED_HOSTS = ['127.0.0.1', '::1']

# Billing settings
IBAN_ACCOUNT_NUMBER = 'FI00 0000 0000 0000 00'
BIC_CODE = 'AAAAAAAA'
BILLING_FROM_EMAIL = 'from@example.com'
BILLING_CC_EMAIL = None
BILL_SUBJECT = 'Test bill, ignore'
REMINDER_SUBJECT = 'Test reminder, ignore'
# Send bills 2 weeks before new cycle starts
BILL_DAYS_BEFORE_CYCLE = 14 # days
# New bills will have a due date 2 weeks in the future
BILL_DAYS_TO_DUE = 14 # days
# Send reminders 2 weeks after previous due day
REMINDER_GRACE_DAYS = 14 # days
ENABLE_REMINDERS = False
# Reminders will be additionally sent to account@domain if specified
UNIX_EMAIL_DOMAIN = 'example.com'

# Generic email settings
FROM_EMAIL = 'from.nonbill@example.com'
SYSADMIN_EMAIL = 'admins@example.com'

# Don't really send email messages, only show them on console
EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
EMAIL_SUBJECT_PREFIX = '[sikteeri-TEST] '

# Paper reminder templates
PAPER_REMINDER_TEMPLATE = '/dev/null'

# When PRODUCTION is true, show final graphics and colours.
# Otherwise indicate that this is a development environment (logo, colour)
PRODUCTION = False
<commit_msg>Further clean up development settings<commit_after># Django development settings for sikteeri project.
# For production use, override with
#
#   --settings=production_settings
#
# and have production_settings.py
#
#   import * from settings
#

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
BASE_DIR = os.path.dirname(os.path.dirname(__file__))

import django.conf.global_settings as DEFAULT_SETTINGS

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
TEMPLATE_DEBUG = True

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'z$g77fzdhwi%lrn-8zezn995*=m!s_nb*k&7=fo(9ppf6(s%_a'

# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.6/ref/settings/#allowed-hosts
ALLOWED_HOSTS = []

# Database
# https://docs.djangoproject.com/en/1.6/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# http://www.i18nguy.com/unicode/language-identifiers.html
TIME_ZONE = 'Europe/Helsinki'
SHORT_DATE_FORMAT = 'd.m.Y'
LANGUAGE_CODE = 'fi-fi'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = False

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
)

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
)

STATIC_URL = '/static/'

STATICFILES_DIRS = (
    os.path.join(BASE_DIR, 'sikteeri/static'),
    os.path.join(BASE_DIR, 'membership/static'),
)

TEMPLATE_DIRS = (
    os.path.join(BASE_DIR, 'sikteeri/templates/'),
)

TEMPLATE_CONTEXT_PROCESSORS = DEFAULT_SETTINGS.TEMPLATE_CONTEXT_PROCESSORS + (
    "sikteeri.context_processors.is_production",
)

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.humanize',
    'django.contrib.comments',
    'django.contrib.staticfiles',
    'membership',
    'services',
    'south',
    'debug_toolbar',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'sikteeri.ForceDefaultLanguageMiddleware.ForceDefaultLanguageMiddleware',
    'sikteeri.GitVersionHeaderMiddleware.GitVersionHeaderMiddleware',
    'django.middleware.locale.LocaleMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'sikteeri.urls'

WSGI_APPLICATION = 'sikteeri.wsgi.application'

LOGIN_URL = 'login'

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

SOUTH_TESTS_MIGRATE = False

# If set, this string will be displayed and sikteeri is disabled
MAINTENANCE_MESSAGE = None

MESSAGE_STORAGE = 'django.contrib.messages.storage.cookie.CookieStorage'

# Show 30 items per page in listview
ENTRIES_PER_PAGE=30

# Hosts allowed to fetch statistics etc. without authentication
TRUSTED_HOSTS = ['127.0.0.1', '::1']

# Billing settings
IBAN_ACCOUNT_NUMBER = 'FI00 0000 0000 0000 00'
BIC_CODE = 'AAAAAAAA'
BILLING_FROM_EMAIL = 'from@example.com'
BILLING_CC_EMAIL = None
BILL_SUBJECT = 'Test bill, ignore'
REMINDER_SUBJECT = 'Test reminder, ignore'
# Send bills 2 weeks before new cycle starts
BILL_DAYS_BEFORE_CYCLE = 14 # days
# New bills will have a due date 2 weeks in the future
BILL_DAYS_TO_DUE = 14 # days
# Send reminders 2 weeks after previous due day
REMINDER_GRACE_DAYS = 14 # days
ENABLE_REMINDERS = False
# Reminders will be additionally sent to account@domain if specified
UNIX_EMAIL_DOMAIN = 'example.com'

# Generic email settings
FROM_EMAIL = 'from.nonbill@example.com'
SYSADMIN_EMAIL = 'admins@example.com'

# Don't really send email messages, only show them on console
EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
EMAIL_SUBJECT_PREFIX = '[sikteeri-TEST] '

# Paper reminder templates
PAPER_REMINDER_TEMPLATE = '/dev/null'

# When PRODUCTION is true, show production graphics and colours.
# Otherwise indicate that this is a development environment (logo, colour)
PRODUCTION = False
<|endoftext|>