<commit_before>import db
import ala
import logging
import multiprocessing
import binascii
import uuid
from sqlalchemy import func, select

log = logging.getLogger(__name__)

class Syncer:

    def __init__(self):
        '''TODO: might pass db into here for unit testing purposes instead of
        using the module directly. Might also do the same for ala module.'''

        row = db.sources.select('id')\
                .where(db.sources.c.name == 'ALA')\
                .execute().fetchone()

        if row is None:
            raise RuntimeError('ALA row missing from sources table in db')

        self.source_row_id = row['id']
        self.cached_upserts = []
        self.num_dirty_records_by_species_id = {}

    def local_species(self):
        '''Returns all db.species rows in the local database.'''
        return db.species.select().execute().fetchall()

    def local_species_by_scientific_name(self):
        '''Returns all species in the local db in a dict. Scientific name is the
        key, the db row is the value.'''

        species = {}
        for row in db.species.select().execute():
            species[row['scientific_name']] = row;
        return species

    def remote_species_by_scientific_name(self):
        '''Returns all species available at ALA in a dict. Scientific name is the
        key, the ala.Species object is the value.'''

        species = {}
        for bird in ala.all_bird_species():
            species[bird.scientific_name] = bird
        return species

    def added_and_deleted_species(self):
        '''Returns (added, deleted) where `added` is an iterable of ala.Species
        objects that are not present in the local db, and `deleted` is an iterable
        of rows from the db.species table that were not found at ALA.'''

        local = self.local_species_by_scientific_name()
        local_set = frozenset(local.keys())
        remote = self.remote_species_by_scientific_name()
        remote_set = frozenset(remote.keys())

        added_set = remote_set - local_set
        deleted_set = local_set - remote_set

        added = [species for name, species in remote.iteritems() if name in added_set]
        deleted = [row for name, row in local.iteritems() if name in deleted_set]

        return (added, deleted)

    def add_species(self, species):
        '''Adds `species` to the local db, where `s` is an ala.Species
        object'''

        log.info('Adding new species "%s"', species.scientific_name)
        db.species.insert().execute(
            scientific_name=species.scientific_name,
            common_name=species.common_name)

    def delete_species(self, row):
        '''Deletes `row` from the local db, where `s` is a row from the
        db.species table'''

        log.info('Deleting species "%s"', row['scientific_name'])
        db.species.delete().where(db.species.c.id == row['id']).execute()

    def sync_occurrences(self, utc_since_date):
        '''Performs all adding, updating, and deleting of rows in the
        db.occurrences table of the database.

        Also updates db.species.c.num_dirty_occurrences.'''

        # insert new, and update existing, occurrences
        for occ in self.occurrences_changed_since(utc_since_date, True):
            self.upsert_occurrence(occ, occ.species_id)
        self.flush_upserts()

        remote_counts = self.remote_occurrence_counts_by_species_id()

        # delete occurrences that have been deleted at ALA
        self.delete_local_occurrences_if_needed(remote_counts)

        # log warnings if the counts dont match up
        self.check_occurrence_counts(remote_counts)

        # increase number in db.species.num_dirty_occurrences
        self.update_num_dirty_occurrences()

    def delete_local_occurrences_if_needed(self, remote_counts):
        '''This must be called as the last step in syncing occurrence records,
        because adding and updating occurrences will alter the number of
        records per species. Checking for deleted records is expensive in terms
        of memory and time, so it only happens with the local occurrence count
        per species is higher than the count at ALA.

        Builds a set of uuid.UUID objects for records that exist at ALA, then
        checks every local record to see if it still exists in the set.'''

        for row, lc, rc in self.species_with_occurrence_counts(remote_counts):
            # don't run unless our count is lower than ALAs count
            if lc <= rc:
                continue

            log.info('Checking for deleted records for species %s',
                     row['scientific_name'])

            # species should never be None, because we already have a count for
            # it from ALA
            species = ala.species_for_scientific_name(row['scientific_name'])
            if species is None:
                log.critical('species should never be None')
                continue

            # build the set of existing uuids
            existing_uuids = set()
            for occurrence in ala.records_for_species(species.lsid):
                existing_uuids.add(occurrence.uuid)

            # loop through every local occurrence, and store the ids of the
            # rows that do not exist at ALA
            row_ids_to_delete = []
            all_rows_for_species = \
                db.occurrences.select().\
                where(db.occurrences.c.species_id == row['id']).\
                where(db.occurrences.c.source_id == self.source_row_id).\
                execute()
            for occ_row in all_rows_for_species:
                occ_uuid = uuid.UUID(bytes=occ_row['source_record_id'])
                if occ_uuid not in existing_uuids:
                    row_ids_to_delete.append(occ_row['id'])

            # free up some memory, because this may be very large
            del existing_uuids

            # delete all the neccessary local occurrences
            for occ_id in row_ids_to_delete:
                db.occurrences.delete().\
                    where(db.occurrences.c.id == occ_id).\
                    execute()

            # free up more memory
            num_deleted = len(row_ids_to_delete)
            del row_ids_to_delete

            # log and keep track of the deletaions
            log.info('Deleted %d records for %s', num_deleted,
                    row['scientific_name'])

            self.increase_dirty_count(row['id'], num_deleted)

    def check_occurrence_counts(self, remote_counts):
        '''Logs warnings if the local and remote occurrence counts per species
        do not match.'''

        for row, lc, rc in self.species_with_occurrence_counts(remote_counts):
            if lc == rc:
                continue #  counts are the same

            log.warning('Occurrence counts differ for species %s,' +
                        '(local count = %d, ALA count = %d)',
                        row['scientific_name'],
                        lc, rc)

    def species_with_occurrence_counts(self, remote_counts):
        '''Checks the number of local occurrences against the number of
        occurrences at ALA, yielding the db.species row, local count and remote
        count if the counts are different.'''

        local_counts = self.local_occurrence_counts_by_species_id()

        for row in self.local_species():
            if row['id'] in remote_counts:
                yield (row, local_counts[row['id']], remote_counts[row['id']])

    def remote_occurrence_counts_by_species_id(self):
        '''Returns a dict with db.species.c.id keys, and the values are the
        number of occurrences present at ALA for that species.'''

        counts = {}

        for row in self.local_species():
            species = ala.species_for_scientific_name(row['scientific_name'])
            if species is not None:
                counts[row['id']] = ala.num_records_for_lsid(species.lsid)

        return counts;

    def local_occurrence_counts_by_species_id(self):
        '''Returns a dict with db.species.c.id keys, and the calues are the
        number of occurrences present in the local database for that
        species.'''

        counts = {}

        for row in self.local_species():
            local_count = \
                db.engine.execute(select(
                    [func.count('*')],
                    #where
                     (db.occurrences.c.species_id == row['id']) &
                     (db.occurrences.c.source_id == self.source_row_id))
                ).scalar()

            counts[row['id']] = local_count

        return counts

    def upsert_occurrence(self, occurrence, species_id):
        '''Looks up whether `occurrence` (an ala.OccurrenceRecord object)
        already exists in the local db. If it does, the db row is updated with
        the information in `occurrence`. If it does not exist, a new row is
        inserted.

        `species_id` must be supplied as an argument because it is not
        obtainable from `occurrence`

        The inserts/updates are cached for performance reasons. The cache is
        flushed every 1000 occurrences. YOU MUST CALL `flush_upserts` AFTER YOU
        HAVE CALLED THIS METHOD FOR THE FINAL TIME.'''

        if len(self.cached_upserts) > 1000:
            self.flush_upserts()

        # TODO: determine rating better using lists of assertions
        rating = 'assumed invalid'
        if occurrence.is_geospatial_kosher:
            rating = 'assumed valid'

        # these should be escaped strings ready for insertion into the SQL
        cols = (str(float(occurrence.latitude)),
                str(float(occurrence.longitude)),
                '"' + rating + '"',
                str(int(species_id)),
                str(int(self.source_row_id)),
                _mysql_encode_binary(occurrence.uuid.bytes))

        self.cached_upserts.append('(' + ','.join(cols) + ')')

    def flush_upserts(self):
        if len(self.cached_upserts) <= 0:
            return

        query = '''INSERT INTO occurrences(
                        latitude, longitude, rating, species_id, source_id,
                        source_record_id)

                   VALUES ''' + \
                \
                ','.join(self.cached_upserts) + \
                \
                ''' ON DUPLICATE KEY UPDATE
                        latitude=VALUES(latitude),
                        longitude=VALUES(longitude),
                        rating=VALUES(rating),
                        species_id=VALUES(species_id);'''

        db.engine.execute(query)

        self.cached_upserts = []

    def occurrences_changed_since(self, since_date, record_dirty=False):
        '''Generator for ala.OccurrenceRecord objects.

        Will use whatever is in the species table of the database, so call
        update the species table before calling this function.

        Uses a pool of processes to fetch occurrence records. The subprocesses
        feed the records into a queue which the original process reads and
        yields. This should let the main process access the database at full
        speed while the subprocesses are waiting for more records to arrive
        over the network.'''

        record_q = multiprocessing.Queue(10000)
        pool = multiprocessing.Pool(8, _mp_init, [record_q])
        active_workers = 0

        # fill the pool full with every species
        for row in db.species.select().execute():
            args = (row['scientific_name'], row['id'], since_date)
            pool.apply_async(_mp_fetch, args)
            active_workers += 1

        pool.close()

        # keep reading from the queue until all the subprocesses are finished
        while active_workers > 0:
            record = record_q.get()
            if isinstance(record, ala.OccurrenceRecord):
                yield record
            elif isinstance(record, tuple):
                active_workers -= 1
                if len(record) == 3:
                    if record[2] > 0:
                        log.info('Finished processing %d records for %s',
                                 record[2],
                                 record[0])
                    if record_dirty:
                        self.increase_dirty_count(record[1], record[2])
                else:
                    raise RuntimeError('Worker process failed: ' + record[0])
            else:
                raise RuntimeError('Unexpected type coming from record_q: ' +
                                   str(type(record)))

        # all the subprocesses should be dead by now
        pool.join()

    def increase_dirty_count(self, species_id, num_dirty):
        if species_id in self.num_dirty_records_by_species_id:
            self.num_dirty_records_by_species_id[species_id] += num_dirty
        else:
            self.num_dirty_records_by_species_id[species_id] = num_dirty

    def local_species_with_no_occurrences(self):
        '''A generator for db.species rows, for rows without any occurrence
        records in the local database'''

        for row in self.local_species():
            q = select(['count(*)'], db.occurrences.c.species_id == row['id'])
            if db.engine.execute(q).scalar() == 0:
                yield row

    def update_num_dirty_occurrences(self):
        '''Updates the species.num_dirty_occurrences column with the number of
        occurrences that have been changed by self.

        TODO: account for deleted records
        '''
        dirty_col = db.species.c.num_dirty_occurrences

        for row in self.local_species():
            if row['id'] not in self.num_dirty_records_by_species_id:
                continue

            newly_dirty = self.num_dirty_records_by_species_id[row['id']]
            if newly_dirty <= 0:
                continue

            db.species.update().\
                values(num_dirty_occurrences=(dirty_col + newly_dirty)).\
                where(db.species.c.id == row['id']).\
                execute()

def _mp_init(record_q):
    '''Called when a subprocess is started. See
    Syncer.occurrences_changed_since'''
    _mp_init.record_q = record_q
    #_mp_init.log = multiprocessing.log_to_stderr()

def _mp_fetch(species_sname, species_id, since_date):
    '''Gets all relevant records for the given species from ALA, and pumps the
    records into _mp_init.record_q.

    If the function finished successfully, will put a len 3 tuple in the
    record_q with (scientific_name, species_id, num_records_found). If the
    function fails, will put a len 1 tuple in the record_q with a failure
    message string in it.

    Adds a `species_id` attribute to each ala.OccurrenceRecord object set to
    the argument given to this function.'''

    num_records = 0
    failure_msg = None
    try:
        num_records = _mp_fetch_inner(species_sname, species_id, since_date)
    except Exception, e:
        failure_msg = str(e)
        if isinstance(e, urllib2.HTTPError):
            failure_msg += '\n\nResponse Headers:\n' + str(dict(e.info()))
            failure_msg += '\n\nResponse Payload:\n' + e.read()

    if failure_msg is None:
        _mp_init.record_q.put((species_sname, species_id, num_records))
    else:
        _mp_init.record_q.put((failure_msg,))

def _mp_fetch_inner(species_sname, species_id, since_date):
    species = ala.species_for_scientific_name(species_sname)
    if species is None:
        raise RuntimeError('Species not found at ALA: ' + species_sname)

    num_records = 0
    for record in ala.records_for_species(species.lsid, since_date):
        record.species_id = species_id
        _mp_init.record_q.put(record)
        num_records += 1

    return num_records

def _mysql_encode_binary(binstr):
    '''
    >>> _mysql_encode_binary('hello')
    "x'68656c6c6f'"
    '''
    return "x'" + binascii.hexlify(binstr) + "'"
<commit_msg>More exception logging from subprocesses<commit_after>import db
import urllib2
import ala
import logging
import multiprocessing
import binascii
import uuid
import traceback
from sqlalchemy import func, select

log = logging.getLogger(__name__)

class Syncer:

    def __init__(self):
        '''TODO: might pass db into here for unit testing purposes instead of
        using the module directly. Might also do the same for ala module.'''

        row = db.sources.select('id')\
                .where(db.sources.c.name == 'ALA')\
                .execute().fetchone()

        if row is None:
            raise RuntimeError('ALA row missing from sources table in db')

        self.source_row_id = row['id']
        self.cached_upserts = []
        self.num_dirty_records_by_species_id = {}

    def local_species(self):
        '''Returns all db.species rows in the local database.'''
        return db.species.select().execute().fetchall()

    def local_species_by_scientific_name(self):
        '''Returns all species in the local db in a dict. Scientific name is the
        key, the db row is the value.'''

        species = {}
        for row in db.species.select().execute():
            species[row['scientific_name']] = row;
        return species

    def remote_species_by_scientific_name(self):
        '''Returns all species available at ALA in a dict. Scientific name is the
        key, the ala.Species object is the value.'''

        species = {}
        for bird in ala.all_bird_species():
            species[bird.scientific_name] = bird
        return species

    def added_and_deleted_species(self):
        '''Returns (added, deleted) where `added` is an iterable of ala.Species
        objects that are not present in the local db, and `deleted` is an iterable
        of rows from the db.species table that were not found at ALA.'''

        local = self.local_species_by_scientific_name()
        local_set = frozenset(local.keys())
        remote = self.remote_species_by_scientific_name()
        remote_set = frozenset(remote.keys())

        added_set = remote_set - local_set
        deleted_set = local_set - remote_set

        added = [species for name, species in remote.iteritems() if name in added_set]
        deleted = [row for name, row in local.iteritems() if name in deleted_set]

        return (added, deleted)

    def add_species(self, species):
        '''Adds `species` to the local db, where `s` is an ala.Species
        object'''

        log.info('Adding new species "%s"', species.scientific_name)
        db.species.insert().execute(
            scientific_name=species.scientific_name,
            common_name=species.common_name)

    def delete_species(self, row):
        '''Deletes `row` from the local db, where `s` is a row from the
        db.species table'''

        log.info('Deleting species "%s"', row['scientific_name'])
        db.species.delete().where(db.species.c.id == row['id']).execute()

    def sync_occurrences(self, utc_since_date):
        '''Performs all adding, updating, and deleting of rows in the
        db.occurrences table of the database.

        Also updates db.species.c.num_dirty_occurrences.'''

        # insert new, and update existing, occurrences
        for occ in self.occurrences_changed_since(utc_since_date, True):
            self.upsert_occurrence(occ, occ.species_id)
        self.flush_upserts()

        remote_counts = self.remote_occurrence_counts_by_species_id()

        # delete occurrences that have been deleted at ALA
        self.delete_local_occurrences_if_needed(remote_counts)

        # log warnings if the counts dont match up
        self.check_occurrence_counts(remote_counts)

        # increase number in db.species.num_dirty_occurrences
        self.update_num_dirty_occurrences()

    def delete_local_occurrences_if_needed(self, remote_counts):
        '''This must be called as the last step in syncing occurrence records,
        because adding and updating occurrences will alter the number of
        records per species. Checking for deleted records is expensive in terms
        of memory and time, so it only happens with the local occurrence count
        per species is higher than the count at ALA.

        Builds a set of uuid.UUID objects for records that exist at ALA, then
        checks every local record to see if it still exists in the set.'''

        for row, lc, rc in self.species_with_occurrence_counts(remote_counts):
            # don't run unless our count is lower than ALAs count
            if lc <= rc:
                continue

            log.info('Checking for deleted records for species %s',
                     row['scientific_name'])

            # species should never be None, because we already have a count for
            # it from ALA
            species = ala.species_for_scientific_name(row['scientific_name'])
            if species is None:
                log.critical('species should never be None')
                continue

            # build the set of existing uuids
            existing_uuids = set()
            for occurrence in ala.records_for_species(species.lsid):
                existing_uuids.add(occurrence.uuid)

            # loop through every local occurrence, and store the ids of the
            # rows that do not exist at ALA
            row_ids_to_delete = []
            all_rows_for_species = \
                db.occurrences.select().\
                where(db.occurrences.c.species_id == row['id']).\
                where(db.occurrences.c.source_id == self.source_row_id).\
                execute()
            for occ_row in all_rows_for_species:
                occ_uuid = uuid.UUID(bytes=occ_row['source_record_id'])
                if occ_uuid not in existing_uuids:
                    row_ids_to_delete.append(occ_row['id'])

            # free up some memory, because this may be very large
            del existing_uuids

            # delete all the neccessary local occurrences
            for occ_id in row_ids_to_delete:
                db.occurrences.delete().\
                    where(db.occurrences.c.id == occ_id).\
                    execute()

            # free up more memory
            num_deleted = len(row_ids_to_delete)
            del row_ids_to_delete

            # log and keep track of the deletaions
            log.info('Deleted %d records for %s', num_deleted,
                    row['scientific_name'])

            self.increase_dirty_count(row['id'], num_deleted)

    def check_occurrence_counts(self, remote_counts):
        '''Logs warnings if the local and remote occurrence counts per species
        do not match.'''

        for row, lc, rc in self.species_with_occurrence_counts(remote_counts):
            if lc == rc:
                continue #  counts are the same

            log.warning('Occurrence counts differ for species %s,' +
                        '(local count = %d, ALA count = %d)',
                        row['scientific_name'],
                        lc, rc)

    def species_with_occurrence_counts(self, remote_counts):
        '''Checks the number of local occurrences against the number of
        occurrences at ALA, yielding the db.species row, local count and remote
        count if the counts are different.'''

        local_counts = self.local_occurrence_counts_by_species_id()

        for row in self.local_species():
            if row['id'] in remote_counts:
                yield (row, local_counts[row['id']], remote_counts[row['id']])

    def remote_occurrence_counts_by_species_id(self):
        '''Returns a dict with db.species.c.id keys, and the values are the
        number of occurrences present at ALA for that species.'''

        counts = {}

        for row in self.local_species():
            species = ala.species_for_scientific_name(row['scientific_name'])
            if species is not None:
                counts[row['id']] = ala.num_records_for_lsid(species.lsid)

        return counts;

    def local_occurrence_counts_by_species_id(self):
        '''Returns a dict with db.species.c.id keys, and the calues are the
        number of occurrences present in the local database for that
        species.'''

        counts = {}

        for row in self.local_species():
            local_count = \
                db.engine.execute(select(
                    [func.count('*')],
                    #where
                     (db.occurrences.c.species_id == row['id']) &
                     (db.occurrences.c.source_id == self.source_row_id))
                ).scalar()

            counts[row['id']] = local_count

        return counts

    def upsert_occurrence(self, occurrence, species_id):
        '''Looks up whether `occurrence` (an ala.OccurrenceRecord object)
        already exists in the local db. If it does, the db row is updated with
        the information in `occurrence`. If it does not exist, a new row is
        inserted.

        `species_id` must be supplied as an argument because it is not
        obtainable from `occurrence`

        The inserts/updates are cached for performance reasons. The cache is
        flushed every 1000 occurrences. YOU MUST CALL `flush_upserts` AFTER YOU
        HAVE CALLED THIS METHOD FOR THE FINAL TIME.'''

        if len(self.cached_upserts) > 1000:
            self.flush_upserts()

        # TODO: determine rating better using lists of assertions
        rating = 'assumed invalid'
        if occurrence.is_geospatial_kosher:
            rating = 'assumed valid'

        # these should be escaped strings ready for insertion into the SQL
        cols = (str(float(occurrence.latitude)),
                str(float(occurrence.longitude)),
                '"' + rating + '"',
                str(int(species_id)),
                str(int(self.source_row_id)),
                _mysql_encode_binary(occurrence.uuid.bytes))

        self.cached_upserts.append('(' + ','.join(cols) + ')')

    def flush_upserts(self):
        if len(self.cached_upserts) <= 0:
            return

        query = '''INSERT INTO occurrences(
                        latitude, longitude, rating, species_id, source_id,
                        source_record_id)

                   VALUES ''' + \
                \
                ','.join(self.cached_upserts) + \
                \
                ''' ON DUPLICATE KEY UPDATE
                        latitude=VALUES(latitude),
                        longitude=VALUES(longitude),
                        rating=VALUES(rating),
                        species_id=VALUES(species_id);'''

        db.engine.execute(query)

        self.cached_upserts = []

    def occurrences_changed_since(self, since_date, record_dirty=False):
        '''Generator for ala.OccurrenceRecord objects.

        Will use whatever is in the species table of the database, so call
        update the species table before calling this function.

        Uses a pool of processes to fetch occurrence records. The subprocesses
        feed the records into a queue which the original process reads and
        yields. This should let the main process access the database at full
        speed while the subprocesses are waiting for more records to arrive
        over the network.'''

        record_q = multiprocessing.Queue(10000)
        pool = multiprocessing.Pool(8, _mp_init, [record_q])
        active_workers = 0

        # fill the pool full with every species
        for row in db.species.select().execute():
            args = (row['scientific_name'], row['id'], since_date)
            pool.apply_async(_mp_fetch, args)
            active_workers += 1

        pool.close()

        # keep reading from the queue until all the subprocesses are finished
        while active_workers > 0:
            record = record_q.get()
            if isinstance(record, ala.OccurrenceRecord):
                yield record
            elif isinstance(record, tuple):
                active_workers -= 1
                if len(record) == 3:
                    if record[2] > 0:
                        log.info('Finished processing %d records for %s',
                                 record[2],
                                 record[0])
                    if record_dirty:
                        self.increase_dirty_count(record[1], record[2])
                else:
                    raise RuntimeError('Worker process failed: ' + record[0])
            else:
                raise RuntimeError('Unexpected type coming from record_q: ' +
                                   str(type(record)))

        # all the subprocesses should be dead by now
        pool.join()

    def increase_dirty_count(self, species_id, num_dirty):
        if species_id in self.num_dirty_records_by_species_id:
            self.num_dirty_records_by_species_id[species_id] += num_dirty
        else:
            self.num_dirty_records_by_species_id[species_id] = num_dirty

    def local_species_with_no_occurrences(self):
        '''A generator for db.species rows, for rows without any occurrence
        records in the local database'''

        for row in self.local_species():
            q = select(['count(*)'], db.occurrences.c.species_id == row['id'])
            if db.engine.execute(q).scalar() == 0:
                yield row

    def update_num_dirty_occurrences(self):
        '''Updates the species.num_dirty_occurrences column with the number of
        occurrences that have been changed by self.

        TODO: account for deleted records
        '''
        dirty_col = db.species.c.num_dirty_occurrences

        for row in self.local_species():
            if row['id'] not in self.num_dirty_records_by_species_id:
                continue

            newly_dirty = self.num_dirty_records_by_species_id[row['id']]
            if newly_dirty <= 0:
                continue

            db.species.update().\
                values(num_dirty_occurrences=(dirty_col + newly_dirty)).\
                where(db.species.c.id == row['id']).\
                execute()

def _mp_init(record_q):
    '''Called when a subprocess is started. See
    Syncer.occurrences_changed_since'''
    _mp_init.record_q = record_q
    #_mp_init.log = multiprocessing.log_to_stderr()

def _mp_fetch(species_sname, species_id, since_date):
    '''Gets all relevant records for the given species from ALA, and pumps the
    records into _mp_init.record_q.

    If the function finished successfully, will put a len 3 tuple in the
    record_q with (scientific_name, species_id, num_records_found). If the
    function fails, will put a len 1 tuple in the record_q with a failure
    message string in it.

    Adds a `species_id` attribute to each ala.OccurrenceRecord object set to
    the argument given to this function.'''

    num_records = 0
    failure_msg = None
    try:
        num_records = _mp_fetch_inner(species_sname, species_id, since_date)
    except Exception, e:
        failure_msg = str(e)
        failure_msg += traceback.format_exc()
        if isinstance(e, urllib2.HTTPError):
            failure_msg += '\n\nResponse Headers:\n' + str(dict(e.info()))
            failure_msg += '\n\nResponse Payload:\n' + e.read()

    if failure_msg is None:
        _mp_init.record_q.put((species_sname, species_id, num_records))
    else:
        _mp_init.record_q.put((failure_msg,))

def _mp_fetch_inner(species_sname, species_id, since_date):
    species = ala.species_for_scientific_name(species_sname)
    if species is None:
        raise RuntimeError('Species not found at ALA: ' + species_sname)

    num_records = 0
    for record in ala.records_for_species(species.lsid, since_date):
        record.species_id = species_id
        _mp_init.record_q.put(record)
        num_records += 1

    return num_records

def _mysql_encode_binary(binstr):
    '''
    >>> _mysql_encode_binary('hello')
    "x'68656c6c6f'"
    '''
    return "x'" + binascii.hexlify(binstr) + "'"
<|endoftext|>


<commit_before># pylint: disable=C0103,C0111

import mock
import unittest
import importlib

import tests.mocks as mocks

from bumblebee.engine import all_modules
from bumblebee.output import Widget
from bumblebee.config import Config

class TestGenericModules(unittest.TestCase):
    def setUp(self):
        engine = mock.Mock()
        engine.input = mock.Mock()
        config = Config()
        self.objects = {}
        for mod in all_modules():
            cls = importlib.import_module("bumblebee.modules.{}".format(mod["name"]))
            self.objects[mod["name"]] = getattr(cls, "Module")(engine, {"config": config})
            for widget in self.objects[mod["name"]].widgets():
                self.assertEquals(widget.get("variable", None), None)

    def test_widgets(self):
        popen = mocks.MockPopen()
        popen.mock.communicate.return_value = (str.encode("1"), "error")
        popen.mock.returncode = 0

        for mod in self.objects:
            widgets = self.objects[mod].widgets()
            for widget in widgets:
                widget.link_module(self.objects[mod])
                self.assertEquals(widget.module, mod)
                self.assertTrue(isinstance(widget, Widget))
                self.assertTrue(hasattr(widget, "full_text"))
                widget.set("variable", "value")
                self.assertEquals(widget.get("variable", None), "value")
                self.assertTrue(isinstance(widget.full_text(), str) or isinstance(widget.full_text(), unicode))
        popen.cleanup()

    def test_update(self):
        popen = mocks.MockPopen()
        popen.mock.communicate.return_value = (str.encode("1"), "error")
        popen.mock.returncode = 0
        for mod in self.objects:
            widgets = self.objects[mod].widgets()
            self.objects[mod].update(widgets)
            self.test_widgets()
            self.assertEquals(widgets, self.objects[mod].widgets())
        popen.cleanup()

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
<commit_msg>[tests] Maybe patching the platform call fixes the issue with Travis<commit_after># pylint: disable=C0103,C0111

import mock
import unittest
import importlib

import tests.mocks as mocks

from bumblebee.engine import all_modules
from bumblebee.output import Widget
from bumblebee.config import Config

class TestGenericModules(unittest.TestCase):
    def setUp(self):
        engine = mock.Mock()
        engine.input = mock.Mock()
        config = Config()
        self.objects = {}

        self.popen = mocks.MockPopen()
        self.popen.mock.communicate.return_value = (str.encode("1"), "error")
        self.popen.mock.returncode = 0

        self._platform = mock.patch("bumblebee.modules.kernel.platform")
        self.platform = self._platform.start()
        self.platform.release.return_value = "unknown linux v1"

        for mod in all_modules():
            name = "bumblebee.modules.{}".format(mod["name"])
            cls = importlib.import_module(name)
            self.objects[mod["name"]] = getattr(cls, "Module")(engine, {"config": config})
            for widget in self.objects[mod["name"]].widgets():
                self.assertEquals(widget.get("variable", None), None)

    def tearDown(self):
        self._platform.stop()
        self.popen.cleanup()

    def test_widgets(self):
        for mod in self.objects:
            widgets = self.objects[mod].widgets()
            for widget in widgets:
                widget.link_module(self.objects[mod])
                self.assertEquals(widget.module, mod)
                self.assertTrue(isinstance(widget, Widget))
                self.assertTrue(hasattr(widget, "full_text"))
                widget.set("variable", "value")
                self.assertEquals(widget.get("variable", None), "value")
                self.assertTrue(isinstance(widget.full_text(), str) or isinstance(widget.full_text(), unicode))

    def test_update(self):
        for mod in self.objects:
            widgets = self.objects[mod].widgets()
            self.objects[mod].update(widgets)
            self.test_widgets()
            self.assertEquals(widgets, self.objects[mod].widgets())

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
<|endoftext|>


<commit_before>from __future__ import unicode_literals
from redis._compat import unicode

def wait_for_command(client, monitor, command):
    # issue a command with a key name that's local to this process.
    # if we find a command with our key before the command we're waiting
    # for, something went wrong
    key = '__REDIS-PY-%s__' % str(client.client_id())
    client.get(key)
    while True:
        monitor_response = monitor.next_command()
        if command in monitor_response['command']:
            return monitor_response
        if key in monitor_response['command']:
            return None

class TestPipeline(object):
    def test_wait_command_not_found(self, r):
        "Make sure the wait_for_command func works when command is not found"
        with r.monitor() as m:
            response = wait_for_command(r, m, 'nothing')
            assert response is None

    def test_response_values(self, r):
        with r.monitor() as m:
            r.ping()
            response = wait_for_command(r, m, 'PING')
            assert isinstance(response['time'], float)
            assert response['db'] == 9
            assert response['client_type'] in ('tcp', 'unix')
            assert isinstance(response['client_address'], unicode)
            assert isinstance(response['client_port'], unicode)
            assert response['command'] == 'PING'

    def test_command_with_quoted_key(self, r):
        with r.monitor() as m:
            r.get('foo"bar')
            response = wait_for_command(r, m, 'GET foo"bar')
            assert response['command'] == 'GET foo"bar'

    def test_command_with_binary_data(self, r):
        with r.monitor() as m:
            byte_string = b'foo\x92'
            r.get(byte_string)
            response = wait_for_command(r, m, 'GET foo\\x92')
            assert response['command'] == 'GET foo\\x92'

    def test_lua_script(self, r):
        with r.monitor() as m:
            script = 'return redis.call("GET", "foo")'
            assert r.eval(script, 0) is None
            response = wait_for_command(r, m, 'GET foo')
            assert response['command'] == 'GET foo'
            assert response['client_type'] == 'lua'
            assert response['client_address'] == 'lua'
            assert response['client_port'] == ''
<commit_msg>Support old EPEL-7 Redis. (#1227)<commit_after>from __future__ import unicode_literals
from redis._compat import unicode
from .conftest import (skip_if_server_version_lt, skip_if_server_version_gte,
                       skip_unless_arch_bits)

def wait_for_command(client, monitor, command):
    # issue a command with a key name that's local to this process.
    # if we find a command with our key before the command we're waiting
    # for, something went wrong
    key = '__REDIS-PY-%s__' % str(client.client_id())
    client.get(key)
    while True:
        monitor_response = monitor.next_command()
        if command in monitor_response['command']:
            return monitor_response
        if key in monitor_response['command']:
            return None

class TestPipeline(object):
    @skip_if_server_version_lt('5.0.0')
    def test_wait_command_not_found(self, r):
        "Make sure the wait_for_command func works when command is not found"
        with r.monitor() as m:
            response = wait_for_command(r, m, 'nothing')
            assert response is None

    @skip_if_server_version_lt('5.0.0')
    def test_response_values(self, r):
        with r.monitor() as m:
            r.ping()
            response = wait_for_command(r, m, 'PING')
            assert isinstance(response['time'], float)
            assert response['db'] == 9
            assert response['client_type'] in ('tcp', 'unix')
            assert isinstance(response['client_address'], unicode)
            assert isinstance(response['client_port'], unicode)
            assert response['command'] == 'PING'

    @skip_if_server_version_lt('5.0.0')
    def test_command_with_quoted_key(self, r):
        with r.monitor() as m:
            r.get('foo"bar')
            response = wait_for_command(r, m, 'GET foo"bar')
            assert response['command'] == 'GET foo"bar'

    @skip_if_server_version_lt('5.0.0')
    def test_command_with_binary_data(self, r):
        with r.monitor() as m:
            byte_string = b'foo\x92'
            r.get(byte_string)
            response = wait_for_command(r, m, 'GET foo\\x92')
            assert response['command'] == 'GET foo\\x92'

    @skip_if_server_version_lt('5.0.0')
    def test_lua_script(self, r):
        with r.monitor() as m:
            script = 'return redis.call("GET", "foo")'
            assert r.eval(script, 0) is None
            response = wait_for_command(r, m, 'GET foo')
            assert response['command'] == 'GET foo'
            assert response['client_type'] == 'lua'
            assert response['client_address'] == 'lua'
            assert response['client_port'] == ''
<|endoftext|>


<commit_before># Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
# For details: https://github.com/nedbat/coveragepy/blob/master/NOTICE.txt

"""Tests for coverage.numbits"""

from hypothesis import given, settings
from hypothesis.strategies import sets, integers

from coverage import env
from coverage.numbits import (
    nums_to_numbits, numbits_to_nums, merge_numbits, numbits_any_intersection,
    num_in_numbits,
    )

from tests.coveragetest import CoverageTest

# Hypothesis-generated line number data
line_number = integers(min_value=1, max_value=9999)
line_numbers = sets(line_number, min_size=1)

# When coverage-testing ourselves, hypothesis complains about a test being
# flaky because the first run exceeds the deadline (and fails), and the second
# run succeeds.  Disable the deadline if we are coverage-testing.
default_settings = settings()
if env.METACOV:
    default_settings = settings(default_settings, deadline=None)

class NumbitsOpTest(CoverageTest):
    """Tests of the numbits operations in numbits.py."""

    run_in_temp_dir = False

    @given(line_numbers)
    @settings(default_settings)
    def test_conversion(self, nums):
        nums2 = numbits_to_nums(nums_to_numbits(nums))
        self.assertEqual(nums, set(nums2))

    @given(line_numbers, line_numbers)
    @settings(default_settings)
    def test_merging(self, nums1, nums2):
        merged = numbits_to_nums(merge_numbits(nums_to_numbits(nums1), nums_to_numbits(nums2)))
        self.assertEqual(nums1 | nums2, set(merged))

    @given(line_numbers, line_numbers)
    @settings(default_settings)
    def test_any_intersection(self, nums1, nums2):
        inter = numbits_any_intersection(nums_to_numbits(nums1), nums_to_numbits(nums2))
        expect = bool(nums1 & nums2)
        self.assertEqual(expect, bool(inter))

    @given(line_number, line_numbers)
    @settings(default_settings)
    def test_num_in_numbits(self, num, nums):
        numbits = nums_to_numbits(nums)
        is_in = num_in_numbits(num, numbits)
        self.assertEqual(num in nums, is_in)
<commit_msg>Use more conventional strategy naming<commit_after># Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
# For details: https://github.com/nedbat/coveragepy/blob/master/NOTICE.txt

"""Tests for coverage.numbits"""

from hypothesis import given, settings
from hypothesis.strategies import sets, integers

from coverage import env
from coverage.numbits import (
    nums_to_numbits, numbits_to_nums, merge_numbits, numbits_any_intersection,
    num_in_numbits,
    )

from tests.coveragetest import CoverageTest

# Hypothesis-generated line number data
line_numbers = integers(min_value=1, max_value=9999)
line_number_sets = sets(line_numbers, min_size=1)

# When coverage-testing ourselves, hypothesis complains about a test being
# flaky because the first run exceeds the deadline (and fails), and the second
# run succeeds.  Disable the deadline if we are coverage-testing.
default_settings = settings()
if env.METACOV:
    default_settings = settings(default_settings, deadline=None)

class NumbitsOpTest(CoverageTest):
    """Tests of the numbits operations in numbits.py."""

    run_in_temp_dir = False

    @given(line_number_sets)
    @settings(default_settings)
    def test_conversion(self, nums):
        nums2 = numbits_to_nums(nums_to_numbits(nums))
        self.assertEqual(nums, set(nums2))

    @given(line_number_sets, line_number_sets)
    @settings(default_settings)
    def test_merging(self, nums1, nums2):
        merged = numbits_to_nums(merge_numbits(nums_to_numbits(nums1), nums_to_numbits(nums2)))
        self.assertEqual(nums1 | nums2, set(merged))

    @given(line_number_sets, line_number_sets)
    @settings(default_settings)
    def test_any_intersection(self, nums1, nums2):
        inter = numbits_any_intersection(nums_to_numbits(nums1), nums_to_numbits(nums2))
        expect = bool(nums1 & nums2)
        self.assertEqual(expect, bool(inter))

    @given(line_numbers, line_number_sets)
    @settings(default_settings)
    def test_num_in_numbits(self, num, nums):
        numbits = nums_to_numbits(nums)
        is_in = num_in_numbits(num, numbits)
        self.assertEqual(num in nums, is_in)
<|endoftext|>


<commit_before>import pandas as pd
from unittest import TestCase  # or `from unittest import ...` if on Python 3.4+
import numpy as np
import tests.helpers as th

import category_encoders as encoders

np_X = th.create_array(n_rows=100)
np_X_t = th.create_array(n_rows=50, extras=True)
np_y = np.random.randn(np_X.shape[0]) > 0.5
np_y_t = np.random.randn(np_X_t.shape[0]) > 0.5
X = th.create_dataset(n_rows=100)
X_t = th.create_dataset(n_rows=50, extras=True)
y = pd.DataFrame(np_y)
y_t = pd.DataFrame(np_y_t)

class TestOneHotEncoderTestCase(TestCase):

    def test_one_hot(self):
        enc = encoders.OneHotEncoder(verbose=1, return_df=False)
        enc.fit(X)
        self.assertEqual(enc.transform(X_t).shape[1],
                         enc.transform(X).shape[1],
                         'We have to get the same count of columns despite the presence of a new value')

        enc = encoders.OneHotEncoder(verbose=1, return_df=True, handle_unknown='indicator')
        enc.fit(X)
        out = enc.transform(X_t)
        self.assertIn('extra_-1', out.columns.values)

        enc = encoders.OneHotEncoder(verbose=1, return_df=True, handle_unknown='return_nan')
        enc.fit(X)
        out = enc.transform(X_t)
        self.assertEqual(len([x for x in out.columns.values if str(x).startswith('extra_')]), 3)

        enc = encoders.OneHotEncoder(verbose=1, return_df=True, handle_unknown='error')
        # The exception is already raised in fit() because transform() is called there to get
        # feature_names right.
        enc.fit(X)
        with self.assertRaises(ValueError):
            enc.transform(X_t)

        enc = encoders.OneHotEncoder(verbose=1, return_df=True, handle_unknown='return_nan', use_cat_names=True)
        enc.fit(X)
        out = enc.transform(X_t)
        self.assertIn('extra_A', out.columns.values)

        enc = encoders.OneHotEncoder(verbose=1, return_df=True, use_cat_names=True, handle_unknown='indicator')
        enc.fit(X)
        out = enc.transform(X_t)
        self.assertIn('extra_-1', out.columns.values)

        # test inverse_transform
        X_i = th.create_dataset(n_rows=100, has_missing=False)
        X_i_t = th.create_dataset(n_rows=50, has_missing=False)
        cols = ['underscore', 'none', 'extra', 321, 'categorical']

        enc = encoders.OneHotEncoder(verbose=1, use_cat_names=True, cols=cols)
        enc.fit(X_i)
        obtained = enc.inverse_transform(enc.transform(X_i_t))
        th.verify_inverse_transform(X_i_t, obtained)

    def test_fit_transform_HaveMissingValuesAndUseCatNames_ExpectCorrectValue(self):
        encoder = encoders.OneHotEncoder(cols=[0], use_cat_names=True, handle_unknown='indicator', return_df=False)

        result = encoder.fit_transform([[-1]])

        self.assertListEqual([[1, 0]], result.tolist())

    def test_inverse_transform_HaveDedupedColumns_ExpectCorrectInverseTransform(self):
        encoder = encoders.OneHotEncoder(cols=['match', 'match_box'], use_cat_names=True)
        value = pd.DataFrame({'match': pd.Series('box_-1'), 'match_box': pd.Series(-1)})

        transformed = encoder.fit_transform(value)
        inverse_transformed = encoder.inverse_transform(transformed)

        assert value.equals(inverse_transformed)

    def test_inverse_transform_HaveNoCatNames_ExpectCorrectInverseTransform(self):
        encoder = encoders.OneHotEncoder(cols=['match', 'match_box'], use_cat_names=False)
        value = pd.DataFrame({'match': pd.Series('box_-1'), 'match_box': pd.Series(-1)})

        transformed = encoder.fit_transform(value)
        inverse_transformed = encoder.inverse_transform(transformed)

        assert value.equals(inverse_transformed)

    def test_fit_transform_HaveColumnAppearTwice_ExpectColumnsDeduped(self):
        encoder = encoders.OneHotEncoder(cols=['match', 'match_box'], use_cat_names=True, handle_unknown='indicator')
        value = pd.DataFrame({'match': pd.Series('box_-1'), 'match_box': pd.Series('-1')})

        result = encoder.fit_transform(value)
        columns = result.columns.tolist()

        self.assertSetEqual({'match_box_-1', 'match_-1', 'match_box_-1#', 'match_box_-1##'}, set(columns))

    def test_fit_transform_HaveHandleUnknownValueAndUnseenValues_ExpectAllZeroes(self):
        train = pd.DataFrame({'city': ['Chicago', 'Seattle']})
        test = pd.DataFrame({'city': ['Chicago', 'Detroit']})
        expected_result = pd.DataFrame({'city_1': [1, 0],
                                        'city_2': [0, 0]},
                                       columns=['city_1', 'city_2'])

        enc = encoders.OneHotEncoder(handle_unknown='value')
        result = enc.fit(train).transform(test)

        pd.testing.assert_frame_equal(expected_result, result)

    def test_fit_transform_HaveHandleUnknownValueAndSeenValues_ExpectMappingUsed(self):
        train = pd.DataFrame({'city': ['Chicago', 'Seattle']})
        expected_result = pd.DataFrame({'city_1': [1, 0],
                                        'city_2': [0, 1]},
                                       columns=['city_1', 'city_2'])

        enc = encoders.OneHotEncoder(handle_unknown='value')
        result = enc.fit(train).transform(train)

        pd.testing.assert_frame_equal(expected_result, result)

    def test_fit_transform_HaveHandleUnknownIndicatorAndNoMissingValue_ExpectExtraColumn(self):
        train = pd.DataFrame({'city': ['Chicago', 'Seattle']})
        expected_result = pd.DataFrame({'city_1': [1, 0],
                                        'city_2': [0, 1],
                                        'city_-1': [0, 0]},
                                       columns=['city_1', 'city_2', 'city_-1'])

        enc = encoders.OneHotEncoder(handle_unknown='indicator')
        result = enc.fit(train).transform(train)

        pd.testing.assert_frame_equal(expected_result, result)

    def test_fit_transform_HaveHandleUnknownIndicatorAndMissingValue_ExpectValueSet(self):
        train = pd.DataFrame({'city': ['Chicago', 'Seattle']})
        test = pd.DataFrame({'city': ['Chicago', 'Detroit']})
        expected_result = pd.DataFrame({'city_1': [1, 0],
                                        'city_2': [0, 0],
                                        'city_-1': [0, 1]},
                                       columns=['city_1', 'city_2', 'city_-1'])

        enc = encoders.OneHotEncoder(handle_unknown='indicator')
        result = enc.fit(train).transform(test)

        pd.testing.assert_frame_equal(expected_result, result)

    def test_HandleMissingError(self):
        data_no_missing = ['A', 'B', 'B']
        data_w_missing = [np.nan, 'B', 'B']
        encoder = encoders.OneHotEncoder(handle_missing="error")

        result = encoder.fit_transform(data_no_missing)
        expected = [[1, 0],
                    [0, 1],
                    [0, 1]]
        self.assertEqual(result.values.tolist(), expected)

        self.assertRaisesRegex(ValueError, '.*null.*', encoder.transform, data_w_missing)

        self.assertRaisesRegex(ValueError, '.*null.*', encoder.fit, data_w_missing)

    def test_HandleMissingReturnNan(self):
        train = pd.DataFrame({'x': ['A', np.nan, 'B']})
        encoder = encoders.OneHotEncoder(handle_missing='return_nan', use_cat_names=True)
        result = encoder.fit_transform(train)
        pd.testing.assert_frame_equal(
            result,
            pd.DataFrame({'x_A': [1, np.nan, 0], 'x_B': [0, np.nan, 1]}),
        )

    def test_HandleMissingIndicator_NanInTrain_ExpectAsColumn(self):
        train = ['A', 'B', np.nan]

        encoder = encoders.OneHotEncoder(handle_missing='indicator', handle_unknown='value')
        result = encoder.fit_transform(train)

        expected = [[1, 0, 0],
                    [0, 1, 0],
                    [0, 0, 1]]
        self.assertEqual(result.values.tolist(), expected)

    def test_HandleMissingIndicator_HaveNoNan_ExpectSecondColumn(self):
        train = ['A', 'B']

        encoder = encoders.OneHotEncoder(handle_missing='indicator', handle_unknown='value')
        result = encoder.fit_transform(train)

        expected = [[1, 0, 0],
                    [0, 1, 0]]
        self.assertEqual(result.values.tolist(), expected)

    def test_HandleMissingIndicator_NanNoNanInTrain_ExpectAsNanColumn(self):
        train = ['A', 'B']
        test = ['A', 'B', np.nan]

        encoder = encoders.OneHotEncoder(handle_missing='indicator', handle_unknown='value')
        encoder.fit(train)
        result = encoder.transform(test)

        expected = [[1, 0, 0],
                    [0, 1, 0],
                    [0, 0, 1]]
        self.assertEqual(result.values.tolist(), expected)

    def test_HandleUnknown_HaveNoUnknownInTrain_ExpectIndicatorInTest(self):
        train = ['A', 'B']
        test = ['A', 'B', 'C']

        encoder = encoders.OneHotEncoder(handle_unknown='indicator', handle_missing='value')
        encoder.fit(train)
        result = encoder.transform(test)

        expected = [[1, 0, 0],
                    [0, 1, 0],
                    [0, 0, 1]]
        self.assertEqual(result.values.tolist(), expected)

    def test_HandleUnknown_HaveOnlyKnown_ExpectSecondColumn(self):
        train = ['A', 'B']

        encoder = encoders.OneHotEncoder(handle_unknown='indicator', handle_missing='value')
        result = encoder.fit_transform(train)

        expected = [[1, 0, 0],
                    [0, 1, 0]]
        self.assertEqual(result.values.tolist(), expected)

    def test_inverse_transform_HaveNanInTrainAndHandleMissingValue_ExpectReturnedWithNan(self):
        train = pd.DataFrame({'city': ['chicago', np.nan]})

        enc = encoders.OneHotEncoder(handle_missing='value', handle_unknown='value')
        result = enc.fit_transform(train)
        original = enc.inverse_transform(result)

        pd.testing.assert_frame_equal(train, original)

    def test_inverse_transform_HaveNanInTrainAndHandleMissingReturnNan_ExpectReturnedWithNan(self):
        train = pd.DataFrame({'city': ['chicago', np.nan]})

        enc = encoders.OneHotEncoder(handle_missing='return_nan', handle_unknown='value')
        result = enc.fit_transform(train)
        original = enc.inverse_transform(result)

        pd.testing.assert_frame_equal(train, original)

    def test_inverse_transform_BothFieldsAreReturnNanWithNan_ExpectValueError(self):
        train = pd.DataFrame({'city': ['chicago', np.nan]})
        test = pd.DataFrame({'city': ['chicago', 'los angeles']})

        enc = encoders.OneHotEncoder(handle_missing='return_nan', handle_unknown='return_nan')
        enc.fit(train)
        result = enc.transform(test)

        message = 'inverse_transform is not supported because transform impute '\
                  'the unknown category nan when encode city'

        with self.assertWarns(UserWarning, msg=message) as w:
            enc.inverse_transform(result)

    def test_inverse_transform_HaveMissingAndNoUnknown_ExpectInversed(self):
        train = pd.DataFrame({'city': ['chicago', np.nan]})
        test = pd.DataFrame({'city': ['chicago', 'los angeles']})

        enc = encoders.OneHotEncoder(handle_missing='value', handle_unknown='return_nan')
        enc.fit(train)
        result = enc.transform(test)
        original = enc.inverse_transform(result)

        pd.testing.assert_frame_equal(train, original)

    def test_inverse_transform_HaveHandleMissingValueAndHandleUnknownReturnNan_ExpectBestInverse(self):
        train = pd.DataFrame({'city': ['chicago', np.nan]})
        test = pd.DataFrame({'city': ['chicago', np.nan, 'los angeles']})
        expected = pd.DataFrame({'city': ['chicago', np.nan, np.nan]})

        enc = encoders.OneHotEncoder(handle_missing='value', handle_unknown='return_nan')
        enc.fit(train)
        result = enc.transform(test)
        original = enc.inverse_transform(result)

        pd.testing.assert_frame_equal(expected, original)
<commit_msg>More explicit test<commit_after>import pandas as pd
from unittest import TestCase  # or `from unittest import ...` if on Python 3.4+
import numpy as np
import tests.helpers as th

import category_encoders as encoders

np_X = th.create_array(n_rows=100)
np_X_t = th.create_array(n_rows=50, extras=True)
np_y = np.random.randn(np_X.shape[0]) > 0.5
np_y_t = np.random.randn(np_X_t.shape[0]) > 0.5
X = th.create_dataset(n_rows=100)
X_t = th.create_dataset(n_rows=50, extras=True)
y = pd.DataFrame(np_y)
y_t = pd.DataFrame(np_y_t)

class TestOneHotEncoderTestCase(TestCase):

    def test_one_hot(self):
        enc = encoders.OneHotEncoder(verbose=1, return_df=False)
        enc.fit(X)
        self.assertEqual(enc.transform(X_t).shape[1],
                         enc.transform(X).shape[1],
                         'We have to get the same count of columns despite the presence of a new value')

        enc = encoders.OneHotEncoder(verbose=1, return_df=True, handle_unknown='indicator')
        enc.fit(X)
        out = enc.transform(X_t)
        self.assertIn('extra_-1', out.columns.values)

        enc = encoders.OneHotEncoder(verbose=1, return_df=True, handle_unknown='return_nan')
        enc.fit(X)
        out = enc.transform(X_t)
        self.assertEqual(len([x for x in out.columns.values if str(x).startswith('extra_')]), 3)

        enc = encoders.OneHotEncoder(verbose=1, return_df=True, handle_unknown='error')
        # The exception is already raised in fit() because transform() is called there to get
        # feature_names right.
        enc.fit(X)
        with self.assertRaises(ValueError):
            enc.transform(X_t)

        enc = encoders.OneHotEncoder(verbose=1, return_df=True, handle_unknown='return_nan', use_cat_names=True)
        enc.fit(X)
        out = enc.transform(X_t)
        self.assertIn('extra_A', out.columns.values)

        enc = encoders.OneHotEncoder(verbose=1, return_df=True, use_cat_names=True, handle_unknown='indicator')
        enc.fit(X)
        out = enc.transform(X_t)
        self.assertIn('extra_-1', out.columns.values)

        # test inverse_transform
        X_i = th.create_dataset(n_rows=100, has_missing=False)
        X_i_t = th.create_dataset(n_rows=50, has_missing=False)
        cols = ['underscore', 'none', 'extra', 321, 'categorical']

        enc = encoders.OneHotEncoder(verbose=1, use_cat_names=True, cols=cols)
        enc.fit(X_i)
        obtained = enc.inverse_transform(enc.transform(X_i_t))
        th.verify_inverse_transform(X_i_t, obtained)

    def test_fit_transform_HaveMissingValuesAndUseCatNames_ExpectCorrectValue(self):
        encoder = encoders.OneHotEncoder(cols=[0], use_cat_names=True, handle_unknown='indicator', return_df=False)

        result = encoder.fit_transform([[-1]])

        self.assertListEqual([[1, 0]], result.tolist())

    def test_inverse_transform_HaveDedupedColumns_ExpectCorrectInverseTransform(self):
        encoder = encoders.OneHotEncoder(cols=['match', 'match_box'], use_cat_names=True)
        value = pd.DataFrame({'match': pd.Series('box_-1'), 'match_box': pd.Series(-1)})

        transformed = encoder.fit_transform(value)
        inverse_transformed = encoder.inverse_transform(transformed)

        assert value.equals(inverse_transformed)

    def test_inverse_transform_HaveNoCatNames_ExpectCorrectInverseTransform(self):
        encoder = encoders.OneHotEncoder(cols=['match', 'match_box'], use_cat_names=False)
        value = pd.DataFrame({'match': pd.Series('box_-1'), 'match_box': pd.Series(-1)})

        transformed = encoder.fit_transform(value)
        inverse_transformed = encoder.inverse_transform(transformed)

        assert value.equals(inverse_transformed)

    def test_fit_transform_HaveColumnAppearTwice_ExpectColumnsDeduped(self):
        encoder = encoders.OneHotEncoder(cols=['match', 'match_box'], use_cat_names=True, handle_unknown='indicator')
        value = pd.DataFrame({'match': pd.Series('box_-1'), 'match_box': pd.Series('-1')})

        result = encoder.fit_transform(value)
        columns = result.columns.tolist()

        self.assertSetEqual({'match_box_-1', 'match_-1', 'match_box_-1#', 'match_box_-1##'}, set(columns))

    def test_fit_transform_HaveHandleUnknownValueAndUnseenValues_ExpectAllZeroes(self):
        train = pd.DataFrame({'city': ['Chicago', 'Seattle']})
        test = pd.DataFrame({'city': ['Chicago', 'Detroit']})
        expected_result = pd.DataFrame({'city_1': [1, 0],
                                        'city_2': [0, 0]},
                                       columns=['city_1', 'city_2'])

        enc = encoders.OneHotEncoder(handle_unknown='value')
        result = enc.fit(train).transform(test)

        pd.testing.assert_frame_equal(expected_result, result)

    def test_fit_transform_HaveHandleUnknownValueAndSeenValues_ExpectMappingUsed(self):
        train = pd.DataFrame({'city': ['Chicago', 'Seattle']})
        expected_result = pd.DataFrame({'city_1': [1, 0],
                                        'city_2': [0, 1]},
                                       columns=['city_1', 'city_2'])

        enc = encoders.OneHotEncoder(handle_unknown='value')
        result = enc.fit(train).transform(train)

        pd.testing.assert_frame_equal(expected_result, result)

    def test_fit_transform_HaveHandleUnknownIndicatorAndNoMissingValue_ExpectExtraColumn(self):
        train = pd.DataFrame({'city': ['Chicago', 'Seattle']})
        expected_result = pd.DataFrame({'city_1': [1, 0],
                                        'city_2': [0, 1],
                                        'city_-1': [0, 0]},
                                       columns=['city_1', 'city_2', 'city_-1'])

        enc = encoders.OneHotEncoder(handle_unknown='indicator')
        result = enc.fit(train).transform(train)

        pd.testing.assert_frame_equal(expected_result, result)

    def test_fit_transform_HaveHandleUnknownIndicatorAndMissingValue_ExpectValueSet(self):
        train = pd.DataFrame({'city': ['Chicago', 'Seattle']})
        test = pd.DataFrame({'city': ['Chicago', 'Detroit']})
        expected_result = pd.DataFrame({'city_1': [1, 0],
                                        'city_2': [0, 0],
                                        'city_-1': [0, 1]},
                                       columns=['city_1', 'city_2', 'city_-1'])

        enc = encoders.OneHotEncoder(handle_unknown='indicator')
        result = enc.fit(train).transform(test)

        pd.testing.assert_frame_equal(expected_result, result)

    def test_HandleMissingError(self):
        data_no_missing = ['A', 'B', 'B']
        data_w_missing = [np.nan, 'B', 'B']
        encoder = encoders.OneHotEncoder(handle_missing="error")

        result = encoder.fit_transform(data_no_missing)
        expected = [[1, 0],
                    [0, 1],
                    [0, 1]]
        self.assertEqual(result.values.tolist(), expected)

        self.assertRaisesRegex(ValueError, '.*null.*', encoder.transform, data_w_missing)

        self.assertRaisesRegex(ValueError, '.*null.*', encoder.fit, data_w_missing)

    def test_HandleMissingReturnNan(self):
        train = pd.DataFrame({'x': ['A', np.nan, 'B']})
        encoder = encoders.OneHotEncoder(handle_missing='return_nan', use_cat_names=True)
        result = encoder.fit_transform(train)
        pd.testing.assert_frame_equal(
            result,
            pd.DataFrame({'x_A': [1, np.nan, 0], 'x_B': [0, np.nan, 1]}),
        )

    def test_HandleMissingIndicator_NanInTrain_ExpectAsColumn(self):
        train = ['A', 'B', np.nan]

        encoder = encoders.OneHotEncoder(handle_missing='indicator', handle_unknown='value')
        result = encoder.fit_transform(train)

        expected = [[1, 0, 0],
                    [0, 1, 0],
                    [0, 0, 1]]
        self.assertEqual(result.values.tolist(), expected)

    def test_HandleMissingIndicator_HaveNoNan_ExpectSecondColumn(self):
        train = ['A', 'B']

        encoder = encoders.OneHotEncoder(handle_missing='indicator', handle_unknown='value')
        result = encoder.fit_transform(train)

        expected = [[1, 0, 0],
                    [0, 1, 0]]
        self.assertEqual(result.values.tolist(), expected)

    def test_HandleMissingIndicator_NanNoNanInTrain_ExpectAsNanColumn(self):
        train = ['A', 'B']
        test = ['A', 'B', np.nan]

        encoder = encoders.OneHotEncoder(handle_missing='indicator', handle_unknown='value')
        encoded_train = encoder.fit_transform(train)
        encoded_test = encoder.transform(test)

        expected_1 = [[1, 0, 0],
                      [0, 1, 0]]
        self.assertEqual(encoded_train.values.tolist(), expected_1)

        expected_2 = [[1, 0, 0],
                      [0, 1, 0],
                      [0, 0, 1]]
        self.assertEqual(encoded_test.values.tolist(), expected_2)

    def test_HandleUnknown_HaveNoUnknownInTrain_ExpectIndicatorInTest(self):
        train = ['A', 'B']
        test = ['A', 'B', 'C']

        encoder = encoders.OneHotEncoder(handle_unknown='indicator', handle_missing='value')
        encoder.fit(train)
        result = encoder.transform(test)

        expected = [[1, 0, 0],
                    [0, 1, 0],
                    [0, 0, 1]]
        self.assertEqual(result.values.tolist(), expected)

    def test_HandleUnknown_HaveOnlyKnown_ExpectSecondColumn(self):
        train = ['A', 'B']

        encoder = encoders.OneHotEncoder(handle_unknown='indicator', handle_missing='value')
        result = encoder.fit_transform(train)

        expected = [[1, 0, 0],
                    [0, 1, 0]]
        self.assertEqual(result.values.tolist(), expected)

    def test_inverse_transform_HaveNanInTrainAndHandleMissingValue_ExpectReturnedWithNan(self):
        train = pd.DataFrame({'city': ['chicago', np.nan]})

        enc = encoders.OneHotEncoder(handle_missing='value', handle_unknown='value')
        result = enc.fit_transform(train)
        original = enc.inverse_transform(result)

        pd.testing.assert_frame_equal(train, original)

    def test_inverse_transform_HaveNanInTrainAndHandleMissingReturnNan_ExpectReturnedWithNan(self):
        train = pd.DataFrame({'city': ['chicago', np.nan]})

        enc = encoders.OneHotEncoder(handle_missing='return_nan', handle_unknown='value')
        result = enc.fit_transform(train)
        original = enc.inverse_transform(result)

        pd.testing.assert_frame_equal(train, original)

    def test_inverse_transform_BothFieldsAreReturnNanWithNan_ExpectValueError(self):
        train = pd.DataFrame({'city': ['chicago', np.nan]})
        test = pd.DataFrame({'city': ['chicago', 'los angeles']})

        enc = encoders.OneHotEncoder(handle_missing='return_nan', handle_unknown='return_nan')
        enc.fit(train)
        result = enc.transform(test)

        message = 'inverse_transform is not supported because transform impute '\
                  'the unknown category nan when encode city'

        with self.assertWarns(UserWarning, msg=message) as w:
            enc.inverse_transform(result)

    def test_inverse_transform_HaveMissingAndNoUnknown_ExpectInversed(self):
        train = pd.DataFrame({'city': ['chicago', np.nan]})
        test = pd.DataFrame({'city': ['chicago', 'los angeles']})

        enc = encoders.OneHotEncoder(handle_missing='value', handle_unknown='return_nan')
        enc.fit(train)
        result = enc.transform(test)
        original = enc.inverse_transform(result)

        pd.testing.assert_frame_equal(train, original)

    def test_inverse_transform_HaveHandleMissingValueAndHandleUnknownReturnNan_ExpectBestInverse(self):
        train = pd.DataFrame({'city': ['chicago', np.nan]})
        test = pd.DataFrame({'city': ['chicago', np.nan, 'los angeles']})
        expected = pd.DataFrame({'city': ['chicago', np.nan, np.nan]})

        enc = encoders.OneHotEncoder(handle_missing='value', handle_unknown='return_nan')
        enc.fit(train)
        result = enc.transform(test)
        original = enc.inverse_transform(result)

        pd.testing.assert_frame_equal(expected, original)
<|endoftext|>


<commit_before>import weatherpy.weather as weather
import unittest
import sys

from weather_mock import MockArgs, MockIO

class TestPrinterFunctions(unittest.TestCase):
    def setUp(self):
        self.spy = MockIO()

    def tearDown(self):
        self.spy.clear()

    def test_initialization(self):
        printer = weather.ResultPrinter()

        # Ensure we've wrapped std.out
        # XXX need to look into better way of testing
        self.assertNotEqual(printer.out, sys.stdout)

        printer = weather.ResultPrinter(sys.stdout)
        self.assertEqual(printer.out, sys.stdout)

        default_settings = weather.Settings()
        self.assertEqual(default_settings.units, printer.settings.units)
        self.assertEqual(default_settings.api_key, printer.settings.api_key)

    def test_print_no_alerts(self):
        """
        Test to ensure we print a message when there are no alerts
        """
        printer = weather.ResultPrinter(self.spy)

        printer.print_alerts({'alerts': [], 'current_observation': {'display_location': { 'full': 'test'} } })

        self.assertEqual(len(self.spy.captured_out), 2)
        self.assertEqual(self.spy.captured_out[0], 'No alerts for test')
        self.assertEqual(self.spy.captured_out[1], '\n')

    def test_print_alerts(self):
        """
        Test to ensure we print the message, followed by a new line, then the expiration time
        """
        printer = weather.ResultPrinter(self.spy)
        printer.print_alerts({'alerts': [{'message': '1234', 'expires': 'never!' }]})

        self.assertEqual(len(self.spy.captured_out), 2)
        self.assertEqual(self.spy.captured_out[0], '\x1b[91m1234\nExpires: never!\x1b[0m')
        self.assertEqual(self.spy.captured_out[1], '\n')

    def test_print_conditions(self):
        printer = weather.ResultPrinter(self.spy)
        printer.print_conditions(self._gen_conditions_dict(metric_larger=False))
        self.assertEqual(len(self.spy.captured_out), 8)
        self.assertEqual(self.spy.captured_out[0], 'Weather for Ann Arbor')
        self.assertEqual(self.spy.captured_out[1], '\n')
        self.assertEqual(self.spy.captured_out[2], u'Currently: 45 \xb0F (44 \xb0C) Light showers')
        self.assertEqual(self.spy.captured_out[3], '\n')
        self.assertEqual(self.spy.captured_out[4], 'Wind: Calm')
        self.assertEqual(self.spy.captured_out[5], '\n')
        self.assertEqual(self.spy.captured_out[6], 'Humidity: 37')
        self.assertEqual(self.spy.captured_out[7], '\n')

        self.spy.clear()
        printer = weather.ResultPrinter(settings=weather.Settings(MockArgs(units="metric")), out=self.spy)
        printer.print_conditions(self._gen_conditions_dict(metric_larger=False))
        self.assertEqual(printer.settings.units, "metric")
        self.assertEqual(len(self.spy.captured_out), 8)
        self.assertEqual(self.spy.captured_out[0], 'Weather for Ann Arbor')
        self.assertEqual(self.spy.captured_out[1], '\n')
        self.assertEqual(self.spy.captured_out[2], u'Currently: 44 \xb0C (45 \xb0F) Light showers')
        self.assertEqual(self.spy.captured_out[3], '\n')
        self.assertEqual(self.spy.captured_out[4], 'Wind: Calm')
        self.assertEqual(self.spy.captured_out[5], '\n')
        self.assertEqual(self.spy.captured_out[6], 'Humidity: 37')
        self.assertEqual(self.spy.captured_out[7], '\n')

    def _gen_conditions_dict(self, city="Ann Arbor", weather="Light showers", temp=45,
                                   metric_larger=False, wind="Calm", humidity=37):
        """
        Generate a dictionary with the data for print_conditions

        C will be one less than F if metric_larger is True.
        """
        temp_key = "temp_f"
        other_temp_key = "temp_c"
        if metric_larger:
            temp_key = "temp_c"
            other_temp_key = "temp_f"

        return { 'display_location': {'full': city }, temp_key: temp, other_temp_key: temp-1,
                 'weather': weather, 'wind_string': wind, 'relative_humidity': humidity }

    def test_print_forecast(self):
        pass

    def test_print_hourly(self):
        pass

    def test_print_weather_data(self):
        pass

<commit_msg>test removing conf file<commit_after>import weatherpy.weather as weather
import unittest
import sys
import os

from weather_mock import MockArgs, MockIO

class TestPrinterFunctions(unittest.TestCase):
    def setUp(self):
        self.conf_path = os.path.expanduser(weather.WEATHER_CONF_FILE)

        if os.path.exists(self.conf_path):
            os.remove(self.conf_path)
        self.spy = MockIO()

    def tearDown(self):
        self.spy.clear()

    def test_initialization(self):
        printer = weather.ResultPrinter()

        # Ensure we've wrapped std.out
        # XXX need to look into better way of testing
        self.assertNotEqual(printer.out, sys.stdout)

        printer = weather.ResultPrinter(sys.stdout)
        self.assertEqual(printer.out, sys.stdout)

        default_settings = weather.Settings()
        self.assertEqual(default_settings.units, printer.settings.units)
        self.assertEqual(default_settings.api_key, printer.settings.api_key)

    def test_print_no_alerts(self):
        """
        Test to ensure we print a message when there are no alerts
        """
        printer = weather.ResultPrinter(self.spy)

        printer.print_alerts({'alerts': [], 'current_observation': {'display_location': { 'full': 'test'} } })

        self.assertEqual(len(self.spy.captured_out), 2)
        self.assertEqual(self.spy.captured_out[0], 'No alerts for test')
        self.assertEqual(self.spy.captured_out[1], '\n')

    def test_print_alerts(self):
        """
        Test to ensure we print the message, followed by a new line, then the expiration time
        """
        printer = weather.ResultPrinter(self.spy)
        printer.print_alerts({'alerts': [{'message': '1234', 'expires': 'never!' }]})

        self.assertEqual(len(self.spy.captured_out), 2)
        self.assertEqual(self.spy.captured_out[0], '\x1b[91m1234\nExpires: never!\x1b[0m')
        self.assertEqual(self.spy.captured_out[1], '\n')

    def test_print_conditions(self):
        printer = weather.ResultPrinter(self.spy)
        printer.print_conditions(self._gen_conditions_dict(metric_larger=False))
        self.assertEqual(len(self.spy.captured_out), 8)
        self.assertEqual(self.spy.captured_out[0], 'Weather for Ann Arbor')
        self.assertEqual(self.spy.captured_out[1], '\n')
        self.assertEqual(self.spy.captured_out[2], u'Currently: 45 \xb0F (44 \xb0C) Light showers')
        self.assertEqual(self.spy.captured_out[3], '\n')
        self.assertEqual(self.spy.captured_out[4], 'Wind: Calm')
        self.assertEqual(self.spy.captured_out[5], '\n')
        self.assertEqual(self.spy.captured_out[6], 'Humidity: 37')
        self.assertEqual(self.spy.captured_out[7], '\n')

        self.spy.clear()
        printer = weather.ResultPrinter(settings=weather.Settings(MockArgs(units="metric")), out=self.spy)
        printer.print_conditions(self._gen_conditions_dict(metric_larger=False))
        self.assertEqual(printer.settings.units, "metric")
        self.assertEqual(len(self.spy.captured_out), 8)
        self.assertEqual(self.spy.captured_out[0], 'Weather for Ann Arbor')
        self.assertEqual(self.spy.captured_out[1], '\n')
        self.assertEqual(self.spy.captured_out[2], u'Currently: 44 \xb0C (45 \xb0F) Light showers')
        self.assertEqual(self.spy.captured_out[3], '\n')
        self.assertEqual(self.spy.captured_out[4], 'Wind: Calm')
        self.assertEqual(self.spy.captured_out[5], '\n')
        self.assertEqual(self.spy.captured_out[6], 'Humidity: 37')
        self.assertEqual(self.spy.captured_out[7], '\n')

    def _gen_conditions_dict(self, city="Ann Arbor", weather="Light showers", temp=45,
                                   metric_larger=False, wind="Calm", humidity=37):
        """
        Generate a dictionary with the data for print_conditions

        C will be one less than F if metric_larger is True.
        """
        temp_key = "temp_f"
        other_temp_key = "temp_c"
        if metric_larger:
            temp_key = "temp_c"
            other_temp_key = "temp_f"

        return { 'display_location': {'full': city }, temp_key: temp, other_temp_key: temp-1,
                 'weather': weather, 'wind_string': wind, 'relative_humidity': humidity }

    def test_print_forecast(self):
        pass

    def test_print_hourly(self):
        pass

    def test_print_weather_data(self):
        pass

<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
# vim:fenc=utf-8
#
# Copyright © 2014 Christopher C. Strelioff <chris.strelioff@gmail.com>
#
# Distributed under terms of the MIT license.

"""test_process.py

Test (non-command line) methods in the process.py module.

"""
import unittest

import os
import tempfile
import shutil

import yaml
from resumepy import process_html_resume
from resumepy import process_pdf_resume
from resumepy import process_text_resume
from resumepy import resumepy_path

class ResumepyProcessTest(unittest.TestCase):
    """Test the elements of process.py"""

    def setUp(self):
        self.tempdir = tempfile.mkdtemp()
        self.cwd = os.getcwd()
        os.chdir(self.tempdir)

        self.yaml = os.path.join(resumepy_path, 'data', 'examples', 'example_resume.yml')

        with open(self.yaml) as f:
            self.resume = yaml.load(f)

    def tearDown(self):
        shutil.rmtree(self.tempdir)
        os.chdir(self.cwd)

    def test_process_html_resume_created(self):
        """
        process: test_process_html_resume_created()
        """
        process_html_resume(self.resume,
                            os.path.join(resumepy_path, 'data', 'templates'),
                            'template.html')
        self.assertTrue(os.path.exists('build/html/resume.html'))

    def test_process_pdf_resume_created(self):
        """
        process: test_process_pdf_resume_created()
        """
        process_pdf_resume(self.resume,
                           os.path.join(resumepy_path, 'data', 'templates'),
                           'template.tex')
        self.assertTrue(os.path.exists('build/pdf/resume.pdf'))

    def test_process_text_resume_created(self):
        """
        process: test_process_text_resume_created()
        """
        process_text_resume(self.resume,
                            os.path.join(resumepy_path, 'data', 'templates'),
                            'template.txt')
        self.assertTrue(os.path.exists('build/text/resume.txt'))
<commit_msg>update tests with new resume template filenames<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
# vim:fenc=utf-8
#
# Copyright © 2014 Christopher C. Strelioff <chris.strelioff@gmail.com>
#
# Distributed under terms of the MIT license.

"""test_process.py

Test (non-command line) methods in the process.py module.

"""
import unittest

import os
import tempfile
import shutil

import yaml
from resumepy import process_html_resume
from resumepy import process_pdf_resume
from resumepy import process_text_resume
from resumepy import resumepy_path

class ResumepyProcessTest(unittest.TestCase):
    """Test the elements of process.py"""

    def setUp(self):
        self.tempdir = tempfile.mkdtemp()
        self.cwd = os.getcwd()
        os.chdir(self.tempdir)

        self.yaml = os.path.join(resumepy_path, 'data', 'examples', 'example_resume.yml')

        with open(self.yaml) as f:
            self.resume = yaml.load(f)

    def tearDown(self):
        shutil.rmtree(self.tempdir)
        os.chdir(self.cwd)

    def test_process_html_resume_created(self):
        """
        process: test_process_html_resume_created()
        """
        process_html_resume(self.resume,
                            os.path.join(resumepy_path, 'data', 'templates'),
                            'resume.html')
        self.assertTrue(os.path.exists('build/html/resume.html'))

    def test_process_pdf_resume_created(self):
        """
        process: test_process_pdf_resume_created()
        """
        process_pdf_resume(self.resume,
                           os.path.join(resumepy_path, 'data', 'templates'),
                           'resume.tex')
        self.assertTrue(os.path.exists('build/pdf/resume.pdf'))

    def test_process_text_resume_created(self):
        """
        process: test_process_text_resume_created()
        """
        process_text_resume(self.resume,
                            os.path.join(resumepy_path, 'data', 'templates'),
                            'resume.txt')
        self.assertTrue(os.path.exists('build/text/resume.txt'))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import psycopg2
from mosql.result2 import Model

class PostgreSQL(Model):
    getconn = classmethod(lambda cls: psycopg2.connect(database='mosky'))
    putconn = classmethod(lambda cls, conn: None)

class Person(PostgreSQL):
    clauses = dict(table='person')
    arrange_by = ('person_id', )

class Detail(PostgreSQL):
    clauses = dict(table='detail')
    arrange_by = ('person_id', 'key')
    squashed = arrange_by
    ident_by = ('detail_id', )

if __name__ == '__main__':

    from pprint import pprint

    d = next(Detail.arrange(where={'person_id': 'mosky'}))
    Detail.dump_sql = True
    Detail.dry_run = True

    print '# original'
    pprint(dict(d))
    print

    print '# setitem'
    d['val', 0] = 'modified@email.com'
    pprint(dict(d))
    print

    print '# append'
    d.append({'val': 'new@email.com'})
    pprint(dict(d))
    print

    print '# pop it'
    d.pop(0)
    pprint(dict(d))
    print

    print '# save it'
    print d.save()
<commit_msg>test modifying the identity column<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import psycopg2
from mosql.result2 import Model

class PostgreSQL(Model):
    getconn = classmethod(lambda cls: psycopg2.connect(database='mosky'))
    putconn = classmethod(lambda cls, conn: None)

class Person(PostgreSQL):
    clauses = dict(table='person')
    arrange_by = ('person_id', )

class Detail(PostgreSQL):
    clauses = dict(table='detail')
    arrange_by = ('person_id', 'key')
    squashed = arrange_by
    ident_by = ('detail_id', )

if __name__ == '__main__':

    from pprint import pprint

    d = next(Detail.arrange(where={'person_id': 'mosky'}))
    Detail.dump_sql = True
    Detail.dry_run = True

    print '# original'
    pprint(dict(d))
    print

    print '# setitem'
    d['val', 0] = 'modified@email.com'
    d['detail_id', 0] = 123
    pprint(dict(d))
    print

    print '# append'
    d.append({'val': 'new@email.com'})
    pprint(dict(d))
    print

    print '# pop it'
    d.pop(0)
    pprint(dict(d))
    print

    print '# save it'
    print d.save()
<|endoftext|>