<commit_before>"""
Class to read DBF files.
"""

import os
import datetime
import collections

from .struct_parser import StructParser
from .field_parser import FieldParser
from .common import parse_string
from .ifiles import ifind
from .fpt import FPT
from .codepages import guess_encoding

DBFHeader = StructParser(
    'DBFHeader',
    '<BBBBLHHHBBLLLBBH',
    ['dbversion',
     'year',
     'month',
     'day',
     'numrecords',
     'headerlen',
     'recordlen',
     'reserved1',
     'incomplete_transaction',
     'encryption_flag',
     'free_record_thread',
     'reserved2',
     'reserved3',
     'mdx_flag',
     'language_driver',
     'reserved4',
     ])

DBFField = StructParser(
    'DBFField',
    '<11scLBBHBBBB7sB',
    ['name',
     'type',
     'address',
     'length',
     'decimal_count',
     'reserved1',
     'workarea_id',
     'reserved2',
     'reserved3',
     'set_fields_flag',
     'reserved4',
     'index_field_flag',
     ])

FieldValue = collections.namedtuple('Field', 'name value')

def expand_year(year):
    """Convert 2-digit year to 4-digit year."""

    if year < 80:
        return 2000 + year
    else:
        return 1900 + year

class RecordIterator(object):
    def __init__(self, table, deleted=False):
        self._deleted = deleted
        self._table = table

    def __iter__(self):
        for record in self._table._iter_records(deleted=self._deleted,
                                                read=True):
            yield record

    def __len__(self):
        num_records = 0
        for _ in self._table._iter_records(deleted=self._deleted,
                                           read=False):
            num_records += 1
        return num_records

class FakeMemoFile(object):
    def __getitem__(self, i):
        return ''

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        return False

_FAKE_MEMOFILE = FakeMemoFile()

class Table(list):
    """
    Class to read DBF files.
    """

    def __init__(self, filename,
                 encoding=None,
                 ignorecase=True,
                 lowernames=False,
                 parserclass=FieldParser,
                 recfactory=dict,
                 load=False,
                 raw=False):

        self.encoding = encoding
        self.ignorecase = ignorecase
        self.lowernames = lowernames
        self.parserclass = parserclass
        self.recfactory = recfactory
        self.raw = raw

        # Name part before .dbf is the table name
        self.name = os.path.basename(filename)
        self.name = os.path.splitext(self.name)[0]
        self.name = self.name.lower()

        self.deleted = []
        self.loaded = False

        if ignorecase:
            self.filename = ifind(filename)
            if not self.filename:
                raise IOError('No such file: {!r}'.format(filename))
        else:
            self.filename = filename
        # Filled in by self._read_headers()
        self.memofilename = None
        self.header = None
        self.fields = []       # namedtuples
        self.field_names = []  # strings

        with open(self.filename, mode='rb') as infile:
            self._read_headers(infile)
            self._field_parser = self.parserclass(self.encoding)

            self._check_headers()

            self.date = datetime.date(expand_year(self.header.year),
                                      self.header.month,
                                      self.header.day)

        if load:
            self.load()
        else:
            self.unload()

    def _get_memofile(self):
        if self.memofilename and not self.raw:
            return FPT(self.memofilename)
        else:
            return _FAKE_MEMOFILE

    def load(self):
        if not self.loaded:
            del self[:]
            for record in RecordIterator(self):
                self.append(record)

            self.deleted = list(RecordIterator(self, deleted=True))

            self.loaded = True

    def unload(self):
        # self.loaded is not checked here because this
        # is called by __init__() where self.loaded=False.
        # Also, unloading twice has no consequences.
        del self[:]
        self.deleted = RecordIterator(self, deleted=True)
        self.loaded = False

    def _read_headers(self, infile):
        #
        # Todo: more checks
        # http://www.clicketyclick.dk/databases/xbase/format/dbf_check.html#CHECK_DBF
        #
        self.header = DBFHeader.read(infile)

        if self.encoding is None:
            try:
                self.encoding = guess_encoding(self.header.language_driver)
            except LookupError as err:
                self.encoding = 'latin1'

        #
        # Read field headers
        #
        while 1:
            sep = infile.read(1)
            if sep in (b'\x0d', '\n', ''):
                # End of field headers
                break

            fh = DBFField.read(infile, prepend=sep)
            # We need to fix the name and type

            fieldname = parse_string(fh.name, self.encoding)
            if self.lowernames:
                fieldname = fieldname.lower()
            fieldtype = parse_string(fh.type, self.encoding)

            fh = fh._replace(name=fieldname,
                             type=fieldtype)

            self.field_names.append(fh.name)

            self.fields.append(fh)

        if len(self.fields) < 1:
            message = 'dbf file must have at least one field: {!r}'
            raise ValueError(message.format(self.filename))

        # Check for memo file
        field_types = set([field.type for field in self.fields])
        if 'M' in field_types:
            fn = os.path.splitext(self.filename)[0] + '.fpt'
            match = ifind(self.filename, ext='.fpt')
            if match:
                self.memofilename = match
            else:
                # Todo: warn and return field as byte string?
                raise IOError('Missing memo file: {!r}'.format(fn))

    def _check_headers(self):
        """Check headers for possible format errors."""
        for field in self.fields:

            if field.type == '0' and field.length != 1:
                message = 'Field of type 0 must have length 1 (was {})'
                raise ValueError(message.format(field.length))

            elif field.type == 'I' and field.length != 4:
                message = 'Field type I must have length 4 (was {})'
                raise ValueError(message.format(field.length))

            elif field.type == 'L' and field.length != 1:
                message = 'Field type L must have length 1 (was {})'
                raise ValueError(message.format(field.length))

            elif not self._field_parser.field_type_supported(field.type):
                # Todo: return as byte string?
                raise ValueError('Unknown field type: {!r}'.format(field.type))

    def _read_record(self, infile, memofile):
        items = []  # List of Field
        for field in self.fields:
            value = infile.read(field.length)
            if not self.raw:
                value = self._field_parser.parse(field, value)

                #
                # Decoding memo fields requires a little more
                # trickery.
                #
                if field.type == 'M':
                    if value is None:
                        value = ''
                    else:
                        memo = memofile[value]
                        if memo.type == 'memo' and not self.raw:
                            # Decode to unicode
                            value = parse_string(memo.data, self.encoding)
                        else:
                            # Byte array
                            value = memo.data

            items.append(FieldValue(name=field.name, value=value))

        return self.recfactory(items)

    def _skip_record(self, infile):
        infile.seek(self.header.recordlen - 1, 1)

    def _iter_records(self, deleted=False, read=False):
        with open(self.filename, 'rb') as infile, \
              self._get_memofile() as memofile:
            # Skip to first record.
            infile.seek(self.header.headerlen, 0)
            while True:
                sep = infile.read(1)

                if sep == b'' or sep == b'\x1a':
                    return  # End of file reached
                elif sep not in b' *':
                    raise IOError("invalid record separator '{}'".format(sep))

                if deleted and sep == b'*':
                    interesting_record = True
                elif not deleted and sep == b' ':
                    interesting_record = True
                else:
                    interesting_record = False

                if interesting_record:
                    if read:
                        yield self._read_record(infile, memofile)
                    else:
                        yield self._skip_record(infile)
                else:
                    self._skip_record(infile)

    def __iter__(self):
        if self.loaded:
            for record in list.__iter__(self):
                yield record
        else:
            for record in RecordIterator(self):
                yield record

    def __len__(self):
        if self.loaded:
            return list.__len__(self)
        else:
            return len(RecordIterator(self))

    def __repr__(self):
        if self.loaded:
            return list.__repr__(self)
        else:
            return '<Unloaded DBF table {!r}>'.format(self.filename)

<commit_msg>Rewrote record iteration and counting. The code is much cleaner.<commit_after>"""
Class to read DBF files.
"""

import os
import datetime
import collections

from .struct_parser import StructParser
from .field_parser import FieldParser
from .common import parse_string
from .ifiles import ifind
from .fpt import FPT
from .codepages import guess_encoding

DBFHeader = StructParser(
    'DBFHeader',
    '<BBBBLHHHBBLLLBBH',
    ['dbversion',
     'year',
     'month',
     'day',
     'numrecords',
     'headerlen',
     'recordlen',
     'reserved1',
     'incomplete_transaction',
     'encryption_flag',
     'free_record_thread',
     'reserved2',
     'reserved3',
     'mdx_flag',
     'language_driver',
     'reserved4',
     ])

DBFField = StructParser(
    'DBFField',
    '<11scLBBHBBBB7sB',
    ['name',
     'type',
     'address',
     'length',
     'decimal_count',
     'reserved1',
     'workarea_id',
     'reserved2',
     'reserved3',
     'set_fields_flag',
     'reserved4',
     'index_field_flag',
     ])

FieldValue = collections.namedtuple('Field', 'name value')

def expand_year(year):
    """Convert 2-digit year to 4-digit year."""

    if year < 80:
        return 2000 + year
    else:
        return 1900 + year

class RecordIterator(object):
    def __init__(self, table, wanted_types):
        self._wanted_types = wanted_types
        self._table = table

    def __iter__(self):
        for _, record in self._table._iter_records(self._wanted_types):
            yield record

    def __len__(self):
        return self._table._count_records(self._wanted_types)

class FakeMemoFile(object):
    def __getitem__(self, i):
        return ''

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        return False

_FAKE_MEMOFILE = FakeMemoFile()

class Table(list):
    """
    Class to read DBF files.
    """

    def __init__(self, filename,
                 encoding=None,
                 ignorecase=True,
                 lowernames=False,
                 parserclass=FieldParser,
                 recfactory=dict,
                 load=False,
                 raw=False):

        self.encoding = encoding
        self.ignorecase = ignorecase
        self.lowernames = lowernames
        self.parserclass = parserclass
        self.recfactory = recfactory
        self.raw = raw

        # Name part before .dbf is the table name
        self.name = os.path.basename(filename)
        self.name = os.path.splitext(self.name)[0]
        self.name = self.name.lower()

        self.deleted = []
        self.loaded = False

        if ignorecase:
            self.filename = ifind(filename)
            if not self.filename:
                raise IOError('No such file: {!r}'.format(filename))
        else:
            self.filename = filename
        # Filled in by self._read_headers()
        self.memofilename = None
        self.header = None
        self.fields = []       # namedtuples
        self.field_names = []  # strings

        with open(self.filename, mode='rb') as infile:
            self._read_headers(infile)
            self._field_parser = self.parserclass(self.encoding)

            self._check_headers()

            self.date = datetime.date(expand_year(self.header.year),
                                      self.header.month,
                                      self.header.day)

        if load:
            self.load()
        else:
            self.unload()

    def _get_memofile(self):
        if self.memofilename and not self.raw:
            return FPT(self.memofilename)
        else:
            return _FAKE_MEMOFILE

    def load(self):
        if not self.loaded:
            del self[:]
            self.deleted = []

            for record_type, record in self._iter_records(b' *'):
                if record_type == b' ':
                    self.append(record)
                else:
                    self.deleted.append(record)

            self.loaded = True

    def unload(self):
        # self.loaded is not checked here because this
        # is called by __init__() where self.loaded=False.
        # Also, unloading twice has no consequences.
        del self[:]
        self.deleted = RecordIterator(self, b'*')
        self.loaded = False

    def _read_headers(self, infile):
        #
        # Todo: more checks
        # http://www.clicketyclick.dk/databases/xbase/format/dbf_check.html#CHECK_DBF
        #
        self.header = DBFHeader.read(infile)

        if self.encoding is None:
            try:
                self.encoding = guess_encoding(self.header.language_driver)
            except LookupError as err:
                self.encoding = 'latin1'

        #
        # Read field headers
        #
        while 1:
            sep = infile.read(1)
            if sep in (b'\x0d', '\n', ''):
                # End of field headers
                break

            fh = DBFField.read(infile, prepend=sep)
            # We need to fix the name and type

            fieldname = parse_string(fh.name, self.encoding)
            if self.lowernames:
                fieldname = fieldname.lower()
            fieldtype = parse_string(fh.type, self.encoding)

            fh = fh._replace(name=fieldname,
                             type=fieldtype)

            self.field_names.append(fh.name)

            self.fields.append(fh)

        if len(self.fields) < 1:
            message = 'dbf file must have at least one field: {!r}'
            raise ValueError(message.format(self.filename))

        # Check for memo file
        field_types = set([field.type for field in self.fields])
        if 'M' in field_types:
            fn = os.path.splitext(self.filename)[0] + '.fpt'
            match = ifind(self.filename, ext='.fpt')
            if match:
                self.memofilename = match
            else:
                # Todo: warn and return field as byte string?
                raise IOError('Missing memo file: {!r}'.format(fn))

    def _check_headers(self):
        """Check headers for possible format errors."""
        for field in self.fields:

            if field.type == '0' and field.length != 1:
                message = 'Field of type 0 must have length 1 (was {})'
                raise ValueError(message.format(field.length))

            elif field.type == 'I' and field.length != 4:
                message = 'Field type I must have length 4 (was {})'
                raise ValueError(message.format(field.length))

            elif field.type == 'L' and field.length != 1:
                message = 'Field type L must have length 1 (was {})'
                raise ValueError(message.format(field.length))

            elif not self._field_parser.field_type_supported(field.type):
                # Todo: return as byte string?
                raise ValueError('Unknown field type: {!r}'.format(field.type))

    def _read_record(self, infile, memofile):
        items = []  # List of Field
        for field in self.fields:
            value = infile.read(field.length)
            if not self.raw:
                value = self._field_parser.parse(field, value)

                #
                # Decoding memo fields requires a little more
                # trickery.
                #
                if field.type == 'M':
                    if value is None:
                        value = ''
                    else:
                        memo = memofile[value]
                        if memo.type == 'memo' and not self.raw:
                            # Decode to unicode
                            value = parse_string(memo.data, self.encoding)
                        else:
                            # Byte array
                            value = memo.data

            items.append(FieldValue(name=field.name, value=value))

        return self.recfactory(items)

    def _skip_record(self, infile):
        infile.seek(self.header.recordlen - 1, 1)

    def _count_records(self, wanted_types=b' '):
        count = 0

        with open(self.filename, 'rb') as infile:
            infile.seek(self.header.headerlen, 0)
            while True:
                sep = infile.read(1)
                if sep in wanted_types:
                    count += 1
                elif sep in b'\x1a':
                    # End of records.
                    # (Separator is b'' or b'\x1a'.)
                    break
                self._skip_record(infile)

        return count

    def _iter_records(self, wanted_types=b' '):
        with open(self.filename, 'rb') as infile, \
              self._get_memofile() as memofile:
            # Skip to first record.
            infile.seek(self.header.headerlen, 0)
            while True:
                sep = infile.read(1)

                if sep in wanted_types:
                    yield (sep, self._read_record(infile, memofile))
                elif sep in b'\x1a':
                    # End of records.
                    # (Separator is b'' or b'\x1a'.)
                    break
                else:
                    self._skip_record(infile)

    def __iter__(self):
        if self.loaded:
            for record in list.__iter__(self):
                yield record
        else:
            for _, record in self._iter_records():
                yield record

    def __len__(self):
        if self.loaded:
            return list.__len__(self)
        else:
            return self._count_records()

    def __repr__(self):
        if self.loaded:
            return list.__repr__(self)
        else:
            return '<Unloaded DBF table {!r}>'.format(self.filename)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os
import itertools
from multiprocessing.dummy import Pool

import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
from requests_futures.sessions import FuturesSession

class Client:

    def __init__(self):
        self.fut_session = FuturesSession(max_workers=8)
        self.req_session = requests.Session()
        self.thread_pool = Pool(processes=8)
        self.retries = Retry(
            total=5,
            backoff_factor=0.1,
            status_forcelist=[500, 502, 503, 504])
        self.req_session.mount('https://', HTTPAdapter(max_retries=self.retries))

    def get(self, url, **kwargs):
        response = self.req_session.get(url, **kwargs)
        return response.json()

    def get_stream(self, url, **kwargs):
        return self.req_session.get(url, stream=True, **kwargs)

    def fut_get(self, url, **kwargs):
        return self.fut_session.get(url, **kwargs)

    def parallel_tasks(self, function, tasks):
        return self.thread_pool.map_async(function, tasks)

    @staticmethod
    def flatten_result_lists(meta_lists):
        return list(itertools.chain.from_iterable(meta_lists))

    @staticmethod
    def chunks(elements, chunck_size=30):
        for i in range(0, len(elements), chunck_size):
            yield elements[i:i+chunck_size]

client = Client()
<commit_msg>[feature] add exception handle in responses<commit_after># -*- coding: utf-8 -*-

import os
import logging
import itertools
from multiprocessing.dummy import Pool

import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
from requests.exceptions import RetryError
from requests_futures.sessions import FuturesSession

logger = logging.getLogger('dcard')

class Client:

    def __init__(self, workers=8):
        self.fut_session = FuturesSession(max_workers=workers)
        self.req_session = requests.Session()
        self.thread_pool = Pool(processes=workers)
        self.retries = Retry(
            total=5,
            backoff_factor=0.1,
            status_forcelist=[500, 502, 503, 504])
        self.req_session.mount('https://', HTTPAdapter(max_retries=self.retries))

    def get(self, url, **kwargs):
        try:
            response = self.req_session.get(url, **kwargs)
            data = response.json()
            if isinstance(data, dict) and data.get('error'):
                logger.error('when get {}, error {}'.format(url, data))
                return {}
            return data
        except RetryError as e:
            logger.error('when get {}, error {}'.format(url, e))

    def get_stream(self, url, **kwargs):
        return self.req_session.get(url, stream=True, **kwargs)

    def fut_get(self, url, **kwargs):
        return self.fut_session.get(url, **kwargs)

    def parallel_tasks(self, function, tasks):
        return self.thread_pool.map_async(function, tasks)

    @staticmethod
    def flatten_result_lists(meta_lists):
        return list(itertools.chain.from_iterable(meta_lists))

    @staticmethod
    def chunks(elements, chunck_size=30):
        for i in range(0, len(elements), chunck_size):
            yield elements[i:i+chunck_size]

client = Client()
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import random
import json
import itertools
import logging
import warnings
import multiprocessing
import Queue
import numpy
import time
import collections
import backport

import lr

def grouper(iterable, n, fillvalue=None): # pragma : no cover
    "Collect data into fixed-length chunks or blocks"
    # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx
    args = [iter(iterable)] * n
    return itertools.izip_longest(fillvalue=fillvalue, *args)

def randomPairsWithReplacement(n_records, sample_size) :
    # If the population is very large relative to the sample
    # size than we'll get very few duplicates by chance
    warnings.warn("There may be duplicates in the sample")

    random_indices = numpy.random.randint(n_records,
                                          size=sample_size*2)
    random_indices = random_indices.reshape((-1, 2))
    random_indices.sort(axis=1)

    return random_indices

def randomPairs(n_records, sample_size):
    """
    Return random combinations of indices for a square matrix of size
    n records
    """

    if n_records < 2 :
        raise ValueError("Needs at least two records")
    n = n_records * (n_records - 1) / 2

    if sample_size >= n:
        if sample_size > n :
            warnings.warn("Requested sample of size %d, only returning %d possible pairs" % (sample_size, n))

        random_indices = numpy.arange(n)
    elif 8 * n > numpy.iinfo('uint').max :
        return randomPairsWithReplacement(n_records, sample_size)
    else :
        try:
            random_indices = numpy.random.randint(n, size=sample_size)
            random_indices.dtype = 'uint'
        except OverflowError:
            return randomPairsWithReplacement(n_records, sample_size)

    b = 1 - 2 * n_records

    x = numpy.trunc((-b - numpy.sqrt(b ** 2 - 8 * random_indices)) / 2)
    y = random_indices + x * (b + x + 2) / 2 + 1

    return numpy.column_stack((x, y)).astype(int)

def randomPairsMatch(n_records_A, n_records_B, sample_size):
    """
    Return random combinations of indices for record list A and B
    """
    if not n_records_A or not n_records_B :
        raise ValueError("There must be at least one record in A and in B")

    if sample_size >= n_records_A * n_records_B :

        if sample_size > n_records_A * n_records_B :
            warnings.warn("Requested sample of size %d, only returning %d possible pairs" % (sample_size, n_records_A * n_records_B))

        return backport.cartesian((numpy.arange(n_records_A),
                                   numpy.arange(n_records_B)))

    A_samples = numpy.random.randint(n_records_A, size=sample_size)
    B_samples = numpy.random.randint(n_records_B, size=sample_size)
    pairs = zip(A_samples,B_samples)
    set_pairs = set(pairs)

    while len(set_pairs) < sample_size:
        set_pairs.update(randomPairsMatch(n_records_A,
                                          n_records_B,
                                          (sample_size-len(set_pairs))))
    else:
        return set_pairs

def trainModel(training_data, data_model, alpha=.001):
    """
    Use logistic regression to train weights for all fields in the data model
    """

    labels = numpy.array(training_data['label'] == 'match', dtype='i4')
    examples = training_data['distances']

    (weight, bias) = lr.lr(labels, examples, alpha)

    for i, name in enumerate(data_model['fields']) :
        data_model['fields'][name]['weight'] = float(weight[i])

    data_model['bias'] = bias

    return data_model

def fieldDistances(record_pairs, data_model):
    # Think about filling this in instead of concatenating

    field_distances = numpy.fromiter((compare(record_pair[0][field],
                                              record_pair[1][field])
                                      for record_pair in record_pairs
                                      for field, compare in data_model.field_comparators
                                      if record_pair),
                                     'f4')

    field_distances = field_distances.reshape(-1,
                                              len(data_model.field_comparators))

    for cat_index, length in data_model.categorical_indices :
        different_sources = field_distances[:, cat_index][...,None] == numpy.arange(2, length)[None,...]

        field_distances[:, cat_index][field_distances[:, cat_index] > 1] = 0
        field_distances = numpy.concatenate((field_distances,
                                             different_sources.astype(float)),
                                            axis=1)

    interaction_distances = numpy.empty((field_distances.shape[0],
                                         len(data_model.interactions)))

    for i, interaction in enumerate(data_model.interactions) :
        a = numpy.prod(field_distances[...,interaction], axis=1)
        interaction_distances[...,i] = a

    field_distances = numpy.concatenate((field_distances,
                                         interaction_distances),
                                        axis=1)

    missing_data = numpy.isnan(field_distances)

    field_distances[missing_data] = 0

    missing_indicators = 1-missing_data[:,data_model.missing_field_indices]

    field_distances = numpy.concatenate((field_distances,
                                         missing_indicators),
                                        axis=1)

    return field_distances

def scorePairs(field_distances, data_model):
    fields = data_model['fields']

    field_weights = [fields[name]['weight'] for name in fields]
    bias = data_model['bias']

    scores = numpy.dot(field_distances, field_weights)

    scores = numpy.exp(scores + bias) / (1 + numpy.exp(scores + bias))

    return scores

class ScoringFunction(object) :
    def __init__(self, data_model, threshold, dtype) :
        self.data_model = data_model
        self.threshold = threshold
        self.dtype = dtype

    def __call__(self, chunk_queue, scored_pairs_queue) :
        while True :
            record_pairs = chunk_queue.get()
            if record_pairs is None :
                chunk_queue.put(None)
                scored_pairs_queue.put(None)
                break
            scored_pairs = self.scoreRecords(record_pairs)
            scored_pairs_queue.put(scored_pairs)

    def scoreRecords(self, record_pairs) :
        ids = []

        def split_records() :
            for pair in record_pairs :
                if pair :
                    ids.append((pair[0][0], pair[1][0]))
                    yield (pair[0][1], pair[1][1])

        scores = scorePairs(fieldDistances(split_records(),
                                           self.data_model),
                            self.data_model)

        filtered_scores = ((pair_id, score)
                           for pair_id, score in itertools.izip(ids, scores)
                           if score > self.threshold)

        filtered_scores = list(filtered_scores)

        scored_pairs = numpy.array(filtered_scores,
                                   dtype=self.dtype)

        return scored_pairs

def collector(scored_pairs_queue, child_conn, num_scorers) :
    all_scored_pairs = None
    while True :
        scored_pairs = scored_pairs_queue.get()
        if scored_pairs == None :
            num_scorers -= 1
            if num_scorers == 0 :
                child_conn.send(all_scored_pairs)
                child_conn.close()
                break
            else :
                continue
        if all_scored_pairs is None :
            all_scored_pairs = scored_pairs
        else :
            all_scored_pairs = numpy.concatenate(scored_pairs,
                                                 all_scored_pairs)

def scoreDuplicates(records, data_model, num_processes, threshold=0):
    record_pairs_queue = multiprocessing.Queue()
    scored_pairs_queue = multiprocessing.JoinableQueue()
    parent_conn, child_conn = multiprocessing.Pipe()

    record, records = peek(records)

    id_type = idType(record)

    score_dtype = [('pairs', id_type, 2), ('score', 'f4', 1)]

    scoring_function = ScoringFunction(data_model,
                                       threshold,
                                       score_dtype)

    # Start processes
    for i in xrange(num_processes) :
        multiprocessing.Process(target=scoring_function,
                                args=(record_pairs_queue,
                                      scored_pairs_queue)).start()

    multiprocessing.Process(target=collector,
                            args=(scored_pairs_queue, child_conn, num_processes)).start()

    for chunk in grouper(records, 100000) :
        record_pairs_queue.put(chunk)

    record_pairs_queue.put(None)

    scored_pairs = parent_conn.recv()

    scored_pairs.sort()
    flag = numpy.ones(len(scored_pairs), dtype=bool)
    numpy.not_equal(scored_pairs[1:],
                    scored_pairs[:-1],
                    out=flag[1:])

    return scored_pairs[flag]

def idType(record) :
    id_type = type(record[0][0])
    if id_type is str or id_type is unicode :
        id_type = (unicode, len(record[0][0]) + 5)

    return numpy.dtype(id_type)

def peek(records) :
    try :
        record = records.next()
    except AttributeError as e:
        if "has no attribute 'next'" not in str(e) :
            raise
        try :
            records = iter(records)
            record = records.next()
        except StopIteration :
            return None, records

    return record, itertools.chain([record], records)

def freezeData(data) : # pragma : no cover
    return [(frozendict(record_1),
             frozendict(record_2))
            for record_1, record_2 in data]

class frozendict(collections.Mapping):
    """Don't forget the docstrings!!"""

    def __init__(self, *args, **kwargs): # pragma : no cover
        self._d = dict(*args, **kwargs)

    def __iter__(self):                  # pragma : no cover
        return iter(self._d)

    def __len__(self):                   # pragma : no cover
        return len(self._d)

    def __getitem__(self, key):          # pragma : no cover
        return self._d[key]

    def __repr__(self) :
        return '<frozendict %s>' % repr(self._d)

    def __hash__(self):
        try:
            return self._cached_hash
        except AttributeError:
            h = self._cached_hash = hash(frozenset(self._d.iteritems()))
            return h
<commit_msg>simpler results handler<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import random
import json
import itertools
import logging
import warnings
import multiprocessing
import Queue
import numpy
import time
import collections
import backport

import lr

def grouper(iterable, n, fillvalue=None): # pragma : no cover
    "Collect data into fixed-length chunks or blocks"
    # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx
    args = [iter(iterable)] * n
    return itertools.izip_longest(fillvalue=fillvalue, *args)

def randomPairsWithReplacement(n_records, sample_size) :
    # If the population is very large relative to the sample
    # size than we'll get very few duplicates by chance
    warnings.warn("There may be duplicates in the sample")

    random_indices = numpy.random.randint(n_records,
                                          size=sample_size*2)
    random_indices = random_indices.reshape((-1, 2))
    random_indices.sort(axis=1)

    return random_indices

def randomPairs(n_records, sample_size):
    """
    Return random combinations of indices for a square matrix of size
    n records
    """

    if n_records < 2 :
        raise ValueError("Needs at least two records")
    n = n_records * (n_records - 1) / 2

    if sample_size >= n:
        if sample_size > n :
            warnings.warn("Requested sample of size %d, only returning %d possible pairs" % (sample_size, n))

        random_indices = numpy.arange(n)
    elif 8 * n > numpy.iinfo('uint').max :
        return randomPairsWithReplacement(n_records, sample_size)
    else :
        try:
            random_indices = numpy.random.randint(n, size=sample_size)
            random_indices.dtype = 'uint'
        except OverflowError:
            return randomPairsWithReplacement(n_records, sample_size)

    b = 1 - 2 * n_records

    x = numpy.trunc((-b - numpy.sqrt(b ** 2 - 8 * random_indices)) / 2)
    y = random_indices + x * (b + x + 2) / 2 + 1

    return numpy.column_stack((x, y)).astype(int)

def randomPairsMatch(n_records_A, n_records_B, sample_size):
    """
    Return random combinations of indices for record list A and B
    """
    if not n_records_A or not n_records_B :
        raise ValueError("There must be at least one record in A and in B")

    if sample_size >= n_records_A * n_records_B :

        if sample_size > n_records_A * n_records_B :
            warnings.warn("Requested sample of size %d, only returning %d possible pairs" % (sample_size, n_records_A * n_records_B))

        return backport.cartesian((numpy.arange(n_records_A),
                                   numpy.arange(n_records_B)))

    A_samples = numpy.random.randint(n_records_A, size=sample_size)
    B_samples = numpy.random.randint(n_records_B, size=sample_size)
    pairs = zip(A_samples,B_samples)
    set_pairs = set(pairs)

    while len(set_pairs) < sample_size:
        set_pairs.update(randomPairsMatch(n_records_A,
                                          n_records_B,
                                          (sample_size-len(set_pairs))))
    else:
        return set_pairs

def trainModel(training_data, data_model, alpha=.001):
    """
    Use logistic regression to train weights for all fields in the data model
    """

    labels = numpy.array(training_data['label'] == 'match', dtype='i4')
    examples = training_data['distances']

    (weight, bias) = lr.lr(labels, examples, alpha)

    for i, name in enumerate(data_model['fields']) :
        data_model['fields'][name]['weight'] = float(weight[i])

    data_model['bias'] = bias

    return data_model

def fieldDistances(record_pairs, data_model):
    # Think about filling this in instead of concatenating

    field_distances = numpy.fromiter((compare(record_pair[0][field],
                                              record_pair[1][field])
                                      for record_pair in record_pairs
                                      for field, compare in data_model.field_comparators
                                      if record_pair),
                                     'f4')

    field_distances = field_distances.reshape(-1,
                                              len(data_model.field_comparators))

    for cat_index, length in data_model.categorical_indices :
        different_sources = field_distances[:, cat_index][...,None] == numpy.arange(2, length)[None,...]

        field_distances[:, cat_index][field_distances[:, cat_index] > 1] = 0
        field_distances = numpy.concatenate((field_distances,
                                             different_sources.astype(float)),
                                            axis=1)

    interaction_distances = numpy.empty((field_distances.shape[0],
                                         len(data_model.interactions)))

    for i, interaction in enumerate(data_model.interactions) :
        a = numpy.prod(field_distances[...,interaction], axis=1)
        interaction_distances[...,i] = a

    field_distances = numpy.concatenate((field_distances,
                                         interaction_distances),
                                        axis=1)

    missing_data = numpy.isnan(field_distances)

    field_distances[missing_data] = 0

    missing_indicators = 1-missing_data[:,data_model.missing_field_indices]

    field_distances = numpy.concatenate((field_distances,
                                         missing_indicators),
                                        axis=1)

    return field_distances

def scorePairs(field_distances, data_model):
    fields = data_model['fields']

    field_weights = [fields[name]['weight'] for name in fields]
    bias = data_model['bias']

    scores = numpy.dot(field_distances, field_weights)

    scores = numpy.exp(scores + bias) / (1 + numpy.exp(scores + bias))

    return scores

class ScoringFunction(object) :
    def __init__(self, data_model, threshold, dtype) :
        self.data_model = data_model
        self.threshold = threshold
        self.dtype = dtype

    def __call__(self, chunk_queue, scored_pairs_queue) :
        while True :
            record_pairs = chunk_queue.get()
            if record_pairs is None :
                chunk_queue.put(None)
                scored_pairs_queue.put(None)
                break
            scored_pairs = self.scoreRecords(record_pairs)
            scored_pairs_queue.put(scored_pairs)

    def scoreRecords(self, record_pairs) :
        ids = []

        def split_records() :
            for pair in record_pairs :
                if pair :
                    ids.append((pair[0][0], pair[1][0]))
                    yield (pair[0][1], pair[1][1])

        scores = scorePairs(fieldDistances(split_records(),
                                           self.data_model),
                            self.data_model)

        filtered_scores = ((pair_id, score)
                           for pair_id, score in itertools.izip(ids, scores)
                           if score > self.threshold)

        filtered_scores = list(filtered_scores)

        scored_pairs = numpy.array(filtered_scores,
                                   dtype=self.dtype)

        return scored_pairs

def scoreDuplicates(records, data_model, num_processes, threshold=0):
    record_pairs_queue = multiprocessing.Queue()
    scored_pairs_queue = multiprocessing.JoinableQueue()

    record, records = peek(records)

    id_type = idType(record)

    score_dtype = [('pairs', id_type, 2), ('score', 'f4', 1)]

    scoring_function = ScoringFunction(data_model,
                                       threshold,
                                       score_dtype)

    # Start processes
    for i in xrange(num_processes) :
        multiprocessing.Process(target=scoring_function,
                                args=(record_pairs_queue,
                                      scored_pairs_queue)).start()

    for chunk in grouper(records, 100000) :
        record_pairs_queue.put(chunk)
    record_pairs_queue.put(None)

    def scored_pairs_generator(num_scorers) :
        while True :
            scored_pairs = scored_pairs_queue.get()
            if scored_pairs == None :
                num_scorers -= 1
                if num_scorers == 0 :
                    break
                else :
                    continue
            yield scored_pairs

    scored_pairs = numpy.concatenate([scored_pairs
                                      for scored_pairs
                                      in scored_pairs_generator(num_processes)])

    scored_pairs.sort()
    flag = numpy.ones(len(scored_pairs), dtype=bool)
    numpy.not_equal(scored_pairs[1:],
                    scored_pairs[:-1],
                    out=flag[1:])

    return scored_pairs[flag]

def idType(record) :
    id_type = type(record[0][0])
    if id_type is str or id_type is unicode :
        id_type = (unicode, len(record[0][0]) + 5)

    return numpy.dtype(id_type)

def peek(records) :
    try :
        record = records.next()
    except AttributeError as e:
        if "has no attribute 'next'" not in str(e) :
            raise
        try :
            records = iter(records)
            record = records.next()
        except StopIteration :
            return None, records

    return record, itertools.chain([record], records)

def freezeData(data) : # pragma : no cover
    return [(frozendict(record_1),
             frozendict(record_2))
            for record_1, record_2 in data]

class frozendict(collections.Mapping):
    """Don't forget the docstrings!!"""

    def __init__(self, *args, **kwargs): # pragma : no cover
        self._d = dict(*args, **kwargs)

    def __iter__(self):                  # pragma : no cover
        return iter(self._d)

    def __len__(self):                   # pragma : no cover
        return len(self._d)

    def __getitem__(self, key):          # pragma : no cover
        return self._d[key]

    def __repr__(self) :
        return '<frozendict %s>' % repr(self._d)

    def __hash__(self):
        try:
            return self._cached_hash
        except AttributeError:
            h = self._cached_hash = hash(frozenset(self._d.iteritems()))
            return h
<|endoftext|>


<commit_before>import collections, csv, dedupe, json, os, pickle, re, uuid
from numpy  import nan
from pprint import pprint
from glob   import glob
import dedupe.serializer as serializer

processed_files = 'processed_files'
settings_file = 'learned_settings'
training_file = 'trained.json'

def sameOrNotComparator(field_1, field_2) :
    if field_1 and field_2 :
        if field_1 == field_2 :
            return 1
        else:
            return 0
    else :
        return nan

def replaceWhitespace(str):
    return re.sub('  +', ' ', str)

def preProcess(column):
    """
    Do a little bit of data cleaning with the help of
    [AsciiDammit](https://github.com/tnajdek/ASCII--Dammit) and
    Regex. Things like casing, extra spaces, quotes and new lines can
    be ignored.
    """

    column = column.encode("ascii","replace")
    column = re.sub('\n', ' ', column)
    column = column.strip().strip('"').strip("'").lower().strip()
    return replaceWhitespace(column)

def processName(name):
    name = replaceWhitespace(re.sub('[-\.,()]', ' ', name))

    Goodbye = {"&": "and",
               "twenty first century": "21st century"}
    for k,v in Goodbye.iteritems():
        if k in name:
            name = re.sub(k,v,name)

    ImmaLetYouFinish = ["on behalf of","livingston group for","livingston group",
                        "capitol strategies ",  "alcalde and fay","white house consulting",
                        "the ickes and enright group","corporation",
                        "dla piper us for", "obo" ]
    for s in ImmaLetYouFinish:
        if s in name:
            a = name.split(s)[-1]
            if len(a) > 2:
                name = replaceWhitespace(a)

    GetOuttaHere = ["american"," l p","corporation","international","national",
                    "association","incorporated"," l l c", "tribe", " u s "
                    "university of", "coalition", "professional","town of",
                    "group","technologies"," u s a","company","limited",
                    "political action committee","city of","informal coalitions",
                    "associates"    ]
    for s in GetOuttaHere:
        if s in name:
            a = re.sub(s,' ',name)
            if len(a) > 2:
                name = replaceWhitespace(a)

    Shwaties =["llc","inc","corp","llp","of","the"]
    for s in Shwaties:
        if s in name.split():
            a = re.sub(s,' ',name)
            if len(a) > 2:
                name = replaceWhitespace(a)

    return name
    #St. Joesph's
    #remove stray end )?
    #strip out
    #tribal reservations
    #fka - formerly known as
    #through
    #prev. rptd - previously reported

def loadFile(f):
    jOb = {}
    corruption = "{http://www.PureEdge.com/XFDL/Custom}"
    try:
        jOb = json.loads(open(f).read())[u'LOBBYINGDISCLOSURE1']
    except KeyError:
        try:
            corruptedjOb = json.loads(open(f).read())[corruption+u'LOBBYINGDISCLOSURE1']
            jOb = {}
            for (k,v) in corruptedjOb.iteritems():
                jOb[k.split(corruption)[-1]]=v
        except KeyError:
            # We'll do it live
            return None

    client = {
        "exact_name":  preProcess(jOb["clientName"]),
        "rough_name":  processName(preProcess(jOb["clientName"])),
        "address":     preProcess(jOb["clientAddress"]),
        "city":        preProcess(jOb["clientCity"]),
        "country":     preProcess(jOb["clientCountry"]),
        "state":       preProcess(jOb["clientState"]),
        "zip":         preProcess(jOb["clientZip"]),

        "alis":        frozenset(filter(lambda x: x != u"",jOb["alis"])),

        "houseID":     preProcess(jOb["houseID"]),
        "senate":      preProcess(jOb["senateID"]),
        "filename": f,
        "description":     preProcess(jOb["clientGeneralDescription"]),
        "specific_issues": preProcess(jOb["specific_issues"]),
    }
    return client

def loadData():
    print 'Reading into clients ...'
    if os.path.exists(processed_files):
        print ("File %s exists reading now" % processed_files)
        with open(processed_files,"r") as f:
            return pickle.load(f)
    else:
        print "Loading and processing files now"
        clients = {}
        for c in map(loadFile,glob(os.environ["HOUSEXML"]+"/LD1/*/*/*.json")):
            if c != None:
                clients[uuid.uuid4().hex] = c
        with open(processed_files,"w") as f:
            pickle.dump(clients,f,2)
        return clients

# Training
def train(clients):

    if os.path.exists(settings_file):
        print 'reading from', settings_file
        return dedupe.StaticDedupe(settings_file,num_processes=8)

    else:
        fields = {'city':        {'type': 'String', 'Has Missing': True},
                  'country':     {'type': 'String', 'Has Missing': True},
                  'zip':         {'type': 'String', 'Has Missing': True},
                  'houseID':     {'type': 'String', 'Has Missing': True},
                  'state':       {'type': 'String', 'Has Missing': True},
                  'address':     {'type': 'Text', 'Has Missing': True,
                                  'corpus': [] #map(lambda x: x['address'],clients.values())
                              },

                  'description': {'type': 'Text', 'Has Missing': True,
                                  'corpus': [] #map(lambda x: x['description'],clients.values())
                              },
                  'specific_issues': {'type': 'Text', 'Has Missing': True,
                                      'corpus': [] #map(lambda x: x['specific_issues'],clients.values())
                                  },
                  'rough_name':  {'type': 'Text',
                                  'corpus': [] #map(lambda x: x['rough_name'],clients.values())
                              },
                  'alis':     {'type': 'Set',
                               'corpus': [] #map(lambda x: x['alis'],clients.values())
                           },

                  'exact_name':  {'type': 'Custom',
                                  'comparator': sameOrNotComparator},
        }
        for k,v in list(fields.iteritems()):
            if k != 'exact_name':
                fields[k+"-exact_name"] = {'type':'Interaction',
                                           'Interaction Fields': ["exact_name", k]}
        # Create a new deduper object and pass our data model to it.
        deduper = dedupe.Dedupe(fields,num_processes=8)

        # To train dedupe, we feed it a random sample of records.
        deduper.sample(clients, 150000)

        # If we have training data saved from a previous run of dedupe,
        # look for it an load it in.
        # __Note:__ if you want to train from scratch, delete the training_file
        if os.path.exists(training_file):
            print 'reading labeled examples from ', training_file
            deduper.readTraining(training_file)

        for f in ["closenames","exactnames"]:
            labels = json.load(open(f+".json"), cls=serializer.dedupe_decoder)
            deduper.markPairs(labels)

        # ## Active learning
        # Dedupe will find the next pair of records
        # it is least certain about and ask you to label them as duplicates
        # or not.
        # use 'y', 'n' and 'u' keys to flag duplicates
        # press 'f' when you are finished
        print 'starting active labeling...'

        #dedupe.consoleLabel(deduper)

        deduper.train()

        # When finished, save our training away to disk
        deduper.writeTraining(training_file)

        # Save our weights and predicates to disk.  If the settings file
        # exists, we will skip all the training and learning next time we run
        # this file.
        #deduper.writeSettings(settings_file)
        return deduper

def main():
    clients = loadData()
    deduper = train(clients)

    print 'threshold...'
    threshold = deduper.threshold(clients, recall_weight=2)
    print 'matchering...'
    clustered_dupes = deduper.match(clients, threshold)

    print '# duplicate sets', len(clustered_dupes)
    print '# duplicate sets', map(len,clustered_dupes)
    # ## Writing Results

    # Write our original data back out to a CSV with a new column called
    # 'Cluster ID' which indicates which records refer to each other.

    for (cluster_id, cluster) in enumerate(clustered_dupes):
         print("Group {}".format(cluster_id))
         for uuid in cluster:
             pprint(clients[uuid])
         print("\n")

if __name__ == "__main__":
    main()

<commit_msg>Pickle up the end results.<commit_after>import collections, csv, dedupe, json, os, pickle, re, time, uuid
from numpy  import nan
from pprint import pprint
from glob   import glob
import dedupe.serializer as serializer

processed_files = 'processed_files'
settings_file = 'learned_settings'
training_file = 'trained.json'
output_pickle = 'clusters.pickle'

def sameOrNotComparator(field_1, field_2) :
    if field_1 and field_2 :
        if field_1 == field_2 :
            return 1
        else:
            return 0
    else :
        return nan

def replaceWhitespace(str):
    return re.sub('  +', ' ', str)

def preProcess(column):
    """
    Do a little bit of data cleaning with the help of
    [AsciiDammit](https://github.com/tnajdek/ASCII--Dammit) and
    Regex. Things like casing, extra spaces, quotes and new lines can
    be ignored.
    """

    column = column.encode("ascii","replace")
    column = re.sub('\n', ' ', column)
    column = column.strip().strip('"').strip("'").lower().strip()
    return replaceWhitespace(column)

def processName(name):
    name = replaceWhitespace(re.sub('[-\.,()]', ' ', name))

    Goodbye = {"&": "and",
               "twenty first century": "21st century"}
    for k,v in Goodbye.iteritems():
        if k in name:
            name = re.sub(k,v,name)

    ImmaLetYouFinish = ["on behalf of","livingston group for","livingston group",
                        "capitol strategies ",  "alcalde and fay","white house consulting",
                        "the ickes and enright group","corporation",
                        "dla piper us for", "obo" ]
    for s in ImmaLetYouFinish:
        if s in name:
            a = name.split(s)[-1]
            if len(a) > 2:
                name = replaceWhitespace(a)

    GetOuttaHere = ["american"," l p","corporation","international","national",
                    "association","incorporated"," l l c", "tribe", " u s "
                    "university of", "coalition", "professional","town of",
                    "group","technologies"," u s a","company","limited",
                    "political action committee","city of","informal coalitions",
                    "associates"    ]
    for s in GetOuttaHere:
        if s in name:
            a = re.sub(s,' ',name)
            if len(a) > 2:
                name = replaceWhitespace(a)

    Shwaties =["llc","inc","corp","llp","of","the"]
    for s in Shwaties:
        if s in name.split():
            a = re.sub(s,' ',name)
            if len(a) > 2:
                name = replaceWhitespace(a)

    return name
    #St. Joesph's
    #remove stray end )?
    #strip out
    #tribal reservations
    #fka - formerly known as
    #through
    #prev. rptd - previously reported

def loadFile(f):
    jOb = {}
    corruption = "{http://www.PureEdge.com/XFDL/Custom}"
    try:
        jOb = json.loads(open(f).read())[u'LOBBYINGDISCLOSURE1']
    except KeyError:
        try:
            corruptedjOb = json.loads(open(f).read())[corruption+u'LOBBYINGDISCLOSURE1']
            jOb = {}
            for (k,v) in corruptedjOb.iteritems():
                jOb[k.split(corruption)[-1]]=v
        except KeyError:
            # We'll do it live
            return None

    client = {
        "exact_name":  preProcess(jOb["clientName"]),
        "rough_name":  processName(preProcess(jOb["clientName"])),
        "address":     preProcess(jOb["clientAddress"]),
        "city":        preProcess(jOb["clientCity"]),
        "country":     preProcess(jOb["clientCountry"]),
        "state":       preProcess(jOb["clientState"]),
        "zip":         preProcess(jOb["clientZip"]),

        "alis":        frozenset(filter(lambda x: x != u"",jOb["alis"])),

        "houseID":     preProcess(jOb["houseID"]),
        "senate":      preProcess(jOb["senateID"]),
        "filename": f,
        "description":     preProcess(jOb["clientGeneralDescription"]),
        "specific_issues": preProcess(jOb["specific_issues"]),
    }
    return client

def loadData():
    print 'Reading into clients ...'
    if os.path.exists(processed_files):
        print ("File %s exists reading now" % processed_files)
        with open(processed_files,"r") as f:
            return pickle.load(f)
    else:
        print "Loading and processing files now"
        clients = {}
        for c in map(loadFile,glob(os.environ["HOUSEXML"]+"/LD1/*/*/*.json")):
            if c != None:
                clients[uuid.uuid4().hex] = c
        with open(processed_files,"w") as f:
            pickle.dump(clients,f,2)
        return clients

# Training
def train(clients):

    if os.path.exists(settings_file):
        print 'reading from', settings_file
        return dedupe.StaticDedupe(settings_file,num_processes=8)

    else:
        fields = {'city':        {'type': 'String', 'Has Missing': True},
                  'country':     {'type': 'String', 'Has Missing': True},
                  'zip':         {'type': 'String', 'Has Missing': True},
                  'houseID':     {'type': 'String', 'Has Missing': True},
                  'state':       {'type': 'String', 'Has Missing': True},
                  'address':     {'type': 'Text', 'Has Missing': True,
                                  'corpus': [] #map(lambda x: x['address'],clients.values())
                              },

                  'description': {'type': 'Text', 'Has Missing': True,
                                  'corpus': [] #map(lambda x: x['description'],clients.values())
                              },
                  'specific_issues': {'type': 'Text', 'Has Missing': True,
                                      'corpus': [] #map(lambda x: x['specific_issues'],clients.values())
                                  },
                  'rough_name':  {'type': 'Text',
                                  'corpus': [] #map(lambda x: x['rough_name'],clients.values())
                              },
                  'alis':     {'type': 'Set',
                               'corpus': [] #map(lambda x: x['alis'],clients.values())
                           },

                  'exact_name':  {'type': 'Custom',
                                  'comparator': sameOrNotComparator},
        }
        for k,v in list(fields.iteritems()):
            if k != 'exact_name':
                fields[k+"-exact_name"] = {'type':'Interaction',
                                           'Interaction Fields': ["exact_name", k]}
        # Create a new deduper object and pass our data model to it.
        deduper = dedupe.Dedupe(fields,num_processes=8)

        # To train dedupe, we feed it a random sample of records.
        deduper.sample(clients, 150000)

        # If we have training data saved from a previous run of dedupe,
        # look for it an load it in.
        # __Note:__ if you want to train from scratch, delete the training_file
        if os.path.exists(training_file):
            print 'reading labeled examples from ', training_file
            deduper.readTraining(training_file)

        for f in ["closenames","exactnames"]:
            labels = json.load(open(f+".json"), cls=serializer.dedupe_decoder)
            deduper.markPairs(labels)

        # ## Active learning
        # Dedupe will find the next pair of records
        # it is least certain about and ask you to label them as duplicates
        # or not.
        # use 'y', 'n' and 'u' keys to flag duplicates
        # press 'f' when you are finished
        print 'starting active labeling...'

        #dedupe.consoleLabel(deduper)

        deduper.train()

        # When finished, save our training away to disk
        deduper.writeTraining(training_file)

        # Save our weights and predicates to disk.  If the settings file
        # exists, we will skip all the training and learning next time we run
        # this file.
        #deduper.writeSettings(settings_file)
        return deduper

def main():
    clients = loadData()
    deduper = train(clients)

    print 'threshold...'
    threshold = deduper.threshold(clients, recall_weight=2)
    print 'matchering...'
    clustered_dupes = deduper.match(clients, threshold)

    print '# duplicate sets', len(clustered_dupes)
    print '# duplicate sets', map(len,clustered_dupes)
    # ## Writing Results

    # Write our original data back out to a CSV with a new column called
    # 'Cluster ID' which indicates which records refer to each other.

    for (cluster_id, cluster) in enumerate(clustered_dupes):
         print("Group {}".format(cluster_id))
         for uuid in cluster:
             pprint(clients[uuid])
         print("\n")

    with open(output_pickle+int(time.time()),"w") as f:
            pickle.dump(clustered_dupes,f,2)

if __name__ == "__main__":
    main()

<|endoftext|>


<commit_before>__author__ = 'Jeanne-Marie'
''' Runs gamepad_interface.py as a subprocess.
Reads output from gamepad_interface.py and maps it to 4-tuples.
Calls motor_command_transmitter, passing a 4-tuple.
So four arguments to motor_command_transmitter with type:
float, float, bool, bool '''

import subprocess
import os

import mctransmitter  # COM WITH ARDUINO
import global_data    # SHARED WITH ui_display AND ui_map
import ui_map         # MAP BUTTON EVENTS TO PIN COMMANDS
import ui_display     # VISUAL DISPLAY

# PASS BUTTON VALUES THROUGH THESE
# BEFORE SENDING THEM TO MAP
scale_analog(val):   # [-1,1] -> [0,255]
    return (((val + 1) * ANALOG_MAX) / 2)

scale_digital(val):  # [anything] -> [False, True]
    if (val > 0):
        return True
    else:
        return False

# WHEN WE GET A BUTTON EVENT FROM gampad_interface() CHECK
# IF ITS THE 'SELECT' BUTTON. IF SO, INCREMENT TO NEXT MAP
# IN THIS LIST. OTHERWISE, PASS IT TO THE CURRENT MAP.
map_list = [simple_map(), toggle_map()]

# TODO python documentation suggests using subprocess32

# based on http://stackoverflow.com/questions/2804543/read-subprocess-stdout-line-by-line
def gamepad_to_motor_command_transmitter(whatever_would_be_input_to_gamepad_subprocess):
    proc = subprocess.Popen(['python', '-u', 'game_interface.py'], stdout=subprocess.PIPE)

   for line in iter(proc.stdout.readline,''):
        #parse line to get variables
        speed = 0
        wavelength = 0
        left = False
        right = False

        mctransmitter.send_motor_command(speed, wavelength, left, right)
<commit_msg>better scale_digital()<commit_after>__author__ = 'Jeanne-Marie'
''' Runs gamepad_interface.py as a subprocess.
Reads output from gamepad_interface.py and maps it to 4-tuples.
Calls motor_command_transmitter, passing a 4-tuple.
So four arguments to motor_command_transmitter with type:
float, float, bool, bool '''

import subprocess
import os

import mctransmitter  # COM WITH ARDUINO
import global_data    # SHARED WITH ui_display AND ui_map
import ui_map         # MAP BUTTON EVENTS TO PIN COMMANDS
import ui_display     # VISUAL DISPLAY

# PASS BUTTON VALUES THROUGH THESE
# BEFORE SENDING THEM TO MAP
scale_analog(val):   # [-1,1] -> [0,255]
    return (((val + 1) * ANALOG_MAX) / 2)

scale_digital(val):  # [anything] -> [False, True]
    return (val > 0)

# WHEN WE GET A BUTTON EVENT FROM gampad_interface() CHECK
# IF ITS THE 'SELECT' BUTTON. IF SO, INCREMENT TO NEXT MAP
# IN THIS LIST. OTHERWISE, PASS IT TO THE CURRENT MAP.
map_list = [simple_map(), toggle_map()]

# TODO python documentation suggests using subprocess32

# based on http://stackoverflow.com/questions/2804543/read-subprocess-stdout-line-by-line
def gamepad_to_motor_command_transmitter(whatever_would_be_input_to_gamepad_subprocess):
    proc = subprocess.Popen(['python', '-u', 'game_interface.py'], stdout=subprocess.PIPE)

   for line in iter(proc.stdout.readline,''):
        #parse line to get variables
        speed = 0
        wavelength = 0
        left = False
        right = False

        mctransmitter.send_motor_command(speed, wavelength, left, right)
<|endoftext|>


<commit_before>#!/usr/bin/python
# Written by: Andrew Jackson
# This is used to send a JSON payload to a webhook.
import json
import logging
import os
import time
import uuid
import boto3
import requests
import decimal

def default(obj):
    if isinstance(obj, decimal.Decimal):
        return int(obj)
    return o.__dict__

def handler(event, context):
    print "event.dump = " + json.dumps(event)
    url = 'https://discordapp.com/api/webhooks/308790391031726082/zGRwUsO66V6UXl4Vb8eg8lDZ2pE7eljfPMQLVF9lSJlytAoYOCDSSsi8wUe9F1shC2IJ/github'
    payload = {json.dumps(event)}
    headers = {'content-type': 'application/json'}

    r = requests.post(url, data=json.dumps(payload), headers=headers, default=default)
<commit_msg>update to not encode json 2x<commit_after>#!/usr/bin/python
# Written by: Andrew Jackson
# This is used to send a JSON payload to a webhook.
import json
import logging
import os
import time
import uuid
import boto3
import requests
import decimal

def default(obj):
    if isinstance(obj, decimal.Decimal):
        return int(obj)
    return o.__dict__

def handler(event, context):
    print "event.dump = " + json.dumps(event)
    url = 'https://discordapp.com/api/webhooks/308790391031726082/zGRwUsO66V6UXl4Vb8eg8lDZ2pE7eljfPMQLVF9lSJlytAoYOCDSSsi8wUe9F1shC2IJ/github'
    payload = {json.dumps(event)}
    headers = {'content-type': 'application/json'}

    r = requests.post(url, data=payload, headers=headers, default=default)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
Kay framework.

:Copyright: (c) 2009 Accense Technology, Inc.
                     Takashi Matsuo <tmatsuo@candit.jp>,
                     Ian Lewis <IanMLewis@gmail.com>
                     All rights reserved.
:license: BSD, see LICENSE for more details.
"""

import os
import sys

__version__ = "0.3.0"

KAY_DIR = os.path.abspath(os.path.dirname(__file__))
PROJECT_DIR = os.path.dirname(KAY_DIR)
LIB_DIR = os.path.join(KAY_DIR, 'lib')

def setup_env(manage_py_env=False):
  """Configures app engine environment for command-line apps."""
  # Try to import the appengine code from the system path.
  try:
    from google.appengine.api import apiproxy_stub_map
  except ImportError, e:
    # Not on the system path. Build a list of alternative paths where it
    # may be. First look within the project for a local copy, then look for
    # where the Mac OS SDK installs it.
    paths = [os.path.join(PROJECT_DIR, '.google_appengine'),
             '/usr/local/google_appengine']
    for path in os.environ.get('PATH', '').replace(';', ':').split(':'):
      path = path.rstrip(os.sep)
      if path.endswith('google_appengine'):
        paths.append(path)
    if os.name in ('nt', 'dos'):
      prefix = '%(PROGRAMFILES)s' % os.environ
      paths.append(prefix + r'\Google\google_appengine')
    # Loop through all possible paths and look for the SDK dir.
    SDK_PATH = None
    for sdk_path in paths:
      sdk_path = os.path.realpath(sdk_path)
      if os.path.exists(sdk_path):
        SDK_PATH = sdk_path
        break
    if SDK_PATH is None:
      # The SDK could not be found in any known location.
      sys.stderr.write('The Google App Engine SDK could not be found!\n'
                       'Please visit http://kay-docs.shehas.net/'
                       ' for installation instructions.\n')
      sys.exit(1)
    # Add the SDK and the libraries within it to the system path.
    EXTRA_PATHS = [SDK_PATH]
    lib = os.path.join(SDK_PATH, 'lib')
    # Automatically add all packages in the SDK's lib folder:
    for dir in os.listdir(lib):
      path = os.path.join(lib, dir)
      # Package can be under 'lib/<pkg>/<pkg>/' or 'lib/<pkg>/lib/<pkg>/'
      detect = (os.path.join(path, dir), os.path.join(path, 'lib', dir))
      for path in detect:
        if os.path.isdir(path):
          EXTRA_PATHS.append(os.path.dirname(path))
          break
    sys.path = EXTRA_PATHS + sys.path
    from google.appengine.api import apiproxy_stub_map
  setup()

  if not manage_py_env:
    return
  print 'Running on Kay-%s' % __version__

def setup():
  setup_syspath()
  from kay.conf import settings
  from google.appengine.ext import db
  from google.appengine.ext.db import polymodel

  class _meta(object):
    __slots__ = ('object_name', 'app_label', 'module_name', '_db_table', 'abstract')
    def __init__(self, model):
      try:
        self.app_label = model.__module__.split('.')[-2]
      except IndexError:
        raise ValueError('Kay expects models (here: %s.%s) to be defined in their own apps!' % (model.__module__, model.__name__))
      self.module_name = model.__name__.lower()
      self.abstract = model is db.Model
      self.object_name = model.__name__

    def _set_db_table(self, db_table):
      self._db_table = db_table

    def _get_db_table(self):
      if getattr(settings, 'ADD_APP_PREFIX_TO_KIND', True):
        if hasattr(self, '_db_table'):
          return self._db_table
        return '%s_%s' % (self.app_label, self.module_name)
      return self.object_name
    db_table = property(_get_db_table, _set_db_table)

  def _initialize_model(cls):
    cls._meta = _meta(cls)

  old_propertied_class_init = db.PropertiedClass.__init__
  def __init__(cls, name, bases, attrs, map_kind=True):
    """
    Just add _meta to db.Model.
    """
    _initialize_model(cls)
    old_propertied_class_init(cls, name, bases, attrs,
                              not cls._meta.abstract)
  db.PropertiedClass.__init__ = __init__

  old_poly_init = polymodel.PolymorphicClass.__init__
  def __init__(cls, name, bases, attrs):
    if polymodel.PolyModel not in bases:
      _initialize_model(cls)
    old_poly_init(cls, name, bases, attrs)
  polymodel.PolymorphicClass.__init__ = __init__

  @classmethod
  def kind(cls):
    return cls._meta.db_table
  db.Model.kind = kind

def setup_syspath():
  if not PROJECT_DIR in sys.path:
    sys.path = [PROJECT_DIR] + sys.path
  if not LIB_DIR in sys.path:
    sys.path = [LIB_DIR] + sys.path
<commit_msg>Added PROJECT_LIB_DIR.<commit_after># -*- coding: utf-8 -*-

"""
Kay framework.

:Copyright: (c) 2009 Accense Technology, Inc.
                     Takashi Matsuo <tmatsuo@candit.jp>,
                     Ian Lewis <IanMLewis@gmail.com>
                     All rights reserved.
:license: BSD, see LICENSE for more details.
"""

import os
import sys

__version__ = "0.3.0"

KAY_DIR = os.path.abspath(os.path.dirname(__file__))
PROJECT_DIR = os.path.dirname(KAY_DIR)
LIB_DIR = os.path.join(KAY_DIR, 'lib')
PROJECT_LIB_DIR = os.path.join(PROJECT_DIR, 'lib')

def setup_env(manage_py_env=False):
  """Configures app engine environment for command-line apps."""
  # Try to import the appengine code from the system path.
  try:
    from google.appengine.api import apiproxy_stub_map
  except ImportError, e:
    # Not on the system path. Build a list of alternative paths where it
    # may be. First look within the project for a local copy, then look for
    # where the Mac OS SDK installs it.
    paths = [os.path.join(PROJECT_DIR, '.google_appengine'),
             '/usr/local/google_appengine']
    for path in os.environ.get('PATH', '').replace(';', ':').split(':'):
      path = path.rstrip(os.sep)
      if path.endswith('google_appengine'):
        paths.append(path)
    if os.name in ('nt', 'dos'):
      prefix = '%(PROGRAMFILES)s' % os.environ
      paths.append(prefix + r'\Google\google_appengine')
    # Loop through all possible paths and look for the SDK dir.
    SDK_PATH = None
    for sdk_path in paths:
      sdk_path = os.path.realpath(sdk_path)
      if os.path.exists(sdk_path):
        SDK_PATH = sdk_path
        break
    if SDK_PATH is None:
      # The SDK could not be found in any known location.
      sys.stderr.write('The Google App Engine SDK could not be found!\n'
                       'Please visit http://kay-docs.shehas.net/'
                       ' for installation instructions.\n')
      sys.exit(1)
    # Add the SDK and the libraries within it to the system path.
    EXTRA_PATHS = [SDK_PATH]
    lib = os.path.join(SDK_PATH, 'lib')
    # Automatically add all packages in the SDK's lib folder:
    for dir in os.listdir(lib):
      path = os.path.join(lib, dir)
      # Package can be under 'lib/<pkg>/<pkg>/' or 'lib/<pkg>/lib/<pkg>/'
      detect = (os.path.join(path, dir), os.path.join(path, 'lib', dir))
      for path in detect:
        if os.path.isdir(path):
          EXTRA_PATHS.append(os.path.dirname(path))
          break
    sys.path = EXTRA_PATHS + sys.path
    from google.appengine.api import apiproxy_stub_map
  setup()

  if not manage_py_env:
    return
  print 'Running on Kay-%s' % __version__

def setup():
  setup_syspath()
  from kay.conf import settings
  from google.appengine.ext import db
  from google.appengine.ext.db import polymodel

  class _meta(object):
    __slots__ = ('object_name', 'app_label', 'module_name', '_db_table', 'abstract')
    def __init__(self, model):
      try:
        self.app_label = model.__module__.split('.')[-2]
      except IndexError:
        raise ValueError('Kay expects models (here: %s.%s) to be defined in their own apps!' % (model.__module__, model.__name__))
      self.module_name = model.__name__.lower()
      self.abstract = model is db.Model
      self.object_name = model.__name__

    def _set_db_table(self, db_table):
      self._db_table = db_table

    def _get_db_table(self):
      if getattr(settings, 'ADD_APP_PREFIX_TO_KIND', True):
        if hasattr(self, '_db_table'):
          return self._db_table
        return '%s_%s' % (self.app_label, self.module_name)
      return self.object_name
    db_table = property(_get_db_table, _set_db_table)

  def _initialize_model(cls):
    cls._meta = _meta(cls)

  old_propertied_class_init = db.PropertiedClass.__init__
  def __init__(cls, name, bases, attrs, map_kind=True):
    """
    Just add _meta to db.Model.
    """
    _initialize_model(cls)
    old_propertied_class_init(cls, name, bases, attrs,
                              not cls._meta.abstract)
  db.PropertiedClass.__init__ = __init__

  old_poly_init = polymodel.PolymorphicClass.__init__
  def __init__(cls, name, bases, attrs):
    if polymodel.PolyModel not in bases:
      _initialize_model(cls)
    old_poly_init(cls, name, bases, attrs)
  polymodel.PolymorphicClass.__init__ = __init__

  @classmethod
  def kind(cls):
    return cls._meta.db_table
  db.Model.kind = kind

def setup_syspath():
  if not PROJECT_DIR in sys.path:
    sys.path = [PROJECT_DIR] + sys.path
  if not LIB_DIR in sys.path:
    sys.path = [LIB_DIR] + sys.path
  if not PROJECT_LIB_DIR in sys.path:
    sys.path = [PROJECT_LIB_DIR] + sys.path
<|endoftext|>


<commit_before>import os
from datetime import date, timedelta
from functools import partial
from PIL import Image
from mock import patch
from operator import attrgetter

import stripe

from django import forms
from django.conf import settings
from django.core.urlresolvers import reverse
from django.test import TestCase

from .exceptions import DonationError
from .forms import PaymentForm
from .models import DjangoHero, Donation
from .utils import shuffle_donations

def _fake_random(*results):
    """
    Return a callable that generates the given results when called.
    Useful for mocking random.random().

    Example:

    >>> r = _fake_random(1, 2, 3)
    >>> r()
    1
    >>> r()
    2
    >>> r()
    3
    """
    return partial(next, iter(results))

class TestIndex(TestCase):
    def test_donors_count(self):
        DjangoHero.objects.create()
        response = self.client.get(reverse('fundraising:index'))
        self.assertEqual(response.context['total_donors'], 1)

    def test_hide_campaign_input(self):
        # Checking if rendered output contains campaign form field
        # to not generate ugly URLs
        response = self.client.get(reverse('fundraising:index'))
        self.assertNotContains(response, 'name="campaign"')

        response = self.client.get(reverse('fundraising:index'), {'campaign': 'test'})
        self.assertContains(response, 'name="campaign"')

    def test_render_donate_form_with_amount(self):
        response = self.client.get(reverse('fundraising:donate'), {'amount': 50})
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.context['form'].fixed_amount, '50')
        self.assertEqual(response.context['publishable_key'], settings.STRIPE_PUBLISHABLE_KEY)

        # Checking if amount field is hidden
        self.assertIsInstance(response.context['form'].fields['amount'].widget, forms.HiddenInput)

        # Checking if campaign field is empty
        self.assertEqual(response.context['form'].initial['campaign'], None)

    def test_render_donate_form_without_amount(self):
        response = self.client.get(reverse('fundraising:donate'))
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.context['form'].fixed_amount, None)
        self.assertEqual(response.context['publishable_key'], settings.STRIPE_PUBLISHABLE_KEY)

        # Checking if amount field is visible
        self.assertIsInstance(response.context['form'].fields['amount'].widget, forms.TextInput)
        # Checking if campaign field is empty
        self.assertEqual(response.context['form'].initial['campaign'], None)

    def test_render_donate_form_with_bad_amount(self):
        # this will trigger a DecimalException exception because the amount can't be
        # converted to a Decimal
        response = self.client.get(reverse('fundraising:donate'), {'amount': 'superbad'})
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.context['form'].fixed_amount, 'superbad')
        self.assertEqual(response.context['publishable_key'], settings.STRIPE_PUBLISHABLE_KEY)

        # Checking if amount field is visible
        self.assertIsInstance(response.context['form'].fields['amount'].widget, forms.TextInput)
        # Checking if campaign field is empty
        self.assertEqual(response.context['form'].initial['campaign'], None)

    def test_render_donate_form_with_campaign(self):
        campaigns = ['sample', '']
        for campaign in campaigns:
            response = self.client.get(reverse('fundraising:donate'), {'amount': 100, 'campaign': campaign})
            self.assertEqual(response.status_code, 200)
            self.assertEqual(response.context['form'].fixed_amount, '100')
            self.assertEqual(response.context['publishable_key'], settings.STRIPE_PUBLISHABLE_KEY)

            # Checking if amount field is hidden
            self.assertIsInstance(response.context['form'].fields['amount'].widget, forms.HiddenInput)
            # Checking if campaign field is same as campaign
            self.assertEqual(response.context['form'].initial['campaign'], campaign)

    def test_submitting_donation_form_missing_token(self):
        url = reverse('fundraising:donate')
        response = self.client.post(url, {'amount': 100})
        self.assertFalse(response.context['form'].is_valid())
        self.assertEquals(200, response.status_code)
        self.assertTemplateUsed(response, 'fundraising/donate.html')

    def test_submitting_donation_form_invalid_amount(self):
        url = reverse('fundraising:donate')
        response = self.client.post(url, {
            'amount': 'superbad',
            'stripe_token': 'test',
        })
        self.assertFalse(response.context['form'].is_valid())
        self.assertEquals(200, response.status_code)
        # Checking if amount field is visible
        self.assertIsInstance(response.context['form'].fields['amount'].widget, forms.TextInput)

    @patch('stripe.Customer.create')
    @patch('stripe.Charge.create')
    def test_submitting_donation_form(self, charge_create, customer_create):
        response = self.client.post(reverse('fundraising:donate'), {'amount': 100})
        self.assertFalse(response.context['form'].is_valid())

        response = self.client.post(reverse('fundraising:donate'), {
            'amount': 100,
            'stripe_token': 'test',
            'receipt_email': 'test@example.com',
        })
        donations = Donation.objects.all()
        self.assertEqual(donations.count(), 1)
        self.assertEqual(donations[0].amount, 100)
        self.assertEqual(donations[0].campaign_name, '')
        self.assertEqual(donations[0].receipt_email, 'test@example.com')

    @patch('stripe.Customer.create')
    @patch('stripe.Charge.create')
    def test_submitting_donation_form_with_campaign(self, charge_create, customer_create):
        response = self.client.post(reverse('fundraising:donate'), {
            'amount': 100,
            'campaign': 'test',
        })
        self.assertFalse(response.context['form'].is_valid())
        response = self.client.post(reverse('fundraising:donate'), {
            'amount': 100,
            'campaign': 'test',
            'stripe_token': 'test',
        })
        donations = Donation.objects.all()
        self.assertEqual(donations.count(), 1)
        self.assertEqual(donations[0].amount, 100)
        self.assertEqual(donations[0].campaign_name, 'test')

    @patch('stripe.Customer.create')
    @patch('stripe.Charge.create')
    def test_submitting_donation_form_error_handling(self, charge_create, customer_create):
        data = {
            'amount': 100,
            'campaign': None,
            'stripe_token': 'xxxx',
        }
        form = PaymentForm(data=data)

        self.assertTrue(form.is_valid())

        # some errors are shows as user facting DonationErrors to the user
        # some are bubbling up to raise a 500 to trigger Sentry reports
        errors = [
            [stripe.error.CardError, DonationError],
            [stripe.error.InvalidRequestError, DonationError],
            [stripe.error.APIConnectionError, DonationError],
            [stripe.error.AuthenticationError, None],
            [stripe.error.StripeError, None],
            [ValueError, None],
        ]

        for backend_exception, user_exception in errors:
            customer_create.side_effect = backend_exception('message', 'param', 'code')

            if user_exception is None:
                self.assertRaises(backend_exception, form.make_donation)
            else:
                response = self.client.post(reverse('fundraising:donate'), data)
                self.assertTrue('donation_error' in response.context)

    @patch('fundraising.forms.PaymentForm.make_donation')
    def test_submitting_donation_form_valid(self, make_donation):
        amount = 100
        donation = Donation.objects.create(
            amount=amount,
            stripe_charge_id='xxxx',
            stripe_customer_id='xxxx',
        )
        make_donation.return_value = donation
        response = self.client.post(reverse('fundraising:donate'), {
            'amount': amount,
            'campaign': None,
            'stripe_token': 'xxxx',
        })
        self.assertRedirects(response, donation.get_absolute_url())

class TestDjangoHero(TestCase):
    def setUp(self):
        kwargs = {
            'approved': True,
            'is_visible': True,
            'is_amount_displayed': True,
        }
        self.h1 = DjangoHero.objects.create(**kwargs)
        Donation.objects.create(donor=self.h1, amount='5')
        self.h2 = DjangoHero.objects.create(**kwargs)
        Donation.objects.create(donor=self.h2, amount='15')
        # hidden donation amount should display last
        kwargs['is_amount_displayed'] = False
        self.h3 = DjangoHero.objects.create(**kwargs)
        Donation.objects.create(donor=self.h3, amount='10')
        self.today = date.today()

    def test_donation_shuffling(self):
        queryset = DjangoHero.objects.in_period(self.today, self.today + timedelta(days=1))

        for random, expected in [
            (lambda: 1, [15, 10, 5]),
            (lambda: -1, [5, 10, 15]),
            (_fake_random(1, 0, 1), [15, 5, 10]),
        ]:
            with patch('fundraising.utils.random', side_effect=random):
                self.assertQuerysetEqual(
                    shuffle_donations(queryset),
                    expected,
                    attrgetter('donated_amount')
                )

    def test_thumbnail(self):
        try:
            os.makedirs(os.path.join(settings.MEDIA_ROOT, 'fundraising/logos/'))
        except OSError:  # directory may already exist
            pass
        image_path = os.path.join(settings.MEDIA_ROOT, 'fundraising/logos/test_logo.jpg')
        image = Image.new('L', (500, 500))
        image.save(image_path)
        self.h1 = DjangoHero.objects.last()
        self.h1.logo = image_path
        self.h1.save()
        thumbnail = self.h1.thumbnail
        self.assertEqual(thumbnail.x, 340)
        self.assertEqual(thumbnail.y, 340)
        os.remove(image_path)

    def test_thumbnail_no_logo(self):
        self.assertIsNone(self.h2.thumbnail)

class TestPaymentForm(TestCase):
    @patch('stripe.Customer.create')
    @patch('stripe.Charge.create')
    def test_make_donation(self, charge_create, customer_create):
        customer_create.return_value.id = 'xxxx'
        charge_create.return_value.id = 'xxxx'
        form = PaymentForm(data={
            'amount': 100,
            'campaign': None,
            'stripe_token': 'xxxx',
        })
        self.assertTrue(form.is_valid())
        donation = form.make_donation()
        self.assertEqual(100, donation.amount)

    @patch('stripe.Customer.create')
    @patch('stripe.Charge.create')
    def test_make_donation_exception(self, charge_create, customer_create):
        customer_create.side_effect = ValueError("Something is wrong")
        form = PaymentForm(data={
            'amount': 100,
            'campaign': None,
            'stripe_token': 'xxxx',
        })
        self.assertTrue(form.is_valid())
        with self.assertRaises(ValueError):
            donation = form.make_donation()
            self.assertIsNone(donation)
<commit_msg>Make sure thumbnail file exist in thumbnail test<commit_after>import os
from datetime import date, timedelta
from functools import partial
from PIL import Image
from mock import patch
from operator import attrgetter

import stripe

from django import forms
from django.conf import settings
from django.core.urlresolvers import reverse
from django.test import TestCase

from .exceptions import DonationError
from .forms import PaymentForm
from .models import DjangoHero, Donation
from .utils import shuffle_donations

def _fake_random(*results):
    """
    Return a callable that generates the given results when called.
    Useful for mocking random.random().

    Example:

    >>> r = _fake_random(1, 2, 3)
    >>> r()
    1
    >>> r()
    2
    >>> r()
    3
    """
    return partial(next, iter(results))

class TestIndex(TestCase):
    def test_donors_count(self):
        DjangoHero.objects.create()
        response = self.client.get(reverse('fundraising:index'))
        self.assertEqual(response.context['total_donors'], 1)

    def test_hide_campaign_input(self):
        # Checking if rendered output contains campaign form field
        # to not generate ugly URLs
        response = self.client.get(reverse('fundraising:index'))
        self.assertNotContains(response, 'name="campaign"')

        response = self.client.get(reverse('fundraising:index'), {'campaign': 'test'})
        self.assertContains(response, 'name="campaign"')

    def test_render_donate_form_with_amount(self):
        response = self.client.get(reverse('fundraising:donate'), {'amount': 50})
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.context['form'].fixed_amount, '50')
        self.assertEqual(response.context['publishable_key'], settings.STRIPE_PUBLISHABLE_KEY)

        # Checking if amount field is hidden
        self.assertIsInstance(response.context['form'].fields['amount'].widget, forms.HiddenInput)

        # Checking if campaign field is empty
        self.assertEqual(response.context['form'].initial['campaign'], None)

    def test_render_donate_form_without_amount(self):
        response = self.client.get(reverse('fundraising:donate'))
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.context['form'].fixed_amount, None)
        self.assertEqual(response.context['publishable_key'], settings.STRIPE_PUBLISHABLE_KEY)

        # Checking if amount field is visible
        self.assertIsInstance(response.context['form'].fields['amount'].widget, forms.TextInput)
        # Checking if campaign field is empty
        self.assertEqual(response.context['form'].initial['campaign'], None)

    def test_render_donate_form_with_bad_amount(self):
        # this will trigger a DecimalException exception because the amount can't be
        # converted to a Decimal
        response = self.client.get(reverse('fundraising:donate'), {'amount': 'superbad'})
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.context['form'].fixed_amount, 'superbad')
        self.assertEqual(response.context['publishable_key'], settings.STRIPE_PUBLISHABLE_KEY)

        # Checking if amount field is visible
        self.assertIsInstance(response.context['form'].fields['amount'].widget, forms.TextInput)
        # Checking if campaign field is empty
        self.assertEqual(response.context['form'].initial['campaign'], None)

    def test_render_donate_form_with_campaign(self):
        campaigns = ['sample', '']
        for campaign in campaigns:
            response = self.client.get(reverse('fundraising:donate'), {'amount': 100, 'campaign': campaign})
            self.assertEqual(response.status_code, 200)
            self.assertEqual(response.context['form'].fixed_amount, '100')
            self.assertEqual(response.context['publishable_key'], settings.STRIPE_PUBLISHABLE_KEY)

            # Checking if amount field is hidden
            self.assertIsInstance(response.context['form'].fields['amount'].widget, forms.HiddenInput)
            # Checking if campaign field is same as campaign
            self.assertEqual(response.context['form'].initial['campaign'], campaign)

    def test_submitting_donation_form_missing_token(self):
        url = reverse('fundraising:donate')
        response = self.client.post(url, {'amount': 100})
        self.assertFalse(response.context['form'].is_valid())
        self.assertEquals(200, response.status_code)
        self.assertTemplateUsed(response, 'fundraising/donate.html')

    def test_submitting_donation_form_invalid_amount(self):
        url = reverse('fundraising:donate')
        response = self.client.post(url, {
            'amount': 'superbad',
            'stripe_token': 'test',
        })
        self.assertFalse(response.context['form'].is_valid())
        self.assertEquals(200, response.status_code)
        # Checking if amount field is visible
        self.assertIsInstance(response.context['form'].fields['amount'].widget, forms.TextInput)

    @patch('stripe.Customer.create')
    @patch('stripe.Charge.create')
    def test_submitting_donation_form(self, charge_create, customer_create):
        response = self.client.post(reverse('fundraising:donate'), {'amount': 100})
        self.assertFalse(response.context['form'].is_valid())

        response = self.client.post(reverse('fundraising:donate'), {
            'amount': 100,
            'stripe_token': 'test',
            'receipt_email': 'test@example.com',
        })
        donations = Donation.objects.all()
        self.assertEqual(donations.count(), 1)
        self.assertEqual(donations[0].amount, 100)
        self.assertEqual(donations[0].campaign_name, '')
        self.assertEqual(donations[0].receipt_email, 'test@example.com')

    @patch('stripe.Customer.create')
    @patch('stripe.Charge.create')
    def test_submitting_donation_form_with_campaign(self, charge_create, customer_create):
        response = self.client.post(reverse('fundraising:donate'), {
            'amount': 100,
            'campaign': 'test',
        })
        self.assertFalse(response.context['form'].is_valid())
        response = self.client.post(reverse('fundraising:donate'), {
            'amount': 100,
            'campaign': 'test',
            'stripe_token': 'test',
        })
        donations = Donation.objects.all()
        self.assertEqual(donations.count(), 1)
        self.assertEqual(donations[0].amount, 100)
        self.assertEqual(donations[0].campaign_name, 'test')

    @patch('stripe.Customer.create')
    @patch('stripe.Charge.create')
    def test_submitting_donation_form_error_handling(self, charge_create, customer_create):
        data = {
            'amount': 100,
            'campaign': None,
            'stripe_token': 'xxxx',
        }
        form = PaymentForm(data=data)

        self.assertTrue(form.is_valid())

        # some errors are shows as user facting DonationErrors to the user
        # some are bubbling up to raise a 500 to trigger Sentry reports
        errors = [
            [stripe.error.CardError, DonationError],
            [stripe.error.InvalidRequestError, DonationError],
            [stripe.error.APIConnectionError, DonationError],
            [stripe.error.AuthenticationError, None],
            [stripe.error.StripeError, None],
            [ValueError, None],
        ]

        for backend_exception, user_exception in errors:
            customer_create.side_effect = backend_exception('message', 'param', 'code')

            if user_exception is None:
                self.assertRaises(backend_exception, form.make_donation)
            else:
                response = self.client.post(reverse('fundraising:donate'), data)
                self.assertTrue('donation_error' in response.context)

    @patch('fundraising.forms.PaymentForm.make_donation')
    def test_submitting_donation_form_valid(self, make_donation):
        amount = 100
        donation = Donation.objects.create(
            amount=amount,
            stripe_charge_id='xxxx',
            stripe_customer_id='xxxx',
        )
        make_donation.return_value = donation
        response = self.client.post(reverse('fundraising:donate'), {
            'amount': amount,
            'campaign': None,
            'stripe_token': 'xxxx',
        })
        self.assertRedirects(response, donation.get_absolute_url())

class TestDjangoHero(TestCase):
    def setUp(self):
        kwargs = {
            'approved': True,
            'is_visible': True,
            'is_amount_displayed': True,
        }
        self.h1 = DjangoHero.objects.create(**kwargs)
        Donation.objects.create(donor=self.h1, amount='5')
        self.h2 = DjangoHero.objects.create(**kwargs)
        Donation.objects.create(donor=self.h2, amount='15')
        # hidden donation amount should display last
        kwargs['is_amount_displayed'] = False
        self.h3 = DjangoHero.objects.create(**kwargs)
        Donation.objects.create(donor=self.h3, amount='10')
        self.today = date.today()

    def test_donation_shuffling(self):
        queryset = DjangoHero.objects.in_period(self.today, self.today + timedelta(days=1))

        for random, expected in [
            (lambda: 1, [15, 10, 5]),
            (lambda: -1, [5, 10, 15]),
            (_fake_random(1, 0, 1), [15, 5, 10]),
        ]:
            with patch('fundraising.utils.random', side_effect=random):
                self.assertQuerysetEqual(
                    shuffle_donations(queryset),
                    expected,
                    attrgetter('donated_amount')
                )

    def test_thumbnail(self):
        image_path = os.path.join(settings.MEDIA_ROOT, 'fundraising/logos/test_logo.jpg')
        image = Image.new('L', (500, 500))
        image.save(image_path)
        self.h1.logo = image_path
        self.h1.save()
        thumbnail = self.h1.thumbnail
        self.assertEqual(thumbnail.x, 340)
        self.assertEqual(thumbnail.y, 340)
        os.remove(image_path)
        self.assertTrue(
            os.path.exists(
                thumbnail.url.replace(settings.MEDIA_URL, '{}/'.format(settings.MEDIA_ROOT))
            )
        )

    def test_thumbnail_no_logo(self):
        self.assertIsNone(self.h2.thumbnail)

class TestPaymentForm(TestCase):
    @patch('stripe.Customer.create')
    @patch('stripe.Charge.create')
    def test_make_donation(self, charge_create, customer_create):
        customer_create.return_value.id = 'xxxx'
        charge_create.return_value.id = 'xxxx'
        form = PaymentForm(data={
            'amount': 100,
            'campaign': None,
            'stripe_token': 'xxxx',
        })
        self.assertTrue(form.is_valid())
        donation = form.make_donation()
        self.assertEqual(100, donation.amount)

    @patch('stripe.Customer.create')
    @patch('stripe.Charge.create')
    def test_make_donation_exception(self, charge_create, customer_create):
        customer_create.side_effect = ValueError("Something is wrong")
        form = PaymentForm(data={
            'amount': 100,
            'campaign': None,
            'stripe_token': 'xxxx',
        })
        self.assertTrue(form.is_valid())
        with self.assertRaises(ValueError):
            donation = form.make_donation()
            self.assertIsNone(donation)
<|endoftext|>