<commit_before>from pkg_resources import resource_filename
from functools import lru_cache
import langcodes
import msgpack
import re
import gzip
import itertools
import pathlib
import random
import logging
logger = logging.getLogger(__name__)

CACHE_SIZE = 100000
DATA_PATH = pathlib.Path(resource_filename('wordfreq', 'data'))

def load_range(filename):
    """
    Load a file from the data path.
    """
    with (DATA_PATH / filename).open() as file:
        return file.read()

EMOJI_RANGE = load_range('emoji.txt')
NON_PUNCT_RANGE = load_range('non_punct.txt')
COMBINING_MARK_RANGE = load_range('combining_mark.txt')

COMBINING_MARK_RE = re.compile(COMBINING_MARK_RANGE)
TOKEN_RE = re.compile("{0}|{1}+(?:'{1}+)*".format(EMOJI_RANGE, NON_PUNCT_RANGE))

def simple_tokenize(text):
    """
    A simple tokenizer that can be applied to most languages.

    It considers a word to be made of a sequence of 'token characters', an
    overly inclusive range that includes letters, Han characters, emoji, and a
    bunch of miscellaneous whatnot, but excludes most punctuation and
    whitespace.

    The single complication for the sake of English is that apostrophes are not
    considered part of the token if they appear on the edge of the character
    sequence, but they are if they appear internally. "cats'" is not a token,
    but "cat's" is.
    """
    return [token.casefold() for token in TOKEN_RE.findall(text)]

mecab_tokenize = None
def tokenize(text, lang):
    """
    Tokenize this text in a way that's straightforward but appropriate for
    the language.

    So far, this means that Japanese is handled by mecab_tokenize, and
    everything else is handled by simple_tokenize. Additionally, Arabic commas
    and combining marks are removed.

    Strings that are looked up in wordfreq will be run through this function
    first, so that they can be expected to match the data.
    """
    if lang == 'ja':
        global mecab_tokenize
        if mecab_tokenize is None:
            from wordfreq.mecab import mecab_tokenize
        return mecab_tokenize(text)

    if lang == 'ar':
        text = COMBINING_MARK_RE.sub('', text.replace('ـ', ''))

    return simple_tokenize(text)

def read_cBpack(filename):
    """
    Read a file from an idiosyncratic format that we use for storing
    approximate word frequencies, called "cBpack".

    The cBpack format is as follows:

    - The file on disk is a gzipped file in msgpack format, which decodes to a
      list whose first element is a header, and whose remaining elements are
      lists of words.

    - The header is a dictionary with 'format' and 'version' keys that make
      sure that we're reading the right thing.

    - Each inner list of words corresponds to a particular word frequency,
      rounded to the nearest centibel -- that is, one tenth of a decibel, or
      a factor of 10 ** .01.

      0 cB represents a word that occurs with probability 1, so it is the only
      word in the data (this of course doesn't happen). -200 cB represents a
      word that occurs once per 100 tokens, -300 cB represents a word that
      occurs once per 1000 tokens, and so on.

    - The index of each list within the overall list (without the header) is
      the negative of its frequency in centibels.

    - Each inner list is sorted in alphabetical order.

    As an example, consider a corpus consisting only of the words "red fish
    blue fish". The word "fish" occurs as 50% of tokens (-30 cB), while "red"
    and "blue" occur as 25% of tokens (-60 cB). The cBpack file of their word
    frequencies would decode to this:

        [
            {'format': 'cB', 'version': 1},
            [], [], [], ...    # 30 empty lists
            ['fish'],
            [], [], [], ...    # 29 more empty lists
            ['blue', 'red']
        ]
    """
    with gzip.open(filename, 'rb') as infile:
        data = msgpack.load(infile, encoding='utf-8')
    header = data[0]
    if (
        not isinstance(header, dict) or header.get('format') != 'cB'
        or header.get('version') != 1
    ):
        raise ValueError("Unexpected header: %r" % header)
    return data[1:]

def available_languages(wordlist='combined'):
    """
    List the languages (as language-code strings) that the wordlist of a given
    name is available in.
    """
    available = {}
    for path in DATA_PATH.glob('*.msgpack.gz'):
        list_name = path.name.split('.')[0]
        name, lang = list_name.split('_')
        if name == wordlist:
            available[lang] = str(path)
    return available

@lru_cache(maxsize=None)
def get_frequency_list(lang, wordlist='combined', match_cutoff=30):
    """
    Read the raw data from a wordlist file, returning it as a list of
    lists. (See `read_cBpack` for what this represents.)

    Because we use the `langcodes` module, we can handle slight
    variations in language codes. For example, looking for 'pt-BR',
    'pt_br', or even 'PT_BR' will get you the 'pt' (Portuguese) list.
    Looking up the alternate code 'por' will also get the same list.
    """
    available = available_languages(wordlist)
    best, score = langcodes.best_match(lang, list(available),
                                       min_score=match_cutoff)
    if score == 0:
        raise LookupError("No wordlist available for language %r" % lang)

    if best != lang:
        logger.warning(
            "You asked for word frequencies in language %r. Using the "
            "nearest match, which is %r (%s)."
            % (lang, best, langcodes.get(best).language_name('en'))
        )

    return read_cBpack(available[best])

def cB_to_freq(cB):
    """
    Convert a word frequency from the logarithmic centibel scale that we use
    internally, to a proportion from 0 to 1.

    On this scale, 0 cB represents the maximum possible frequency of
    1.0. -100 cB represents a word that happens 1 in 10 times,
    -200 cB represents something that happens 1 in 100 times, and so on.

    In general, x cB represents a frequency of 10 ** (x/100).
    """
    if cB > 0:
        raise ValueError(
            "A frequency cannot be a positive number of centibels."
        )
    return 10 ** (cB / 100)

@lru_cache(maxsize=None)
def get_frequency_dict(lang, wordlist='combined', match_cutoff=30):
    """
    Get a word frequency list as a dictionary, mapping tokens to
    frequencies as floating-point probabilities.
    """
    freqs = {}
    pack = get_frequency_list(lang, wordlist, match_cutoff)
    for index, bucket in enumerate(pack):
        freq = cB_to_freq(-index)
        for word in bucket:
            freqs[word] = freq
    return freqs

def iter_wordlist(lang, wordlist='combined'):
    """
    Yield the words in a wordlist in approximate descending order of
    frequency.

    Because wordfreq rounds off its frequencies, the words will form 'bands'
    with the same rounded frequency, appearing in alphabetical order within
    each band.
    """
    return itertools.chain(*get_frequency_list(lang, wordlist))

# This dict and inner function are used to implement a "drop everything" cache
# for word_frequency(); the overheads of lru_cache() are comparable to the time
# it takes to look up frequencies from scratch, so something faster is needed.
_wf_cache = {}

def _word_frequency(word, lang, wordlist, minimum):
    tokens = tokenize(word, lang)
    if not tokens:
        return minimum

    # Frequencies for multiple tokens are combined using the formula
    #     1 / f = 1 / f1 + 1 / f2 + ...
    # Thus the resulting frequency is less than any individual frequency, and
    # the smallest frequency dominates the sum.
    freqs = get_frequency_dict(lang, wordlist)
    one_over_result = 0.0
    for token in tokens:
        if token not in freqs:
            # If any word is missing, just return the default value
            return minimum
        one_over_result += 1.0 / freqs[token]

    return max(1.0 / one_over_result, minimum)

def word_frequency(word, lang, wordlist='combined', minimum=0.):
    """
    Get the frequency of `word` in the language with code `lang`, from the
    specified `wordlist`. The default wordlist is 'combined', built from
    whichever of these five sources have sufficient data for the language:

      - Full text of Wikipedia
      - A sample of 72 million tweets collected from Twitter in 2014,
        divided roughly into languages using automatic language detection
      - Frequencies extracted from OpenSubtitles
      - The Leeds Internet Corpus
      - Google Books Syntactic Ngrams 2013

    Another available wordlist is 'twitter', which uses only the data from
    Twitter.

    Words that we believe occur at least once per million tokens, based on
    the average of these lists, will appear in the word frequency list.

    The value returned will always be at least as large as `minimum`.

    If a word decomposes into multiple tokens, we'll return a smoothed estimate
    of the word frequency that is no greater than the frequency of any of its
    individual tokens.
    """
    args = (word, lang, wordlist, minimum)
    try:
        return _wf_cache[args]
    except KeyError:
        if len(_wf_cache) >= CACHE_SIZE:
            _wf_cache.clear()
        _wf_cache[args] = _word_frequency(*args)
        return _wf_cache[args]

@lru_cache(maxsize=100)
def top_n_list(lang, n, wordlist='combined', ascii_only=False):
    """
    Return a frequency list of length `n` in descending order of frequency.
    This list contains words from `wordlist`, of the given language.
    If `ascii_only`, then only ascii words are considered.
    """
    results = []
    for word in iter_wordlist(lang, wordlist):
        if (not ascii_only) or max(word) <= '~':
            results.append(word)
            if len(results) >= n:
                break
    return results

def random_words(lang='en', wordlist='combined', nwords=5, bits_per_word=12,
                 ascii_only=False):
    """
    Returns a string of random, space separated words.

    These words are of the given language and from the given wordlist.
    There will be `nwords` words in the string.

    `bits_per_word` determines the amount of entropy provided by each word;
    when it's higher, this function will choose from a larger list of
    words, some of which are more rare.

    You can restrict the selection of words to those written in ASCII
    characters by setting `ascii_only` to True.
    """
    n_choices = 2 ** bits_per_word
    choices = top_n_list(lang, n_choices, wordlist, ascii_only=ascii_only)
    if len(choices) < n_choices:
        raise ValueError(
            "There aren't enough words in the wordlist to provide %d bits of "
            "entropy per word." % bits_per_word
        )
    return ' '.join([random.choice(choices) for i in range(nwords)])

def random_ascii_words(lang='en', wordlist='combined', nwords=5,
                       bits_per_word=12):
    """
    Returns a string of random, space separated, ASCII words.

    These words are of the given language and from the given wordlist.
    There will be `nwords` words in the string.

    `bits_per_word` determines the amount of entropy provided by each word;
    when it's higher, this function will choose from a larger list of
    words, some of which are more rare.
    """
    return random_words(lang, wordlist, nwords, bits_per_word, ascii_only=True)
<commit_msg>updated word_frequency docstring<commit_after>from pkg_resources import resource_filename
from functools import lru_cache
import langcodes
import msgpack
import re
import gzip
import itertools
import pathlib
import random
import logging
logger = logging.getLogger(__name__)

CACHE_SIZE = 100000
DATA_PATH = pathlib.Path(resource_filename('wordfreq', 'data'))

def load_range(filename):
    """
    Load a file from the data path.
    """
    with (DATA_PATH / filename).open() as file:
        return file.read()

EMOJI_RANGE = load_range('emoji.txt')
NON_PUNCT_RANGE = load_range('non_punct.txt')
COMBINING_MARK_RANGE = load_range('combining_mark.txt')

COMBINING_MARK_RE = re.compile(COMBINING_MARK_RANGE)
TOKEN_RE = re.compile("{0}|{1}+(?:'{1}+)*".format(EMOJI_RANGE, NON_PUNCT_RANGE))

def simple_tokenize(text):
    """
    A simple tokenizer that can be applied to most languages.

    It considers a word to be made of a sequence of 'token characters', an
    overly inclusive range that includes letters, Han characters, emoji, and a
    bunch of miscellaneous whatnot, but excludes most punctuation and
    whitespace.

    The single complication for the sake of English is that apostrophes are not
    considered part of the token if they appear on the edge of the character
    sequence, but they are if they appear internally. "cats'" is not a token,
    but "cat's" is.
    """
    return [token.casefold() for token in TOKEN_RE.findall(text)]

mecab_tokenize = None
def tokenize(text, lang):
    """
    Tokenize this text in a way that's straightforward but appropriate for
    the language.

    So far, this means that Japanese is handled by mecab_tokenize, and
    everything else is handled by simple_tokenize. Additionally, Arabic commas
    and combining marks are removed.

    Strings that are looked up in wordfreq will be run through this function
    first, so that they can be expected to match the data.
    """
    if lang == 'ja':
        global mecab_tokenize
        if mecab_tokenize is None:
            from wordfreq.mecab import mecab_tokenize
        return mecab_tokenize(text)

    if lang == 'ar':
        text = COMBINING_MARK_RE.sub('', text.replace('ـ', ''))

    return simple_tokenize(text)

def read_cBpack(filename):
    """
    Read a file from an idiosyncratic format that we use for storing
    approximate word frequencies, called "cBpack".

    The cBpack format is as follows:

    - The file on disk is a gzipped file in msgpack format, which decodes to a
      list whose first element is a header, and whose remaining elements are
      lists of words.

    - The header is a dictionary with 'format' and 'version' keys that make
      sure that we're reading the right thing.

    - Each inner list of words corresponds to a particular word frequency,
      rounded to the nearest centibel -- that is, one tenth of a decibel, or
      a factor of 10 ** .01.

      0 cB represents a word that occurs with probability 1, so it is the only
      word in the data (this of course doesn't happen). -200 cB represents a
      word that occurs once per 100 tokens, -300 cB represents a word that
      occurs once per 1000 tokens, and so on.

    - The index of each list within the overall list (without the header) is
      the negative of its frequency in centibels.

    - Each inner list is sorted in alphabetical order.

    As an example, consider a corpus consisting only of the words "red fish
    blue fish". The word "fish" occurs as 50% of tokens (-30 cB), while "red"
    and "blue" occur as 25% of tokens (-60 cB). The cBpack file of their word
    frequencies would decode to this:

        [
            {'format': 'cB', 'version': 1},
            [], [], [], ...    # 30 empty lists
            ['fish'],
            [], [], [], ...    # 29 more empty lists
            ['blue', 'red']
        ]
    """
    with gzip.open(filename, 'rb') as infile:
        data = msgpack.load(infile, encoding='utf-8')
    header = data[0]
    if (
        not isinstance(header, dict) or header.get('format') != 'cB'
        or header.get('version') != 1
    ):
        raise ValueError("Unexpected header: %r" % header)
    return data[1:]

def available_languages(wordlist='combined'):
    """
    List the languages (as language-code strings) that the wordlist of a given
    name is available in.
    """
    available = {}
    for path in DATA_PATH.glob('*.msgpack.gz'):
        list_name = path.name.split('.')[0]
        name, lang = list_name.split('_')
        if name == wordlist:
            available[lang] = str(path)
    return available

@lru_cache(maxsize=None)
def get_frequency_list(lang, wordlist='combined', match_cutoff=30):
    """
    Read the raw data from a wordlist file, returning it as a list of
    lists. (See `read_cBpack` for what this represents.)

    Because we use the `langcodes` module, we can handle slight
    variations in language codes. For example, looking for 'pt-BR',
    'pt_br', or even 'PT_BR' will get you the 'pt' (Portuguese) list.
    Looking up the alternate code 'por' will also get the same list.
    """
    available = available_languages(wordlist)
    best, score = langcodes.best_match(lang, list(available),
                                       min_score=match_cutoff)
    if score == 0:
        raise LookupError("No wordlist available for language %r" % lang)

    if best != lang:
        logger.warning(
            "You asked for word frequencies in language %r. Using the "
            "nearest match, which is %r (%s)."
            % (lang, best, langcodes.get(best).language_name('en'))
        )

    return read_cBpack(available[best])

def cB_to_freq(cB):
    """
    Convert a word frequency from the logarithmic centibel scale that we use
    internally, to a proportion from 0 to 1.

    On this scale, 0 cB represents the maximum possible frequency of
    1.0. -100 cB represents a word that happens 1 in 10 times,
    -200 cB represents something that happens 1 in 100 times, and so on.

    In general, x cB represents a frequency of 10 ** (x/100).
    """
    if cB > 0:
        raise ValueError(
            "A frequency cannot be a positive number of centibels."
        )
    return 10 ** (cB / 100)

@lru_cache(maxsize=None)
def get_frequency_dict(lang, wordlist='combined', match_cutoff=30):
    """
    Get a word frequency list as a dictionary, mapping tokens to
    frequencies as floating-point probabilities.
    """
    freqs = {}
    pack = get_frequency_list(lang, wordlist, match_cutoff)
    for index, bucket in enumerate(pack):
        freq = cB_to_freq(-index)
        for word in bucket:
            freqs[word] = freq
    return freqs

def iter_wordlist(lang, wordlist='combined'):
    """
    Yield the words in a wordlist in approximate descending order of
    frequency.

    Because wordfreq rounds off its frequencies, the words will form 'bands'
    with the same rounded frequency, appearing in alphabetical order within
    each band.
    """
    return itertools.chain(*get_frequency_list(lang, wordlist))

# This dict and inner function are used to implement a "drop everything" cache
# for word_frequency(); the overheads of lru_cache() are comparable to the time
# it takes to look up frequencies from scratch, so something faster is needed.
_wf_cache = {}

def _word_frequency(word, lang, wordlist, minimum):
    tokens = tokenize(word, lang)
    if not tokens:
        return minimum

    # Frequencies for multiple tokens are combined using the formula
    #     1 / f = 1 / f1 + 1 / f2 + ...
    # Thus the resulting frequency is less than any individual frequency, and
    # the smallest frequency dominates the sum.
    freqs = get_frequency_dict(lang, wordlist)
    one_over_result = 0.0
    for token in tokens:
        if token not in freqs:
            # If any word is missing, just return the default value
            return minimum
        one_over_result += 1.0 / freqs[token]

    return max(1.0 / one_over_result, minimum)

def word_frequency(word, lang, wordlist='combined', minimum=0.):
    """
    Get the frequency of `word` in the language with code `lang`, from the
    specified `wordlist`. The default wordlist is 'combined', built from
    whichever of these five sources have sufficient data for the language:

      - Full text of Wikipedia
      - A sample of 72 million tweets collected from Twitter in 2014,
        divided roughly into languages using automatic language detection
      - Frequencies extracted from OpenSubtitles
      - The Leeds Internet Corpus
      - Google Books Syntactic Ngrams 2013

    Another available wordlist is 'twitter', which uses only the data from
    Twitter.

    Words that we believe occur at least once per million tokens, based on
    the average of these lists, will appear in the word frequency list.

    The value returned will always be at least as large as `minimum`.

    If a word decomposes into multiple tokens, we'll return a smoothed estimate
    of the word frequency that is no greater than the frequency of any of its
    individual tokens.

    It should be noted that the current tokenizer does not support
    multi-character Chinese terms.
    """
    args = (word, lang, wordlist, minimum)
    try:
        return _wf_cache[args]
    except KeyError:
        if len(_wf_cache) >= CACHE_SIZE:
            _wf_cache.clear()
        _wf_cache[args] = _word_frequency(*args)
        return _wf_cache[args]

@lru_cache(maxsize=100)
def top_n_list(lang, n, wordlist='combined', ascii_only=False):
    """
    Return a frequency list of length `n` in descending order of frequency.
    This list contains words from `wordlist`, of the given language.
    If `ascii_only`, then only ascii words are considered.
    """
    results = []
    for word in iter_wordlist(lang, wordlist):
        if (not ascii_only) or max(word) <= '~':
            results.append(word)
            if len(results) >= n:
                break
    return results

def random_words(lang='en', wordlist='combined', nwords=5, bits_per_word=12,
                 ascii_only=False):
    """
    Returns a string of random, space separated words.

    These words are of the given language and from the given wordlist.
    There will be `nwords` words in the string.

    `bits_per_word` determines the amount of entropy provided by each word;
    when it's higher, this function will choose from a larger list of
    words, some of which are more rare.

    You can restrict the selection of words to those written in ASCII
    characters by setting `ascii_only` to True.
    """
    n_choices = 2 ** bits_per_word
    choices = top_n_list(lang, n_choices, wordlist, ascii_only=ascii_only)
    if len(choices) < n_choices:
        raise ValueError(
            "There aren't enough words in the wordlist to provide %d bits of "
            "entropy per word." % bits_per_word
        )
    return ' '.join([random.choice(choices) for i in range(nwords)])

def random_ascii_words(lang='en', wordlist='combined', nwords=5,
                       bits_per_word=12):
    """
    Returns a string of random, space separated, ASCII words.

    These words are of the given language and from the given wordlist.
    There will be `nwords` words in the string.

    `bits_per_word` determines the amount of entropy provided by each word;
    when it's higher, this function will choose from a larger list of
    words, some of which are more rare.
    """
    return random_words(lang, wordlist, nwords, bits_per_word, ascii_only=True)
<|endoftext|>


<commit_before>import sklearn, re, nltk, base64, json, urllib2, os
import numpy as np
import cPickle as pickle
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import CountVectorizer

MIN_RESULTS = 30 # Minimum number of results needed for valid user input
BASE_SEARCH_URL = 'https://api.twitter.com/1.1/search/tweets.json?'

class TweetMining(object):
    def __init__(self, method = 'tf_idf_old'):
        nltk.data.path.append('nltk_data/')
        self.method = method
        self.setup()

    # Sets up Twitter API connection
    def setup(self):
        consumer_key = os.getenv('CONSUMER_KEY')
        consumer_secret = os.getenv('CONSUMER_SECRET')

        bearer_token = '%s:%s' % (consumer_key, consumer_secret)
        bearer_token_64 = base64.b64encode(bearer_token)

        token_request = urllib2.Request('https://api.twitter.com/oauth2/token')
        token_request.add_header('Content-Type', 'application/x-www-form-urlencoded;charset=UTF-8')
        token_request.add_header('Authorization', 'Basic %s' % bearer_token_64)
        token_request.data = 'grant_type=client_credentials'

        token_response = urllib2.urlopen(token_request)
        token_contents = token_response.read()
        token_data = json.loads(token_contents)
        self.access_token = token_data['access_token']

        with open('project_template/pho_dict.p', 'rb') as handle:
            self.dict = pickle.load(handle)

        if self.method == 'tf_idf_new':
            with open('project_template/idf.pickle', 'rb') as handle:
                self.idf = pickle.load(handle)

    # Returns list of at most num_words topical words for the given hashtag_set
    def get_topical_words(self, hashtag_set, num_words = 20):
        hashtag_set = self.cleanup_tags(hashtag_set)
        statuses = [t['text'] for t in self.get_tweets(hashtag_set, 200 * len(hashtag_set))]
        if len(statuses) < MIN_RESULTS:
            return []

        if self.method == 'tf_idf_old':
            self.process_tweets(statuses)
            vect = TfidfVectorizer(min_df = 2, stop_words = 'english', strip_accents = 'ascii')
            matrix = vect.fit_transform(statuses)
            top_indices = np.argsort(vect.idf_)[::-1]
            features = vect.get_feature_names()
            return [features[i] for i in top_indices[:num_words]]

        elif self.method == 'tf_idf_new':
            self.process_tweets(statuses, nouns_only = False)

            getIDF = lambda word : self.idf[word] if word in self.idf else 0
            vect = CountVectorizer(stop_words = 'english', strip_accents = 'ascii')

            tf = vect.fit_transform([' '.join(statuses)]).toarray()
            features = vect.get_feature_names()
            idf_vals = np.array([np.log(1600000.0 / (1 + getIDF(word))) for word in features])
            tfidf = np.multiply(tf, idf_vals)

            top_tf_inds = np.argsort(tf[0])[::-1]
            word_frequencies = [(features[i], tf[0][i]) for i in top_tf_inds[:50]]

            top_words = [word for word in hashtag_set if word.upper() in self.dict]
            top_indices = np.argsort(tfidf[0])[::-1]
            for i in top_indices:
                word = features[i]
                if word not in top_words and word.upper() in self.dict:
                    top_words.append(word)
                if len(top_words) == num_words:
                    break
            return top_words, word_frequencies

        else:
            raise Exception('Error: Invalid method specified')

    # Helper function for get_topical_words
    # Cleans up hashtag list input by stripping hashtags if they exist
    def cleanup_tags(self, hashtags):
        return [h.strip('#').strip() for h in hashtags]

    # Helper function for get_topical_words
    # Returns dict of keys "status_metadata" and "statuses" from Twitter API
    #   => "statuses" maps to a list of dicts; access "text" key to get status text
    # hashtag_set is a list of hashtags to search for (don't include #)
    def get_tweets(self, hashtag_set, num_tweets = 500):
        num_queries = num_tweets / 100
        extra_tweets = num_tweets % 100

        base_query = BASE_SEARCH_URL + 'q='
        for i in range(len(hashtag_set)):
            base_query += '%23' + hashtag_set[i]
            if i < len(hashtag_set) - 1:
                base_query += '%20OR%20'
        base_query += '&lang=en&result_type=recent&count=100'

        def callAPI(query_url):
            request = urllib2.Request(query_url)
            request.add_header('Authorization', 'Bearer %s' % self.access_token)
            response = urllib2.urlopen(request)
            contents = response.read()
            return json.loads(contents)

        result = []
        query = base_query
        for q in range(num_queries):
            statuses = callAPI(query)['statuses']
            result.extend(statuses)
            minID = min([status['id'] for status in statuses])
            query = base_query + '&max_id=' + str(minID)

        if extra_tweets > 0 and not out_of_tweets:
            query = re.sub(r'&count=\d+', '', query) + '&count=' + str(extra_tweets)
            result.extend(callAPI(query)['statuses'])

        return result

    # Helper method for get_topical_words
    # Processes statuses in-place by removing irrelevant components
    def process_tweets(self, statuses, nouns_only = True):
        for i in range(len(statuses)):
            statuses[i] = re.sub(r'\S*/\S*', '', statuses[i]) # Links
            statuses[i] = re.sub(r'htt\S*', '', statuses[i]) # Hanging https
            statuses[i] = re.sub(r'#\S*', '', statuses[i]) # Hashtag symbols
            statuses[i] = re.sub(r'(RT)*( )?@\S*', '', statuses[i]) # RT, @user
            statuses[i] = re.sub(r'(RT |rt[^a-z])', '', statuses[i]) # RT/rt
            statuses[i] = re.sub(r'\S*\d+\S*', '', statuses[i]) # Numerical
            statuses[i] = re.sub(r"\w+'[^s ]+", '', statuses[i]) # Contractions
            statuses[i] = re.sub(r'&\S+;', '', statuses[i]) # HTML entities

            if nouns_only:
                pos_info = nltk.pos_tag(nltk.word_tokenize(statuses[i]))
                statuses[i] = ' '.join([word[0] for word in pos_info if 'NN' in word[1]])
<commit_msg>Strip out commas and only use top 30 words<commit_after>import sklearn, re, nltk, base64, json, urllib2, os
import numpy as np
import cPickle as pickle
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import CountVectorizer

MIN_RESULTS = 30 # Minimum number of results needed for valid user input
BASE_SEARCH_URL = 'https://api.twitter.com/1.1/search/tweets.json?'

class TweetMining(object):
    def __init__(self, method = 'tf_idf_old'):
        nltk.data.path.append('nltk_data/')
        self.method = method
        self.setup()

    # Sets up Twitter API connection
    def setup(self):
        consumer_key = os.getenv('CONSUMER_KEY')
        consumer_secret = os.getenv('CONSUMER_SECRET')

        bearer_token = '%s:%s' % (consumer_key, consumer_secret)
        bearer_token_64 = base64.b64encode(bearer_token)

        token_request = urllib2.Request('https://api.twitter.com/oauth2/token')
        token_request.add_header('Content-Type', 'application/x-www-form-urlencoded;charset=UTF-8')
        token_request.add_header('Authorization', 'Basic %s' % bearer_token_64)
        token_request.data = 'grant_type=client_credentials'

        token_response = urllib2.urlopen(token_request)
        token_contents = token_response.read()
        token_data = json.loads(token_contents)
        self.access_token = token_data['access_token']

        with open('project_template/pho_dict.p', 'rb') as handle:
            self.dict = pickle.load(handle)

        if self.method == 'tf_idf_new':
            with open('project_template/idf.pickle', 'rb') as handle:
                self.idf = pickle.load(handle)

    # Returns list of at most num_words topical words for the given hashtag_set
    def get_topical_words(self, hashtag_set, num_words = 20):
        hashtag_set = self.cleanup_tags(hashtag_set)
        statuses = [t['text'] for t in self.get_tweets(hashtag_set, 200 * len(hashtag_set))]
        if len(statuses) < MIN_RESULTS:
            return []

        if self.method == 'tf_idf_old':
            self.process_tweets(statuses)
            vect = TfidfVectorizer(min_df = 2, stop_words = 'english', strip_accents = 'ascii')
            matrix = vect.fit_transform(statuses)
            top_indices = np.argsort(vect.idf_)[::-1]
            features = vect.get_feature_names()
            return [features[i] for i in top_indices[:num_words]]

        elif self.method == 'tf_idf_new':
            self.process_tweets(statuses, nouns_only = False)

            getIDF = lambda word : self.idf[word] if word in self.idf else 0
            vect = CountVectorizer(stop_words = 'english', strip_accents = 'ascii')

            tf = vect.fit_transform([' '.join(statuses)]).toarray()
            features = vect.get_feature_names()
            idf_vals = np.array([np.log(1600000.0 / (1 + getIDF(word))) for word in features])
            tfidf = np.multiply(tf, idf_vals)

            top_tf_inds = np.argsort(tf[0])[::-1]
            word_frequencies = [(features[i], tf[0][i]) for i in top_tf_inds[:30]]

            top_words = [word for word in hashtag_set if word.upper() in self.dict]
            top_indices = np.argsort(tfidf[0])[::-1]
            for i in top_indices:
                word = features[i]
                if word not in top_words and word.upper() in self.dict:
                    top_words.append(word)
                if len(top_words) == num_words:
                    break
            return top_words, word_frequencies

        else:
            raise Exception('Error: Invalid method specified')

    # Helper function for get_topical_words
    # Cleans up hashtag list input by stripping hashtags if they exist
    def cleanup_tags(self, hashtags):
        return [h.strip(',').strip('#').strip() for h in hashtags]

    # Helper function for get_topical_words
    # Returns dict of keys "status_metadata" and "statuses" from Twitter API
    #   => "statuses" maps to a list of dicts; access "text" key to get status text
    # hashtag_set is a list of hashtags to search for (don't include #)
    def get_tweets(self, hashtag_set, num_tweets = 500):
        num_queries = num_tweets / 100
        extra_tweets = num_tweets % 100

        base_query = BASE_SEARCH_URL + 'q='
        for i in range(len(hashtag_set)):
            base_query += '%23' + hashtag_set[i]
            if i < len(hashtag_set) - 1:
                base_query += '%20OR%20'
        base_query += '&lang=en&result_type=recent&count=100'

        def callAPI(query_url):
            request = urllib2.Request(query_url)
            request.add_header('Authorization', 'Bearer %s' % self.access_token)
            response = urllib2.urlopen(request)
            contents = response.read()
            return json.loads(contents)

        result = []
        query = base_query
        for q in range(num_queries):
            statuses = callAPI(query)['statuses']
            result.extend(statuses)
            minID = min([status['id'] for status in statuses])
            query = base_query + '&max_id=' + str(minID)

        if extra_tweets > 0 and not out_of_tweets:
            query = re.sub(r'&count=\d+', '', query) + '&count=' + str(extra_tweets)
            result.extend(callAPI(query)['statuses'])

        return result

    # Helper method for get_topical_words
    # Processes statuses in-place by removing irrelevant components
    def process_tweets(self, statuses, nouns_only = True):
        for i in range(len(statuses)):
            statuses[i] = re.sub(r'\S*/\S*', '', statuses[i]) # Links
            statuses[i] = re.sub(r'htt\S*', '', statuses[i]) # Hanging https
            statuses[i] = re.sub(r'#\S*', '', statuses[i]) # Hashtag symbols
            statuses[i] = re.sub(r'(RT)*( )?@\S*', '', statuses[i]) # RT, @user
            statuses[i] = re.sub(r'(RT |rt[^a-z])', '', statuses[i]) # RT/rt
            statuses[i] = re.sub(r'\S*\d+\S*', '', statuses[i]) # Numerical
            statuses[i] = re.sub(r"\w+'[^s ]+", '', statuses[i]) # Contractions
            statuses[i] = re.sub(r'&\S+;', '', statuses[i]) # HTML entities

            if nouns_only:
                pos_info = nltk.pos_tag(nltk.word_tokenize(statuses[i]))
                statuses[i] = ' '.join([word[0] for word in pos_info if 'NN' in word[1]])
<|endoftext|>


<commit_before>#In this project the user adds the Ball class, which handles
#most of the game physics, and adds the win and lose functions.

#Imports
from rstem import led_matrix
import accel
import sys
import time

#Initialize hardware
led_matrix.init_grid(2, 2)
accel.init(1)

#Initialize game data
bricks = []
score = 0
start_time = time.time()
ball_tick = 0
MAX_BALL_TICK = 8
#Useful clamp function
def clamp(value, minimum, maximum):
	return min(maximum, max(value, minimum))

#Display losing score
def lose():
	string = "You lost! Score: %i" % (score)
	msg = led_matrix.LEDText(string, font_name='large')
	for i in range(len(string)*6 + 15):
		led_matrix.fill(0)
		led_matrix.text(msg, (15 - i, 7))
		led_matrix.show()
		time.sleep(0.1)
	sys.exit(0)

#If all bricks are destroyed, display winning score
def win():
	string = "You won! Score: %i" % (score)
	msg = led_matrix.LEDText(string, font_name='large')
	for i in range(len(string)*6 + 15):
		led_matrix.fill(0)
		led_matrix.text(msg, (15 - i, 7))
		led_matrix.show()
	sys.exit(0)

#Simple class for player Paddle
class Paddle:

	def __init__(self, pos, size):
		self.pos = pos
		self.size = size

	def move(self, x):
		self.pos[0] = int(x)

#Declare paddle for use in ball class
p = Paddle([7, 0], [4, 0])

#Class for ball which handles most game physics
class Ball:

	def __init__(self, pos, dir):
		self.pos = pos
		self.dir = dir

#	Update game physics
	def update(self):

#		Move ball
		self.pos[0] = self.pos[0] + self.dir[0]
		self.pos[1] = self.pos[1] + self.dir[1]
#		Player wins if there are no more bricks
		if len(bricks) == 0:
			win()
#		Loop through bricks and check for collisions
		for b in bricks:
			if b.xCol(self) and b.yCol(self):
#				If the ball hits a side edge reverse its x direction
				self.dir[0] = self.dir[0]*(1 - 2*((self.pos[1] >= b.pos[1]) and (self.pos[1] < b.pos[1] + b.size[1])))
#				If the ball hits a side edge reverse its y direction
				self.dir[1] = self.dir[1]*(1 - 2*((self.pos[0] >= b.pos[0]) and (self.pos[0] < b.pos[0] + b.size[0])))
#				Update score
				b.hit()

#		If the ball hits the paddle reverse its y direction
		if (self.pos[1] + self.dir[1] == p.pos[1]) and (self.pos[0] + self.dir[0] >= p.pos[0]) and (self.pos[0] + self.dir[0] < p.pos[0] + p.size[0]):
			self.dir[1] = -self.dir[1]
#		Bounce off edge of screen
		if self.pos[1] >= 15:
			self.dir[1] = -self.dir[1]
		if self.pos[0] <= 0 or self.pos[0] >= 15:
			self.dir[0] = -self.dir[0]

#		Lose if the ball passes the paddle
		if self.pos[1] <= 0:
			for i in range(4):
				led_matrix.point(self.pos[0], self.pos[1])
				led_matrix.show()
				time.sleep(0.1)
				led_matrix.point(self.pos[0], self.pos[1], 0)
				led_matrix.show()
				time.sleep(0.1)
				lose()

#Class for brick
class Brick:
	def __init__(self, pos, size):
		self.pos = pos
		self.size = size
		self.brightness = 15

#	If hit increment score and decrement brightness
	def hit(self):
		global score
		self.brightness = self.brightness - 5
#		If the brightness is 0, remove brick
		if(self.brightness <= 0):
			bricks.remove(self)
		score = score + 1

#	Functions for collision detection
	def xCol(self, ball):
		return (ball.pos[0] + ball.dir[0] >= self.pos[0]) and (ball.pos[0] < self.pos[0] + self.size[0])

	def yCol(self, ball):
		return ((ball.pos[1] + ball.dir[1] >= self.pos[1]) and (ball.pos[1] < self.pos[1] + self.size[1]))

#Initialize bricks
for x in range(4):
	for y in range(3):
		bricks.append(Brick([x*4, 15 - y*3 - 1],[3, 2]))

#Initialize ball
ball = Ball([8, 0],[1, 1])

#Initialize player movement data
velocity = 0.0
player_pos = 7.0
alpha = 0.1

while True:
#	Clear frame buffer
	led_matrix.fill(0)
#	Get angle data
	angles = accel.angles()
#	Simple lowpass filter for velocity data
	velocity = velocity*alpha + (angles[0]*2*8/90)*(1 - alpha)
#	Move player's paddle
	player_pos = player_pos + velocity
	player_pos = clamp(player_pos, 0, 15 - (p.size[0] - 1))
#	If the paddle hits the edge the velocity can't be into the edge
	if int(player_pos) <= 0 and velocity < 0:
		velocity = 0
	elif int(player_pos) >= 15 and velocity > 0:
		velcoity = 0
#	Move player
	p.move(player_pos)
#	Update physics
	if ball_tick == 0:
		ball.update()
#	Display player, bricks, and the ball
	led_matrix.point(ball.pos[0], ball.pos[1])
	led_matrix.line(p.pos, [p.pos[0] + p.size[0] - 1, 0])
	for b in bricks:
		for y in range(b.size[1]):
			led_matrix.line((b.pos[0], b.pos[1]+y),(b.pos[0] + b.size[0] - 1, b.pos[1] + y), b.brightness)
	led_matrix.show()

#	Slowly speed up, start with delay at 0.1 and reach 0 at 60s
	time.sleep(0.01)
	ball_tick = (ball_tick + 1) & (MAX_BALL_TICK - 1)
<commit_msg>Changed imports<commit_after>#In this project the user adds the Ball class, which handles
#most of the game physics, and adds the win and lose functions.

#Imports
from rstem import led_matrix
from rstem import accel
import sys
import time

#Initialize hardware
led_matrix.init_grid(2, 2)
accel.init(1)

#Initialize game data
bricks = []
score = 0
start_time = time.time()
ball_tick = 0
MAX_BALL_TICK = 8
#Useful clamp function
def clamp(value, minimum, maximum):
	return min(maximum, max(value, minimum))

#Display losing score
def lose():
	string = "You lost! Score: %i" % (score)
	msg = led_matrix.LEDText(string, font_name='large')
	for i in range(len(string)*6 + 15):
		led_matrix.fill(0)
		led_matrix.text(msg, (15 - i, 7))
		led_matrix.show()
		time.sleep(0.1)
	sys.exit(0)

#If all bricks are destroyed, display winning score
def win():
	string = "You won! Score: %i" % (score)
	msg = led_matrix.LEDText(string, font_name='large')
	for i in range(len(string)*6 + 15):
		led_matrix.fill(0)
		led_matrix.text(msg, (15 - i, 7))
		led_matrix.show()
	sys.exit(0)

#Simple class for player Paddle
class Paddle:

	def __init__(self, pos, size):
		self.pos = pos
		self.size = size

	def move(self, x):
		self.pos[0] = int(x)

#Declare paddle for use in ball class
p = Paddle([7, 0], [4, 0])

#Class for ball which handles most game physics
class Ball:

	def __init__(self, pos, dir):
		self.pos = pos
		self.dir = dir

#	Update game physics
	def update(self):

#		Move ball
		self.pos[0] = self.pos[0] + self.dir[0]
		self.pos[1] = self.pos[1] + self.dir[1]
#		Player wins if there are no more bricks
		if len(bricks) == 0:
			win()
#		Loop through bricks and check for collisions
		for b in bricks:
			if b.xCol(self) and b.yCol(self):
#				If the ball hits a side edge reverse its x direction
				self.dir[0] = self.dir[0]*(1 - 2*((self.pos[1] >= b.pos[1]) and (self.pos[1] < b.pos[1] + b.size[1])))
#				If the ball hits a side edge reverse its y direction
				self.dir[1] = self.dir[1]*(1 - 2*((self.pos[0] >= b.pos[0]) and (self.pos[0] < b.pos[0] + b.size[0])))
#				Update score
				b.hit()

#		If the ball hits the paddle reverse its y direction
		if (self.pos[1] + self.dir[1] == p.pos[1]) and (self.pos[0] + self.dir[0] >= p.pos[0]) and (self.pos[0] + self.dir[0] < p.pos[0] + p.size[0]):
			self.dir[1] = -self.dir[1]
#		Bounce off edge of screen
		if self.pos[1] >= 15:
			self.dir[1] = -self.dir[1]
		if self.pos[0] <= 0 or self.pos[0] >= 15:
			self.dir[0] = -self.dir[0]

#		Lose if the ball passes the paddle
		if self.pos[1] <= 0:
			for i in range(4):
				led_matrix.point(self.pos[0], self.pos[1])
				led_matrix.show()
				time.sleep(0.1)
				led_matrix.point(self.pos[0], self.pos[1], 0)
				led_matrix.show()
				time.sleep(0.1)
				lose()

#Class for brick
class Brick:
	def __init__(self, pos, size):
		self.pos = pos
		self.size = size
		self.brightness = 15

#	If hit increment score and decrement brightness
	def hit(self):
		global score
		self.brightness = self.brightness - 5
#		If the brightness is 0, remove brick
		if(self.brightness <= 0):
			bricks.remove(self)
		score = score + 1

#	Functions for collision detection
	def xCol(self, ball):
		return (ball.pos[0] + ball.dir[0] >= self.pos[0]) and (ball.pos[0] < self.pos[0] + self.size[0])

	def yCol(self, ball):
		return ((ball.pos[1] + ball.dir[1] >= self.pos[1]) and (ball.pos[1] < self.pos[1] + self.size[1]))

#Initialize bricks
for x in range(4):
	for y in range(3):
		bricks.append(Brick([x*4, 15 - y*3 - 1],[3, 2]))

#Initialize ball
ball = Ball([8, 0],[1, 1])

#Initialize player movement data
velocity = 0.0
player_pos = 7.0
alpha = 0.1

while True:
#	Clear frame buffer
	led_matrix.fill(0)
#	Get angle data
	angles = accel.angles()
#	Simple lowpass filter for velocity data
	velocity = velocity*alpha + (angles[0]*2*8/90)*(1 - alpha)
#	Move player's paddle
	player_pos = player_pos + velocity
	player_pos = clamp(player_pos, 0, 15 - (p.size[0] - 1))
#	If the paddle hits the edge the velocity can't be into the edge
	if int(player_pos) <= 0 and velocity < 0:
		velocity = 0
	elif int(player_pos) >= 15 and velocity > 0:
		velcoity = 0
#	Move player
	p.move(player_pos)
#	Update physics
	if ball_tick == 0:
		ball.update()
#	Display player, bricks, and the ball
	led_matrix.point(ball.pos[0], ball.pos[1])
	led_matrix.line(p.pos, [p.pos[0] + p.size[0] - 1, 0])
	for b in bricks:
		for y in range(b.size[1]):
			led_matrix.line((b.pos[0], b.pos[1]+y),(b.pos[0] + b.size[0] - 1, b.pos[1] + y), b.brightness)
	led_matrix.show()

#	Slowly speed up, start with delay at 0.1 and reach 0 at 60s
	time.sleep(0.01)
	ball_tick = (ball_tick + 1) & (MAX_BALL_TICK - 1)
<|endoftext|>


<commit_before># Mrs
# Copyright 2008 Brigham Young University
#
# This file is part of Mrs.
#
# Mrs is free software: you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any later
# version.
#
# Mrs is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# Mrs.  If not, see <http://www.gnu.org/licenses/>.
#
# Inquiries regarding any further use of the Materials contained on this site,
# please contact the Copyright Licensing Office, Brigham Young University,
# 3760 HBLL, Provo, UT 84602, (801) 422-9339 or 422-3821, e-mail
# copyright@byu.edu.

"""Mrs Slave

The Mrs Slave runs in three threads: the main thread, the event thread, and
the worker thread.

The main thread doesn't really do anything.  It starts the other two threads
and waits for the other two to finish.  If the user hits CTRL-C, the main
thread will be interrupted, and it will shut down the event thread.  The only
reason that the main thread exists at all is to deal with signals.

The event thread does all of the work.  It connects to the server, listens for
RPC requests, and downloads files.  Its control flow is asynchronous.  This
can make some things easier to understand and other things harder to
understand.  It's a tradeoff, but in the end, it's easier and more efficient
to deal with network IO asynchronously than with a heap of threads.  Anyway,
the event loop is managed by the Twisted reactor.

The worker thread executes the user's map function and reduce function.
That's it.  It just does what the event thread tells it to.  Note that since
the worker thread is a daemon thread, it quits implicitly once the other two
threads complete.
"""

# Timeout for RPC requests (including pings):
RPC_TIMEOUT = 15
# Number of ping timeouts before giving up:
PING_ATTEMPTS = 50

COOKIE_LEN = 8
QUIT_DELAY = 0.5

import threading
from twisted.internet import reactor, defer
from twist import TwistedThread, GrimReaper, PingTask, ErrbackException
from twistrpc import RequestXMLRPC, TimeoutProxy

def slave_main(registry, user_run, user_setup, args, opts):
    """Run Mrs Slave

    Slave Main is called directly from Mrs Main.  On exit, the process
    will return slave_main's return value.
    """

    slave = Slave(registry, user_setup, opts.mrs_master)

    # Create the other threads:
    worker = Worker(slave)
    event_thread = SlaveEventThread(slave)

    # Start the other threads:
    event_thread.start()
    worker.start()

    try:
        # Note: under normal circumstances, the reactor (in the event
        # thread) will quit on its own.
        slave.reaper.wait()
    except KeyboardInterrupt:
        pass

    event_thread.shutdown()
    event_thread.join()

    if slave.reaper.traceback:
        print slave.reaper.traceback

    return 0

# TODO: when we stop supporting Python older than 2.5, use inlineCallbacks:
class SlaveEventThread(TwistedThread):
    """Thread on slave that runs the Twisted reactor

    We don't trust the Twisted reactor's signal handlers, so we run it with
    the handlers disabled.  As a result, it shouldn't be in the primary
    thread.
    """

    def __init__(self, slave):
        TwistedThread.__init__(self)
        self.slave = slave

    def run(self):
        """Called when the thread is started.

        It starts the reactor and schedules signin() to be called.
        """
        reactor.callLater(0, self.slave.signin)
        TwistedThread.run(self)

        # Let other threads know that we are quitting.
        self.slave.quit()

class Slave(object):
    """Mrs Slave"""

    def __init__(self, registry, user_setup, mrs_master):
        self.mrs_master = mrs_master
        self.registry = registry
        self.user_setup = user_setup

        self.reaper = GrimReaper()
        self.worker = None
        self.ping_task = None
        self.timeouts = 0

        self.master_rpc = TimeoutProxy(self.mrs_master, RPC_TIMEOUT)

        self.id = None
        self.alive = True
        self.cookie = self.rand_cookie()

    # State 1
    def signin(self):
        """Start Slave RPC Server and sign in to master."""
        from twisted.web import server

        resource = SlaveInterface(self)
        site = server.Site(resource)
        tcpport = reactor.listenTCP(0, site)
        address = tcpport.getHost()

        # Initiate signin to master
        from version import VERSION
        cookie = self.cookie
        port = address.port
        source_hash = self.registry.source_hash()
        reg_hash = self.registry.reg_hash()

        deferred = self.master_rpc.callRemote('signin', VERSION, cookie, port,
                source_hash, reg_hash)

        deferred.addCallbacks(self.signin_callback, self.signin_errback)

    def signin_errback(self, failure):
        print failure
        self.quit('Unable to contact master.')

    # State 2
    def signin_callback(self, value):
        """Process the results from the signin.

        This is the callback after signin.  After saving the return values,
        schedule the user_setup function to be run in the Worker.
        """
        slave_id, optdict = value

        if slave_id < 0:
            self.quit('Master rejected signin.')

        # Save the slave id given by the master.
        self.id = slave_id

        # Create the Ping Task
        ping_args = ('ping', self.id, self.cookie)
        self.ping_task = PingTask(ping_args, self.master_rpc,
                self.ping_success, self.ping_failure, self.get_timestamp)

        # Tell the Worker to run the user_setup function.
        import optparse
        options = optparse.Values(optdict)
        callback = self.report_ready
        self.worker.start_setup(options, callback)

    def report_ready(self):
        """Report to the master that we are ready to accept tasks.

        This is the callback after user_setup is called.
        """
        # Report for duty.
        deferred = self.master_rpc.callRemote('ready', self.id, self.cookie)
        deferred.addCallback(self.ready_success)
        deferred.addErrback(self.ready_failure)

    def ready_success(self, value):
        """Called when reporting in as ready succeeded."""
        import sys
        print >>sys.stderr, "Connected to master."
        self.update_timestamp()
        self.timeouts = 0
        if not self.ping_task.running:
            self.ping_task.start()

    def ready_failure(self, err):
        """Called when reporting in as ready failed."""
        import sys
        give_up = False
        if err.check(defer.TimeoutError):
            self.timeouts += 1
            if self.timeouts > PING_ATTEMPTS:
                print >>sys.stderr, "Too many ping timeouts.  Giving up."
                give_up = True
            else:
                # Try to connect again
                print >>sys.stderr, "Reconnect timed out.  Trying again."
                self.report_ready()
        else:
            print >>sys.stderr, "Couldn't report in due to network error."
            print >>sys.stderr, err
            give_up = True

        if give_up:
            self.reaper.reap()

    def ping_failure(self, err):
        """Report that the master failed to respond to an RPC request.

        This may be either a ping or some other request.  At the moment,
        we aren't very lenient, but in the future we could allow a few
        failures before disconnecting.
        """
        import sys

        give_up = False
        if err.check(defer.TimeoutError):
            self.timeouts += 1
            if self.timeouts > PING_ATTEMPTS:
                print >>sys.stderr, "Too many ping timeouts.  Giving up."
                give_up = True
            else:
                print >>sys.stderr, 'Ping timeout.  Trying again.'
        else:
            print >>sys.stderr, 'Lost master due to network error.'
            print >>sys.stderr, err

        if give_up:
            self.ping_task.stop()
            self.reaper.reap()

    def ping_success(self):
        """Called when a ping successfully completes."""
        self.update_timestamp()
        self.timeouts = 0

    def update_timestamp(self):
        """Set the timestamp to the current time."""
        from datetime import datetime
        self.timestamp = datetime.utcnow()

    def get_timestamp(self):
        """Report the most recent timestamp."""
        return self.timestamp

    # Miscellaneous
    @classmethod
    def rand_cookie(cls):
        # Generate a cookie so that mostly only authorized people can connect.
        from random import choice
        import string
        possible = string.letters + string.digits
        return ''.join(choice(possible) for i in xrange(COOKIE_LEN))

    def check_cookie(self, cookie):
        if cookie != self.cookie:
            raise CookieValidationError

    def register_worker(self, worker):
        """Called by the worker so the Slave can have a reference to it."""
        self.worker = worker

    def die(self, exception):
        """Die with an exception."""
        self.reaper.reap(exception)

    def quit(self, message=None):
        """Called to quit the slave."""
        if message:
            import sys
            print >>sys.stderr, message
        self.reaper.reap()

    def reconnect(self):
        """Called by the worker thread when it can't contact the master."""
        reactor.callFromThread(self.ping_task.stop)
        reactor.callFromThread(self.report_ready)

class SlaveInterface(RequestXMLRPC):
    """Public XML RPC Interface

    Note that any method not beginning with "xmlrpc_" will be exposed to
    remote hosts.  Any of these can return either a result or a deferred.
    """
    def __init__(self, slave):
        RequestXMLRPC.__init__(self)
        self.slave = slave

    def xmlrpc_start_map(self, source, inputs, func_name, part_name, nparts,
            output, extension, cookie):
        self.slave.check_cookie(cookie)
        self.slave.update_timestamp()
        return self.slave.worker.start_map(source, inputs, func_name,
                part_name, nparts, output, extension)

    def xmlrpc_start_reduce(self, source, inputs, func_name, part_name,
            nparts, output, extension, cookie):
        self.slave.check_cookie(cookie)
        self.slave.update_timestamp()
        return self.slave.worker.start_reduce(source, inputs, func_name,
                part_name, nparts, output, extension)

    def xmlrpc_quit(self, cookie):
        self.slave.check_cookie(cookie)
        self.slave.update_timestamp()
        self.slave.alive = False
        # We delay before actually stopping because we need to make sure that
        # the response gets sent back.
        reactor.callLater(QUIT_DELAY, self.slave.quit,
                "Quitting as requested by an RPC call.")
        return True

    def xmlrpc_ping(self, cookie):
        """Master checking if we're still here.
        """
        self.slave.check_cookie(cookie)
        self.slave.update_timestamp()
        return True

class CookieValidationError(Exception):
    pass

class Worker(threading.Thread):
    """Execute map tasks and reduce tasks.

    The worker waits for other threads to make assignments by calling
    start_map and start_reduce.

    This needs to run in a daemon thread rather than in the main thread so
    that it can be killed by other threads.
    """
    def __init__(self, slave):
        threading.Thread.__init__(self)
        # Die when all other non-daemon threads have exited:
        self.setDaemon(True)

        self.slave = slave
        self.slave.register_worker(self)

        self._task = None
        self._cond = threading.Condition()
        self.active = False
        self.exception = None

        self.options = None
        self._setup_ready = threading.Event()
        self._setup_callback = None

    def start_setup(self, options, callback):
        """Start running the user_setup function.

        The given callback function will be invoked in the reactor thread.
        """
        self.options = options
        self._setup_ready.set()
        self._setup_callback = callback

    def start_map(self, source, inputs, map_name, part_name, nparts, output,
            extension):
        """Tell this worker to start working on a map task.

        This will ordinarily be called from some other thread.
        """
        from task import MapTask
        from datasets import FileData
        from io import writerformat

        input_data = FileData(inputs, splits=1)
        format = writerformat(extension)

        success = False
        self._cond.acquire()
        if self._task is None:
            registry = self.slave.registry
            self._task = MapTask(input_data, 0, source, map_name, part_name,
                    nparts, output, format, registry)
            success = True
            self._cond.notify()
        self._cond.release()
        return success

    def start_reduce(self, source, inputs, reduce_name, part_name, nparts,
            output, extension):
        """Tell this worker to start working on a reduce task.

        This will ordinarily be called from some other thread.
        """
        from task import ReduceTask
        from datasets import FileData
        from io import writerformat

        input_data = FileData(inputs, splits=1)
        format = writerformat(extension)

        success = False
        self._cond.acquire()
        if self._task is None:
            registry = self.slave.registry
            self._task = ReduceTask(input_data, 0, source, reduce_name,
                    part_name, nparts, output, format, registry)
            success = True
            self._cond.notify()
        self._cond.release()
        return success

    def run(self):
        """Run the worker."""
        import sys

        # Run user_setup if requested:
        self._setup_ready.wait()
        user_setup = self.slave.user_setup
        if user_setup:
            try:
                user_setup(self.options)
            except Exception, e:
                print "Caught an exception in the Worker thread (in setup)!"
                self.slave.die(e)
                return

        # Alert the Twisted thread that user_setup is done.
        reactor.callFromThread(self._setup_callback)

        # Process tasks:
        while True:
            self._cond.acquire()
            while self._task is None:
                self._cond.wait()
            task = self._task
            self._cond.release()

            try:
                task.run()
            except Exception, e:
                print >>sys.stderr, "Caught an exception in the Worker thread!"
                self.slave.die(e)
                return

            self._cond.acquire()
            self._task = None
            self._cond.release()

            # TODO: right now, dataset.dump() happens in task.run().  Instead,
            # we should tell the dataset to become available() here, and the
            # data should automatically be dumped.
            try:
                self.slave.master_rpc.blocking_call('done', self.slave.id,
                        task.outurls(), self.slave.cookie)
            except ErrbackException, e:
                # TODO: We should be able to retry calling done, but this will
                # only be possible if done uses some unique id.  For now,
                # retrying done can be very destructive.
                #print >>sys.stderr, ("RPC error when reporting back."
                #        "  Trying again.")
                print >>sys.stderr, ("RPC error when reporting back."
                        "  Giving up.")
                self.slave.reconnect()

# vim: et sw=4 sts=4
<commit_msg>added some more comments to the Slave class<commit_after># Mrs
# Copyright 2008 Brigham Young University
#
# This file is part of Mrs.
#
# Mrs is free software: you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation, either version 3 of the License, or (at your option) any later
# version.
#
# Mrs is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# Mrs.  If not, see <http://www.gnu.org/licenses/>.
#
# Inquiries regarding any further use of the Materials contained on this site,
# please contact the Copyright Licensing Office, Brigham Young University,
# 3760 HBLL, Provo, UT 84602, (801) 422-9339 or 422-3821, e-mail
# copyright@byu.edu.

"""Mrs Slave

The Mrs Slave runs in three threads: the main thread, the event thread, and
the worker thread.

The main thread doesn't really do anything.  It starts the other two threads
and waits for the other two to finish.  If the user hits CTRL-C, the main
thread will be interrupted, and it will shut down the event thread.  The only
reason that the main thread exists at all is to deal with signals.

The event thread does all of the work.  It connects to the server, listens for
RPC requests, and downloads files.  Its control flow is asynchronous.  This
can make some things easier to understand and other things harder to
understand.  It's a tradeoff, but in the end, it's easier and more efficient
to deal with network IO asynchronously than with a heap of threads.  Anyway,
the event loop is managed by the Twisted reactor.

The worker thread executes the user's map function and reduce function.
That's it.  It just does what the event thread tells it to.  Note that since
the worker thread is a daemon thread, it quits implicitly once the other two
threads complete.
"""

# Timeout for RPC requests (including pings):
RPC_TIMEOUT = 15
# Number of ping timeouts before giving up:
PING_ATTEMPTS = 50

COOKIE_LEN = 8
QUIT_DELAY = 0.5

import threading
from twisted.internet import reactor, defer
from twist import TwistedThread, GrimReaper, PingTask, ErrbackException
from twistrpc import RequestXMLRPC, TimeoutProxy

def slave_main(registry, user_run, user_setup, args, opts):
    """Run Mrs Slave

    Slave Main is called directly from Mrs Main.  On exit, the process
    will return slave_main's return value.
    """

    slave = Slave(registry, user_setup, opts.mrs_master)

    # Create the other threads:
    worker = Worker(slave)
    event_thread = SlaveEventThread(slave)

    # Start the other threads:
    event_thread.start()
    worker.start()

    try:
        # Note: under normal circumstances, the reactor (in the event
        # thread) will quit on its own.
        slave.reaper.wait()
    except KeyboardInterrupt:
        pass

    event_thread.shutdown()
    event_thread.join()

    if slave.reaper.traceback:
        print slave.reaper.traceback

    return 0

# TODO: when we stop supporting Python older than 2.5, use inlineCallbacks:
class SlaveEventThread(TwistedThread):
    """Thread on slave that runs the Twisted reactor

    We don't trust the Twisted reactor's signal handlers, so we run it with
    the handlers disabled.  As a result, it shouldn't be in the primary
    thread.
    """

    def __init__(self, slave):
        TwistedThread.__init__(self)
        self.slave = slave

    def run(self):
        """Called when the thread is started.

        It starts the reactor and schedules signin() to be called.
        """
        reactor.callLater(0, self.slave.signin)
        TwistedThread.run(self)

        # Let other threads know that we are quitting.
        self.slave.quit()

class Slave(object):
    """State of a Mrs slave

    Since execution flow is event-driven, a little overview might be helpful.
    When the slave starts up, the `signin` method is called.  If it fails, the
    `signin_errback` method shuts everything down, but if it succeeds, the
    `signin_callback` method starts some initialization in the Worker thread.
    When this initialization finishes, the Worker thread will trigger the
    `report_ready` method, which causes the slave to report in to the master.
    Later on, the Worker may trigger this method again if it can't contact the
    master; this gives us a chance to reconnect if the network hiccups for a
    few minutes.
    """

    def __init__(self, registry, user_setup, mrs_master):
        self.mrs_master = mrs_master
        self.registry = registry
        self.user_setup = user_setup

        self.reaper = GrimReaper()
        self.worker = None
        self.ping_task = None
        self.timeouts = 0

        self.master_rpc = TimeoutProxy(self.mrs_master, RPC_TIMEOUT)

        self.id = None
        self.alive = True
        self.cookie = self.rand_cookie()

    # State 1
    def signin(self):
        """Start Slave RPC Server and sign in to master."""
        from twisted.web import server

        resource = SlaveInterface(self)
        site = server.Site(resource)
        tcpport = reactor.listenTCP(0, site)
        address = tcpport.getHost()

        # Initiate signin to master
        from version import VERSION
        cookie = self.cookie
        port = address.port
        source_hash = self.registry.source_hash()
        reg_hash = self.registry.reg_hash()

        deferred = self.master_rpc.callRemote('signin', VERSION, cookie, port,
                source_hash, reg_hash)

        deferred.addCallbacks(self.signin_callback, self.signin_errback)

    def signin_errback(self, failure):
        print failure
        self.quit('Unable to contact master.')

    # State 2
    def signin_callback(self, value):
        """Process the results from the signin.

        This is the callback after signin.  After saving the return values,
        schedule the user_setup function to be run in the Worker.
        """
        slave_id, optdict = value

        if slave_id < 0:
            self.quit('Master rejected signin.')

        # Save the slave id given by the master.
        self.id = slave_id

        # Create the Ping Task
        ping_args = ('ping', self.id, self.cookie)
        self.ping_task = PingTask(ping_args, self.master_rpc,
                self.ping_success, self.ping_failure, self.get_timestamp)

        # Tell the Worker to run the user_setup function.
        import optparse
        options = optparse.Values(optdict)
        callback = self.report_ready
        self.worker.start_setup(options, callback)

    def report_ready(self):
        """Report to the master that we are ready to accept tasks.

        This is the callback after user_setup is called.
        """
        # Report for duty.
        deferred = self.master_rpc.callRemote('ready', self.id, self.cookie)
        deferred.addCallback(self.ready_success)
        deferred.addErrback(self.ready_failure)

    def ready_success(self, value):
        """Called when reporting in as ready succeeded."""
        import sys
        print >>sys.stderr, "Connected to master."
        self.update_timestamp()
        self.timeouts = 0
        if not self.ping_task.running:
            self.ping_task.start()

    def ready_failure(self, err):
        """Called when reporting in as ready failed."""
        import sys
        give_up = False
        if err.check(defer.TimeoutError):
            self.timeouts += 1
            if self.timeouts > PING_ATTEMPTS:
                print >>sys.stderr, "Too many ping timeouts.  Giving up."
                give_up = True
            else:
                # Try to connect again
                print >>sys.stderr, "Reconnect timed out.  Trying again."
                self.report_ready()
        else:
            print >>sys.stderr, "Couldn't report in due to network error."
            print >>sys.stderr, err
            give_up = True

        if give_up:
            self.reaper.reap()

    def ping_failure(self, err):
        """Report that the master failed to respond to an RPC request.

        This may be either a ping or some other request.  At the moment,
        we aren't very lenient, but in the future we could allow a few
        failures before disconnecting.
        """
        import sys

        give_up = False
        if err.check(defer.TimeoutError):
            self.timeouts += 1
            if self.timeouts > PING_ATTEMPTS:
                print >>sys.stderr, "Too many ping timeouts.  Giving up."
                give_up = True
            else:
                print >>sys.stderr, 'Ping timeout.  Trying again.'
        else:
            print >>sys.stderr, 'Lost master due to network error.'
            print >>sys.stderr, err

        if give_up:
            self.ping_task.stop()
            self.reaper.reap()

    def ping_success(self):
        """Called when a ping successfully completes."""
        self.update_timestamp()
        self.timeouts = 0

    def update_timestamp(self):
        """Set the timestamp to the current time."""
        from datetime import datetime
        self.timestamp = datetime.utcnow()

    def get_timestamp(self):
        """Report the most recent timestamp."""
        return self.timestamp

    # Miscellaneous
    @classmethod
    def rand_cookie(cls):
        # Generate a cookie so that mostly only authorized people can connect.
        from random import choice
        import string
        possible = string.letters + string.digits
        return ''.join(choice(possible) for i in xrange(COOKIE_LEN))

    def check_cookie(self, cookie):
        if cookie != self.cookie:
            raise CookieValidationError

    def register_worker(self, worker):
        """Called by the worker so the Slave can have a reference to it."""
        self.worker = worker

    def die(self, exception):
        """Die with an exception."""
        self.reaper.reap(exception)

    def quit(self, message=None):
        """Called to quit the slave."""
        if message:
            import sys
            print >>sys.stderr, message
        self.reaper.reap()

    def reconnect(self):
        """Called by the worker thread when it can't contact the master."""
        reactor.callFromThread(self.ping_task.stop)
        reactor.callFromThread(self.report_ready)

class SlaveInterface(RequestXMLRPC):
    """Public XML RPC Interface

    Note that any method not beginning with "xmlrpc_" will be exposed to
    remote hosts.  Any of these can return either a result or a deferred.
    """
    def __init__(self, slave):
        RequestXMLRPC.__init__(self)
        self.slave = slave

    def xmlrpc_start_map(self, source, inputs, func_name, part_name, nparts,
            output, extension, cookie):
        self.slave.check_cookie(cookie)
        self.slave.update_timestamp()
        return self.slave.worker.start_map(source, inputs, func_name,
                part_name, nparts, output, extension)

    def xmlrpc_start_reduce(self, source, inputs, func_name, part_name,
            nparts, output, extension, cookie):
        self.slave.check_cookie(cookie)
        self.slave.update_timestamp()
        return self.slave.worker.start_reduce(source, inputs, func_name,
                part_name, nparts, output, extension)

    def xmlrpc_quit(self, cookie):
        self.slave.check_cookie(cookie)
        self.slave.update_timestamp()
        self.slave.alive = False
        # We delay before actually stopping because we need to make sure that
        # the response gets sent back.
        reactor.callLater(QUIT_DELAY, self.slave.quit,
                "Quitting as requested by an RPC call.")
        return True

    def xmlrpc_ping(self, cookie):
        """Master checking if we're still here.
        """
        self.slave.check_cookie(cookie)
        self.slave.update_timestamp()
        return True

class CookieValidationError(Exception):
    pass

class Worker(threading.Thread):
    """Execute map tasks and reduce tasks.

    The worker waits for other threads to make assignments by calling
    start_map and start_reduce.

    This needs to run in a daemon thread rather than in the main thread so
    that it can be killed by other threads.
    """
    def __init__(self, slave):
        threading.Thread.__init__(self)
        # Die when all other non-daemon threads have exited:
        self.setDaemon(True)

        self.slave = slave
        self.slave.register_worker(self)

        self._task = None
        self._cond = threading.Condition()
        self.active = False
        self.exception = None

        self.options = None
        self._setup_ready = threading.Event()
        self._setup_callback = None

    def start_setup(self, options, callback):
        """Start running the user_setup function.

        The given callback function will be invoked in the reactor thread.
        """
        self.options = options
        self._setup_ready.set()
        self._setup_callback = callback

    def start_map(self, source, inputs, map_name, part_name, nparts, output,
            extension):
        """Tell this worker to start working on a map task.

        This will ordinarily be called from some other thread.
        """
        from task import MapTask
        from datasets import FileData
        from io import writerformat

        input_data = FileData(inputs, splits=1)
        format = writerformat(extension)

        success = False
        self._cond.acquire()
        if self._task is None:
            registry = self.slave.registry
            self._task = MapTask(input_data, 0, source, map_name, part_name,
                    nparts, output, format, registry)
            success = True
            self._cond.notify()
        self._cond.release()
        return success

    def start_reduce(self, source, inputs, reduce_name, part_name, nparts,
            output, extension):
        """Tell this worker to start working on a reduce task.

        This will ordinarily be called from some other thread.
        """
        from task import ReduceTask
        from datasets import FileData
        from io import writerformat

        input_data = FileData(inputs, splits=1)
        format = writerformat(extension)

        success = False
        self._cond.acquire()
        if self._task is None:
            registry = self.slave.registry
            self._task = ReduceTask(input_data, 0, source, reduce_name,
                    part_name, nparts, output, format, registry)
            success = True
            self._cond.notify()
        self._cond.release()
        return success

    def run(self):
        """Run the worker."""
        import sys

        # Run user_setup if requested:
        self._setup_ready.wait()
        user_setup = self.slave.user_setup
        if user_setup:
            try:
                user_setup(self.options)
            except Exception, e:
                print "Caught an exception in the Worker thread (in setup)!"
                self.slave.die(e)
                return

        # Alert the Twisted thread that user_setup is done.
        reactor.callFromThread(self._setup_callback)

        # Process tasks:
        while True:
            self._cond.acquire()
            while self._task is None:
                self._cond.wait()
            task = self._task
            self._cond.release()

            try:
                task.run()
            except Exception, e:
                print >>sys.stderr, "Caught an exception in the Worker thread!"
                self.slave.die(e)
                return

            self._cond.acquire()
            self._task = None
            self._cond.release()

            # TODO: right now, dataset.dump() happens in task.run().  Instead,
            # we should tell the dataset to become available() here, and the
            # data should automatically be dumped.
            try:
                self.slave.master_rpc.blocking_call('done', self.slave.id,
                        task.outurls(), self.slave.cookie)
            except ErrbackException, e:
                # TODO: We should be able to retry calling done, but this will
                # only be possible if done uses some unique id.  For now,
                # retrying done can be very destructive.
                #print >>sys.stderr, ("RPC error when reporting back."
                #        "  Trying again.")
                print >>sys.stderr, ("RPC error when reporting back."
                        "  Giving up.")
                self.slave.reconnect()

# vim: et sw=4 sts=4
<|endoftext|>


<commit_before>try:
    import queue
except ImportError:
    import Queue as queue

from elasticsearch_raven import configuration
from elasticsearch_raven import transport
from elasticsearch_raven.udp_server import Sender

class HttpUtils:
    def __init__(self):
        self._pending_logs = queue.Queue(configuration['queue_maxsize'])
        self._exception_queue = queue.Queue()

    def start_sender(self):
        log_transport = transport.get_configured_log_transport()
        sender = Sender(log_transport, self._pending_logs,
                        self._exception_queue).as_thread()
        sender.start()

    def get_application(self):
        def application(environ, start_response):
            try:
                exception = self._exception_queue.get_nowait()
            except queue.Empty:
                pass
            else:
                raise exception
            length = int(environ.get('CONTENT_LENGTH', '0'))
            data = environ['wsgi.input'].read(length)
            self._pending_logs.put(transport.SentryMessage.create_from_http(
                environ['HTTP_X_SENTRY_AUTH'], data))

            status = '200 OK'
            response_headers = [('Content-Type', 'text/plain')]
            start_response(status, response_headers)
            return [''.encode('utf-8')]
        return application
<commit_msg>Refactor.<commit_after>try:
    import queue
except ImportError:
    import Queue as queue

from elasticsearch_raven import configuration
from elasticsearch_raven import transport
from elasticsearch_raven.udp_server import Sender

class HttpUtils:
    def __init__(self):
        self._pending_logs = queue.Queue(configuration['queue_maxsize'])
        self._exception_queue = queue.Queue()

    def start_sender(self):
        log_transport = transport.get_configured_log_transport()
        sender = Sender(log_transport, self._pending_logs,
                        self._exception_queue).as_thread()
        sender.start()

    def get_application(self):
        def application(environ, start_response):
            try:
                exception = self._exception_queue.get_nowait()
            except queue.Empty:
                pass
            else:
                raise exception
            length = int(environ.get('CONTENT_LENGTH', '0'))
            data = environ['wsgi.input'].read(length)
            self._pending_logs.put(transport.SentryMessage.create_from_http(
                environ['HTTP_X_SENTRY_AUTH'], data))

            status = '200 OK'
            response_headers = [('Content-Type', 'text/plain')]
            start_response(status, response_headers)
            return [b'']
        return application
<|endoftext|>


<commit_before>from rest_framework import serializers

from . import models
from clients import serializers as client_serializers
from users import serializers as user_serializers

class WorkTypeSerializer(serializers.ModelSerializer):

    class Meta:
        model = models.WorkType
        fields = ('id', 'name',)

class ArtTypeSerializer(serializers.ModelSerializer):

    class Meta:
        model = models.ArtType
        fields = ('id', 'work_type', 'name',)

class ArtIgualaSerializer(serializers.ModelSerializer):

    art_type_complete = ArtTypeSerializer(source='art_type', read_only=True)

    class Meta:
        model = models.ArtIguala
        fields = ('id', 'iguala', 'art_type', 'quantity', 'art_type_complete')

class IgualaSerializer(serializers.ModelSerializer):

    client_complete = client_serializers.ClientSerializer(source='client', read_only=True)
    art_iguala = ArtIgualaSerializer(many=True, read_only=True)

    class Meta:
        model = models.Iguala
        fields = ('id', 'client', 'client_complete', 'name', 'start_date', 'end_date',
                  'art_iguala',)

class StatusSerializer(serializers.ModelSerializer):

    name = serializers.CharField(source='__str__', read_only=True)

    class Meta:
        model = models.Status
        fields = ('id', 'status_id', 'name',)

class WorkSerializer(serializers.ModelSerializer):

    creation_date = serializers.DateField(read_only=True)
    executive_complete = user_serializers.UserSerializer(source='executive', read_only=True)
    contact_complete = client_serializers.ContactSerializer(source='contact', read_only=True)
    current_status_complete = StatusSerializer(source='current_status', read_only=True)
    work_type_complete = WorkTypeSerializer(source='work_type', read_only=True)
    iguala_complete = IgualaSerializer(source='iguala', read_only=True)

    class Meta:
        model = models.Work
        fields = ('id',
                  'executive',
                  'executive_complete',
                  'contact',
                  'contact_complete',
                  'current_status',
                  'current_status_complete',
                  'work_type',
                  'work_type_complete',
                  'iguala',
                  'iguala_complete',
                  'creation_date',
                  'name',
                  'expected_delivery_date',
                  'brief',
                  'final_link',
                  )

class ArtWorkSerializer(serializers.ModelSerializer):

    class Meta:
        model = models.ArtWork
        fields = ('id', 'work', 'art_type', 'quantity',)

class FileSerializer(serializers.ModelSerializer):

    class Meta:
        model = models.File
        fields = ('id', 'work', 'upload',)

class WorkDesignerSerializer(serializers.ModelSerializer):

    start_date = serializers.DateTimeField(read_only=True)
    end_date = serializers.DateTimeField(read_only=True)

    class Meta:
        model = models.WorkDesigner
        fields = ('id', 'designer', 'work', 'start_date', 'end_date', 'active_work',)

class StatusChangeSerializer(serializers.ModelSerializer):

    date = serializers.DateTimeField(read_only=True)

    class Meta:
        model = models.StatusChange
        fields = ('id', 'work', 'status', 'user', 'date',)
<commit_msg>Add art_type name to iguala's WS<commit_after>from rest_framework import serializers

from . import models
from clients import serializers as client_serializers
from users import serializers as user_serializers

class WorkTypeSerializer(serializers.ModelSerializer):

    class Meta:
        model = models.WorkType
        fields = ('id', 'name',)

class ArtTypeSerializer(serializers.ModelSerializer):

    class Meta:
        model = models.ArtType
        fields = ('id', 'work_type', 'name',)

class ArtIgualaSerializer(serializers.ModelSerializer):

    art_type_name = serializers.CharField(source='art_type.name', read_only=True)

    class Meta:
        model = models.ArtIguala
        fields = ('id', 'iguala', 'art_type', 'quantity', 'art_type_name')

class IgualaSerializer(serializers.ModelSerializer):

    client_complete = client_serializers.ClientSerializer(source='client', read_only=True)
    art_iguala = ArtIgualaSerializer(many=True, read_only=True)

    class Meta:
        model = models.Iguala
        fields = ('id', 'client', 'client_complete', 'name', 'start_date', 'end_date',
                  'art_iguala',)

class StatusSerializer(serializers.ModelSerializer):

    name = serializers.CharField(source='__str__', read_only=True)

    class Meta:
        model = models.Status
        fields = ('id', 'status_id', 'name',)

class WorkSerializer(serializers.ModelSerializer):

    creation_date = serializers.DateField(read_only=True)
    executive_complete = user_serializers.UserSerializer(source='executive', read_only=True)
    contact_complete = client_serializers.ContactSerializer(source='contact', read_only=True)
    current_status_complete = StatusSerializer(source='current_status', read_only=True)
    work_type_complete = WorkTypeSerializer(source='work_type', read_only=True)
    iguala_complete = IgualaSerializer(source='iguala', read_only=True)

    class Meta:
        model = models.Work
        fields = ('id',
                  'executive',
                  'executive_complete',
                  'contact',
                  'contact_complete',
                  'current_status',
                  'current_status_complete',
                  'work_type',
                  'work_type_complete',
                  'iguala',
                  'iguala_complete',
                  'creation_date',
                  'name',
                  'expected_delivery_date',
                  'brief',
                  'final_link',
                  )

class ArtWorkSerializer(serializers.ModelSerializer):

    class Meta:
        model = models.ArtWork
        fields = ('id', 'work', 'art_type', 'quantity',)

class FileSerializer(serializers.ModelSerializer):

    class Meta:
        model = models.File
        fields = ('id', 'work', 'upload',)

class WorkDesignerSerializer(serializers.ModelSerializer):

    start_date = serializers.DateTimeField(read_only=True)
    end_date = serializers.DateTimeField(read_only=True)

    class Meta:
        model = models.WorkDesigner
        fields = ('id', 'designer', 'work', 'start_date', 'end_date', 'active_work',)

class StatusChangeSerializer(serializers.ModelSerializer):

    date = serializers.DateTimeField(read_only=True)

    class Meta:
        model = models.StatusChange
        fields = ('id', 'work', 'status', 'user', 'date',)
<|endoftext|>


<commit_before>from __future__ import with_statement

from datetime import datetime
from BeautifulSoup import BeautifulStoneSoup

class BaseWordpressImporter(object):

    default_mapping = [
        ('title', 'title'),
        ('dc:creator', 'creator'),
        ('description', 'description'),
        ('content:encoded', 'content'),
        ('wp:post_id', ('post_id', 'convert_int')),
        ('wp:post_date', ('post_date', 'convert_date')),
        ('wp:post_date_gmt', ('post_date_gmt', 'convert_date')),
        ('wp:comment_status', 'comment_status'),
        ('wp:post_name', 'slug'),
        ('wp:status', 'status'),
        ('wp:post_parent', 'parent'),
        ('excerpt:encoded', 'excerpt'),
        ('wp:postmeta', ('postmeta', 'convert_meta', 'dict')),
        ('category', ('categories', 'convert_category', 'list')),
        ('wp:comment', ('comments', 'convert_comment', 'list')),
        ('wp:post_type', 'post_type'),
        ('wp:attachment_url', 'attachment_url'),
        ('wp:ping_status', 'ping_status'),
        ('wp:menu_order', ('menu_order', 'convert_int')),
        ('wp:post_password', 'post_password'),
        ('wp:is_sticky', ('is_sticky', 'convert_bool')),
        ('link', 'link'),
        ('pubdate', 'pubdate'),
        ('guid', ('guid', 'convert_guid')),

        ('wp:tag_slug', 'tag_slug'),
        ('wp:tag_name', 'tag_name'),
        ('wp:term_id', ('term_id', 'convert_int')),
        ('wp:term_parent', 'term_parent'),
        ('wp:term_taxonomy', 'term_taxonomy'),
        ('wp:term_slug', 'term_slug'),
        ('wp:term_name', 'term_name'),

        ('wp:cat_name', 'category_name'),
        ('wp:category_nicename', 'category_slug'),
        ('wp:category_parent', 'category_parent'),

        ('wp:comment_id', ('id', 'convert_int')),
        ('wp:comment_author', 'author'),
        ('wp:comment_author_email', 'author_email'),
        ('wp:comment_author_url', 'author_url'),
        ('wp:comment_author_ip', 'author_ip'),
        ('wp:comment_date', ('date', 'convert_date')),
        ('wp:comment_date_gmt', ('date_gmt', 'convert_date')),
        ('wp:comment_content', 'content'),
        ('wp:comment_approved', ('approved', 'convert_bool')),
        ('wp:comment_parent', ('parent', 'convert_int')),
        ('wp:comment_user_id', ('user_id', 'convert_int')),
        ('wp:comment_type', 'type'),
        ('wp:author_id', ('author_id', 'convert_int')),
        ('wp:author_login', 'author_login'),
        ('wp:author_email', 'author_email'),
        ('wp:author_display_name', 'author_display_name'),
        ('wp:author_first_name', 'author_first_name'),
        ('wp:author_last_name', 'author_last_name'),
    ]

    def __init__(self, filename, verbosity=0):
        self.verbosity = verbosity
        with open(filename) as fh:
            self.xml = BeautifulStoneSoup(fh.read())

    def convert_int(self, tag):
        return int(tag.string)

    def convert_bool(self, tag):
        return bool(int(tag.string))

    def convert_guid(self, tag):
        return {'is_permalink': tag['ispermalink'] == 'true', 'guid': tag.string}

    def convert_date(self, tag):
        try:
            return datetime.strptime(tag.string, "%Y-%m-%d %H:%M:%S")
        except ValueError:
            return None

    def convert_meta(self, tag):
        key = tag.find('wp:meta_key').string
        value = tag.find('wp:meta_value').string
        return (key, value)

    def convert_category(self, tag):
        return {
            'slug': tag['nicename'],
            'name': tag.string,
            'domain': tag['domain'],
        }

    def convert_comment(self, tag):
        return self.handle_tag(tag, dict(self.default_mapping))

    def parse(self):
        items = self.xml.findAll(lambda x: x.parent.name == 'channel')
        for item in items:
            post_type = item.find('wp:post_type') and item.find('wp:post_type').string
            if post_type and hasattr(self, 'handle_' + post_type):
                getattr(self, 'handle_' + post_type)(item)
            else:
                self.handle_default(item)

    def handle_tag(self, tag, mapping):
        result = {}
        children = tag.findAll(lambda x: x.parent == tag)
        for child in children:
            attr = mapping.get(child.name, None)
            if attr:
                if isinstance(attr, str):
                    result[attr] = child.string
                elif isinstance(attr, tuple):
                    if len(attr) == 2:
                        attr_name, converter = attr
                        attr_type = None
                    elif len(attr) == 3:
                        attr_name, converter, attr_type = attr
                    if isinstance(converter, str):
                        converter = getattr(self, converter)
                    value = converter(child)
                    if attr_type == 'list':
                        attr_value = result.get(attr_name, [])
                        attr_value.append(value)
                        result[attr_name] = attr_value
                    elif attr_type == 'dict':
                        attr_value = result.get(attr_name, {})
                        attr_value[value[0]] = value[1]
                        result[attr_name] = attr_value
                    else:
                        result[attr_name] = value
            elif self.verbosity > 0:
                print "No match: ", child
        return result

    def handle_default(self, tag):
        mp = dict(self.default_mapping)
        result = self.handle_tag(tag, mp)
        if not result:
            result = dict([(tag.name, tag.string)])
        if self.verbosity > 1:
            from pprint import pprint
            pprint(result)
        return tag, result
<commit_msg>Don't return tag, since we already pass it as a parameter to the method<commit_after>from __future__ import with_statement

from datetime import datetime
from BeautifulSoup import BeautifulStoneSoup

class BaseWordpressImporter(object):

    default_mapping = [
        ('title', 'title'),
        ('dc:creator', 'creator'),
        ('description', 'description'),
        ('content:encoded', 'content'),
        ('wp:post_id', ('post_id', 'convert_int')),
        ('wp:post_date', ('post_date', 'convert_date')),
        ('wp:post_date_gmt', ('post_date_gmt', 'convert_date')),
        ('wp:comment_status', 'comment_status'),
        ('wp:post_name', 'slug'),
        ('wp:status', 'status'),
        ('wp:post_parent', 'parent'),
        ('excerpt:encoded', 'excerpt'),
        ('wp:postmeta', ('postmeta', 'convert_meta', 'dict')),
        ('category', ('categories', 'convert_category', 'list')),
        ('wp:comment', ('comments', 'convert_comment', 'list')),
        ('wp:post_type', 'post_type'),
        ('wp:attachment_url', 'attachment_url'),
        ('wp:ping_status', 'ping_status'),
        ('wp:menu_order', ('menu_order', 'convert_int')),
        ('wp:post_password', 'post_password'),
        ('wp:is_sticky', ('is_sticky', 'convert_bool')),
        ('link', 'link'),
        ('pubdate', 'pubdate'),
        ('guid', ('guid', 'convert_guid')),

        ('wp:tag_slug', 'tag_slug'),
        ('wp:tag_name', 'tag_name'),
        ('wp:term_id', ('term_id', 'convert_int')),
        ('wp:term_parent', 'term_parent'),
        ('wp:term_taxonomy', 'term_taxonomy'),
        ('wp:term_slug', 'term_slug'),
        ('wp:term_name', 'term_name'),

        ('wp:cat_name', 'category_name'),
        ('wp:category_nicename', 'category_slug'),
        ('wp:category_parent', 'category_parent'),

        ('wp:comment_id', ('id', 'convert_int')),
        ('wp:comment_author', 'author'),
        ('wp:comment_author_email', 'author_email'),
        ('wp:comment_author_url', 'author_url'),
        ('wp:comment_author_ip', 'author_ip'),
        ('wp:comment_date', ('date', 'convert_date')),
        ('wp:comment_date_gmt', ('date_gmt', 'convert_date')),
        ('wp:comment_content', 'content'),
        ('wp:comment_approved', ('approved', 'convert_bool')),
        ('wp:comment_parent', ('parent', 'convert_int')),
        ('wp:comment_user_id', ('user_id', 'convert_int')),
        ('wp:comment_type', 'type'),
        ('wp:author_id', ('author_id', 'convert_int')),
        ('wp:author_login', 'author_login'),
        ('wp:author_email', 'author_email'),
        ('wp:author_display_name', 'author_display_name'),
        ('wp:author_first_name', 'author_first_name'),
        ('wp:author_last_name', 'author_last_name'),
    ]

    def __init__(self, filename, verbosity=0):
        self.verbosity = verbosity
        with open(filename) as fh:
            self.xml = BeautifulStoneSoup(fh.read())

    def convert_int(self, tag):
        return int(tag.string)

    def convert_bool(self, tag):
        return bool(int(tag.string))

    def convert_guid(self, tag):
        return {'is_permalink': tag['ispermalink'] == 'true', 'guid': tag.string}

    def convert_date(self, tag):
        try:
            return datetime.strptime(tag.string, "%Y-%m-%d %H:%M:%S")
        except ValueError:
            return None

    def convert_meta(self, tag):
        key = tag.find('wp:meta_key').string
        value = tag.find('wp:meta_value').string
        return (key, value)

    def convert_category(self, tag):
        return {
            'slug': tag['nicename'],
            'name': tag.string,
            'domain': tag['domain'],
        }

    def convert_comment(self, tag):
        return self.handle_tag(tag, dict(self.default_mapping))

    def parse(self):
        items = self.xml.findAll(lambda x: x.parent.name == 'channel')
        for item in items:
            post_type = item.find('wp:post_type') and item.find('wp:post_type').string
            if post_type and hasattr(self, 'handle_' + post_type):
                getattr(self, 'handle_' + post_type)(item)
            else:
                self.handle_default(item)

    def handle_tag(self, tag, mapping):
        result = {}
        children = tag.findAll(lambda x: x.parent == tag)
        for child in children:
            attr = mapping.get(child.name, None)
            if attr:
                if isinstance(attr, str):
                    result[attr] = child.string
                elif isinstance(attr, tuple):
                    if len(attr) == 2:
                        attr_name, converter = attr
                        attr_type = None
                    elif len(attr) == 3:
                        attr_name, converter, attr_type = attr
                    if isinstance(converter, str):
                        converter = getattr(self, converter)
                    value = converter(child)
                    if attr_type == 'list':
                        attr_value = result.get(attr_name, [])
                        attr_value.append(value)
                        result[attr_name] = attr_value
                    elif attr_type == 'dict':
                        attr_value = result.get(attr_name, {})
                        attr_value[value[0]] = value[1]
                        result[attr_name] = attr_value
                    else:
                        result[attr_name] = value
            elif self.verbosity > 0:
                print "No match: ", child
        return result

    def handle_default(self, tag):
        mp = dict(self.default_mapping)
        result = self.handle_tag(tag, mp)
        if not result:
            result = dict([(tag.name, tag.string)])
        if self.verbosity > 1:
            from pprint import pprint
            pprint(result)
        return result
<|endoftext|>