<commit_before>from tornado import (
    httpserver,
    ioloop,
    options,
    web
    )
from tornado.options import (
    define,
    options
    )
from datetime import date
import tornado
import asyncmongo
import urllib2
from urllib2 import URLError
import json

define("port", default=8888, help="run on the given port", type=int)
define("mongodb_port", default=27017, help="run MongoDB on the given port", type=int)
define("mongodb_host", default='localhost', help="run MongoDB on the given hostname")
define("mongodb_database", default='analytics', help="Record accesses on the given database")
define("mongodb_max_connections", default=200, help="run MongoDB with the given max connections", type=int)
define("mongodb_max_cached", default=20, help="run MongoDB with the given max cached", type=int)
define("resources", default=None, help="indicates a txt file with api resources. Once this parameter is defined, the API will just work as accesses delivery.", type=str)
define("allowed_hosts", default=None, help="indicates a txt file with hostnames allowed to post data.", type=str)

class Application(tornado.web.Application):
    def __init__(self):
        handlers = [
            (r"/", RootHandler),
            (r"/api/v1/journal", JournalHandler),
            (r"/api/v1/issue", IssueHandler),
            (r"/api/v1/article", ArticleHandler),
            (r"/api/v1/pdf", PdfHandler),
        ]

        self.db = asyncmongo.Client(
            pool_id='accesses',
            host=options.mongodb_host,
            port=options.mongodb_port,
            maxcached=options.mongodb_max_cached,
            maxconnections=options.mongodb_max_connections,
            dbname=options.mongodb_database
        )

        # Local is the default the default way that ratchet works.
        self.api_style = 'local'

        self.resources = []

        if options.resources:
            with open(options.resources) as f:
                for line in f:
                    self.resources.append(line)

            if len(self.resources) > 0:
                self.api_style = 'global'
                handlers.append((r"/api/v1/resources", ResourcesHandler))

        tornado.web.Application.__init__(self, handlers)

class RootHandler(tornado.web.RequestHandler):
    def get(self):
        if self.application.api_style == 'global':
            self.write("Another Ratchet Global Resource")
        else:
            self.write("Another Ratchet Local Resource")
        self.finish()

class ResourcesHandler(tornado.web.RequestHandler):

    def get(self):
        response = {}
        for resource in self.application.resources:
            url = "http://%s/" % resource.strip()
            try:
                urllib2.urlopen(url)
                response[url] = 'online'
            except URLError:
                response[url] = 'offline'
                continue

        self.write(str(response))
        self.finish()

class PdfHandler(tornado.web.RequestHandler):
    @property
    def db(self):
        self._db = self.application.db
        return self._db

    def post(self):
        code = self.get_argument('code')
        region = self.get_argument('region')
        journal = self.get_argument('journal')
        access_date = self.get_argument('access_date')
        iso_date = access_date
        month_date = iso_date[:7]

        self.db.accesses.update(
            {'code': code},
            {'$set': {'type': 'article', 'journal': journal}, '$inc': {region: 1, iso_date: 1, month_date: 1, 'total': 1}},
            safe=False,
            upsert=True
        )

class ArticleHandler(tornado.web.RequestHandler):

    def _remove_callback(self, response, error):
        pass

    def _on_get_response(self, response, error):
        if error:
            raise tornado.web.HTTPError(500)

        if len(response) > 0:
            self.write(str(response[0]))

        self.finish()

    @property
    def db(self):
        self._db = self.application.db
        return self._db

    def post(self):
        code = self.get_argument('code')
        region = self.get_argument('region')
        journal = self.get_argument('journal')
        issue = self.get_argument('issue')
        iso_date = date.isoformat(date.today())
        month_date = iso_date[:7]

        self.db.accesses.update(
            {'code': code},
            {'$set': {'type': 'article', 'journal': journal, 'issue': issue}, '$inc': {region: 1, iso_date: 1, month_date: 1, 'total': 1}},
            safe=False,
            upsert=True
        )

    @tornado.web.asynchronous
    def get(self):
        code = self.get_argument('code')

        if self.application.api_style == 'global':
            self.db.accesses.remove({'code': code}, callback=self._remove_callback)

            for resource in self.application.resources:
                url = "http://%s/api/v1/article?code=%s" % (resource, code)
                try:
                    data = urllib2.urlopen(url).read().replace("'", '"').replace('u"', '"')
                    if data:
                        data = json.loads(data)
                        str_data = {'journal': data['journal'], 'issue': data['issue'], 'type': 'article'}
                        del(data['code'])
                        del(data['type'])
                        del(data['journal'])
                        del(data['issue'])
                        self.db.accesses.update(
                            {'code': code},
                            {'$inc': data, '$set': str_data},
                            safe=False,
                            upsert=True
                        )
                except URLError:
                    continue
                    # must register error log

        self.db.accesses.find({"code": code, "type": "article"}, {"_id": 0}, limit=1, callback=self._on_get_response)

class IssueHandler(tornado.web.RequestHandler):

    def _remove_callback(self, response, error):
        pass

    def _on_get_response(self, response, error):
        if error:
            raise tornado.web.HTTPError(500)

        if len(response) > 0:
            self.write(str(response[0]))

        self.finish()

    @property
    def db(self):
        self._db = self.application.db
        return self._db

    def post(self):
        code = self.get_argument('code')
        region = self.get_argument('region')
        journal = self.get_argument('journal')
        iso_date = date.isoformat(date.today())
        month_date = iso_date[:7]

        self.db.accesses.update(
            {'code': code},
            {'$set': {'type': 'issue', 'journal': journal}, '$inc': {region: 1, iso_date: 1, month_date: 1, 'total': 1}},
            safe=False,
            upsert=True
        )

    @tornado.web.asynchronous
    def get(self):
        code = self.get_argument('code')

        if self.application.api_style == 'global':
            self.db.accesses.remove({'code': code}, callback=self._remove_callback)

            for resource in self.application.resources:
                url = "http://%s/api/v1/issue?code=%s" % (resource, code)
                try:
                    data = urllib2.urlopen(url).read().replace("'", '"').replace('u"', '"')
                    if data:
                        data = json.loads(data)
                        str_data = {'journal': data['journal'], 'type': 'issue'}
                        del(data['code'])
                        del(data['type'])
                        del(data['journal'])
                        self.db.accesses.update(
                            {'code': code},
                            {'$inc': data, '$set': str_data},
                            safe=False,
                            upsert=True
                        )
                except URLError:
                    continue
                    # must register error log

        self.db.accesses.find({"code": code, "type": "issue"}, {"_id": 0}, limit=1, callback=self._on_get_response)

class JournalHandler(tornado.web.RequestHandler):

    def _remove_callback(self, response, error):
        pass

    def _on_get_response(self, response, error):
        if error:
            raise tornado.web.HTTPError(500)

        if len(response) > 0:
            self.write(str(response[0]))

        self.finish()

    @property
    def db(self):
        self._db = self.application.db
        return self._db

    def post(self):
        code = self.get_argument('code')
        region = self.get_argument('region')
        iso_date = date.isoformat(date.today())
        month_date = iso_date[:7]

        self.db.accesses.update(
            {'code': code},
            {'$set': {'type': 'journal'}, '$inc': {region: 1, iso_date: 1, month_date: 1, 'total': 1}},
            safe=False,
            upsert=True
        )

    @tornado.web.asynchronous
    def get(self):
        code = self.get_argument('code')

        if self.application.api_style == 'global':
            self.db.accesses.remove({'code': code}, callback=self._remove_callback)

            for resource in self.application.resources:
                url = "http://%s/api/v1/journal?code=%s" % (resource, code)
                try:
                    data = urllib2.urlopen(url).read().replace("'", '"').replace('u"', '"')
                    if data:
                        data = json.loads(data)
                        del(data['code'])
                        del(data['type'])
                        self.db.accesses.update(
                            {'code': code},
                            {'$set': {'type': 'journal'}, '$inc': data},
                            safe=False,
                            upsert=True
                        )
                except URLError:
                    continue
                    # must register error log

        self.db.accesses.find({"code": code, "type": "journal"}, {"_id": 0}, limit=1, callback=self._on_get_response)

if __name__ == '__main__':
    tornado.options.parse_command_line()
    http_server = tornado.httpserver.HTTPServer(Application())
    http_server.listen(options.port)
    tornado.ioloop.IOLoop.instance().start()
<commit_msg>Tornando broadcast asincrono<commit_after>import urllib
import urllib2
import json
from datetime import date

from tornado import (
    httpserver,
    httpclient,
    ioloop,
    options,
    web,
    gen
    )
from tornado.options import (
    define,
    options
    )
import tornado
import asyncmongo

define("port", default=8888, help="run on the given port", type=int)
define("mongodb_port", default=27017, help="run MongoDB on the given port", type=int)
define("mongodb_host", default='localhost', help="run MongoDB on the given hostname")
define("mongodb_database", default='analytics', help="Record accesses on the given database")
define("mongodb_max_connections", default=200, help="run MongoDB with the given max connections", type=int)
define("mongodb_max_cached", default=20, help="run MongoDB with the given max cached", type=int)
define("resources", default=None, help="indicates a txt file with api resources. Once this parameter is defined, the API will just work as accesses delivery.", type=str)
define("allowed_hosts", default=None, help="indicates a txt file with hostnames allowed to post data.", type=str)
define("broadcast_timeout", default=1, help="indicates the max timeout in seconds that the broadcast must finish.", type=str)

class Application(tornado.web.Application):
    def __init__(self):
        handlers = [
            (r"/", RootHandler),
            (r"/api/v1/journal", JournalHandler),
            (r"/api/v1/issue", IssueHandler),
            (r"/api/v1/article", ArticleHandler),
            (r"/api/v1/pdf", PdfHandler),
        ]

        self.db = asyncmongo.Client(
            pool_id='accesses',
            host=options.mongodb_host,
            port=options.mongodb_port,
            maxcached=options.mongodb_max_cached,
            maxconnections=options.mongodb_max_connections,
            dbname=options.mongodb_database
        )

        # Local is the default the default way that ratchet works.
        self.broadcast_timeout = options.broadcast_timeout
        self.api_style = 'local'
        self.resources = []
        if options.resources:
            with open(options.resources) as f:
                for line in f:
                    self.resources.append(line)

            if len(self.resources) > 0:
                self.api_style = 'global'
                handlers.append((r"/api/v1/resources", ResourcesHandler))

        tornado.web.Application.__init__(self, handlers)

class RootHandler(tornado.web.RequestHandler):
    def get(self):
        if self.application.api_style == 'global':
            self.write("Another Ratchet Global Resource")
        else:
            self.write("Another Ratchet Local Resource")
        self.finish()

class ResourcesHandler(tornado.web.RequestHandler):

    def get(self):
        response = {}
        for resource in self.application.resources:
            url = "http://%s/" % resource.strip()
            try:
                urllib2.urlopen(url)
                response[url] = 'online'
            except urllib2.URLError:
                response[url] = 'offline'
                continue

        self.write(str(response))
        self.finish()

class PdfHandler(tornado.web.RequestHandler):
    @property
    def db(self):
        self._db = self.application.db
        return self._db

    def post(self):
        code = self.get_argument('code')
        region = self.get_argument('region')
        journal = self.get_argument('journal')
        access_date = self.get_argument('access_date')
        iso_date = access_date
        month_date = iso_date[:7]

        self.db.accesses.update(
            {'code': code},
            {'$set': {'type': 'article', 'journal': journal}, '$inc': {region: 1, iso_date: 1, month_date: 1, 'total': 1}},
            safe=False,
            upsert=True
        )

class ArticleHandler(tornado.web.RequestHandler):

    def _remove_callback(self, response, error):
        pass

    def _on_get_response(self, response, error):
        if error:
            raise tornado.web.HTTPError(500)

        if len(response) > 0:
            self.write(str(response[0]))

        self.finish()

    @property
    def db(self):
        self._db = self.application.db
        return self._db

    def post(self):
        code = self.get_argument('code')
        region = self.get_argument('region')
        journal = self.get_argument('journal')
        issue = self.get_argument('issue')
        iso_date = date.isoformat(date.today())
        month_date = iso_date[:7]

        self.db.accesses.update(
            {'code': code},
            {'$set': {'type': 'article', 'journal': journal, 'issue': issue}, '$inc': {region: 1, iso_date: 1, month_date: 1, 'total': 1}},
            safe=False,
            upsert=True
        )

    @tornado.web.asynchronous
    def get(self):
        code = self.get_argument('code')

        if self.application.api_style == 'global':
            self.db.accesses.remove({'code': code}, callback=self._remove_callback)

            for resource in self.application.resources:
                url = "http://%s/api/v1/article?code=%s" % (resource, code)
                try:
                    data = urllib2.urlopen(url).read().replace("'", '"').replace('u"', '"')
                    if data:
                        data = json.loads(data)
                        str_data = {'journal': data['journal'], 'issue': data['issue'], 'type': 'article'}
                        del(data['code'])
                        del(data['type'])
                        del(data['journal'])
                        del(data['issue'])
                        self.db.accesses.update(
                            {'code': code},
                            {'$inc': data, '$set': str_data},
                            safe=False,
                            upsert=True
                        )
                except urllib2.URLError:
                    continue
                    # must register error log

        self.db.accesses.find({"code": code, "type": "article"}, {"_id": 0}, limit=1, callback=self._on_get_response)

class IssueHandler(tornado.web.RequestHandler):

    def _remove_callback(self, response, error):
        pass

    def _on_get_response(self, response, error):
        if error:
            raise tornado.web.HTTPError(500)

        if len(response) > 0:
            self.write(str(response[0]))

        self.finish()

    @property
    def db(self):
        self._db = self.application.db
        return self._db

    def post(self):
        code = self.get_argument('code')
        region = self.get_argument('region')
        journal = self.get_argument('journal')
        iso_date = date.isoformat(date.today())
        month_date = iso_date[:7]

        self.db.accesses.update(
            {'code': code},
            {'$set': {'type': 'issue', 'journal': journal}, '$inc': {region: 1, iso_date: 1, month_date: 1, 'total': 1}},
            safe=False,
            upsert=True
        )

    @tornado.web.asynchronous
    def get(self):
        code = self.get_argument('code')

        if self.application.api_style == 'global':
            self.db.accesses.remove({'code': code}, callback=self._remove_callback)

            for resource in self.application.resources:
                url = "http://%s/api/v1/issue?code=%s" % (resource, code)
                try:
                    data = urllib2.urlopen(url).read().replace("'", '"').replace('u"', '"')
                    if data:
                        data = json.loads(data)
                        str_data = {'journal': data['journal'], 'type': 'issue'}
                        del(data['code'])
                        del(data['type'])
                        del(data['journal'])
                        self.db.accesses.update(
                            {'code': code},
                            {'$inc': data, '$set': str_data},
                            safe=False,
                            upsert=True
                        )
                except urllib2.URLError:
                    continue
                    # must register error log

        self.db.accesses.find({"code": code, "type": "issue"}, {"_id": 0}, limit=1, callback=self._on_get_response)

class JournalHandler(tornado.web.RequestHandler):

    def _remove_callback(self, response, error):
        pass

    def _on_get_response(self, response, error):
        if error:
            raise tornado.web.HTTPError(500)

        if len(response) > 0:
            self.write(str(response[0]))

        self.finish()

    @property
    def db(self):
        self._db = self.application.db
        return self._db

    def post(self):
        code = self.get_argument('code')
        region = self.get_argument('region')
        iso_date = date.isoformat(date.today())
        month_date = iso_date[:7]

        self.db.accesses.update(
            {'code': code},
            {'$set': {'type': 'journal'}, '$inc': {region: 1, iso_date: 1, month_date: 1, 'total': 1}},
            safe=False,
            upsert=True
        )

    @tornado.web.asynchronous
    @tornado.gen.engine
    def get(self):
        code = self.get_argument('code')

        if self.application.api_style == 'global':
            self.db.accesses.remove({'code': code}, callback=self._remove_callback)
            http_client = httpclient.AsyncHTTPClient()
            for resource in self.application.resources:
                resource = resource.strip()
                url = "http://%s/api/v1/journal?%s" % (resource, urllib.urlencode({'code': code}))
                response = yield tornado.gen.Task(http_client.fetch, url)

                if not response.error:
                    data = response.body.replace("'", '"').replace('u"', '"')
                    data = json.loads(data)
                    del(data['code'])
                    del(data['type'])
                    self.db.accesses.update(
                        {'code': code},
                        {'$set': {'type': 'journal'}, '$inc': data},
                        safe=False,
                        upsert=True
                    )

        self.db.accesses.find({"code": code, "type": "journal"}, {"_id": 0}, limit=1, callback=self._on_get_response)

if __name__ == '__main__':
    tornado.options.parse_command_line()
    http_server = tornado.httpserver.HTTPServer(Application())
    http_server.listen(options.port)
    tornado.ioloop.IOLoop.instance().start()
<|endoftext|>


<commit_before>"""Retrieve the dictionary definitions."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import logging
import time
import shutil
import traceback
from collections import Counter, defaultdict

import numpy
import json
import nltk
from wordnik import swagger, WordApi, AccountApi

from dictlearn.util import vec2str
from dictlearn.corenlp import StanfordCoreNLP

logger = logging.getLogger(__name__)

_MIN_REMAINING_CALLS = 100
_SAVE_EVERY_CALLS = 100

class Dictionary(object):
    """The dictionary of definitions.

    The native representation of the dictionary is a mapping from a word
    to a list of definitions, each of which is a sequence of words. All
    the words are stored as strings.

    """
    def __init__(self, path=None):

        if not path.endswith("json"):
            raise Exception("Please pass path ending in .json")

        self._data = {}
        self._meta_data = {}
        self._path = path
        self._meta_path = path.replace(".json", "_meta.json")
        self._tmp_path = os.path.join(os.path.dirname(path),
                                         self._path + '.tmp')
        self._meta_tmp_path = os.path.join(os.path.dirname(path),
            self._meta_path + '.tmp')
        if self._path and os.path.exists(self._path):
            self.load()

    def load(self):
        with open(self._path, 'r') as src:
            self._data = json.load(src)
        if os.path.exists(self._meta_path):
            with open(self._meta_path, 'r') as src:
                self._meta_data = json.load(src)

    def save(self):
        logger.debug("saving...")
        with open(self._tmp_path, 'w') as dst:
            json.dump(self._data, dst, indent=2)
        shutil.move(self._tmp_path, self._path)
        logger.debug("saving meta...")
        with open(self._meta_tmp_path, 'w') as dst:
            json.dump(self._meta_data, dst, indent=2)
        shutil.move(self._meta_tmp_path, self._meta_path)
        logger.debug("saved")

    def _wait_until_quota_reset(self):
        while True:
            status = self._account_api.getApiTokenStatus()
            logger.debug("{} remaining calls".format(status.remainingCalls))
            if status.remainingCalls >= _MIN_REMAINING_CALLS:
                logger.debug("Wordnik quota was resetted")
                self._remaining_calls = status.remainingCalls
                return
            logger.debug("sleep until quota reset")
            time.sleep(60.)

    def setup_identity_mapping(self, vocab):
        for word in vocab.words:
            self._data[word] = [[word]]
        self.save()

    def setup_spelling(self, vocab):
        for word in vocab.words:
            def_ = word.decode('utf-8')
            if len(def_) > 10:
                def_ = u"{}-{}".format(def_[:5], def_[-5:])
            self._data[word] = [list(def_)]
        self.save()

    def add_from_lemma_definitions(self, vocab):
        """Add lemma definitions for non-lemmas.

        This code covers the following scenario: supposed a dictionary is crawled,
        but only for word lemmas.

        """
        lemmatizer = nltk.WordNetLemmatizer()
        for word in vocab.words:
            try:
                for part_of_speech in ['a', 's', 'r', 'n', 'v']:
                    lemma = lemmatizer.lemmatize(word, part_of_speech)
                    lemma_defs = self._data.get(lemma)
                    if lemma != word and lemma_defs:
                        # This can be quite slow. But this code will not be used
                        # very often.
                        for def_ in lemma_defs:
                            if not def_ in self._data[word]:
                                self._data[word].append(def_)
            except:
                logger.error("lemmatizer crashed on {}".format(word))
        self.save()

    def crawl_lemmas(self, vocab):
        """Add Wordnet lemmas as definitions."""
        lemmatizer = nltk.WordNetLemmatizer()
        for word in vocab.words:
            definitions = []
            try:
                for part_of_speech in ['a', 's', 'r', 'n', 'v']:
                    lemma = lemmatizer.lemmatize(word, part_of_speech)
                    if lemma != word and not [lemma] in definitions:
                        definitions.append([lemma])
            except:
                logger.error("lemmatizer crashed on {}".format(word))
            if definitions:
                self._data[word] = definitions
        self.save()

    def crawl_lowercase(self, vocab):
        """Add Wordnet lemmas as definitions."""
        for word in vocab.words:
            self._data[word] = [[word.lower()]]
        self.save()

    def crawl_wordnik(self, vocab, api_key, corenlp_url,
                      call_quota=15000, crawl_also_lowercase=False, crawl_also_lemma=False):

        """Download and preprocess definitions from Wordnik.

        vocab
            Vocabulary for which the definitions should be found.
        api_key
            The API key to use in communications with Wordnik.
        call_quota
            Maximum number of calls per hour.
        crawl_also_lowercase
            If true will add lowercase version of each word to crawl list
        crawl_also_lemma
            If true will also crawl lemma versions of words
            WARNING: Lemma of Cat is Cat! So if you want to have definition of "cat"
            you have to also pass crawl_also_lowercase!

        """
        corenlp = StanfordCoreNLP(corenlp_url)

        self._remaining_calls = call_quota
        self._last_saved = 0

        client = swagger.ApiClient(
            api_key, 'https://api.wordnik.com/v4')
        self._word_api = WordApi.WordApi(client)
        self._account_api = AccountApi.AccountApi(client)

        words = list(vocab.words)

        # Note(kudkudak): for SNLI it adds 3k words
        if crawl_also_lowercase:
            words_set = set(words)  # For efficiency

            logger.info("Adding lowercase words to crawl")
            lowercased = []
            for w in words:
                if w.lower() not in words_set:
                    lowercased.append(w.lower())
            logger.info("Crawling additional {} words".format(len(lowercased)))
            words.extend(sorted(lowercased))

        # Note(kudkudak): for SNLI it adds 2k words, so we can expect
        # like sabotage,sabotaging
        # Note that lemma crawling is *after* lowercasing
        if crawl_also_lemma:
            words_set = set(words) # For efficiency

            logger.info("Adding lemmatized vrsions to crawl")
            lemmas = []
            original = []
            lemmatizer = nltk.WordNetLemmatizer()
            for w in words:
                # Note(kudkudak): more words don't hurt too much.
                # so making sure here we really have lemma of word
                # even if it was capitalized in sentence
                for part_of_speech in ['a', 's', 'r', 'n', 'v']:
                    lemma = lemmatizer.lemmatize(w, part_of_speech)
                    if lemma not in words_set:
                        lemmas.append(lemma)
                        original.append(w)
            logger.info("Crawling additional {} words".format(len(lemmas)))
            for id in numpy.random.choice(len(lemmas), 100):
                logger.info("Example:" + lemmas[id] + "," + original[id])
                words.extend(sorted(lemmas))

        # Here, for now, we don't do any stemming or lemmatization.
        # Stemming is useless because the dictionary is not indexed with
        # lemmas, not stems. Lemmatizers, on the other hand, can not be
        # fully trusted when it comes to unknown words.
        for word in words:
            if word in self._data:
                logger.debug("a known word {}, skip".format(word))
                continue

            if self._last_saved >= _SAVE_EVERY_CALLS:
                self.save()
                self._last_saved = 0

            # 100 is a safery margin, I don't want to DDoS Wordnik :)
            if self._remaining_calls < _MIN_REMAINING_CALLS:
                self._wait_until_quota_reset()
            try:
                # NOTE(kudkudak): We fetch all dictionaries, but retrieval can filter them based on meta info
                definitions = self._word_api.getDefinitions(word)
            except Exception:
                logger.error("error during fetching '{}'".format(word))
                logger.error(traceback.format_exc())
                continue
            self._remaining_calls -= 1
            self._last_saved += 1

            if not definitions:
                definitions = []
            self._data[word] = []
            self._meta_data[word] = []
            for def_ in definitions:
                try:
                    # seems like definition text can be both str and unicode
                    text = def_.text
                    if isinstance(text, str):
                        text = text.decode('utf-8')
                    tokenized_def = corenlp.tokenize(text)[0]
                    self._data[word].append(tokenized_def)
                    # Note(kudkudak): I don't think there is much more useful meta data for us
                    self._meta_data[word].append({"sourceDictionary": def_.sourceDictionary})
                except Exception:
                    logger.error("error during tokenizing '{}'".format(text))
                    logger.error(traceback.format_exc())
            logger.debug("definitions for '{}' fetched {} remaining".format(word, self._remaining_calls))
        self.save()
        self._last_saved = 0

    def num_entries(self):
        return len(self._data)

    def get_definitions(self, key):
        return self._data.get(key, [])

class Retrieval(object):

    def __init__(self, vocab, dictionary,
                 max_def_length=1000, exclude_top_k=None, max_def_per_word=1000000,
                 try_lowercase=False, lemmatization_strategy="none"):
        """Retrieves the definitions.

        vocab
            The vocabulary.
        dictionary
            The dictionary of the definitions.
        max_def_length
            Disregard definitions that are longer than that.
        exclude_top_k
            Do not provide defitions for the first top k
            words of the vocabulary (typically the most frequent ones).
        max_def_per_word
            Pick at most max_n_def definitions for each word
        try_lowercase
            If True will always first prioritize uppercase, then lowercase.

            WARNING: There is perhaps unintuitive interplay with lemmatization_strategy.
            Lemma doesn't change capitalization! So if you want to resolve Cat -> cat you need
            to pass try_lowercase=True and lemmatization_strategy!="none"

        lemmatization_strategy
            * "none" - does nothing
            * "try" - if query word is not in dict, it will try lemma
            * "prioritize" - will check first lemma, then resolve to query word
            * "force_lemma" - will use only lemma
                will raise error if query word is in dict, while lemma is not.
                This is useful for asserting --add_lemas from crawl_dict.py
                did its job
        """
        self._vocab = vocab
        self._dictionary = dictionary
        self._max_def_length = max_def_length
        self._exclude_top_k = exclude_top_k
        self._max_def_per_word = max_def_per_word
        self._try_lowercase = try_lowercase
        self._lemmatization_strategy = lemmatization_strategy

    def retrieve(self, batch):
        """Retrieves all definitions for a batch of words sequences.

        TODO: definitions of phrases, phrasal verbs, etc.

        Returns
        -------
        defs
            A list of word definitions, each definition is a list of words.
        def_map
            A list of triples (batch_index, time_step, def_index). Maps
            words to their respective definitions from `defs`.

        """
        definitions = []
        def_map = []
        word_def_indices = defaultdict(list)

        for seq_pos, sequence in enumerate(batch):
            for word_pos, word in enumerate(sequence):
                if isinstance(word, numpy.ndarray):
                    word = vec2str(word)
                word_id = self._vocab.word_to_id(word)
                if (self._exclude_top_k
                    and word_id != self._vocab.unk
                    and word_id < self._exclude_top_k):
                    continue

                if word not in word_def_indices:
                    # The first time a word is encountered in a batch
                    word_defs = self._dictionary.get_definitions(word)
                    if not word_defs:
                        if self._try_lowercase:
                            word_defs = self._dictionary.get_definitions(word.lower())

                        if not word_defs:
                            continue

                    if self._max_def_per_word < len(word_defs):
                        word_defs_ids = numpy.random.choice(range(len(word_defs)), self._max_def_per_word, replace=False)
                        word_defs = [word_defs[i] for i in word_defs_ids]

                    for i, def_ in enumerate(word_defs):
                        if len(def_) > self._max_def_length:
                            continue
                        final_def_ = [self._vocab.bod]
                        for token in def_:
                            final_def_.append(self._vocab.word_to_id(token))
                        final_def_.append(self._vocab.eod)
                        word_def_indices[word].append(len(definitions))
                        definitions.append(final_def_)
                for def_index in word_def_indices[word]:
                    def_map.append((seq_pos, word_pos, def_index))

        return definitions, def_map

    def retrieve_and_pad(self, batch):
        defs, def_map = self.retrieve(batch)
        if not defs:
            defs.append(self.sentinel_definition())
        # `defs` have variable length and have to be padded
        max_def_length = max(map(len, defs))
        def_array = numpy.zeros((len(defs), max_def_length), dtype='int64')
        def_mask = numpy.ones_like(def_array, dtype='float32')
        for i, def_ in enumerate(defs):
            def_array[i, :len(def_)] = def_
            def_mask[i, len(def_):] = 0.
        def_map = (numpy.array(def_map)
                   if def_map
                   else numpy.zeros((0, 3), dtype='int64'))
        return def_array, def_mask, def_map

    def sentinel_definition(self):
        """An empty definition.

        If you ever need a definition that is syntactically correct but
        doesn't mean a thing, call me.

        """
        return [self._vocab.bod, self._vocab.eod]
<commit_msg>Add lowercase_strategy<commit_after>"""Retrieve the dictionary definitions."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import logging
import time
import shutil
import traceback
from collections import Counter, defaultdict

import numpy
import json
import nltk
from wordnik import swagger, WordApi, AccountApi

from dictlearn.util import vec2str
from dictlearn.corenlp import StanfordCoreNLP

logger = logging.getLogger(__name__)

_MIN_REMAINING_CALLS = 100
_SAVE_EVERY_CALLS = 100

class Dictionary(object):
    """The dictionary of definitions.

    The native representation of the dictionary is a mapping from a word
    to a list of definitions, each of which is a sequence of words. All
    the words are stored as strings.

    """
    def __init__(self, path=None):

        if not path.endswith("json"):
            raise Exception("Please pass path ending in .json")

        self._data = {}
        self._meta_data = {}
        self._path = path
        self._meta_path = path.replace(".json", "_meta.json")
        self._tmp_path = os.path.join(os.path.dirname(path),
                                         self._path + '.tmp')
        self._meta_tmp_path = os.path.join(os.path.dirname(path),
            self._meta_path + '.tmp')
        if self._path and os.path.exists(self._path):
            self.load()

    def load(self):
        with open(self._path, 'r') as src:
            self._data = json.load(src)
        if os.path.exists(self._meta_path):
            with open(self._meta_path, 'r') as src:
                self._meta_data = json.load(src)

    def save(self):
        logger.debug("saving...")
        with open(self._tmp_path, 'w') as dst:
            json.dump(self._data, dst, indent=2)
        shutil.move(self._tmp_path, self._path)
        logger.debug("saving meta...")
        with open(self._meta_tmp_path, 'w') as dst:
            json.dump(self._meta_data, dst, indent=2)
        shutil.move(self._meta_tmp_path, self._meta_path)
        logger.debug("saved")

    def _wait_until_quota_reset(self):
        while True:
            status = self._account_api.getApiTokenStatus()
            logger.debug("{} remaining calls".format(status.remainingCalls))
            if status.remainingCalls >= _MIN_REMAINING_CALLS:
                logger.debug("Wordnik quota was resetted")
                self._remaining_calls = status.remainingCalls
                return
            logger.debug("sleep until quota reset")
            time.sleep(60.)

    def setup_identity_mapping(self, vocab):
        for word in vocab.words:
            self._data[word] = [[word]]
        self.save()

    def setup_spelling(self, vocab):
        for word in vocab.words:
            def_ = word.decode('utf-8')
            if len(def_) > 10:
                def_ = u"{}-{}".format(def_[:5], def_[-5:])
            self._data[word] = [list(def_)]
        self.save()

    def add_from_lemma_definitions(self, vocab):
        """Add lemma definitions for non-lemmas.

        This code covers the following scenario: supposed a dictionary is crawled,
        but only for word lemmas.

        """
        lemmatizer = nltk.WordNetLemmatizer()
        for word in vocab.words:
            try:
                for part_of_speech in ['a', 's', 'r', 'n', 'v']:
                    lemma = lemmatizer.lemmatize(word, part_of_speech)
                    lemma_defs = self._data.get(lemma)
                    if lemma != word and lemma_defs:
                        # This can be quite slow. But this code will not be used
                        # very often.
                        for def_ in lemma_defs:
                            if not def_ in self._data[word]:
                                self._data[word].append(def_)
            except:
                logger.error("lemmatizer crashed on {}".format(word))
        self.save()

    def crawl_lemmas(self, vocab):
        """Add Wordnet lemmas as definitions."""
        lemmatizer = nltk.WordNetLemmatizer()
        for word in vocab.words:
            definitions = []
            try:
                for part_of_speech in ['a', 's', 'r', 'n', 'v']:
                    lemma = lemmatizer.lemmatize(word, part_of_speech)
                    if lemma != word and not [lemma] in definitions:
                        definitions.append([lemma])
            except:
                logger.error("lemmatizer crashed on {}".format(word))
            if definitions:
                self._data[word] = definitions
        self.save()

    def crawl_lowercase(self, vocab):
        """Add Wordnet lemmas as definitions."""
        for word in vocab.words:
            self._data[word] = [[word.lower()]]
        self.save()

    def crawl_wordnik(self, vocab, api_key, corenlp_url,
                      call_quota=15000, crawl_also_lowercase=False, crawl_also_lemma=False):

        """Download and preprocess definitions from Wordnik.

        vocab
            Vocabulary for which the definitions should be found.
        api_key
            The API key to use in communications with Wordnik.
        call_quota
            Maximum number of calls per hour.
        crawl_also_lowercase
            If true will add lowercase version of each word to crawl list
        crawl_also_lemma
            If true will also crawl lemma versions of words
            WARNING: Lemma of Cat is Cat! So if you want to have definition of "cat"
            you have to also pass crawl_also_lowercase!

        """
        corenlp = StanfordCoreNLP(corenlp_url)

        self._remaining_calls = call_quota
        self._last_saved = 0

        client = swagger.ApiClient(
            api_key, 'https://api.wordnik.com/v4')
        self._word_api = WordApi.WordApi(client)
        self._account_api = AccountApi.AccountApi(client)

        words = list(vocab.words)

        # Note(kudkudak): for SNLI it adds 3k words
        if crawl_also_lowercase:
            words_set = set(words)  # For efficiency

            logger.info("Adding lowercase words to crawl")
            lowercased = []
            for w in words:
                if w.lower() not in words_set:
                    lowercased.append(w.lower())
            logger.info("Crawling additional {} words".format(len(lowercased)))
            words.extend(sorted(lowercased))

        # Note(kudkudak): for SNLI it adds 2k words, so we can expect
        # like sabotage,sabotaging
        # Note that lemma crawling is *after* lowercasing
        if crawl_also_lemma:
            words_set = set(words) # For efficiency

            logger.info("Adding lemmatized vrsions to crawl")
            lemmas = []
            original = []
            lemmatizer = nltk.WordNetLemmatizer()
            for w in words:
                for part_of_speech in ['a', 's', 'r', 'n', 'v']:
                    lemma = lemmatizer.lemmatize(w, part_of_speech)
                    if lemma not in words_set:
                        lemmas.append(lemma)
                        original.append(w)
            logger.info("Crawling additional {} words".format(len(lemmas)))
            for id in numpy.random.choice(len(lemmas), 100):
                logger.info("Example:" + lemmas[id] + "," + original[id])
                words.extend(sorted(lemmas))

        # Here, for now, we don't do any stemming or lemmatization.
        # Stemming is useless because the dictionary is not indexed with
        # lemmas, not stems. Lemmatizers, on the other hand, can not be
        # fully trusted when it comes to unknown words.
        for word in words:
            if word in self._data:
                logger.debug("a known word {}, skip".format(word))
                continue

            if self._last_saved >= _SAVE_EVERY_CALLS:
                self.save()
                self._last_saved = 0

            # 100 is a safery margin, I don't want to DDoS Wordnik :)
            if self._remaining_calls < _MIN_REMAINING_CALLS:
                self._wait_until_quota_reset()
            try:
                # NOTE(kudkudak): We fetch all dictionaries, but retrieval can filter them based on meta info
                definitions = self._word_api.getDefinitions(word)
            except Exception:
                logger.error("error during fetching '{}'".format(word))
                logger.error(traceback.format_exc())
                continue
            self._remaining_calls -= 1
            self._last_saved += 1

            if not definitions:
                definitions = []
            self._data[word] = []
            self._meta_data[word] = []
            for def_ in definitions:
                try:
                    # seems like definition text can be both str and unicode
                    text = def_.text
                    if isinstance(text, str):
                        text = text.decode('utf-8')
                    tokenized_def = corenlp.tokenize(text)[0]
                    self._data[word].append(tokenized_def)
                    # Note(kudkudak): I don't think there is much more useful meta data for us
                    self._meta_data[word].append({"sourceDictionary": def_.sourceDictionary})
                except Exception:
                    logger.error("error during tokenizing '{}'".format(text))
                    logger.error(traceback.format_exc())
            logger.debug("definitions for '{}' fetched {} remaining".format(word, self._remaining_calls))
        self.save()
        self._last_saved = 0

    def num_entries(self):
        return len(self._data)

    def get_definitions(self, key):
        return self._data.get(key, [])

class Retrieval(object):

    def __init__(self, vocab, dictionary,
                 max_def_length=1000, exclude_top_k=None, max_def_per_word=1000000,
                 lowercase_strategy="none", lemmatization_strategy="none"):
        """Retrieves the definitions.

        vocab
            The vocabulary.
        dictionary
            The dictionary of the definitions.
        max_def_length
            Disregard definitions that are longer than that.
        exclude_top_k
            Do not provide defitions for the first top k
            words of the vocabulary (typically the most frequent ones).
        max_def_per_word
            Pick at most max_n_def definitions for each word
        lowercase_strategy
            If "try" will always first prioritize uppercase, then lowercase
            If "concat" will add definitions of normal and lowercase version.
                Model then could learn which defs to trust, I imagine
            If "none" will do nothing

            WARNING: There is perhaps unintuitive interplay with lemmatization_strategy.
            Lemma doesn't change capitalization! So if you want to resolve Cat -> cat you need
            to pass try_lowercase=True and lemmatization_strategy!="none"

        lemmatization_strategy
            * "none" - does nothing
            * "try" - if query word is not in dict, it will try lemma
            * "prioritize" - will check first lemma, then resolve to query word
            * "force_lemma" - will use only lemma
                will raise error if query word is in dict, while lemma is not.
                This is useful for asserting --add_lemas from crawl_dict.py
                did its job
        """
        self._vocab = vocab
        self._dictionary = dictionary
        self._max_def_length = max_def_length
        self._exclude_top_k = exclude_top_k
        self._max_def_per_word = max_def_per_word

        if lowercase_strategy not in {"none", "try", "concat"}:
            raise NotImplementedError("Not implemented " + lowercase_strategy)

        self._lowercase_strategy = lowercase_strategy

        if lemmatization_strategy not in {"none", "try", "prioritize", "force_lemma"}:
            raise NotImplementedError("Not implemented " + lemmatization_strategy)

        self._lemmatization_strategy = lemmatization_strategy
        self._lemmatizer = nltk.WordNetLemmatizer()

    def _resolve_word(self, word):
        word_defs = [] # First that resolves returns

        if self._lemmatization_strategy == "prioritize" or self._lemmatizer == "force_lemma":
            word_defs = self._dictionary.get_definitions(self._lemmatizer.lemmatize(word))

        # TODO(kudkudak): Lemmas might be a miss for some words. Do we care?
        # probably not so much because otherwise we would just pass empty list
        # so it has to help anyway
        if not word_defs:
            word_defs = self._dictionary.get_definitions(word)

        if not word_defs and self._lemmatization_strategy == "try":
            word_defs = self._dictionary.get_definitions(self._lemmatizer.lemmatize(word))

        return word_defs

    def retrieve(self, batch):
        """Retrieves all definitions for a batch of words sequences.

        TODO: definitions of phrases, phrasal verbs, etc.

        Returns
        -------
        defs
            A list of word definitions, each definition is a list of words.
        def_map
            A list of triples (batch_index, time_step, def_index). Maps
            words to their respective definitions from `defs`.

        """
        definitions = []
        def_map = []
        word_def_indices = defaultdict(list)

        for seq_pos, sequence in enumerate(batch):
            for word_pos, word in enumerate(sequence):
                if isinstance(word, numpy.ndarray):
                    word = vec2str(word)
                word_id = self._vocab.word_to_id(word)
                if (self._exclude_top_k
                    and word_id != self._vocab.unk
                    and word_id < self._exclude_top_k):
                    continue

                if word not in word_def_indices:
                    # The first time a word is encountered in a batch
                    if self._lowercase_strategy == "none":
                        word_defs = self._resolve_word(word)
                    elif self._lowercase_strategy == "try":
                        word_defs = self._resolve_word(word)
                        if not word_defs:
                            word_defs = self._resolve_word(word.lower())
                    elif self._lowercase_strategy == "concat":
                        word_defs = self._resolve_word(word)
                        if word != word.lower():
                            word_defs += self._resolve_word(word.lower())
                    else:
                        raise NotImplementedError()

                    if not word_defs:
                        continue

                    if self._max_def_per_word < len(word_defs):
                        word_defs_ids = numpy.random.choice(range(len(word_defs)), self._max_def_per_word, replace=False)
                        word_defs = [word_defs[i] for i in word_defs_ids]

                    for i, def_ in enumerate(word_defs):
                        if len(def_) > self._max_def_length:
                            continue
                        final_def_ = [self._vocab.bod]
                        for token in def_:
                            final_def_.append(self._vocab.word_to_id(token))
                        final_def_.append(self._vocab.eod)
                        word_def_indices[word].append(len(definitions))
                        definitions.append(final_def_)
                for def_index in word_def_indices[word]:
                    def_map.append((seq_pos, word_pos, def_index))

        return definitions, def_map

    def retrieve_and_pad(self, batch):
        defs, def_map = self.retrieve(batch)
        if not defs:
            defs.append(self.sentinel_definition())
        # `defs` have variable length and have to be padded
        max_def_length = max(map(len, defs))
        def_array = numpy.zeros((len(defs), max_def_length), dtype='int64')
        def_mask = numpy.ones_like(def_array, dtype='float32')
        for i, def_ in enumerate(defs):
            def_array[i, :len(def_)] = def_
            def_mask[i, len(def_):] = 0.
        def_map = (numpy.array(def_map)
                   if def_map
                   else numpy.zeros((0, 3), dtype='int64'))
        return def_array, def_mask, def_map

    def sentinel_definition(self):
        """An empty definition.

        If you ever need a definition that is syntactically correct but
        doesn't mean a thing, call me.

        """
        return [self._vocab.bod, self._vocab.eod]
<|endoftext|>


<commit_before>from base import (BaseDocument, DictPunch, DocumentMetaclass, TopLevelDocumentMetaclass)

from base import json

__all__ = ['Document', 'EmbeddedDocument', 'DictPunch']

class EmbeddedDocument(BaseDocument):
    """A :class:`~dictshield.Document` that isn't stored in its own
    collection.  :class:`~dictshield.EmbeddedDocument`\ s should be used as
    fields on :class:`~dictshield.Document`\ s through the
    :class:`~dictshield.EmbeddedDocumentField` field type.
    """

    __metaclass__ = DocumentMetaclass

class Document(BaseDocument):
    """The base class used for defining the structure and properties of
    collections of documents modeled in DictShield. Inherit from this class,
    and add fields as class attributes to define a document's structure.
    Individual documents may then be created by making instances of the
    :class:`~dictshield.Document` subclass.

    A :class:`~dictshield.Document` subclass may be itself subclassed, to
    create a specialised version of the document that can be stored in the
    same collection. To facilitate this behaviour, `_cls` and `_types`
    fields are added to documents to specify the install class and the types
    in the Document class hierarchy. To disable this behaviour and remove
    the dependence on the presence of `_cls` and `_types`, set
    :attr:`allow_inheritance` to ``False`` in the :attr:`meta` dictionary.

    :attr:`_internal_fields` class field is used to list fields which are
    private and not meant to leave the system. This consists of `_id`, `_cls`
    and `_types` but a user can extend the lis of internal fields by defining
    a class level list field called _private_fields.

    :attr:`make_json_ownersafe` is defined to remove the keys listed in both
    fields, make it safe for sending as json.

    If :attr:`_public_fields` is defined, the `make_json_publicsafe` method
    will use it as a whitelist for which fields to not erase.
    """

    __metaclass__ = TopLevelDocumentMetaclass

    _internal_fields = [
        '_id', 'id', '_cls', '_types',
    ]

    _public_fields = None

    @classmethod
    def _get_internal_fields(cls):
        """Helper function that determines the union of :attr:`_internal_fields`
        and :attr:`_private_fields`, else returns just :attr:`_internal_fields`.
        """
        internal_fields = set(cls._internal_fields)
        if hasattr(cls, '_private_fields'):
            private_fields = set(cls._private_fields)
            internal_fields = internal_fields.union(private_fields)
        return internal_fields

    @classmethod
    def _safe_data_from_input(cls, fun, data):
        """Helper function for handling variable inputs to make_json_*safe
        functions.

        Returns a safe doc if given one element.

        Returns an list if given a list. Can handle a list of dicts or
        list of docs.
        """
        # single cls instance
        if isinstance(data, cls):
            return fun(data.to_python())

        # single dict instance
        elif isinstance(data, dict):
            return fun(data)

        # list of cls instances or list of dicts
        elif isinstance(data, list):
            if len(data) < 1:
                return list()
            elif isinstance(data[0], dict):
                pass # written for clarity
            elif isinstance(data[0], cls):
                data = [d.to_python() for d in data]
            return map(fun, data)

    @classmethod
    def make_json_ownersafe(cls, doc_dict_or_dicts):
        """This function removes internal fields and handles any steps
        required for making the data stucture (list, dict or Document)
        safe for transmission to the owner of the data.

        It attempts to handle multiple inputs types to avoid as many
        translation steps as possible.
        """
        internal_fields = cls._get_internal_fields()

        def handle_doc(doc_dict):
            # internal_fields is a blacklist
            for f in internal_fields:
                if doc_dict.has_key(f):
                    del doc_dict[f]
            return doc_dict

        trimmed = cls._safe_data_from_input(handle_doc, doc_dict_or_dicts)
        return json.dumps(trimmed)

    @classmethod
    def make_json_publicsafe(cls, doc_dict_or_dicts):
        """This funciton ensures found_data only contains the keys as
        listed in cls._public_fields.

        This function can be safely called without calling make_json_ownersafe
        first because it treats cls._public_fields as a whitelist and
        removes anything not listed.
        """
        if cls._public_fields is None:
            raise DictPunch('make_json_publicsafe called cls with no _public_fields')

        def handle_doc(doc_dict):
            # public_fields is a whitelist
            for f in doc_dict.keys():
                if f not in cls._public_fields:
                    del doc_dict[f]
            return doc_dict

        trimmed = cls._safe_data_from_input(handle_doc, doc_dict_or_dicts)
        return json.dumps(trimmed)

    @classmethod
    def _validate_helper(cls, fun, values, validate_all=False):
        """This is a convenience function that loops over the given values
        and attempts to validate them against the class definition. It only
        validates the data in values and does not guarantee a complete document
        is present.

        'not present' is defined as not having a value OR having '' (or u'')
        as a value.
        """
        if not hasattr(cls, '_fields'):
            raise ValueError('cls is not a DictShield')

        internal_fields = cls._get_internal_fields()

        if validate_all:
            exceptions = list()
            handle_exception = lambda e: exceptions.append(e)
        else:
            def handle_exception(e):
                raise e

        for k,v in cls._fields.items():
            # handle common id name
            if k is 'id':
                k = '_id'

            # we don't accept internal fields from users
            if k in internal_fields and k in values:
                value_is_default = (values[k] is v.default)
                if not value_is_default:
                    e = DictPunch('Overwrite of internal fields attempted', k, v)
                    handle_exception(e)
                    continue

            if fun(k, v):
                datum = values[k]
                # if datum is None, skip
                if datum is None:
                    continue
                # treat empty strings as empty values and skip
                if isinstance(datum, (str, unicode)) and len(datum.strip()) == 0:
                    continue
                try:
                    v.validate(datum)
                except DictPunch, e:
                    handle_exception(e)

        if validate_all:
            return exceptions
        else:
            return True

    @classmethod
    def validate_class_fields(cls, values, validate_all=False):
        """This is a convenience function that loops over _fields in
        cls to validate them. If the field is not required AND not present,
        it is skipped.
        """
        fun = lambda k,v: v.required or k in values
        return cls._validate_helper(fun, values, validate_all=validate_all)

    @classmethod
    def validate_class_partial(cls, values, validate_all=False):
        """This is a convenience function that loops over _fields in
        cls to validate them. This function is a partial validatation
        only, meaning the values given and does not check if the document
        is complete.
        """
        fun = lambda k,v: k in values
        return cls._validate_helper(fun, values, validate_all=validate_all)
<commit_msg>Added functions for make_*safe that outputs to Python<commit_after>from base import (BaseDocument, DictPunch, DocumentMetaclass, TopLevelDocumentMetaclass)

from base import json

__all__ = ['Document', 'EmbeddedDocument', 'DictPunch']

class EmbeddedDocument(BaseDocument):
    """A :class:`~dictshield.Document` that isn't stored in its own
    collection.  :class:`~dictshield.EmbeddedDocument`\ s should be used as
    fields on :class:`~dictshield.Document`\ s through the
    :class:`~dictshield.EmbeddedDocumentField` field type.
    """

    __metaclass__ = DocumentMetaclass

class Document(BaseDocument):
    """The base class used for defining the structure and properties of
    collections of documents modeled in DictShield. Inherit from this class,
    and add fields as class attributes to define a document's structure.
    Individual documents may then be created by making instances of the
    :class:`~dictshield.Document` subclass.

    A :class:`~dictshield.Document` subclass may be itself subclassed, to
    create a specialised version of the document that can be stored in the
    same collection. To facilitate this behaviour, `_cls` and `_types`
    fields are added to documents to specify the install class and the types
    in the Document class hierarchy. To disable this behaviour and remove
    the dependence on the presence of `_cls` and `_types`, set
    :attr:`allow_inheritance` to ``False`` in the :attr:`meta` dictionary.

    :attr:`_internal_fields` class field is used to list fields which are
    private and not meant to leave the system. This consists of `_id`, `_cls`
    and `_types` but a user can extend the lis of internal fields by defining
    a class level list field called _private_fields.

    :attr:`make_json_ownersafe` is defined to remove the keys listed in both
    fields, make it safe for sending as json.

    If :attr:`_public_fields` is defined, the `make_json_publicsafe` method
    will use it as a whitelist for which fields to not erase.
    """

    __metaclass__ = TopLevelDocumentMetaclass

    _internal_fields = [
        '_id', 'id', '_cls', '_types',
    ]

    _public_fields = None

    @classmethod
    def _get_internal_fields(cls):
        """Helper function that determines the union of :attr:`_internal_fields`
        and :attr:`_private_fields`, else returns just :attr:`_internal_fields`.
        """
        internal_fields = set(cls._internal_fields)
        if hasattr(cls, '_private_fields'):
            private_fields = set(cls._private_fields)
            internal_fields = internal_fields.union(private_fields)
        return internal_fields

    @classmethod
    def _safe_data_from_input(cls, fun, data):
        """Helper function for handling variable inputs to make_json_*safe
        functions.

        Returns a safe doc if given one element.

        Returns an list if given a list. Can handle a list of dicts or
        list of docs.
        """
        # single cls instance
        if isinstance(data, cls):
            return fun(data.to_python())

        # single dict instance
        elif isinstance(data, dict):
            return fun(data)

        # list of cls instances or list of dicts
        elif isinstance(data, list):
            if len(data) < 1:
                return list()
            elif isinstance(data[0], dict):
                pass # written for clarity
            elif isinstance(data[0], cls):
                data = [d.to_python() for d in data]
            return map(fun, data)

    @classmethod
    def make_ownersafe(cls, doc_dict_or_dicts):
        """This function removes internal fields and handles any steps
        required for making the data stucture (list, dict or Document)
        safe for transmission to the owner of the data.

        It attempts to handle multiple inputs types to avoid as many
        translation steps as possible.
        """
        internal_fields = cls._get_internal_fields()

        def handle_doc(doc_dict):
            # internal_fields is a blacklist
            for f in internal_fields:
                if doc_dict.has_key(f):
                    del doc_dict[f]
            return doc_dict

        trimmed = cls._safe_data_from_input(handle_doc, doc_dict_or_dicts)
        return trimmed

    @classmethod
    def make_json_ownersafe(cls, doc_dict_or_dicts):
        """Trims the object using make_ownersafe and dumps to JSON
        """
        trimmed = cls.make_ownersafe(doc_dict_or_dicts)
        return json.dumps(trimmed)

    @classmethod
    def make_publicsafe(cls, doc_dict_or_dicts):
        """This funciton ensures found_data only contains the keys as
        listed in cls._public_fields.

        This function can be safely called without calling make_json_ownersafe
        first because it treats cls._public_fields as a whitelist and
        removes anything not listed.
        """
        if cls._public_fields is None:
            raise DictPunch('make_json_publicsafe called cls with no _public_fields')

        def handle_doc(doc_dict):
            # public_fields is a whitelist
            for f in doc_dict.keys():
                if f not in cls._public_fields:
                    del doc_dict[f]
            return doc_dict

        trimmed = cls._safe_data_from_input(handle_doc, doc_dict_or_dicts)
        return trimmed

    @classmethod
    def make_json_publicsafe(cls, doc_dict_or_dicts):
        """Trims the object using make_publicsafe and dumps to JSON
        """
        trimmed = cls.make_publicsafe(doc_dict_or_dicts)
        return json.dumps(trimmed)

    @classmethod
    def _validate_helper(cls, fun, values, validate_all=False):
        """This is a convenience function that loops over the given values
        and attempts to validate them against the class definition. It only
        validates the data in values and does not guarantee a complete document
        is present.

        'not present' is defined as not having a value OR having '' (or u'')
        as a value.
        """
        if not hasattr(cls, '_fields'):
            raise ValueError('cls is not a DictShield')

        internal_fields = cls._get_internal_fields()

        if validate_all:
            exceptions = list()
            handle_exception = lambda e: exceptions.append(e)
        else:
            def handle_exception(e):
                raise e

        for k,v in cls._fields.items():
            # handle common id name
            if k is 'id':
                k = '_id'

            # we don't accept internal fields from users
            if k in internal_fields and k in values:
                value_is_default = (values[k] is v.default)
                if not value_is_default:
                    e = DictPunch('Overwrite of internal fields attempted', k, v)
                    handle_exception(e)
                    continue

            if fun(k, v):
                datum = values[k]
                # if datum is None, skip
                if datum is None:
                    continue
                # treat empty strings as empty values and skip
                if isinstance(datum, (str, unicode)) and len(datum.strip()) == 0:
                    continue
                try:
                    v.validate(datum)
                except DictPunch, e:
                    handle_exception(e)

        if validate_all:
            return exceptions
        else:
            return True

    @classmethod
    def validate_class_fields(cls, values, validate_all=False):
        """This is a convenience function that loops over _fields in
        cls to validate them. If the field is not required AND not present,
        it is skipped.
        """
        fun = lambda k,v: v.required or k in values
        return cls._validate_helper(fun, values, validate_all=validate_all)

    @classmethod
    def validate_class_partial(cls, values, validate_all=False):
        """This is a convenience function that loops over _fields in
        cls to validate them. This function is a partial validatation
        only, meaning the values given and does not check if the document
        is complete.
        """
        fun = lambda k,v: k in values
        return cls._validate_helper(fun, values, validate_all=validate_all)
<|endoftext|>


<commit_before># Core types and Python boilerplate for defining pixels, panels, thread-safe
# core objects, etc. I would have used collections, but I've prefer to have
# some data safety.

from collections import MutableMapping
import colorsys
import sys
try:
    from scipy.spatial import cKDTree
except ImportError as err:
    sys.stderr.write("Error: failed to import scipy.spatial module ({})".format(err))

class Color(object):
    """ Color representation, stored as RGB. Saturates for invalid values.
    """

    __min = 0
    __max = 255
    __size = 4

    def __init__(self, red = 0, green = 0, blue = 0, alpha = 0):
        self.__color = bytearray([0, 0, 0, 0])
        self.red = red
        self.green = green
        self.blue = blue
        self.alpha = alpha

    @property
    def rgba(self):
        """ Raw RGB-Alpha tuple.
        """
        return (self.red, self.green, self.blue, self.alpha)

    @property
    def hsv(self):
        """ HSV tuple.
        """
        norm = __max
        h, s, v = colorsys.rgb_to_hsv(self.red/norm, self.green/norm, self.blue/norm)
        return (360.*h, s, v)

    def set_rgb(self, red, green, blue, alpha = 0):
        """ Set RGB convenience method.
        """
        self.red = red
        self.green = green
        self.blue = blue
        self.alpha = alpha

    def set_hsv(self, hue, saturation, value):
        """ Set HSV convenience method.
        """
        red, green, blue = colorsys.hsv_to_rgb(hue, saturation, value)
        self.red = int(round(red*__max)))
        self.green = int(round(green*__max))
        self.blue = int(round(blue*__max))

    # Red channel
    def __get_r(self):
        return self.__color[0]
    def __set_r(self, val):
        self.__set_val(0, val)

    # Green channel
    def __get_g(self):
        return self.__color[1]
    def __set_g(self, val):
        self.__set_val(1, val)

    # Blue channel
    def __get_b(self):
        return self.__color[2]
    def __set_b(self, val):
        self.__set_val(2, val)

    # Alpha channel
    def __get_a(self):
        return self.__color[3]
    def __set_a(self, val):
        self.__set_val(3, val)

    def __set_val(self, index, val):
        assert index < Color.__size
        try:
            self.__color[index] = val
        except ValueError:
            if val > Color.__max:
                self.__color[index] = Color.__max
            else:
                self.__color[index] = Color.__min

    red = property(__get_r, __set_r, None, "Red channel.")
    green = property(__get_g, __set_g, None, "Green channel.")
    blue = property(__get_b, __set_b, None, "Blue channel.")
    alpha = property(__get_a, __set_a, None, "Alpha channel.")

    def __repr__(self):
        val = (self.red, self.green, self.blue, self.alpha)
        formatted = "<Color (r = %0.3f, g = %0.3f, b = %0.3f, alpha = %0.3f)>"
        return formatted % val

class Location(object):
    def __init__(self, x = 0, y = 0):
        self.__x = x
        self.__y = y

    @property
    def raw(self):
        return (self.x, self.y)

    def set_loc(self, x, y):
        self.x = x
        self.y = y

    def __get_x(self):
        return self.__x
    def __set_x(self, val):
        self.__x = val
    def __del_x(self):
        del self.__x

    def __get_y(self):
        return self.__y
    def __set_y(self, val):
        self.__y = val
    def __del_y(self):
        del self.__y

    x = property(__get_x, __set_x, __del_x, "Horizontal x.")
    y = property(__get_y, __set_y, __del_y, "Vertical y.")

    def __repr__(self):
        formatted = "<Location (x = %0.3f, y = %0.3f)>"
        return formatted % (self.x, self.y)

class DMXAddress(object):
    """Defines the DMX address for a live pixel.
    """
    __invalid_address = -1
    __dmx_lower = 0
    __dmx_upper = 512

    def __init__(self, universe = 0, address = 0):
        self.__universe = universe
        self.__address = 0
        self.address = address

    def is_valid(self):
        return self.address is not DMXAddress.__invalid_address

    def __get_universe(self):
        return self.__universe
    def __set_universe(self, val):
        self.__universe = val
    def __del_universe(self):
        del self.__universe

    def __get_address(self):
        return self.__address
    def __set_address(self, val):
        if DMXAddress.__dmx_lower <= val <= DMXAddress.__dmx_upper:
            self.__address = val
        else:
            self.__address = DMXAddress.__invalid_address
    def __del_address(self):
        del self.__address

    universe = property(__get_universe, __set_universe, __del_universe, "DMX universe.")
    address = property(__get_address, __set_address, __del_address, "DMX address.")

    def __repr__(self):
        formatted = "<DMXAddress (universe = %0.3f, address = %0.3f)>"
        return formatted % (self.universe, self.address)

class Pixel(object):
    """ A pixel has a color and location. If it is live, it must have a valid
    DMX address.
    """

    def __init__(self,
                 color = Color(),
                 location = Location(),
                 address = DMXAddress(),
                 live = False):
        self.__color = color
        self.__location = location
        self.__address = address
        self.__live = live
        assert not live or (live and address.is_valid())

    def __get_color(self):
        return self.__color
    def __set_color(self, val):
        self.__color = val
    def __del_color(self):
        del self.__color

    def __get_location(self):
        return self.__location
    def __set_location(self, val):
        self.__location = val
    def __del_location(self):
        del self.__location

    def __get_address(self):
        return self.__address
    def __set_address(self, val):
        self.__address = val
    def __del_address(self):
        del self.__address

    def __get_live(self):
        return self.__live
    def __set_live(self, val):
        self.__live = val
    def __del_live(self):
        del self.__live

    color = property(__get_color, __set_color, __del_color, "RGB color.")
    location = property(__get_location, __set_location, __del_location, "Location (x, y).")
    address = property(__get_address, __set_address, __del_address, "DMX address.")
    live = property(__get_live, __set_live, __del_live, "Is live pixel?")

    def __repr__(self):
        return "".join(["<Pixel ",
                        str(self.color), ",",
                        str(self.location), ", live = ",
                        str(self.live), ">"])

class Panel(MutableMapping):
    """ Panel represents a collection of pixels, representing a climbing wall.
    It's structured as a dictionary with fast k-nearest neighbor access of
    pixels.
    """

    def __init__(self):
        self.__pixels = dict()
        if len(self.locations) > 0:
            self.__tree = cKDTree(self.locations)
        else:
            self.__tree = None

    @property
    def locations(self):
        """ Returns an array of (x, y) tuple locations for pixels.
        """
        return self.__pixels.keys()

    def get_nearest(self, location, num_pixels):
        """ Returns the num_pixels closests pixels to a location.
        """
        assert self.__tree, "__tree is empty."
        return self.__tree.query_ball_point(location, num_pixels)

    def __contains__(self, key):
        return key in self.__pixels

    def __getitem__(self, key):
        return self.__pixels[key]

    def __setitem__(self, key, value):
        if key not in self.__pixels:
            self.__tree = cKDTree(self.locations + [key])
        self.__pixels[key] = value

    def __delitem__(self, key):
        del self.__pixels[key]

    def __len__(self):
        return len(self.__pixels)

    def __iter__(self):
        return iter(self.__pixels)

    def __repr__(self):
        return self.__pixels.__repr__()
<commit_msg>Fucking shitcock.<commit_after># Core types and Python boilerplate for defining pixels, panels, thread-safe
# core objects, etc. I would have used collections, but I've prefer to have
# some data safety.

from collections import MutableMapping
import colorsys
import sys
try:
    from scipy.spatial import cKDTree
except ImportError as err:
    sys.stderr.write("Error: failed to import scipy.spatial module ({})".format(err))

class Color(object):
    """ Color representation, stored as RGB. Saturates for invalid values.
    """

    __min = 0
    __max = 255
    __size = 4

    def __init__(self, red = 0, green = 0, blue = 0, alpha = 0):
        self.__color = bytearray([0, 0, 0, 0])
        self.red = red
        self.green = green
        self.blue = blue
        self.alpha = alpha

    @property
    def rgba(self):
        """ Raw RGB-Alpha tuple.
        """
        return (self.red, self.green, self.blue, self.alpha)

    @property
    def hsv(self):
        """ HSV tuple.
        """
        norm = Color.__max
        h, s, v = colorsys.rgb_to_hsv(self.red/norm, self.green/norm, self.blue/norm)
        return (360.*h, s, v)

    def set_rgb(self, red, green, blue, alpha = 0):
        """ Set RGB convenience method.
        """
        self.red = red
        self.green = green
        self.blue = blue
        self.alpha = alpha

    def set_hsv(self, hue, saturation, value):
        """ Set HSV convenience method.
        """
        red, green, blue = colorsys.hsv_to_rgb(hue, saturation, value)
        self.red = int(round(red*Color.__max))
        self.green = int(round(green*Color.__max))
        self.blue = int(round(blue*Color.__max))

    # Red channel
    def __get_r(self):
        return self.__color[0]
    def __set_r(self, val):
        self.__set_val(0, val)

    # Green channel
    def __get_g(self):
        return self.__color[1]
    def __set_g(self, val):
        self.__set_val(1, val)

    # Blue channel
    def __get_b(self):
        return self.__color[2]
    def __set_b(self, val):
        self.__set_val(2, val)

    # Alpha channel
    def __get_a(self):
        return self.__color[3]
    def __set_a(self, val):
        self.__set_val(3, val)

    def __set_val(self, index, val):
        assert index < Color.__size
        try:
            self.__color[index] = val
        except ValueError:
            if val > Color.__max:
                self.__color[index] = Color.__max
            else:
                self.__color[index] = Color.__min

    red = property(__get_r, __set_r, None, "Red channel.")
    green = property(__get_g, __set_g, None, "Green channel.")
    blue = property(__get_b, __set_b, None, "Blue channel.")
    alpha = property(__get_a, __set_a, None, "Alpha channel.")

    def __repr__(self):
        val = (self.red, self.green, self.blue, self.alpha)
        formatted = "<Color (r = %0.3f, g = %0.3f, b = %0.3f, alpha = %0.3f)>"
        return formatted % val

class Location(object):
    def __init__(self, x = 0, y = 0):
        self.__x = x
        self.__y = y

    @property
    def raw(self):
        return (self.x, self.y)

    def set_loc(self, x, y):
        self.x = x
        self.y = y

    def __get_x(self):
        return self.__x
    def __set_x(self, val):
        self.__x = val
    def __del_x(self):
        del self.__x

    def __get_y(self):
        return self.__y
    def __set_y(self, val):
        self.__y = val
    def __del_y(self):
        del self.__y

    x = property(__get_x, __set_x, __del_x, "Horizontal x.")
    y = property(__get_y, __set_y, __del_y, "Vertical y.")

    def __repr__(self):
        formatted = "<Location (x = %0.3f, y = %0.3f)>"
        return formatted % (self.x, self.y)

class DMXAddress(object):
    """Defines the DMX address for a live pixel.
    """
    __invalid_address = -1
    __dmx_lower = 0
    __dmx_upper = 512

    def __init__(self, universe = 0, address = 0):
        self.__universe = universe
        self.__address = 0
        self.address = address

    def is_valid(self):
        return self.address is not DMXAddress.__invalid_address

    def __get_universe(self):
        return self.__universe
    def __set_universe(self, val):
        self.__universe = val
    def __del_universe(self):
        del self.__universe

    def __get_address(self):
        return self.__address
    def __set_address(self, val):
        if DMXAddress.__dmx_lower <= val <= DMXAddress.__dmx_upper:
            self.__address = val
        else:
            self.__address = DMXAddress.__invalid_address
    def __del_address(self):
        del self.__address

    universe = property(__get_universe, __set_universe, __del_universe, "DMX universe.")
    address = property(__get_address, __set_address, __del_address, "DMX address.")

    def __repr__(self):
        formatted = "<DMXAddress (universe = %0.3f, address = %0.3f)>"
        return formatted % (self.universe, self.address)

class Pixel(object):
    """ A pixel has a color and location. If it is live, it must have a valid
    DMX address.
    """

    def __init__(self,
                 color = Color(),
                 location = Location(),
                 address = DMXAddress(),
                 live = False):
        self.__color = color
        self.__location = location
        self.__address = address
        self.__live = live
        assert not live or (live and address.is_valid())

    def __get_color(self):
        return self.__color
    def __set_color(self, val):
        self.__color = val
    def __del_color(self):
        del self.__color

    def __get_location(self):
        return self.__location
    def __set_location(self, val):
        self.__location = val
    def __del_location(self):
        del self.__location

    def __get_address(self):
        return self.__address
    def __set_address(self, val):
        self.__address = val
    def __del_address(self):
        del self.__address

    def __get_live(self):
        return self.__live
    def __set_live(self, val):
        self.__live = val
    def __del_live(self):
        del self.__live

    color = property(__get_color, __set_color, __del_color, "RGB color.")
    location = property(__get_location, __set_location, __del_location, "Location (x, y).")
    address = property(__get_address, __set_address, __del_address, "DMX address.")
    live = property(__get_live, __set_live, __del_live, "Is live pixel?")

    def __repr__(self):
        return "".join(["<Pixel ",
                        str(self.color), ",",
                        str(self.location), ", live = ",
                        str(self.live), ">"])

class Panel(MutableMapping):
    """ Panel represents a collection of pixels, representing a climbing wall.
    It's structured as a dictionary with fast k-nearest neighbor access of
    pixels.
    """

    def __init__(self):
        self.__pixels = dict()
        if len(self.locations) > 0:
            self.__tree = cKDTree(self.locations)
        else:
            self.__tree = None

    @property
    def locations(self):
        """ Returns an array of (x, y) tuple locations for pixels.
        """
        return self.__pixels.keys()

    def get_nearest(self, location, num_pixels):
        """ Returns the num_pixels closests pixels to a location.
        """
        assert self.__tree, "__tree is empty."
        return self.__tree.query_ball_point(location, num_pixels)

    def __contains__(self, key):
        return key in self.__pixels

    def __getitem__(self, key):
        return self.__pixels[key]

    def __setitem__(self, key, value):
        if key not in self.__pixels:
            self.__tree = cKDTree(self.locations + [key])
        self.__pixels[key] = value

    def __delitem__(self, key):
        del self.__pixels[key]

    def __len__(self):
        return len(self.__pixels)

    def __iter__(self):
        return iter(self.__pixels)

    def __repr__(self):
        return self.__pixels.__repr__()
<|endoftext|>