<commit_before>#pylint: skip-file

import unittest
import datetime
import shutil
import os
import subprocess
import yawt
from yawtext.vc import YawtVersionControl
from yawt.article import Article, ArticleInfo
from yawt.utils import save_file, remove_file, load_file
from yawt.test import TempFolder
from yawt import create_app
from flask import g, current_app
from flask.ext.testing import TestCase

from yawtext.vc import vc_status, vc_add_tracked,\
    vc_add_tracked_and_new, vc_commit, post_commit, ChangedFiles
from yawtext.git import _git_cmd
from yawtext import Plugin

class TestYawtGitNewSite(TestCase):
    YAWT_EXTENSIONS = ['yawtext.vc.YawtVersionControl']
    YAWT_VERSION_CONTROL_IFC = 'yawtext.git'

    def create_app(self):
        self.root_dir = '/tmp/blah'
        return create_app(self.root_dir, config=self)

    def setUp(self):
        self.app.preprocess_request()

    def test_git_saves_ignore_file(self):
        self.assertFalse(os.path.isfile(os.path.join(self.root_dir,
                                                     '.gitignore')))
        g.site.initialize()
        self.assertTrue(os.path.isfile(os.path.join(self.root_dir,
                                                    '.gitignore')))

    def tearDown(self):
        shutil.rmtree(self.root_dir)

class TempGitFolder(TempFolder):
    def __init__(self):
        super(TempGitFolder, self).__init__()
        self.files = {
            'content/index.txt': 'index text',
            'content/entry.txt': 'entry text',
            'content/random.txt': 'random text',
            'content/food.txt': 'blabs',
        }

    def initialize_git(self):
        cmd = _git_cmd(['init'])
        out = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
        cmd = _git_cmd(['add', '-A'])
        out = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
        cmd = _git_cmd(['commit', '-m', 'initialcommit'])
        try:
            subprocess.check_output(cmd, stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as e:
            print "return code: "+e.returncode
            print "cmd: "+e.cmd
            print "output: "+e.output

class TestGitPlugin(TestCase):
    YAWT_EXTENSIONS = ['yawtext.vc.YawtVersionControl']
    YAWT_VERSION_CONTROL_IFC = 'yawtext.git'

    def create_app(self):
        self.site = TempGitFolder()
        self.site.initialize()
        return yawt.create_app(self.site.site_root, config=self)

    def setUp(self):
        self.app.preprocess_request()
        self.site.initialize_git()

    def test_status_gives_changed_files(self):
        # modify one file
        save_file(os.path.join(self.site.site_root, 'content/index.txt'),
                  'different stuff')

        # add another new file
        save_file(os.path.join(self.site.site_root, 'content/newfile.txt'),
                  'blah')

        # remove yet another
        remove_file(os.path.join(self.site.site_root, 'content/random.txt'))

        # move a file
        filename1 = os.path.join(self.site.site_root, 'content/food.txt')
        filename2 = os.path.join(self.site.site_root, 'content/newfood.txt')
        save_file(filename2, load_file(filename1))
        remove_file(filename1)

        # add all changes to index
        cmd = _git_cmd(['add', '-A'])
        subprocess.check_output(cmd, stderr=subprocess.STDOUT)

        changed = vc_status()
        expected = ChangedFiles(added=['content/newfile.txt'],
                                modified=['content/index.txt'],
                                deleted=['content/random.txt'],
                                renamed={'content/food.txt':
                                         'content/newfood.txt'})
        self.assertEquals(expected, changed)

    def test_add_tracked(self):
        # modify one file
        save_file(os.path.join(self.site.site_root, 'content/index.txt'),
                  'different stuff')

        # add another new file
        save_file(os.path.join(self.site.site_root, 'content/newfile.txt'),
                  'blah')

        vc_add_tracked()
        changed = vc_status()
        expected = ChangedFiles(modified=['content/index.txt'])
        self.assertEquals(expected, changed)

    def test_add_tracked_and_new(self):
        # modify one file
        save_file(os.path.join(self.site.site_root, 'content/index.txt'),
                  'different stuff')

        # add another new file
        save_file(os.path.join(self.site.site_root, 'content/newfile.txt'),
                  'blah')

        vc_add_tracked_and_new()
        changed = vc_status()
        expected = ChangedFiles(added=['content/newfile.txt'],
                                modified=['content/index.txt'])
        self.assertEquals(expected, changed)

    def test_commit(self):
        # modify one file
        save_file(os.path.join(self.site.site_root, 'content/index.txt'),
                  'different stuff')
        vc_add_tracked()
        vc_commit('hello')
        changed = vc_status()
        expected = ChangedFiles()
        self.assertEquals(expected, changed)

    def tearDown(self):
        self.site.remove()

class TestPlugin(Plugin):
    def __init__(self):
        self.changed = None

    def on_files_changed(self, changed):
        self.changed = changed

class TestGitHooks(TestCase):
    YAWT_EXTENSIONS = ['yawtext.vc.YawtVersionControl',
                       'yawtext.test.test_git.TestPlugin']
    YAWT_VERSION_CONTROL_IFC = 'yawtext.git'

    def create_app(self):
        self.site = TempGitFolder()
        self.site.initialize()
        return yawt.create_app(self.site.site_root, config=self)

    def setUp(self):
        self.app.preprocess_request()
        self.site.initialize_git()

    def test_post_commit(self):
        # modify one file
        save_file(os.path.join(self.site.site_root, 'content/index.txt'),
                  'different stuff')

        # add another new file
        save_file(os.path.join(self.site.site_root, 'content/newfile.txt'),
                  'blah')

        # remove yet another
        remove_file(os.path.join(self.site.site_root, 'content/random.txt'))

        # move a file
        filename1 = os.path.join(self.site.site_root, 'content/food.txt')
        filename2 = os.path.join(self.site.site_root, 'content/newfood.txt')
        save_file(filename2, load_file(filename1))
        remove_file(filename1)

        # add all changes to index
        vc_add_tracked_and_new()
        vc_commit('hello')
        post_commit(self.site.site_root, self.app)
        expected = ChangedFiles(added=['content/newfile.txt'],
                                modified=['content/index.txt'],
                                deleted=['content/random.txt'],
                                renamed={'content/food.txt':
                                         'content/newfood.txt'})

        test_plugin_name = 'yawtext.test.test_git.TestPlugin'
        plugin = self.app.extension_info[0][test_plugin_name]
        self.assertEquals(expected, plugin.changed)

    def tearDown(self):
        self.site.remove()
<commit_msg>try catching exception for git commit (second try)<commit_after>#pylint: skip-file

import unittest
import datetime
import shutil
import os
import subprocess
import yawt
from yawtext.vc import YawtVersionControl
from yawt.article import Article, ArticleInfo
from yawt.utils import save_file, remove_file, load_file
from yawt.test import TempFolder
from yawt import create_app
from flask import g, current_app
from flask.ext.testing import TestCase

from yawtext.vc import vc_status, vc_add_tracked,\
    vc_add_tracked_and_new, vc_commit, post_commit, ChangedFiles
from yawtext.git import _git_cmd
from yawtext import Plugin

class TestYawtGitNewSite(TestCase):
    YAWT_EXTENSIONS = ['yawtext.vc.YawtVersionControl']
    YAWT_VERSION_CONTROL_IFC = 'yawtext.git'

    def create_app(self):
        self.root_dir = '/tmp/blah'
        return create_app(self.root_dir, config=self)

    def setUp(self):
        self.app.preprocess_request()

    def test_git_saves_ignore_file(self):
        self.assertFalse(os.path.isfile(os.path.join(self.root_dir,
                                                     '.gitignore')))
        g.site.initialize()
        self.assertTrue(os.path.isfile(os.path.join(self.root_dir,
                                                    '.gitignore')))

    def tearDown(self):
        shutil.rmtree(self.root_dir)

class TempGitFolder(TempFolder):
    def __init__(self):
        super(TempGitFolder, self).__init__()
        self.files = {
            'content/index.txt': 'index text',
            'content/entry.txt': 'entry text',
            'content/random.txt': 'random text',
            'content/food.txt': 'blabs',
        }

    def initialize_git(self):
        cmd = _git_cmd(['init'])
        out = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
        cmd = _git_cmd(['add', '-A'])
        out = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
        cmd = _git_cmd(['commit', '-m', 'initialcommit'])
        try:
            subprocess.check_output(cmd, stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as e:
            print "return code: "+str(e.returncode)
            print "cmd: "+str(e.cmd)
            print "output: "+str(e.output)

class TestGitPlugin(TestCase):
    YAWT_EXTENSIONS = ['yawtext.vc.YawtVersionControl']
    YAWT_VERSION_CONTROL_IFC = 'yawtext.git'

    def create_app(self):
        self.site = TempGitFolder()
        self.site.initialize()
        return yawt.create_app(self.site.site_root, config=self)

    def setUp(self):
        self.app.preprocess_request()
        self.site.initialize_git()

    def test_status_gives_changed_files(self):
        # modify one file
        save_file(os.path.join(self.site.site_root, 'content/index.txt'),
                  'different stuff')

        # add another new file
        save_file(os.path.join(self.site.site_root, 'content/newfile.txt'),
                  'blah')

        # remove yet another
        remove_file(os.path.join(self.site.site_root, 'content/random.txt'))

        # move a file
        filename1 = os.path.join(self.site.site_root, 'content/food.txt')
        filename2 = os.path.join(self.site.site_root, 'content/newfood.txt')
        save_file(filename2, load_file(filename1))
        remove_file(filename1)

        # add all changes to index
        cmd = _git_cmd(['add', '-A'])
        subprocess.check_output(cmd, stderr=subprocess.STDOUT)

        changed = vc_status()
        expected = ChangedFiles(added=['content/newfile.txt'],
                                modified=['content/index.txt'],
                                deleted=['content/random.txt'],
                                renamed={'content/food.txt':
                                         'content/newfood.txt'})
        self.assertEquals(expected, changed)

    def test_add_tracked(self):
        # modify one file
        save_file(os.path.join(self.site.site_root, 'content/index.txt'),
                  'different stuff')

        # add another new file
        save_file(os.path.join(self.site.site_root, 'content/newfile.txt'),
                  'blah')

        vc_add_tracked()
        changed = vc_status()
        expected = ChangedFiles(modified=['content/index.txt'])
        self.assertEquals(expected, changed)

    def test_add_tracked_and_new(self):
        # modify one file
        save_file(os.path.join(self.site.site_root, 'content/index.txt'),
                  'different stuff')

        # add another new file
        save_file(os.path.join(self.site.site_root, 'content/newfile.txt'),
                  'blah')

        vc_add_tracked_and_new()
        changed = vc_status()
        expected = ChangedFiles(added=['content/newfile.txt'],
                                modified=['content/index.txt'])
        self.assertEquals(expected, changed)

    def test_commit(self):
        # modify one file
        save_file(os.path.join(self.site.site_root, 'content/index.txt'),
                  'different stuff')
        vc_add_tracked()
        vc_commit('hello')
        changed = vc_status()
        expected = ChangedFiles()
        self.assertEquals(expected, changed)

    def tearDown(self):
        self.site.remove()

class TestPlugin(Plugin):
    def __init__(self):
        self.changed = None

    def on_files_changed(self, changed):
        self.changed = changed

class TestGitHooks(TestCase):
    YAWT_EXTENSIONS = ['yawtext.vc.YawtVersionControl',
                       'yawtext.test.test_git.TestPlugin']
    YAWT_VERSION_CONTROL_IFC = 'yawtext.git'

    def create_app(self):
        self.site = TempGitFolder()
        self.site.initialize()
        return yawt.create_app(self.site.site_root, config=self)

    def setUp(self):
        self.app.preprocess_request()
        self.site.initialize_git()

    def test_post_commit(self):
        # modify one file
        save_file(os.path.join(self.site.site_root, 'content/index.txt'),
                  'different stuff')

        # add another new file
        save_file(os.path.join(self.site.site_root, 'content/newfile.txt'),
                  'blah')

        # remove yet another
        remove_file(os.path.join(self.site.site_root, 'content/random.txt'))

        # move a file
        filename1 = os.path.join(self.site.site_root, 'content/food.txt')
        filename2 = os.path.join(self.site.site_root, 'content/newfood.txt')
        save_file(filename2, load_file(filename1))
        remove_file(filename1)

        # add all changes to index
        vc_add_tracked_and_new()
        vc_commit('hello')
        post_commit(self.site.site_root, self.app)
        expected = ChangedFiles(added=['content/newfile.txt'],
                                modified=['content/index.txt'],
                                deleted=['content/random.txt'],
                                renamed={'content/food.txt':
                                         'content/newfood.txt'})

        test_plugin_name = 'yawtext.test.test_git.TestPlugin'
        plugin = self.app.extension_info[0][test_plugin_name]
        self.assertEquals(expected, plugin.changed)

    def tearDown(self):
        self.site.remove()
<|endoftext|>


<commit_before>import os
from Cookie import SimpleCookie
from pprint import pformat
from urllib import urlencode, quote
from django.utils.datastructures import MultiValueDict

RESERVED_CHARS="!*'();:@&=+$,/?%#[]"

try:
    # The mod_python version is more efficient, so try importing it first.
    from mod_python.util import parse_qsl
except ImportError:
    from cgi import parse_qsl

class Http404(Exception):
    pass

class HttpRequest(object):
    "A basic HTTP request"
    def __init__(self):
        self.GET, self.POST, self.COOKIES, self.META, self.FILES = {}, {}, {}, {}, {}
        self.path = ''
        self.method = None

    def __repr__(self):
        return '<HttpRequest\nGET:%s,\nPOST:%s,\nCOOKIES:%s,\nMETA:%s>' % \
            (pformat(self.GET), pformat(self.POST), pformat(self.COOKIES),
            pformat(self.META))

    def __getitem__(self, key):
        for d in (self.POST, self.GET):
            if d.has_key(key):
                return d[key]
        raise KeyError, "%s not found in either POST or GET" % key

    def has_key(self, key):
        return self.GET.has_key(key) or self.POST.has_key(key)

    def get_full_path(self):
        return ''

    def is_secure(self):
        return os.environ.get("HTTPS") == "on"

def parse_file_upload(header_dict, post_data):
    "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)"
    import email, email.Message
    from cgi import parse_header
    raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()])
    raw_message += '\r\n\r\n' + post_data
    msg = email.message_from_string(raw_message)
    POST = MultiValueDict()
    FILES = MultiValueDict()
    for submessage in msg.get_payload():
        if submessage and isinstance(submessage, email.Message.Message):
            name_dict = parse_header(submessage['Content-Disposition'])[1]
            # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads
            # or {'name': 'blah'} for POST fields
            # We assume all uploaded files have a 'filename' set.
            if name_dict.has_key('filename'):
                assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported"
                if not name_dict['filename'].strip():
                    continue
                # IE submits the full path, so trim everything but the basename.
                # (We can't use os.path.basename because it expects Linux paths.)
                filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:]
                FILES.appendlist(name_dict['name'], {
                    'filename': filename,
                    'content-type': (submessage.has_key('Content-Type') and submessage['Content-Type'] or None),
                    'content': submessage.get_payload(),
                })
            else:
                POST.appendlist(name_dict['name'], submessage.get_payload())
    return POST, FILES

class QueryDict(MultiValueDict):
    """A specialized MultiValueDict that takes a query string when initialized.
    This is immutable unless you create a copy of it."""
    def __init__(self, query_string, mutable=False):
        MultiValueDict.__init__(self)
        self._mutable = True
        for key, value in parse_qsl((query_string or ''), True): # keep_blank_values=True
            self.appendlist(key, value)
        self._mutable = mutable

    def _assert_mutable(self):
        if not self._mutable:
            raise AttributeError, "This QueryDict instance is immutable"

    def __setitem__(self, key, value):
        self._assert_mutable()
        MultiValueDict.__setitem__(self, key, value)

    def __copy__(self):
        result = self.__class__('', mutable=True)
        for key, value in dict.items(self):
            dict.__setitem__(result, key, value)
        return result

    def __deepcopy__(self, memo={}):
        import copy
        result = self.__class__('', mutable=True)
        memo[id(self)] = result
        for key, value in dict.items(self):
            dict.__setitem__(result, copy.deepcopy(key, memo), copy.deepcopy(value, memo))
        return result

    def setlist(self, key, list_):
        self._assert_mutable()
        MultiValueDict.setlist(self, key, list_)

    def appendlist(self, key, value):
        self._assert_mutable()
        MultiValueDict.appendlist(self, key, value)

    def update(self, other_dict):
        self._assert_mutable()
        MultiValueDict.update(self, other_dict)

    def pop(self, key):
        self._assert_mutable()
        return MultiValueDict.pop(self, key)

    def popitem(self):
        self._assert_mutable()
        return MultiValueDict.popitem(self)

    def clear(self):
        self._assert_mutable()
        MultiValueDict.clear(self)

    def setdefault(self, *args):
        self._assert_mutable()
        return MultiValueDict.setdefault(self, *args)

    def copy(self):
        "Returns a mutable copy of this object."
        return self.__deepcopy__()

    def urlencode(self):
        output = []
        for k, list_ in self.lists():
            output.extend([urlencode({k: v}) for v in list_])
        return '&'.join(output)

def parse_cookie(cookie):
    if cookie == '':
        return {}
    c = SimpleCookie()
    c.load(cookie)
    cookiedict = {}
    for key in c.keys():
        cookiedict[key] = c.get(key).value
    return cookiedict

class HttpResponse(object):
    "A basic HTTP response, with content and dictionary-accessed headers"
    def __init__(self, content='', mimetype=None):
        from django.conf import settings
        self._charset = settings.DEFAULT_CHARSET
        if not mimetype:
            mimetype = "%s; charset=%s" % (settings.DEFAULT_CONTENT_TYPE, settings.DEFAULT_CHARSET)
        if not isinstance(content, basestring) and hasattr(content, '__iter__'):
            self._container = content
            self._is_string = False
        else:
            self._container = [content]
            self._is_string = True
        self.headers = {'Content-Type': mimetype}
        self.cookies = SimpleCookie()
        self.status_code = 200

    def __str__(self):
        "Full HTTP message, including headers"
        return '\n'.join(['%s: %s' % (key, value)
            for key, value in self.headers.items()]) \
            + '\n\n' + self.content

    def __setitem__(self, header, value):
        self.headers[header] = value

    def __delitem__(self, header):
        try:
            del self.headers[header]
        except KeyError:
            pass

    def __getitem__(self, header):
        return self.headers[header]

    def has_header(self, header):
        "Case-insensitive check for a header"
        header = header.lower()
        for key in self.headers.keys():
            if key.lower() == header:
                return True
        return False

    def set_cookie(self, key, value='', max_age=None, expires=None, path='/', domain=None, secure=None):
        self.cookies[key] = value
        for var in ('max_age', 'path', 'domain', 'secure', 'expires'):
            val = locals()[var]
            if val is not None:
                self.cookies[key][var.replace('_', '-')] = val

    def delete_cookie(self, key, path='/', domain=None):
        self.cookies[key] = ''
        if path is not None:
            self.cookies[key]['path'] = path
        if domain is not None:
            self.cookies[key]['domain'] = domain
        self.cookies[key]['expires'] = 0
        self.cookies[key]['max-age'] = 0

    def _get_content(self):
        content = ''.join(self._container)
        if isinstance(content, unicode):
            content = content.encode(self._charset)
        return content

    def _set_content(self, value):
        self._container = [value]
        self._is_string = True

    content = property(_get_content, _set_content)

    def __iter__(self):
        self._iterator = self._container.__iter__()
        return self

    def next(self):
        chunk = self._iterator.next()
        if isinstance(chunk, unicode):
            chunk = chunk.encode(self._charset)
        return chunk

    def close(self):
        if hasattr(self._container, 'close'):
            self._container.close()

    # The remaining methods partially implement the file-like object interface.
    # See http://docs.python.org/lib/bltin-file-objects.html
    def write(self, content):
        if not self._is_string:
            raise Exception, "This %s instance is not writable" % self.__class__
        self._container.append(content)

    def flush(self):
        pass

    def tell(self):
        if not self._is_string:
            raise Exception, "This %s instance cannot tell its position" % self.__class__
        return sum([len(chunk) for chunk in self._container])

class HttpResponseRedirect(HttpResponse):
    def __init__(self, redirect_to):
        HttpResponse.__init__(self)
        self['Location'] = quote(redirect_to, safe=RESERVED_CHARS)
        self.status_code = 302

class HttpResponsePermanentRedirect(HttpResponse):
    def __init__(self, redirect_to):
        HttpResponse.__init__(self)
        self['Location'] = quote(redirect_to, safe=RESERVED_CHARS)
        self.status_code = 301

class HttpResponseNotModified(HttpResponse):
    def __init__(self):
        HttpResponse.__init__(self)
        self.status_code = 304

class HttpResponseNotFound(HttpResponse):
    def __init__(self, *args, **kwargs):
        HttpResponse.__init__(self, *args, **kwargs)
        self.status_code = 404

class HttpResponseForbidden(HttpResponse):
    def __init__(self, *args, **kwargs):
        HttpResponse.__init__(self, *args, **kwargs)
        self.status_code = 403

class HttpResponseNotAllowed(HttpResponse):
    def __init__(self, permitted_methods):
        HttpResponse.__init__(self)
        self['Allow'] = ', '.join(permitted_methods)
        self.status_code = 405

class HttpResponseGone(HttpResponse):
    def __init__(self, *args, **kwargs):
        HttpResponse.__init__(self, *args, **kwargs)
        self.status_code = 410

class HttpResponseServerError(HttpResponse):
    def __init__(self, *args, **kwargs):
        HttpResponse.__init__(self, *args, **kwargs)
        self.status_code = 500

def get_host(request):
    "Gets the HTTP host from the environment or request headers."
    host = request.META.get('HTTP_X_FORWARDED_HOST', '')
    if not host:
        host = request.META.get('HTTP_HOST', '')
    return host
<commit_msg>Refactored the HttpResponse sub-classes so that adding a subclass that only changes the HTTP status code requires less code (no need to duplicate the __init__ method).<commit_after>import os
from Cookie import SimpleCookie
from pprint import pformat
from urllib import urlencode, quote
from django.utils.datastructures import MultiValueDict

RESERVED_CHARS="!*'();:@&=+$,/?%#[]"

try:
    # The mod_python version is more efficient, so try importing it first.
    from mod_python.util import parse_qsl
except ImportError:
    from cgi import parse_qsl

class Http404(Exception):
    pass

class HttpRequest(object):
    "A basic HTTP request"
    def __init__(self):
        self.GET, self.POST, self.COOKIES, self.META, self.FILES = {}, {}, {}, {}, {}
        self.path = ''
        self.method = None

    def __repr__(self):
        return '<HttpRequest\nGET:%s,\nPOST:%s,\nCOOKIES:%s,\nMETA:%s>' % \
            (pformat(self.GET), pformat(self.POST), pformat(self.COOKIES),
            pformat(self.META))

    def __getitem__(self, key):
        for d in (self.POST, self.GET):
            if d.has_key(key):
                return d[key]
        raise KeyError, "%s not found in either POST or GET" % key

    def has_key(self, key):
        return self.GET.has_key(key) or self.POST.has_key(key)

    def get_full_path(self):
        return ''

    def is_secure(self):
        return os.environ.get("HTTPS") == "on"

def parse_file_upload(header_dict, post_data):
    "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)"
    import email, email.Message
    from cgi import parse_header
    raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()])
    raw_message += '\r\n\r\n' + post_data
    msg = email.message_from_string(raw_message)
    POST = MultiValueDict()
    FILES = MultiValueDict()
    for submessage in msg.get_payload():
        if submessage and isinstance(submessage, email.Message.Message):
            name_dict = parse_header(submessage['Content-Disposition'])[1]
            # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads
            # or {'name': 'blah'} for POST fields
            # We assume all uploaded files have a 'filename' set.
            if name_dict.has_key('filename'):
                assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported"
                if not name_dict['filename'].strip():
                    continue
                # IE submits the full path, so trim everything but the basename.
                # (We can't use os.path.basename because it expects Linux paths.)
                filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:]
                FILES.appendlist(name_dict['name'], {
                    'filename': filename,
                    'content-type': (submessage.has_key('Content-Type') and submessage['Content-Type'] or None),
                    'content': submessage.get_payload(),
                })
            else:
                POST.appendlist(name_dict['name'], submessage.get_payload())
    return POST, FILES

class QueryDict(MultiValueDict):
    """A specialized MultiValueDict that takes a query string when initialized.
    This is immutable unless you create a copy of it."""
    def __init__(self, query_string, mutable=False):
        MultiValueDict.__init__(self)
        self._mutable = True
        for key, value in parse_qsl((query_string or ''), True): # keep_blank_values=True
            self.appendlist(key, value)
        self._mutable = mutable

    def _assert_mutable(self):
        if not self._mutable:
            raise AttributeError, "This QueryDict instance is immutable"

    def __setitem__(self, key, value):
        self._assert_mutable()
        MultiValueDict.__setitem__(self, key, value)

    def __copy__(self):
        result = self.__class__('', mutable=True)
        for key, value in dict.items(self):
            dict.__setitem__(result, key, value)
        return result

    def __deepcopy__(self, memo={}):
        import copy
        result = self.__class__('', mutable=True)
        memo[id(self)] = result
        for key, value in dict.items(self):
            dict.__setitem__(result, copy.deepcopy(key, memo), copy.deepcopy(value, memo))
        return result

    def setlist(self, key, list_):
        self._assert_mutable()
        MultiValueDict.setlist(self, key, list_)

    def appendlist(self, key, value):
        self._assert_mutable()
        MultiValueDict.appendlist(self, key, value)

    def update(self, other_dict):
        self._assert_mutable()
        MultiValueDict.update(self, other_dict)

    def pop(self, key):
        self._assert_mutable()
        return MultiValueDict.pop(self, key)

    def popitem(self):
        self._assert_mutable()
        return MultiValueDict.popitem(self)

    def clear(self):
        self._assert_mutable()
        MultiValueDict.clear(self)

    def setdefault(self, *args):
        self._assert_mutable()
        return MultiValueDict.setdefault(self, *args)

    def copy(self):
        "Returns a mutable copy of this object."
        return self.__deepcopy__()

    def urlencode(self):
        output = []
        for k, list_ in self.lists():
            output.extend([urlencode({k: v}) for v in list_])
        return '&'.join(output)

def parse_cookie(cookie):
    if cookie == '':
        return {}
    c = SimpleCookie()
    c.load(cookie)
    cookiedict = {}
    for key in c.keys():
        cookiedict[key] = c.get(key).value
    return cookiedict

class HttpResponse(object):
    "A basic HTTP response, with content and dictionary-accessed headers"

    status_code = 200

    def __init__(self, content='', mimetype=None):
        from django.conf import settings
        self._charset = settings.DEFAULT_CHARSET
        if not mimetype:
            mimetype = "%s; charset=%s" % (settings.DEFAULT_CONTENT_TYPE, settings.DEFAULT_CHARSET)
        if not isinstance(content, basestring) and hasattr(content, '__iter__'):
            self._container = content
            self._is_string = False
        else:
            self._container = [content]
            self._is_string = True
        self.headers = {'Content-Type': mimetype}
        self.cookies = SimpleCookie()

    def __str__(self):
        "Full HTTP message, including headers"
        return '\n'.join(['%s: %s' % (key, value)
            for key, value in self.headers.items()]) \
            + '\n\n' + self.content

    def __setitem__(self, header, value):
        self.headers[header] = value

    def __delitem__(self, header):
        try:
            del self.headers[header]
        except KeyError:
            pass

    def __getitem__(self, header):
        return self.headers[header]

    def has_header(self, header):
        "Case-insensitive check for a header"
        header = header.lower()
        for key in self.headers.keys():
            if key.lower() == header:
                return True
        return False

    def set_cookie(self, key, value='', max_age=None, expires=None, path='/', domain=None, secure=None):
        self.cookies[key] = value
        for var in ('max_age', 'path', 'domain', 'secure', 'expires'):
            val = locals()[var]
            if val is not None:
                self.cookies[key][var.replace('_', '-')] = val

    def delete_cookie(self, key, path='/', domain=None):
        self.cookies[key] = ''
        if path is not None:
            self.cookies[key]['path'] = path
        if domain is not None:
            self.cookies[key]['domain'] = domain
        self.cookies[key]['expires'] = 0
        self.cookies[key]['max-age'] = 0

    def _get_content(self):
        content = ''.join(self._container)
        if isinstance(content, unicode):
            content = content.encode(self._charset)
        return content

    def _set_content(self, value):
        self._container = [value]
        self._is_string = True

    content = property(_get_content, _set_content)

    def __iter__(self):
        self._iterator = self._container.__iter__()
        return self

    def next(self):
        chunk = self._iterator.next()
        if isinstance(chunk, unicode):
            chunk = chunk.encode(self._charset)
        return chunk

    def close(self):
        if hasattr(self._container, 'close'):
            self._container.close()

    # The remaining methods partially implement the file-like object interface.
    # See http://docs.python.org/lib/bltin-file-objects.html
    def write(self, content):
        if not self._is_string:
            raise Exception, "This %s instance is not writable" % self.__class__
        self._container.append(content)

    def flush(self):
        pass

    def tell(self):
        if not self._is_string:
            raise Exception, "This %s instance cannot tell its position" % self.__class__
        return sum([len(chunk) for chunk in self._container])

class HttpResponseRedirect(HttpResponse):
    status_code = 302

    def __init__(self, redirect_to):
        HttpResponse.__init__(self)
        self['Location'] = quote(redirect_to, safe=RESERVED_CHARS)

class HttpResponsePermanentRedirect(HttpResponse):
    status_code = 301

    def __init__(self, redirect_to):
        HttpResponse.__init__(self)
        self['Location'] = quote(redirect_to, safe=RESERVED_CHARS)

class HttpResponseNotModified(HttpResponse):
    status_code = 304

class HttpResponseNotFound(HttpResponse):
    status_code = 404

class HttpResponseForbidden(HttpResponse):
    status_code = 403

class HttpResponseNotAllowed(HttpResponse):
    status_code = 405

    def __init__(self, permitted_methods):
        HttpResponse.__init__(self)
        self['Allow'] = ', '.join(permitted_methods)

class HttpResponseGone(HttpResponse):
    status_code = 410

    def __init__(self, *args, **kwargs):
        HttpResponse.__init__(self, *args, **kwargs)

class HttpResponseServerError(HttpResponse):
    status_code = 500

    def __init__(self, *args, **kwargs):
        HttpResponse.__init__(self, *args, **kwargs)

def get_host(request):
    "Gets the HTTP host from the environment or request headers."
    host = request.META.get('HTTP_X_FORWARDED_HOST', '')
    if not host:
        host = request.META.get('HTTP_HOST', '')
    return host
<|endoftext|>


<commit_before>"""
Contains the :class:`base class <tinydb.middlewares.Middleware>` for
middlewares and implementations.
"""
from tinydb import TinyDB
from tinydb.storages import JSONStorage

class Middleware(object):
    """
    The base class for all Middlewares.

    Middlewares hook into the read/write process of TinyDB allowing you to
    extend the behaviour by adding caching, logging, ...

    Your middleware's ``__init__`` method has to accept exactly one
    argument which is the class of the "real" storage. It has to be stored as
    ``_storage_cls`` (see :class:`~tinydb.middlewares.CachingMiddleware` for an
    example).
    """

    def __init__(self, storage_cls=TinyDB.DEFAULT_STORAGE):
        self._storage_cls = storage_cls
        self.storage = None

    def __call__(self, *args, **kwargs):
        """
        Create the storage instance and store it as self.storage.

        Usually a user creates a new TinyDB instance like this::

            TinyDB(storage=StorageClass)

        The storage kwarg is used by TinyDB this way::

            self._storage = storage(*args, **kwargs)

        As we can see, ``storage(...)`` runs the constructor and returns the
        new storage instance.

        Using Middlewares, the user will call::

                                       The 'real' storage class
                                       v
            TinyDB(storage=Middleware(StorageClass))
                       ^
                       Already an instance!

        So, when running ``self._storage = storage(*args, **kwargs)`` Python
        now will call ``__call__`` and TinyDB will expect the return value to
        be the storage (or Middleware) instance. Returning the instance is
        simple, but we also got the underlying (*real*) StorageClass as an
        __init__ argument that still is not an instance.
        So, we initialize it in __call__ forwarding any arguments we recieve
        from TinyDB (``TinyDB(arg1, kwarg1=value, storage=...)``).

        In case of nested Middlewares, calling the instance as if it was an
        class results in calling ``__call__`` what initializes the next
        nested Middleware that itself will initialize the next Middleware and
        so on.
        """

        self.storage = self._storage_cls(*args, **kwargs)

        return self

    def __getattr__(self, name):
        """
        Forward all unknown attribute calls to the underlying storage so we
        remain as transparent as possible.
        """

        return getattr(self.__dict__['storage'], name)

class CachingMiddleware(Middleware):
    """
    Add some caching to TinyDB.

    This Middleware aims to improve the performance of TinyDB by writing only
    the last DB state every :attr:`WRITE_CACHE_SIZE` time and reading always
    from cache.
    """

    #: The number of write operations to cache before writing to disc
    WRITE_CACHE_SIZE = 1000

    def __init__(self, storage_cls=TinyDB.DEFAULT_STORAGE):
        super(CachingMiddleware, self).__init__(storage_cls)

        self.cache = None
        self._cache_modified_count = 0

    def __del__(self):
        self.flush()  # Flush potentially unwritten data

    def read(self):
        if self.cache is None:
            self.cache = self.storage.read()
        return self.cache

    def write(self, data):
        self.cache = data
        self._cache_modified_count += 1

        if self._cache_modified_count >= self.WRITE_CACHE_SIZE:
            self.flush()

    def flush(self):
        """
        Flush all unwritten data to disk.
        """

        self.storage.write(self.cache)
        self._cache_modified_count = 0

    def close(self):
        self.flush()
        self.storage.close()

class SerializationMiddleware(Middleware):
    """
    Provide custom serialization for TinyDB.

    This middleware allows users of TinyDB to register custom serializations.
    The serialized data will be passed to the wrapped storage and data that
    is read from the storage will be deserialized.
    """

    def __init__(self, storage_cls=TinyDB.DEFAULT_STORAGE):
        super(SerializationMiddleware, self).__init__(storage_cls)

        self._serializers = {}

    def register_serializer(self, serializer, name):
        """
        Register a new Serializer.

        When reading from/writing to the underlying storage, TinyDB
        will run all objects through the list of registered serializers
        allowing each one to handle objects it recognizes.

        .. note:: The name has to be unique among this database instance.
                  Re-using the same name will overwrite the old serializer.
                  Also, registering a serializer will be reflected in all
                  tables when reading/writing them.

        :param serializer: an instance of the serializer
        :type serializer: tinydb.serialize.Serializer
        """
        self._serializers[name] = serializer

    def read(self):
        data = self.storage.read()

        for serializer_name in self._serializers:
            serializer = self._serializers[serializer_name]
            tag = '{{{0}}}:'.format(serializer_name)  # E.g: '{TinyDate}:'

            for table_name in data:
                table = data[table_name]

                for eid in table:
                    item = data[table_name][eid]

                    for field in item:
                        try:
                            if item[field].startswith(tag):
                                print('Okay!')
                                encoded = item[field][len(tag):]
                                item[field] = serializer.decode(encoded)
                        except AttributeError:
                            pass  # Not a string

        return data

    def write(self, data):
        for serializer_name in self._serializers:
            # If no serializers are registered, this code will just look up
            # the serializer list and continue. But if there are serializers,
            # the inner loop will run very often.
            # For that reason, the lookup of the serialized class is pulled
            # out into the outer loop:

            serializer = self._serializers[serializer_name]
            serializer_class = serializer.OBJ_CLASS

            for table_name in data:
                table = data[table_name]

                for eid in table:
                    item = data[table_name][eid]

                    for field in item:
                        if isinstance(item[field], serializer_class):
                            encoded = serializer.encode(item[field])
                            tagged = '{{{0}}}:{1}'.format(serializer_name,
                                                          encoded)

                            item[field] = tagged

        self.storage.write(data)
<commit_msg>Remove debugging output (closes #55)<commit_after>"""
Contains the :class:`base class <tinydb.middlewares.Middleware>` for
middlewares and implementations.
"""
from tinydb import TinyDB
from tinydb.storages import JSONStorage

class Middleware(object):
    """
    The base class for all Middlewares.

    Middlewares hook into the read/write process of TinyDB allowing you to
    extend the behaviour by adding caching, logging, ...

    Your middleware's ``__init__`` method has to accept exactly one
    argument which is the class of the "real" storage. It has to be stored as
    ``_storage_cls`` (see :class:`~tinydb.middlewares.CachingMiddleware` for an
    example).
    """

    def __init__(self, storage_cls=TinyDB.DEFAULT_STORAGE):
        self._storage_cls = storage_cls
        self.storage = None

    def __call__(self, *args, **kwargs):
        """
        Create the storage instance and store it as self.storage.

        Usually a user creates a new TinyDB instance like this::

            TinyDB(storage=StorageClass)

        The storage kwarg is used by TinyDB this way::

            self._storage = storage(*args, **kwargs)

        As we can see, ``storage(...)`` runs the constructor and returns the
        new storage instance.

        Using Middlewares, the user will call::

                                       The 'real' storage class
                                       v
            TinyDB(storage=Middleware(StorageClass))
                       ^
                       Already an instance!

        So, when running ``self._storage = storage(*args, **kwargs)`` Python
        now will call ``__call__`` and TinyDB will expect the return value to
        be the storage (or Middleware) instance. Returning the instance is
        simple, but we also got the underlying (*real*) StorageClass as an
        __init__ argument that still is not an instance.
        So, we initialize it in __call__ forwarding any arguments we recieve
        from TinyDB (``TinyDB(arg1, kwarg1=value, storage=...)``).

        In case of nested Middlewares, calling the instance as if it was an
        class results in calling ``__call__`` what initializes the next
        nested Middleware that itself will initialize the next Middleware and
        so on.
        """

        self.storage = self._storage_cls(*args, **kwargs)

        return self

    def __getattr__(self, name):
        """
        Forward all unknown attribute calls to the underlying storage so we
        remain as transparent as possible.
        """

        return getattr(self.__dict__['storage'], name)

class CachingMiddleware(Middleware):
    """
    Add some caching to TinyDB.

    This Middleware aims to improve the performance of TinyDB by writing only
    the last DB state every :attr:`WRITE_CACHE_SIZE` time and reading always
    from cache.
    """

    #: The number of write operations to cache before writing to disc
    WRITE_CACHE_SIZE = 1000

    def __init__(self, storage_cls=TinyDB.DEFAULT_STORAGE):
        super(CachingMiddleware, self).__init__(storage_cls)

        self.cache = None
        self._cache_modified_count = 0

    def __del__(self):
        self.flush()  # Flush potentially unwritten data

    def read(self):
        if self.cache is None:
            self.cache = self.storage.read()
        return self.cache

    def write(self, data):
        self.cache = data
        self._cache_modified_count += 1

        if self._cache_modified_count >= self.WRITE_CACHE_SIZE:
            self.flush()

    def flush(self):
        """
        Flush all unwritten data to disk.
        """

        self.storage.write(self.cache)
        self._cache_modified_count = 0

    def close(self):
        self.flush()
        self.storage.close()

class SerializationMiddleware(Middleware):
    """
    Provide custom serialization for TinyDB.

    This middleware allows users of TinyDB to register custom serializations.
    The serialized data will be passed to the wrapped storage and data that
    is read from the storage will be deserialized.
    """

    def __init__(self, storage_cls=TinyDB.DEFAULT_STORAGE):
        super(SerializationMiddleware, self).__init__(storage_cls)

        self._serializers = {}

    def register_serializer(self, serializer, name):
        """
        Register a new Serializer.

        When reading from/writing to the underlying storage, TinyDB
        will run all objects through the list of registered serializers
        allowing each one to handle objects it recognizes.

        .. note:: The name has to be unique among this database instance.
                  Re-using the same name will overwrite the old serializer.
                  Also, registering a serializer will be reflected in all
                  tables when reading/writing them.

        :param serializer: an instance of the serializer
        :type serializer: tinydb.serialize.Serializer
        """
        self._serializers[name] = serializer

    def read(self):
        data = self.storage.read()

        for serializer_name in self._serializers:
            serializer = self._serializers[serializer_name]
            tag = '{{{0}}}:'.format(serializer_name)  # E.g: '{TinyDate}:'

            for table_name in data:
                table = data[table_name]

                for eid in table:
                    item = data[table_name][eid]

                    for field in item:
                        try:
                            if item[field].startswith(tag):
                                encoded = item[field][len(tag):]
                                item[field] = serializer.decode(encoded)
                        except AttributeError:
                            pass  # Not a string

        return data

    def write(self, data):
        for serializer_name in self._serializers:
            # If no serializers are registered, this code will just look up
            # the serializer list and continue. But if there are serializers,
            # the inner loop will run very often.
            # For that reason, the lookup of the serialized class is pulled
            # out into the outer loop:

            serializer = self._serializers[serializer_name]
            serializer_class = serializer.OBJ_CLASS

            for table_name in data:
                table = data[table_name]

                for eid in table:
                    item = data[table_name][eid]

                    for field in item:
                        if isinstance(item[field], serializer_class):
                            encoded = serializer.encode(item[field])
                            tagged = '{{{0}}}:{1}'.format(serializer_name,
                                                          encoded)

                            item[field] = tagged

        self.storage.write(data)
<|endoftext|>


<commit_before>#! /usr/bin/env python2
"""CherryPy Server to provide recommendations of semantic similarity."""

import cherrypy
import codecs
import os
import ConfigParser
import json
import cPickle
from gensim.models import Word2Vec
from jinja2 import Environment, FileSystemLoader
from modules.vector_space_mapper.vector_space_mapper import VectorSpaceMapper

class Server():
    """Server Configuration for t-vex."""

    def __init__(self):
        """Initialization the Language and Model."""
        self.language = None
        self.cross_lang1 = None
        self.cross_lang2 = None
        self.vm = None
        self.model = None

    @cherrypy.expose
    def index(self):
        """Semantic spac visualization html returned."""
        return file(os.path.join('visualization', 'static', 'index.html'))

    @cherrypy.expose
    def multivariate_analysis(self):
        """Parallel Coordinates for multivariate analysis html page return."""
        return file(os.path.join(
            'visualization', 'static', 'multivariate.html')
        )

    @cherrypy.expose
    def cross_lingual(self):
        """Cross Lingual recommender html returned."""
        return file(os.path.join(
            'visualization', 'static', 'cross_lingual.html')
        )

    @cherrypy.expose
    def retrieve_recommendations(self, language, word, limit=10):
        """
        Retrieve number of semantically similar recommendations.

        For specified word in the given language retrieve limit recommendations
        """
        cherrypy.response.headers["Access-Control-Allow-Origin"] = "*"
        if self.language is None or self.language != language:
            try:
                self.model = Word2Vec.load(
                    os.path.join(
                        'data', 'models', 't-vex-%s-model' % (language)
                    )
                )
                self.language = language
                data = self._recommend(
                    word, int(limit), fn=self.model.most_similar
                )
            except (IOError, OSError):
                data = json.dumps(None)
        else:
            data = self._recommend(
                word, int(limit), fn=self.model.most_similar
            )
        return data

    @cherrypy.expose
    def get_cross_lingual_recommendations(
        self,
        lang1,
        lang2,
        word,
        topn=10
    ):
        """Provide cross lingual recommendations."""
        cherrypy.response.headers["Access-Control-Allow-Origin"] = "*"
        if self.cross_lang1 is not lang1 and self.cross_lang2 is not lang2:
            try:
                f = open(os.path.join(
                    'visualization', 'vector_space_mapper', '%s_%s' % (
                        lang1, lang2
                    )
                ), "r")
                self.vm = cPickle.load(f)
                self.cross_lang1 = lang1
                self.cross_lang2 = lang2
                data = self._recommend(
                    word, int(topn), fn=self.vm.get_recommendations_from_word
                )
            except (IOError, OSError):
                data = json.dumps(None)
        else:
            data = self._recommend(
                word, int(topn), fn=self.vm.get_recommendations_from_word
            )
        return data

    @cherrypy.expose
    def create_vector_space_mapper(self, lang1, lang2):
        """Create Vector Space Mapper between Languages."""
        cherrypy.response.headers["Access-Control-Allow-Origin"] = "*"
        if os.path.exists(os.path.join(
            'visualization', 'vector_space_mapper', '%s_%s' % (lang1, lang2)
        )) is False:
            try:
                model_1 = Word2Vec.load(
                    os.path.join('data', 'models', 't-vex-%s-model' % lang1)
                )
                model_2 = Word2Vec.load(
                    os.path.join('data', 'models', 't-vex-%s-model' % lang2)
                )
                with codecs.open(
                    os.path.join(
                        'data', 'bilingual_dictionary', '%s_%s_train_bd' % (
                            lang1, lang2
                        )
                    ), 'r', encoding='utf-8'
                ) as file:
                    data = file.read().split('\n')
                    bilingual_dict = [
                        (line.split(' ')[0], line.split(' ')[1])
                        for line in data
                    ]
                    vm = VectorSpaceMapper(model_1, model_2, bilingual_dict)
                    vm.map_vector_spaces()
                    with open(os.path.join(
                        'visualization', 'vector_space_mapper', '%s_%s' % (
                            lang1, lang2
                        )
                    ), "w") as file:
                        cPickle.dump(vm, file)
                        return json.dumps({'msg': 'Success'})
            except (IOError, OSError):
                return json.dumps({'msg': 'Failure'})
        else:
            return json.dumps({'msg': 'Success'})

    def _recommend(self, word, limit, fn):
        """Vector Space Mapper recommend functionality."""
        try:
            vec_list = fn(word, topn=limit)
            data = json.dumps([
                {
                    'word': tup[0],
                    'weight': tup[1]
                } for tup in vec_list
            ])
        except KeyError:
            data = json.dumps(None)
        return data

if __name__ == '__main__':
    ''' Setting up the Server with Specified Configuration'''

    server_config = ConfigParser.RawConfigParser()
    env = Environment(loader=FileSystemLoader('static'))
    conf = {
        '/': {
            'tools.staticdir.root': os.path.abspath(os.getcwd())
        },
        '/js': {
            'tools.staticdir.on': True,
            'tools.staticdir.dir': os.path.join(
                'visualization', 'static', 'js'
            )
        },
        '/css': {
            'tools.staticdir.on': True,
            'tools.staticdir.dir': os.path.join(
                'visualization', 'static', 'css'
            )
        },
        '/images': {
            'tools.staticdir.on': True,
            'tools.staticdir.dir': os.path.join(
                'visualization', 'static', 'images'
            )
        },
        '/resources': {
            'tools.staticdir.on': True,
            'tools.staticdir.dir': os.path.join(
                'visualization', 'static', 'resources'
            )
        }
    }
    server_config.read(os.path.join('visualization', 'server.conf'))
    server_port = server_config.get('Server', 'port')
    server_host = server_config.get('Server', 'host')
    thread_pool = server_config.get('Server', 'thread_pool')
    queue_size = server_config.get('Server', 'queue_size')
    cherrypy.config.update({'server.socket_host': server_host})
    cherrypy.config.update({'server.thread_pool': int(thread_pool)})
    cherrypy.config.update({'server.socket_queue_size': int(queue_size)})
    cherrypy.config.update({'server.socket_port': int(
        os.environ.get('PORT', server_port)
    )})
    cherrypy.quickstart(Server(), '/', conf)
<commit_msg>Visualization: Create vector_space_mapper dir if non existant<commit_after>#! /usr/bin/env python2
"""CherryPy Server to provide recommendations of semantic similarity."""

import cherrypy
import codecs
import os
import ConfigParser
import json
import cPickle
from gensim.models import Word2Vec
from jinja2 import Environment, FileSystemLoader
from modules.vector_space_mapper.vector_space_mapper import VectorSpaceMapper

class Server():
    """Server Configuration for t-vex."""

    def __init__(self):
        """Initialization the Language and Model."""
        self.language = None
        self.cross_lang1 = None
        self.cross_lang2 = None
        self.vm = None
        self.model = None

    @cherrypy.expose
    def index(self):
        """Semantic spac visualization html returned."""
        return file(os.path.join('visualization', 'static', 'index.html'))

    @cherrypy.expose
    def multivariate_analysis(self):
        """Parallel Coordinates for multivariate analysis html page return."""
        return file(os.path.join(
            'visualization', 'static', 'multivariate.html')
        )

    @cherrypy.expose
    def cross_lingual(self):
        """Cross Lingual recommender html returned."""
        return file(os.path.join(
            'visualization', 'static', 'cross_lingual.html')
        )

    @cherrypy.expose
    def retrieve_recommendations(self, language, word, limit=10):
        """
        Retrieve number of semantically similar recommendations.

        For specified word in the given language retrieve limit recommendations
        """
        cherrypy.response.headers["Access-Control-Allow-Origin"] = "*"
        if self.language is None or self.language != language:
            try:
                self.model = Word2Vec.load(
                    os.path.join(
                        'data', 'models', 't-vex-%s-model' % (language)
                    )
                )
                self.language = language
                data = self._recommend(
                    word, int(limit), fn=self.model.most_similar
                )
            except (IOError, OSError):
                data = json.dumps(None)
        else:
            data = self._recommend(
                word, int(limit), fn=self.model.most_similar
            )
        return data

    @cherrypy.expose
    def get_cross_lingual_recommendations(
        self,
        lang1,
        lang2,
        word,
        topn=10
    ):
        """Provide cross lingual recommendations."""
        cherrypy.response.headers["Access-Control-Allow-Origin"] = "*"
        if self.cross_lang1 is not lang1 and self.cross_lang2 is not lang2:
            try:
                f = open(os.path.join(
                    'visualization', 'vector_space_mapper', '%s_%s' % (
                        lang1, lang2
                    )
                ), "r")
                self.vm = cPickle.load(f)
                self.cross_lang1 = lang1
                self.cross_lang2 = lang2
                data = self._recommend(
                    word, int(topn), fn=self.vm.get_recommendations_from_word
                )
            except (IOError, OSError):
                data = json.dumps(None)
        else:
            data = self._recommend(
                word, int(topn), fn=self.vm.get_recommendations_from_word
            )
        return data

    @cherrypy.expose
    def create_vector_space_mapper(self, lang1, lang2):
        """Create Vector Space Mapper between Languages."""
        cherrypy.response.headers["Access-Control-Allow-Origin"] = "*"
        dir_path = os.path.join(
            'visualization', 'vector_space_mapper'
        )
        if not os.path.exists(dir_path):
            os.makedirs(dir_path)
        if os.path.exists(os.path.join(
            dir_path, '%s_%s' % (lang1, lang2)
        )) is False:
            try:
                model_1 = Word2Vec.load(
                    os.path.join('data', 'models', 't-vex-%s-model' % lang1)
                )
                model_2 = Word2Vec.load(
                    os.path.join('data', 'models', 't-vex-%s-model' % lang2)
                )
                with codecs.open(
                    os.path.join(
                        'data', 'bilingual_dictionary', '%s_%s_train_bd' % (
                            lang1, lang2
                        )
                    ), 'r', encoding='utf-8'
                ) as file:
                    data = file.read().split('\n')
                    bilingual_dict = [
                        (line.split(' ')[0], line.split(' ')[1])
                        for line in data
                    ]
                    vm = VectorSpaceMapper(model_1, model_2, bilingual_dict)
                    vm.map_vector_spaces()
                    with open(os.path.join(
                        'visualization', 'vector_space_mapper', '%s_%s' % (
                            lang1, lang2
                        )
                    ), "w") as file:
                        cPickle.dump(vm, file)
                        return json.dumps({'msg': 'Success'})
            except (IOError, OSError):
                return json.dumps({'msg': 'Failure'})
        else:
            return json.dumps({'msg': 'Success'})

    def _recommend(self, word, limit, fn):
        """Vector Space Mapper recommend functionality."""
        try:
            vec_list = fn(word, topn=limit)
            data = json.dumps([
                {
                    'word': tup[0],
                    'weight': tup[1]
                } for tup in vec_list
            ])
        except KeyError:
            data = json.dumps(None)
        return data

if __name__ == '__main__':
    ''' Setting up the Server with Specified Configuration'''

    server_config = ConfigParser.RawConfigParser()
    env = Environment(loader=FileSystemLoader('static'))
    conf = {
        '/': {
            'tools.staticdir.root': os.path.abspath(os.getcwd())
        },
        '/js': {
            'tools.staticdir.on': True,
            'tools.staticdir.dir': os.path.join(
                'visualization', 'static', 'js'
            )
        },
        '/css': {
            'tools.staticdir.on': True,
            'tools.staticdir.dir': os.path.join(
                'visualization', 'static', 'css'
            )
        },
        '/images': {
            'tools.staticdir.on': True,
            'tools.staticdir.dir': os.path.join(
                'visualization', 'static', 'images'
            )
        },
        '/resources': {
            'tools.staticdir.on': True,
            'tools.staticdir.dir': os.path.join(
                'visualization', 'static', 'resources'
            )
        }
    }
    server_config.read(os.path.join('visualization', 'server.conf'))
    server_port = server_config.get('Server', 'port')
    server_host = server_config.get('Server', 'host')
    thread_pool = server_config.get('Server', 'thread_pool')
    queue_size = server_config.get('Server', 'queue_size')
    cherrypy.config.update({'server.socket_host': server_host})
    cherrypy.config.update({'server.thread_pool': int(thread_pool)})
    cherrypy.config.update({'server.socket_queue_size': int(queue_size)})
    cherrypy.config.update({'server.socket_port': int(
        os.environ.get('PORT', server_port)
    )})
    cherrypy.quickstart(Server(), '/', conf)
<|endoftext|>


<commit_before>from __future__ import unicode_literals
from utils import CanadianScraper, CanadianPerson as Person

import re

COUNCIL_PAGE = 'http://www.ville.brossard.qc.ca/Ma-ville/conseil-municipal.aspx?lang=en-CA'
CONTACT_PAGE = 'http://www.ville.brossard.qc.ca/Ma-ville/conseil-municipal/Municipal-council/Municipal-council-members-%E2%80%93-Contact-information.aspx'

class BrossardPersonScraper(CanadianScraper):

    def scrape(self):
        page = self.lxmlize(COUNCIL_PAGE)
        contact_page = self.lxmlize(CONTACT_PAGE)

        councillor_elems = page.xpath('//a[contains(@class, "slide item-")]')
        email_links = contact_page.xpath('//a[contains(@href, "mailto:")]')
        for elem in councillor_elems:
            name = elem.xpath('.//div[@class="titre"]/text()')[0]
            if name == 'Poste vacant':
                continue
            if name == 'Francyne Raymond':
                name = 'Francine Raymond'  # her name is Francine, not Francyne
            position = elem.xpath('.//div[@class="poste"]/text()')[0]
            role = 'Conseiller'
            if 'Mayor' in position:
                district = 'Brossard'
                role = 'Maire'
            else:
                district = re.sub(r'(?<=[0-9]).+', '', position).strip()

            photo = re.search(r'url\((.+)\)', elem.attrib['style']).group(1)

            p = Person(primary_org='legislature', name=name, district=district, role=role, image=photo)
            p.add_source(COUNCIL_PAGE)
            p.add_source(CONTACT_PAGE)

            email_elem = [link for link in email_links
                          if name in link.text_content().replace('\u2019', "'")][0]
            email = re.match('mailto:(.+@brossard.ca)', email_elem.attrib['href']).group(1)
            p.add_contact('email', email)
            phone = email_elem.xpath('./following-sibling::text()[contains(., "450")]')[0]
            p.add_contact('voice', phone, 'legislature')

            yield p
<commit_msg>ca_qc_brossard: fix deputy mayor<commit_after>from __future__ import unicode_literals
from utils import CanadianScraper, CanadianPerson as Person

import re

COUNCIL_PAGE = 'http://www.ville.brossard.qc.ca/Ma-ville/conseil-municipal.aspx?lang=en-CA'
CONTACT_PAGE = 'http://www.ville.brossard.qc.ca/Ma-ville/conseil-municipal/Municipal-council/Municipal-council-members-%E2%80%93-Contact-information.aspx'

class BrossardPersonScraper(CanadianScraper):

    def scrape(self):
        page = self.lxmlize(COUNCIL_PAGE)
        contact_page = self.lxmlize(CONTACT_PAGE)

        councillor_elems = page.xpath('//a[contains(@class, "slide item-")]')
        email_links = contact_page.xpath('//a[contains(@href, "mailto:")]')
        for elem in councillor_elems:
            name = elem.xpath('.//div[@class="titre"]/text()')[0]
            if name == 'Poste vacant':
                continue
            if name == 'Francyne Raymond':
                name = 'Francine Raymond'  # her name is Francine on the contact page, not Francyne
            position = elem.xpath('.//div[@class="poste"]/text()')[0]
            role = 'Conseiller'
            district = re.search(r'District \d+', position)
            if district:
                district = district.group(0)
            elif 'Mayor' in position:
                district = 'Brossard'
                role = 'Maire'

            photo = re.search(r'url\((.+)\)', elem.attrib['style']).group(1)

            p = Person(primary_org='legislature', name=name, district=district, role=role, image=photo)
            p.add_source(COUNCIL_PAGE)
            p.add_source(CONTACT_PAGE)

            email_elem = [link for link in email_links
                          if name in link.text_content().replace('\u2019', "'")][0]
            email = re.match('mailto:(.+@brossard.ca)', email_elem.attrib['href']).group(1)
            p.add_contact('email', email)
            phone = email_elem.xpath('./following-sibling::text()[contains(., "450")]')[0]
            p.add_contact('voice', phone, 'legislature')

            yield p
<|endoftext|>


<commit_before>"""Module defining the Topic model and its managers."""

from django.db import models

from tmapi.constants import XSD_ANY_URI, XSD_STRING

from construct import Construct
from construct_fields import ConstructFields
from locator import Locator
from name import Name
from occurrence import Occurrence

class Topic (Construct, ConstructFields):

    """Represents a topic item."""

    types = models.ManyToManyField('self', symmetrical=False, blank=True,
                                   related_name='type_instances')

    class Meta:
        app_label = 'tmapi'

    def add_subject_identifier (self, subject_identifier):
        """Adds a subject identifier to this topic.

        :param subject_identifier: the subject identifier to be added

        """
        raise Exception('Not yet implemented')

    def add_subject_locator (self, subject_locator):
        """Adds a subject locator to this topic.

        :param subject_locator: the subject locator to be added

        """
        raise Exception('Not yet implemented')

    def add_type (self, topic_type):
        """Adds a type to this topic."""
        self.types.add(topic_type)

    def create_name (self, value, type=None, scope=None):
        """Creates a `Name` for this topic with the specified `value`,
        `type` and `scope`.

        If `type` is None, the created `Name` will have the default
        name type (a `Topic` with the subject identifier
        http://psi.topicmaps.org/iso13250/model/topic-name).

        If `scope` is None or an empty list, the name will be in the
        unconstrained scope.

        :param value: the string value of the name
        :type value: string
        :param type: the name type
        :type type: `Topic`
        :param scope: a list of themes
        :type scope: list
        :rtype: `Name`

        """
        if type is None:
            type = self.topic_map.create_topic_by_subject_identifier(
                Locator('http://psi.topicmaps.org/iso13250/model/topic-name'))
        name = Name(topic=self, value=value, topic_map=self.topic_map,
                    type=type)
        name.save()
        if scope is not None:
            for theme in scope:
                name.scope.add(theme)
        return name

    def create_occurrence (self, type, value, scope=None, datatype=None):
        """Creates an `Occurrence` for this topic with the specified
        `type`, `value`, and `scope`.

        If `datatype` is not None, the newly created `Occurrence` will
        have the datatype specified by `datatype`.

        :param type: the occurrence type
        :type type: `Topic`
        :param value: the value of the occurrence
        :type value: String or `Locator`
        :param scope: optional list of themes
        :type scope: list of `Topic`s
        :param datatype: optional locator indicating the datatype of `value`
        :type datatype: `Locator`
        :rtype: `Occurrence`

        """
        if datatype is None:
            if isinstance(value, Locator):
                datatype = Locator(XSD_ANY_URI)
            else:
                datatype = Locator(XSD_STRING)
        occurrence = Occurrence(type=type, value=value,
                                datatype=datatype.to_external_form(),
                                topic=self, topic_map=self.topic_map)
        occurrence.save()
        return occurrence

    def get_names (self, name_type=None):
        """Returns a QuerySet of the names of this topic.

        If `name_type` is not None, only names of the specified
        type are returned.

        :param name_type: the type of the `Name`s to be returned

        """
        if name_type is None:
            return self.names.all()
        else:
            return self.names.filter(type=name_type)

    def get_occurrences (self, occurrence_type=None):
        """Returns the `Occurrence`s of this topic.

        If occurrence_type is not None, returns the `Occurrence`s of
        this topic where the occurrence type is `occurrence_type`.

        :param occurrence_type: the type of the `Occurrence`s to be returned

        """
        if occurrence_type is None:
            return self.occurrences.all()
        else:
            return self.occurrences.filter(type__pk=occurrence_type.id)

    def get_parent (self):
        """Returns the `TopicMap` to which this topic belongs.

        :rtype: `TopicMap`

        """
        return self.topic_map

    def get_reified (self):
        """Returns the `Construct` which is reified by this topic.

        :rtype: `Construct` or None

        """
        reified = None
        reifiable_types = (
            'reified_association', 'reified_name', 'reified_occurrence',
            'reified_role', 'reified_topicmap', 'reified_variant')
        for reifiable_type in reifiable_types:
            try:
                reified = getattr(self, reifiable_type)
                break
            except:
                pass
        return reified

    def get_roles_played (self, role_type=None, association_type=None):
        """Returns the roles played by this topic.

        If `role_type` is not None, returns the roles played by this
        topic where the role type is `role_type`.

        If `role_type` and `association_type` are not None, returns
        the roles played by this topic where the role type is
        `role_type` and the association type is `association_type`.

        :param role_type: the type of the `Role`s to be returned
        :type role_type: `Topic` or None
        :param association_type: the type of the `Association` of
          which the returned roles must be part
        :type association_type: `Topic` or None

        """
        roles = []
        if role_type is not None:
            roles = self.role_players.filter(type__pk=role_type.id)
            if association_type is not None:
                roles = roles.filter()
        elif association_type is not None:
            raise Exception('This is a broken call to get_roles_played, specifying an assocation type but not a role type')
        else:
            roles = self.role_players.all()
        return roles

    def get_subject_identifiers (self):
        """Returns the subject identifiers assigned to this topic.

        :rtype: `QuerySet` of `Locator`s

        """
        return self.subject_identifiers.all()

    def get_subject_locators (self):
        """Returns the subject locators assigned to this topic.

        :rtype: `QuerySet` of `Locator`s

        """
        return self.subject_locators.all()

    def get_types (self):
        """Returns a QuerySet containing the types of which this topic
        is an instance."""
        return self.types.all()

    def remove_type (self, topic_type):
        """Removes a type from this topic.

        :param topic_type: the type to be removed from this topic
        :type topic_type: `Topic`

        """
        self.types.remove(topic_type)
<commit_msg>Added exception raising when 'same topic map' constraint is violated in topic.add_type().<commit_after>"""Module defining the Topic model and its managers."""

from django.db import models

from tmapi.constants import XSD_ANY_URI, XSD_STRING
from tmapi.exceptions import ModelConstraintException

from construct import Construct
from construct_fields import ConstructFields
from locator import Locator
from name import Name
from occurrence import Occurrence

class Topic (Construct, ConstructFields):

    """Represents a topic item."""

    types = models.ManyToManyField('self', symmetrical=False, blank=True,
                                   related_name='type_instances')

    class Meta:
        app_label = 'tmapi'

    def add_subject_identifier (self, subject_identifier):
        """Adds a subject identifier to this topic.

        :param subject_identifier: the subject identifier to be added

        """
        raise Exception('Not yet implemented')

    def add_subject_locator (self, subject_locator):
        """Adds a subject locator to this topic.

        :param subject_locator: the subject locator to be added

        """
        raise Exception('Not yet implemented')

    def add_type (self, type):
        """Adds a type to this topic.

        :param type: the type of which this topic should become an instance
        :type type: `Topic`

        """
        if self.topic_map != type.topic_map:
            raise ModelConstraintException
        self.types.add(type)

    def create_name (self, value, type=None, scope=None):
        """Creates a `Name` for this topic with the specified `value`,
        `type` and `scope`.

        If `type` is None, the created `Name` will have the default
        name type (a `Topic` with the subject identifier
        http://psi.topicmaps.org/iso13250/model/topic-name).

        If `scope` is None or an empty list, the name will be in the
        unconstrained scope.

        :param value: the string value of the name
        :type value: string
        :param type: the name type
        :type type: `Topic`
        :param scope: a list of themes
        :type scope: list
        :rtype: `Name`

        """
        if type is None:
            type = self.topic_map.create_topic_by_subject_identifier(
                Locator('http://psi.topicmaps.org/iso13250/model/topic-name'))
        name = Name(topic=self, value=value, topic_map=self.topic_map,
                    type=type)
        name.save()
        if scope is not None:
            for theme in scope:
                name.scope.add(theme)
        return name

    def create_occurrence (self, type, value, scope=None, datatype=None):
        """Creates an `Occurrence` for this topic with the specified
        `type`, `value`, and `scope`.

        If `datatype` is not None, the newly created `Occurrence` will
        have the datatype specified by `datatype`.

        :param type: the occurrence type
        :type type: `Topic`
        :param value: the value of the occurrence
        :type value: String or `Locator`
        :param scope: optional list of themes
        :type scope: list of `Topic`s
        :param datatype: optional locator indicating the datatype of `value`
        :type datatype: `Locator`
        :rtype: `Occurrence`

        """
        if datatype is None:
            if isinstance(value, Locator):
                datatype = Locator(XSD_ANY_URI)
            else:
                datatype = Locator(XSD_STRING)
        occurrence = Occurrence(type=type, value=value,
                                datatype=datatype.to_external_form(),
                                topic=self, topic_map=self.topic_map)
        occurrence.save()
        return occurrence

    def get_names (self, name_type=None):
        """Returns a QuerySet of the names of this topic.

        If `name_type` is not None, only names of the specified
        type are returned.

        :param name_type: the type of the `Name`s to be returned

        """
        if name_type is None:
            return self.names.all()
        else:
            return self.names.filter(type=name_type)

    def get_occurrences (self, occurrence_type=None):
        """Returns the `Occurrence`s of this topic.

        If occurrence_type is not None, returns the `Occurrence`s of
        this topic where the occurrence type is `occurrence_type`.

        :param occurrence_type: the type of the `Occurrence`s to be returned

        """
        if occurrence_type is None:
            return self.occurrences.all()
        else:
            return self.occurrences.filter(type__pk=occurrence_type.id)

    def get_parent (self):
        """Returns the `TopicMap` to which this topic belongs.

        :rtype: `TopicMap`

        """
        return self.topic_map

    def get_reified (self):
        """Returns the `Construct` which is reified by this topic.

        :rtype: `Construct` or None

        """
        reified = None
        reifiable_types = (
            'reified_association', 'reified_name', 'reified_occurrence',
            'reified_role', 'reified_topicmap', 'reified_variant')
        for reifiable_type in reifiable_types:
            try:
                reified = getattr(self, reifiable_type)
                break
            except:
                pass
        return reified

    def get_roles_played (self, role_type=None, association_type=None):
        """Returns the roles played by this topic.

        If `role_type` is not None, returns the roles played by this
        topic where the role type is `role_type`.

        If `role_type` and `association_type` are not None, returns
        the roles played by this topic where the role type is
        `role_type` and the association type is `association_type`.

        :param role_type: the type of the `Role`s to be returned
        :type role_type: `Topic` or None
        :param association_type: the type of the `Association` of
          which the returned roles must be part
        :type association_type: `Topic` or None

        """
        roles = []
        if role_type is not None:
            roles = self.role_players.filter(type__pk=role_type.id)
            if association_type is not None:
                roles = roles.filter()
        elif association_type is not None:
            raise Exception('This is a broken call to get_roles_played, specifying an assocation type but not a role type')
        else:
            roles = self.role_players.all()
        return roles

    def get_subject_identifiers (self):
        """Returns the subject identifiers assigned to this topic.

        :rtype: `QuerySet` of `Locator`s

        """
        return self.subject_identifiers.all()

    def get_subject_locators (self):
        """Returns the subject locators assigned to this topic.

        :rtype: `QuerySet` of `Locator`s

        """
        return self.subject_locators.all()

    def get_types (self):
        """Returns a QuerySet containing the types of which this topic
        is an instance."""
        return self.types.all()

    def remove_type (self, topic_type):
        """Removes a type from this topic.

        :param topic_type: the type to be removed from this topic
        :type topic_type: `Topic`

        """
        self.types.remove(topic_type)
<|endoftext|>