<commit_before>#!/usr/bin/env python
import subprocess
import os.path
import tempfile
import shutil
import sys
import datetime
import re
import argparse

TEMP_FOLDER = '/temp/acd_archive'
ARCHIVE_HOME = '/AutoArchive'

class TempDir(object):

    def __init__(self):
        self._dir = tempfile.mkdtemp()

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        self.clean_up()

    def __del__(self):
        self.clean_up()

    def clean_up(self):
        if self._dir != None:
            shutil.rmtree(self._dir, ignore_errors=True)
            self._dir = None

    @property
    def path(self):
        return self._dir

class CloudStorageBackend(object):
    """ Storage backend base class. """
    def upload(self, path, dest_path, n_retry):
        raise NotImplementedError

class AcdCliBackend(CloudStorageBackend):
    """ Storage backend using acd_cli. """
    def __init__(self):
        args = ['acdcli', 'sync']
        if subprocess.call(args, stdout=subprocess.PIPE) != 0:
            raise Exception('Cannot sync with Amazon cloud drive')

    def upload(self, path, dest_path, n_retry=5):
        args = ['acdcli', 'upload', path, dest_path]
        for _ in xrange(n_retry):
            if subprocess.call(args, shell=False) == 0:
                return
        raise Exception('Cannot upload file')

class RCloneBackend(CloudStorageBackend):

    def __init__(self):
        self._remote = 'acd'

    def upload(self, path, dest_path, n_retry=5):
        dest_path = '{}:{}'.format(self._remote, dest_path)
        args = ['rclone', 'copy', path, dest_path]
        for _ in xrange(n_retry):
            if subprocess.call(args, shell=False) == 0:
                return
        raise Exception('Cannot upload file')

def GenerateZipFileName(dir_name, name, no_prefix=False):
    if no_prefix:
        filename = name + '.7z'
    else:
        filename = name + datetime.datetime.now().strftime('.%Y_%m_%d__%H_%M_%S.7z')
    return os.path.join(dir_name, filename)

def ZipFile(input_path, output_path):
    args = ['7za', 'a', '-m0=lzma2', '-mmt=4', '-mx9', '-t7z', output_path, input_path]
    if subprocess.call(args) != 0:
        raise Exception('Cannot create 7zip archive')

if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='Process some integers.')

    parser.add_argument('path', metavar='path', type=str,
                       help='file path to archive')
    parser.add_argument('-f', '--name', type=str, help='prefix name for archived file')
    parser.add_argument('-d', '--dest', type=str, default=ARCHIVE_HOME, help='destination dir')
    parser.add_argument('--retry', type=int, default=5, help='number of times to retry uploading')
    parser.add_argument('--no_date', action='store_true',
                        help='disable auto prefix for archive name')
    parser.add_argument('-b', '--backend', type=str, default='rclone', help='destination dir')

    args = parser.parse_args()

    input_path = args.path

    if args.name:
        name = args.name
    else:
        name = re.findall('/*([^/]+)/*$', input_path)[0]

    if args.backend == 'acdcli':
        backend = AcdCliBackend()
    elif args.backend == 'rclone':
        backend = RCloneBackend()
    else:
        raise NotImplementedError('Unsupported backend!')

    with TempDir() as temp_dir:
        try:
            output_path = GenerateZipFileName(temp_dir.path, name, args.no_date)
            ZipFile(input_path, output_path)
            backend.upload(output_path, args.dest, args.retry)
        except:
            print "Error!"
            # We need to clean up the temp dir in case of any exceptions.
            pass
<commit_msg>changed display status<commit_after>#!/usr/bin/env python
import subprocess
import os.path
import tempfile
import shutil
import sys
import datetime
import re
import argparse

TEMP_FOLDER = '/temp/acd_archive'
ARCHIVE_HOME = '/AutoArchive'

class TempDir(object):

    def __init__(self):
        self._dir = tempfile.mkdtemp()

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        self.clean_up()

    def __del__(self):
        self.clean_up()

    def clean_up(self):
        if self._dir != None:
            shutil.rmtree(self._dir, ignore_errors=True)
            self._dir = None

    @property
    def path(self):
        return self._dir

class CloudStorageBackend(object):
    """ Storage backend base class. """
    def upload(self, path, dest_path, n_retry):
        raise NotImplementedError

class AcdCliBackend(CloudStorageBackend):
    """ Storage backend using acd_cli. """
    def __init__(self):
        args = ['acdcli', 'sync']
        if subprocess.call(args, stdout=subprocess.PIPE) != 0:
            raise Exception('Cannot sync with Amazon cloud drive')

    def upload(self, path, dest_path, n_retry=5):
        args = ['acdcli', 'upload', path, dest_path]
        for _ in xrange(n_retry):
            if subprocess.call(args, shell=False) == 0:
                return
        raise Exception('Cannot upload file')

class RCloneBackend(CloudStorageBackend):

    def __init__(self):
        self._remote = 'acd'

    def upload(self, path, dest_path, n_retry=5):
        dest_path = '{}:{}'.format(self._remote, dest_path)
        args = ['rclone', 'copy', path, dest_path, '--stats=1s']
        print args
        for _ in xrange(n_retry):
            if subprocess.call(args, shell=False) == 0:
                return
        raise Exception('Cannot upload file')

def GenerateZipFileName(dir_name, name, no_prefix=False):
    if no_prefix:
        filename = name + '.7z'
    else:
        filename = name + datetime.datetime.now().strftime('.%Y_%m_%d__%H_%M_%S.7z')
    return os.path.join(dir_name, filename)

def ZipFile(input_path, output_path):
    args = ['7za', 'a', '-m0=lzma2', '-mmt=4', '-mx9', '-t7z', output_path, input_path]
    if subprocess.call(args) != 0:
        raise Exception('Cannot create 7zip archive')

if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='Process some integers.')

    parser.add_argument('path', metavar='path', type=str,
                       help='file path to archive')
    parser.add_argument('-f', '--name', type=str, help='prefix name for archived file')
    parser.add_argument('-d', '--dest', type=str, default=ARCHIVE_HOME, help='destination dir')
    parser.add_argument('--retry', type=int, default=1, help='number of times to retry uploading')
    parser.add_argument('--no_date', action='store_true',
                        help='disable auto prefix for archive name')
    parser.add_argument('-b', '--backend', type=str, default='rclone', help='destination dir')

    args = parser.parse_args()

    input_path = args.path

    if args.name:
        name = args.name
    else:
        name = re.findall('/*([^/]+)/*$', input_path)[0]

    if args.backend == 'acdcli':
        backend = AcdCliBackend()
    elif args.backend == 'rclone':
        backend = RCloneBackend()
    else:
        raise NotImplementedError('Unsupported backend!')

    with TempDir() as temp_dir:
        try:
            output_path = GenerateZipFileName(temp_dir.path, name, args.no_date)
            ZipFile(input_path, output_path)
            backend.upload(output_path, args.dest, args.retry)
        except:
            print "Error!"
            # We need to clean up the temp dir in case of any exceptions.
            pass
<|endoftext|>


<commit_before>from __future__ import unicode_literals

from django.contrib.auth import get_user_model
from django.contrib.sites.models import Site
from django.contrib.syndication.views import Feed
from django.core.urlresolvers import reverse
from django.shortcuts import get_object_or_404
from django.utils.feedgenerator import Atom1Feed
from django.utils.html import strip_tags

from mezzanine.core.templatetags.mezzanine_tags import richtext_filters
from mezzanine.blog.models import BlogPost, BlogCategory
from mezzanine.generic.models import Keyword
from mezzanine.pages.models import Page
from mezzanine.conf import settings
from mezzanine.utils.html import absolute_urls

User = get_user_model()

class PostsRSS(Feed):
    """
    RSS feed for all blog posts.
    """

    def __init__(self, *args, **kwargs):
        """
        Use the title and description of the Blog page for the feed's
        title and description. If the blog page has somehow been
        removed, fall back to the ``SITE_TITLE`` and ``SITE_TAGLINE``
        settings.
        """
        self.tag = kwargs.pop("tag", None)
        self.category = kwargs.pop("category", None)
        self.username = kwargs.pop("username", None)
        super(PostsRSS, self).__init__(*args, **kwargs)
        self._public = True
        try:
            page = Page.objects.published().get(slug=settings.BLOG_SLUG)
        except Page.DoesNotExist:
            page = None
        else:
            self._public = not page.login_required
        if self._public:
            settings.use_editable()
            if page is not None:
                self._title = "%s | %s" % (page.title, settings.SITE_TITLE)
                self._description = strip_tags(page.description)
            else:
                self._title = settings.SITE_TITLE
                self._description = settings.SITE_TAGLINE

    def title(self):
        return self._title

    def description(self):
        return self._description

    def link(self):
        return reverse("blog_post_list")

    def items(self):
        if not self._public:
            return []
        blog_posts = BlogPost.objects.published().select_related("user")
        if self.tag:
            tag = get_object_or_404(Keyword, slug=self.tag)
            blog_posts = blog_posts.filter(keywords__keyword=tag)
        if self.category:
            category = get_object_or_404(BlogCategory, slug=self.category)
            blog_posts = blog_posts.filter(categories=category)
        if self.username:
            author = get_object_or_404(User, username=self.username)
            blog_posts = blog_posts.filter(user=author)
        limit = settings.BLOG_RSS_LIMIT
        if limit is not None:
            blog_posts = blog_posts[:settings.BLOG_RSS_LIMIT]
        return blog_posts

    def item_description(self, item):
        description = richtext_filters(item.content)
        absolute_urls_name = "mezzanine.utils.html.absolute_urls"
        if absolute_urls_name not in settings.RICHTEXT_FILTERS:
            description = absolute_urls(description)
        return description

    def categories(self):
        if not self._public:
            return []
        return BlogCategory.objects.all()

    def item_author_name(self, item):
        return item.user.get_full_name() or item.user.username

    def item_author_link(self, item):
        username = item.user.username
        return reverse("blog_post_list_author", kwargs={"username": username})

    def item_pubdate(self, item):
        return item.publish_date

    def item_categories(self, item):
        return item.categories.all()

    def item_enclosure_url(self, item):
        if item.featured_image:
            domain = Site.objects.get_current().domain
            enclosure_url = "http://%s%s" % (domain, item.featured_image.url)
            return enclosure_url

class PostsAtom(PostsRSS):
    """
    Atom feed for all blog posts.
    """

    feed_type = Atom1Feed

    def subtitle(self):
        return self.description()
<commit_msg>Prefetch blog categories in RSS feed.<commit_after>from __future__ import unicode_literals

from django.contrib.auth import get_user_model
from django.contrib.sites.models import Site
from django.contrib.syndication.views import Feed
from django.core.urlresolvers import reverse
from django.shortcuts import get_object_or_404
from django.utils.feedgenerator import Atom1Feed
from django.utils.html import strip_tags

from mezzanine.core.templatetags.mezzanine_tags import richtext_filters
from mezzanine.blog.models import BlogPost, BlogCategory
from mezzanine.generic.models import Keyword
from mezzanine.pages.models import Page
from mezzanine.conf import settings
from mezzanine.utils.html import absolute_urls

User = get_user_model()

class PostsRSS(Feed):
    """
    RSS feed for all blog posts.
    """

    def __init__(self, *args, **kwargs):
        """
        Use the title and description of the Blog page for the feed's
        title and description. If the blog page has somehow been
        removed, fall back to the ``SITE_TITLE`` and ``SITE_TAGLINE``
        settings.
        """
        self.tag = kwargs.pop("tag", None)
        self.category = kwargs.pop("category", None)
        self.username = kwargs.pop("username", None)
        super(PostsRSS, self).__init__(*args, **kwargs)
        self._public = True
        try:
            page = Page.objects.published().get(slug=settings.BLOG_SLUG)
        except Page.DoesNotExist:
            page = None
        else:
            self._public = not page.login_required
        if self._public:
            settings.use_editable()
            if page is not None:
                self._title = "%s | %s" % (page.title, settings.SITE_TITLE)
                self._description = strip_tags(page.description)
            else:
                self._title = settings.SITE_TITLE
                self._description = settings.SITE_TAGLINE

    def title(self):
        return self._title

    def description(self):
        return self._description

    def link(self):
        return reverse("blog_post_list")

    def items(self):
        if not self._public:
            return []
        blog_posts = BlogPost.objects.published().select_related("user"
            ).prefetch_related("categories")
        if self.tag:
            tag = get_object_or_404(Keyword, slug=self.tag)
            blog_posts = blog_posts.filter(keywords__keyword=tag)
        if self.category:
            category = get_object_or_404(BlogCategory, slug=self.category)
            blog_posts = blog_posts.filter(categories=category)
        if self.username:
            author = get_object_or_404(User, username=self.username)
            blog_posts = blog_posts.filter(user=author)
        limit = settings.BLOG_RSS_LIMIT
        if limit is not None:
            blog_posts = blog_posts[:settings.BLOG_RSS_LIMIT]
        return blog_posts

    def item_description(self, item):
        description = richtext_filters(item.content)
        absolute_urls_name = "mezzanine.utils.html.absolute_urls"
        if absolute_urls_name not in settings.RICHTEXT_FILTERS:
            description = absolute_urls(description)
        return description

    def categories(self):
        if not self._public:
            return []
        return BlogCategory.objects.all()

    def item_author_name(self, item):
        return item.user.get_full_name() or item.user.username

    def item_author_link(self, item):
        username = item.user.username
        return reverse("blog_post_list_author", kwargs={"username": username})

    def item_pubdate(self, item):
        return item.publish_date

    def item_categories(self, item):
        return item.categories.all()

    def item_enclosure_url(self, item):
        if item.featured_image:
            domain = Site.objects.get_current().domain
            enclosure_url = "http://%s%s" % (domain, item.featured_image.url)
            return enclosure_url

class PostsAtom(PostsRSS):
    """
    Atom feed for all blog posts.
    """

    feed_type = Atom1Feed

    def subtitle(self):
        return self.description()
<|endoftext|>


<commit_before>import re
import traceback
import csv
import logging

from owmeta_core.context import Context
from owmeta_core.datasource import GenericTranslation
from owmeta_core.dataobject import ObjectProperty
from owmeta_core.data_trans.csv_ds import CSVDataTranslator, CSVDataSource

from .. import CONTEXT
from ..utils import normalize_cell_name
from ..connection import Connection
from ..cell import Cell
from ..document import Document
from ..evidence import Evidence
from ..neuron import Neuron
from ..muscle import Muscle
from ..worm import Worm
from ..network import Network

from .common_data import DSMixin, DTMixin
from .data_with_evidence_ds import DataWithEvidenceDataSource

L = logging.getLogger(__name__)

class ConnectomeCSVDataSource(DSMixin, CSVDataSource):
    '''
    A CSV data source whose CSV file describes a neural connectome

    Basically, this is just a marker type to indicate what's described in the CSV --
    there's no consistent schema
    '''
    class_context = CONTEXT

class NeuronConnectomeCSVTranslation(GenericTranslation):
    class_context = CONTEXT

    neurons_source = ObjectProperty(value_type=DataWithEvidenceDataSource)
    muscles_source = ObjectProperty(value_type=DataWithEvidenceDataSource)

    key_properties = (GenericTranslation.source, muscles_source, neurons_source)

class NeuronConnectomeCSVTranslator(DTMixin, CSVDataTranslator):
    class_context = CONTEXT

    input_type = (ConnectomeCSVDataSource, DataWithEvidenceDataSource)
    output_type = DataWithEvidenceDataSource
    translation_type = NeuronConnectomeCSVTranslation

    def make_translation(self, sources):
        tr = super(NeuronConnectomeCSVTranslator, self).make_translation()
        tr.source(sources[0])
        tr.neurons_source(sources[1])
        tr.muscles_source(sources[2])
        return tr

    def translate(self, data_source, neurons_source, muscles_source):
        # muscle cells that are generically defined in source and need to be broken
        # into pair of L and R before being added to owmeta

        # muscle cells that have different names in connectome source and cell list.
        # Their wormbase cell list names will be used in owmeta
        changed_muscles = ['ANAL', 'INTR', 'INTL', 'SPH']

        # counters for terminal printing
        neuron_connections = 0
        muscle_connections = 0
        other_connections = 0

        res = self.make_new_output(sources=(data_source, neurons_source, muscles_source))

        try:
            # XXX: should there be a different src for muscles?
            w_q = muscles_source.data_context.stored(Worm)()
            n_q = neurons_source.data_context.stored(Network)()
            i_n = next(n_q.load(), n_q)
            i_w = next(w_q.load(), w_q)
            # XXX: If the context we query for is the same ID as the default
            # context, then we should query the merge of all graphs (using the
            # contract of the ConjunctiveGraph with 'default_is_union==True').
            # Otherwise, we should query only for what's in that graph.
            #
            neuron_objs = list(set(i_n.neurons()))
            muscle_objs = list(i_w.muscles())
            # get lists of neuron and muscles names
            neurons = [neuron.name() for neuron in neuron_objs]
            muscles = [muscle.name() for muscle in muscle_objs]
            o_w = res.data_context(Worm)()
            o_n = res.data_context(Network)(worm=o_w)
            # Evidence object to assert each connection
            ctxd_Document = res.evidence_context(Document)
            doc = ctxd_Document(key="emmons2015",
                                title='Whole-animal C. elegans connectomes',
                                year=2015,
                                uri='http://abstracts.genetics-gsa.org/cgi-bin/'
                                'celegans15s/wsrch15.pl?author=emmons&sort=ptimes&'
                                'sbutton=Detail&absno=155110844&sid=668862',
                                rdfs_comment="Data related by personal communication")
            doc.author('Emmons, S.')
            doc.author('Cook, S.')
            doc.author('Jarrell, T.')
            doc.author('Wang, Y.')
            doc.author('Yakolev, M.')
            doc.author('Nguyen, K.')
            doc.author('Hall, D.')
            e = res.evidence_context(Evidence)(key="emmons2015", reference=doc)
            docctx = res.evidence_context(Context)(ident=self.identifier + '/emmons2015-context')
            e.supports(docctx.rdf_object)
            with docctx(Neuron, Muscle, Cell, Connection) as ctx:
                res.data_context.add_import(ctx.context)
                with open(data_source.csv_file_name.onedef()) as csvfile:
                    edge_reader = csv.reader(csvfile)
                    next(edge_reader)  # skip header row
                    for row in edge_reader:
                        source, target, weight, syn_type = map(str.strip, row)

                        # set synapse type to something the Connection object
                        # expects, and normalize the source and target names
                        if syn_type == 'electrical':
                            syn_type = 'gapJunction'
                        elif syn_type == 'chemical':
                            syn_type = 'send'

                        source = normalize_cell_name(source).upper()
                        target = normalize_cell_name(target).upper()

                        weight = int(weight)

                        # remove BMW from Body Wall Muscle cells
                        if 'BWM' in source:
                            source = normalize_muscle(source)
                        if 'BWM' in target:
                            target = normalize_muscle(target)

                        # change certain muscle names to names in wormbase
                        if source in changed_muscles:
                            source = changed_muscle(source)
                        if target in changed_muscles:
                            target = changed_muscle(target)

                        sources = convert_to_cell(ctx, source, muscles, neurons)
                        targets = convert_to_cell(ctx, target, muscles, neurons)

                        for s in sources:
                            for t in targets:
                                conn = add_synapse(ctx, s, t, weight, syn_type)
                                o_n.synapse(conn)
                                kind = conn.termination.onedef()
                                if kind == 'muscle':
                                    muscle_connections += 1
                                elif kind == 'neuron':
                                    neuron_connections += 1
                                else:
                                    other_connections += 1

            print('Total neuron to neuron connections added = %i' % neuron_connections)
            print('Total neuron to muscle connections added = %i' % muscle_connections)
            print('Total other connections added = %i' % other_connections)
            print('uploaded connections')

        except Exception:
            traceback.print_exc()
        return res

class NeuronConnectomeSynapseClassTranslation(GenericTranslation):
    class_context = CONTEXT

    neurotransmitter_source = ObjectProperty()

    key_properties = (GenericTranslation.source, neurotransmitter_source)

class NeuronConnectomeSynapseClassTranslator(DTMixin, CSVDataTranslator):
    '''
    Adds synapse classes to existing connections
    '''
    class_context = CONTEXT

    translation_type = NeuronConnectomeSynapseClassTranslation

    input_type = (DataWithEvidenceDataSource, ConnectomeCSVDataSource)
    output_type = DataWithEvidenceDataSource

    def make_translation(self, sources):
        tr = super(NeuronConnectomeSynapseClassTranslator, self).make_translation()
        tr.source(sources[0])
        tr.neurotransmitter_source(sources[1])
        return tr

    def translate(self, data_source, neurotransmitter_source):
        res = self.make_new_output(sources=(data_source, neurotransmitter_source))
        doc = res.evidence_context(Document)(author=['Dimitar Sht. Shterionov', 'Gerda Janssens'],
                       title='Data acquisition and modeling for learning and'
                             ' reasoning in probabilistic logic environment',
                       year=2011,
                       uri='https://groups.google.com/forum/m/#!topic/openworm-discuss/G9wKoR8N-l0/discussion')
        e = res.evidence_context(Evidence)(key="shterionov2011", reference=doc)

        # We don't use doc.as_context for the statements' context because the document
        # itself doesn't actually make the statements below, although it does *support*
        # them by describing the process by which they are derived
        docctx = res.evidence_context(Context)(ident=self.identifier + '#shterionov2011-context')

        # There are many entries in the Shterionov data where the number of synapses
        # doesn't match that in, say, the Emmons (2015) data. We distinguish these so it's
        # easy to look back at the differences later, but the different number of synapses
        # isn't recorded here.
        docctx_anynum = res.evidence_context(Context)(ident=self.identifier
                                                      + '#shterionov2011-context-any-number')
        e.supports(docctx.rdf_object)
        res.data_context.add_import(docctx)
        res.data_context.add_import(docctx_anynum)
        with self.make_reader(neurotransmitter_source,
                              skipheader=False,
                              delimiter=';') as reader:
            for row in reader:
                pre, post, typ, number, nt = row
                with data_source.data_context.stored(Connection, Neuron) as srcctx:
                    conn = srcctx.Connection.query(pre_cell=srcctx.Neuron.query(pre),
                                                   post_cell=srcctx.Neuron.query(post),
                                                   number=int(number),
                                                   syntype=typ)
                    hit = False
                    for c in conn.load():
                        docctx(Connection)(ident=c.identifier).synclass(nt)
                        hit = True

                    if not hit:
                        conn = srcctx.Connection.query(pre_cell=srcctx.Neuron(pre),
                                                       post_cell=srcctx.Neuron(post),
                                                       syntype=typ)
                        hit = False
                        for c in conn.load():
                            docctx_anynum(Connection)(ident=c.identifier).synclass(nt)
                            hit = True
                        if not hit:
                            L.warning("Didn't find any connections matching: {}".format(conn))
                        else:
                            break
        return res

def convert_to_cell(ctx, name, muscles, neurons):
    ret = []
    res = None
    res2 = None
    if name in neurons:
        res = ctx.Neuron(name)
    elif name in muscles:
        res = ctx.Muscle(name)
    elif name in TO_EXPAND_MUSCLES:
        res, res2 = expand_muscle(ctx, name)
    elif name in OTHER_CELLS:
        res = ctx.Cell(name)

    if res is not None:
        ret.append(res)
    if res2 is not None:
        ret.append(res2)

    return ret

def add_synapse(ctx, source, target, weight, syn_type):
    c = ctx.Connection(pre_cell=source, post_cell=target,
                       number=weight, syntype=syn_type)

    if isinstance(source, ctx.Neuron) and isinstance(target, ctx.Neuron):
        c.termination('neuron')
    elif isinstance(source, ctx.Neuron) and isinstance(target, ctx.Muscle) or \
            isinstance(source, ctx.Muscle) and isinstance(target, ctx.Neuron):
        c.termination('muscle')

    return c

# to normalize certian body wall muscle cell names
SEARCH_STRING_MUSCLE = re.compile(r'\w+[BWM]+\w+')
REPLACE_STRING_MUSCLE = re.compile(r'[BWM]+')

def normalize_muscle(name):
    # normalize names of Body Wall Muscles
    # if there is 'BWM' in the name, remove it
    if re.match(SEARCH_STRING_MUSCLE, name):
        name = REPLACE_STRING_MUSCLE.sub('', name)
    return name

MUSCLES = {
    'ANAL': 'MU_ANAL',
    'INTR': 'MU_INT_R',
    'INTL': 'MU_INT_L',
    'SPH': 'MU_SPH'
}

TO_EXPAND_MUSCLES = ['PM1D', 'PM2D', 'PM3D', 'PM4D', 'PM5D']

#
# cells that are neither neurons or muscles. These are marked as
# 'Other Cells' in the wormbase cell list but are still part of the new
# connectome.
#
# TODO: In future work these should be uploaded seperately to
# owmeta in a new upload function and should be referred from there
# instead of this list.
OTHER_CELLS = ['MC1DL', 'MC1DR', 'MC1V', 'MC2DL', 'MC2DR', 'MC2V', 'MC3DL',
               'MC3DR', 'MC3V']

def changed_muscle(x):
    return MUSCLES[x]

def expand_muscle(ctx, name):
    return ctx(Muscle)(name + 'L'), ctx(Muscle)(name + 'R')
<commit_msg>Fixing CSVDataSource path for NeuronConnectomeCSVTranslator<commit_after>import re
import traceback
import csv
import logging

from owmeta_core.context import Context
from owmeta_core.datasource import GenericTranslation
from owmeta_core.dataobject import ObjectProperty
from owmeta_core.data_trans.csv_ds import CSVDataTranslator, CSVDataSource

from .. import CONTEXT
from ..utils import normalize_cell_name
from ..connection import Connection
from ..cell import Cell
from ..document import Document
from ..evidence import Evidence
from ..neuron import Neuron
from ..muscle import Muscle
from ..worm import Worm
from ..network import Network

from .common_data import DSMixin, DTMixin
from .data_with_evidence_ds import DataWithEvidenceDataSource

L = logging.getLogger(__name__)

class ConnectomeCSVDataSource(DSMixin, CSVDataSource):
    '''
    A CSV data source whose CSV file describes a neural connectome

    Basically, this is just a marker type to indicate what's described in the CSV --
    there's no consistent schema
    '''
    class_context = CONTEXT

class NeuronConnectomeCSVTranslation(GenericTranslation):
    class_context = CONTEXT

    neurons_source = ObjectProperty(value_type=DataWithEvidenceDataSource)
    muscles_source = ObjectProperty(value_type=DataWithEvidenceDataSource)

    key_properties = (GenericTranslation.source, muscles_source, neurons_source)

class NeuronConnectomeCSVTranslator(DTMixin, CSVDataTranslator):
    class_context = CONTEXT

    input_type = (ConnectomeCSVDataSource, DataWithEvidenceDataSource)
    output_type = DataWithEvidenceDataSource
    translation_type = NeuronConnectomeCSVTranslation

    def make_translation(self, sources):
        tr = super(NeuronConnectomeCSVTranslator, self).make_translation()
        tr.source(sources[0])
        tr.neurons_source(sources[1])
        tr.muscles_source(sources[2])
        return tr

    def translate(self, data_source, neurons_source, muscles_source):
        # muscle cells that are generically defined in source and need to be broken
        # into pair of L and R before being added to owmeta

        # muscle cells that have different names in connectome source and cell list.
        # Their wormbase cell list names will be used in owmeta
        changed_muscles = ['ANAL', 'INTR', 'INTL', 'SPH']

        # counters for terminal printing
        neuron_connections = 0
        muscle_connections = 0
        other_connections = 0

        res = self.make_new_output(sources=(data_source, neurons_source, muscles_source))

        try:
            # XXX: should there be a different src for muscles?
            w_q = muscles_source.data_context.stored(Worm)()
            n_q = neurons_source.data_context.stored(Network)()
            i_n = next(n_q.load(), n_q)
            i_w = next(w_q.load(), w_q)
            # XXX: If the context we query for is the same ID as the default
            # context, then we should query the merge of all graphs (using the
            # contract of the ConjunctiveGraph with 'default_is_union==True').
            # Otherwise, we should query only for what's in that graph.
            #
            neuron_objs = list(set(i_n.neurons()))
            muscle_objs = list(i_w.muscles())
            # get lists of neuron and muscles names
            neurons = [neuron.name() for neuron in neuron_objs]
            muscles = [muscle.name() for muscle in muscle_objs]
            o_w = res.data_context(Worm)()
            o_n = res.data_context(Network)(worm=o_w)
            # Evidence object to assert each connection
            ctxd_Document = res.evidence_context(Document)
            doc = ctxd_Document(key="emmons2015",
                                title='Whole-animal C. elegans connectomes',
                                year=2015,
                                uri='http://abstracts.genetics-gsa.org/cgi-bin/'
                                'celegans15s/wsrch15.pl?author=emmons&sort=ptimes&'
                                'sbutton=Detail&absno=155110844&sid=668862',
                                rdfs_comment="Data related by personal communication")
            doc.author('Emmons, S.')
            doc.author('Cook, S.')
            doc.author('Jarrell, T.')
            doc.author('Wang, Y.')
            doc.author('Yakolev, M.')
            doc.author('Nguyen, K.')
            doc.author('Hall, D.')
            e = res.evidence_context(Evidence)(key="emmons2015", reference=doc)
            docctx = res.evidence_context(Context)(ident=self.identifier + '/emmons2015-context')
            e.supports(docctx.rdf_object)
            with docctx(Neuron, Muscle, Cell, Connection) as ctx:
                res.data_context.add_import(ctx.context)
                with open(data_source.full_path()) as csvfile:
                    edge_reader = csv.reader(csvfile)
                    next(edge_reader)  # skip header row
                    for row in edge_reader:
                        source, target, weight, syn_type = map(str.strip, row)

                        # set synapse type to something the Connection object
                        # expects, and normalize the source and target names
                        if syn_type == 'electrical':
                            syn_type = 'gapJunction'
                        elif syn_type == 'chemical':
                            syn_type = 'send'

                        source = normalize_cell_name(source).upper()
                        target = normalize_cell_name(target).upper()

                        weight = int(weight)

                        # remove BMW from Body Wall Muscle cells
                        if 'BWM' in source:
                            source = normalize_muscle(source)
                        if 'BWM' in target:
                            target = normalize_muscle(target)

                        # change certain muscle names to names in wormbase
                        if source in changed_muscles:
                            source = changed_muscle(source)
                        if target in changed_muscles:
                            target = changed_muscle(target)

                        sources = convert_to_cell(ctx, source, muscles, neurons)
                        targets = convert_to_cell(ctx, target, muscles, neurons)

                        for s in sources:
                            for t in targets:
                                conn = add_synapse(ctx, s, t, weight, syn_type)
                                o_n.synapse(conn)
                                kind = conn.termination.onedef()
                                if kind == 'muscle':
                                    muscle_connections += 1
                                elif kind == 'neuron':
                                    neuron_connections += 1
                                else:
                                    other_connections += 1

            print('Total neuron to neuron connections added = %i' % neuron_connections)
            print('Total neuron to muscle connections added = %i' % muscle_connections)
            print('Total other connections added = %i' % other_connections)
            print('uploaded connections')

        except Exception:
            traceback.print_exc()
        return res

class NeuronConnectomeSynapseClassTranslation(GenericTranslation):
    class_context = CONTEXT

    neurotransmitter_source = ObjectProperty()

    key_properties = (GenericTranslation.source, neurotransmitter_source)

class NeuronConnectomeSynapseClassTranslator(DTMixin, CSVDataTranslator):
    '''
    Adds synapse classes to existing connections
    '''
    class_context = CONTEXT

    translation_type = NeuronConnectomeSynapseClassTranslation

    input_type = (DataWithEvidenceDataSource, ConnectomeCSVDataSource)
    output_type = DataWithEvidenceDataSource

    def make_translation(self, sources):
        tr = super(NeuronConnectomeSynapseClassTranslator, self).make_translation()
        tr.source(sources[0])
        tr.neurotransmitter_source(sources[1])
        return tr

    def translate(self, data_source, neurotransmitter_source):
        res = self.make_new_output(sources=(data_source, neurotransmitter_source))
        doc = res.evidence_context(Document)(author=['Dimitar Sht. Shterionov', 'Gerda Janssens'],
                       title='Data acquisition and modeling for learning and'
                             ' reasoning in probabilistic logic environment',
                       year=2011,
                       uri='https://groups.google.com/forum/m/#!topic/openworm-discuss/G9wKoR8N-l0/discussion')
        e = res.evidence_context(Evidence)(key="shterionov2011", reference=doc)

        # We don't use doc.as_context for the statements' context because the document
        # itself doesn't actually make the statements below, although it does *support*
        # them by describing the process by which they are derived
        docctx = res.evidence_context(Context)(ident=self.identifier + '#shterionov2011-context')

        # There are many entries in the Shterionov data where the number of synapses
        # doesn't match that in, say, the Emmons (2015) data. We distinguish these so it's
        # easy to look back at the differences later, but the different number of synapses
        # isn't recorded here.
        docctx_anynum = res.evidence_context(Context)(ident=self.identifier
                                                      + '#shterionov2011-context-any-number')
        e.supports(docctx.rdf_object)
        res.data_context.add_import(docctx)
        res.data_context.add_import(docctx_anynum)
        with self.make_reader(neurotransmitter_source,
                              skipheader=False,
                              delimiter=';') as reader:
            for row in reader:
                pre, post, typ, number, nt = row
                with data_source.data_context.stored(Connection, Neuron) as srcctx:
                    conn = srcctx.Connection.query(pre_cell=srcctx.Neuron.query(pre),
                                                   post_cell=srcctx.Neuron.query(post),
                                                   number=int(number),
                                                   syntype=typ)
                    hit = False
                    for c in conn.load():
                        docctx(Connection)(ident=c.identifier).synclass(nt)
                        hit = True

                    if not hit:
                        conn = srcctx.Connection.query(pre_cell=srcctx.Neuron(pre),
                                                       post_cell=srcctx.Neuron(post),
                                                       syntype=typ)
                        hit = False
                        for c in conn.load():
                            docctx_anynum(Connection)(ident=c.identifier).synclass(nt)
                            hit = True
                        if not hit:
                            L.warning("Didn't find any connections matching: {}".format(conn))
                        else:
                            break
        return res

def convert_to_cell(ctx, name, muscles, neurons):
    ret = []
    res = None
    res2 = None
    if name in neurons:
        res = ctx.Neuron(name)
    elif name in muscles:
        res = ctx.Muscle(name)
    elif name in TO_EXPAND_MUSCLES:
        res, res2 = expand_muscle(ctx, name)
    elif name in OTHER_CELLS:
        res = ctx.Cell(name)

    if res is not None:
        ret.append(res)
    if res2 is not None:
        ret.append(res2)

    return ret

def add_synapse(ctx, source, target, weight, syn_type):
    c = ctx.Connection(pre_cell=source, post_cell=target,
                       number=weight, syntype=syn_type)

    if isinstance(source, ctx.Neuron) and isinstance(target, ctx.Neuron):
        c.termination('neuron')
    elif isinstance(source, ctx.Neuron) and isinstance(target, ctx.Muscle) or \
            isinstance(source, ctx.Muscle) and isinstance(target, ctx.Neuron):
        c.termination('muscle')

    return c

# to normalize certian body wall muscle cell names
SEARCH_STRING_MUSCLE = re.compile(r'\w+[BWM]+\w+')
REPLACE_STRING_MUSCLE = re.compile(r'[BWM]+')

def normalize_muscle(name):
    # normalize names of Body Wall Muscles
    # if there is 'BWM' in the name, remove it
    if re.match(SEARCH_STRING_MUSCLE, name):
        name = REPLACE_STRING_MUSCLE.sub('', name)
    return name

MUSCLES = {
    'ANAL': 'MU_ANAL',
    'INTR': 'MU_INT_R',
    'INTL': 'MU_INT_L',
    'SPH': 'MU_SPH'
}

TO_EXPAND_MUSCLES = ['PM1D', 'PM2D', 'PM3D', 'PM4D', 'PM5D']

#
# cells that are neither neurons or muscles. These are marked as
# 'Other Cells' in the wormbase cell list but are still part of the new
# connectome.
#
# TODO: In future work these should be uploaded seperately to
# owmeta in a new upload function and should be referred from there
# instead of this list.
OTHER_CELLS = ['MC1DL', 'MC1DR', 'MC1V', 'MC2DL', 'MC2DR', 'MC2V', 'MC3DL',
               'MC3DR', 'MC3V']

def changed_muscle(x):
    return MUSCLES[x]

def expand_muscle(ctx, name):
    return ctx(Muscle)(name + 'L'), ctx(Muscle)(name + 'R')
<|endoftext|>


<commit_before>"""Unit tests for the github3.gists module."""
import github3

from .helper import (create_example_data_helper, create_url_helper,
                     UnitIteratorHelper)

gist_example_data = create_example_data_helper('gist_example_data')

url_for = create_url_helper(
    'https://api.github.com/gists/b4c7ac7be6e591d0d155'
)

class TestGistIterators(UnitIteratorHelper):

    """Test Gist methods that return Iterators."""

    described_class = github3.gists.Gist
    example_data = gist_example_data()

    def test_comments(self):
        """Show a user can iterate over the comments on a gist."""
        i = self.instance.comments()
        self.get_next(i)

        self.session.get.assert_called_once_with(
            url_for('comments'),
            params={'per_page': 100},
            headers={}
        )

    def test_commits(self):
        """Show a user can iterate over the commits on a gist."""
        i = self.instance.commits()
        self.get_next(i)

        self.session.get.assert_called_once_with(
            url_for('commits'),
            params={'per_page': 100},
            headers={}
        )

    def test_files(self):
        """Show that iterating over a gist's files does not make a request."""
        files = list(self.instance.files())
        assert len(files) > 0

        assert self.session.get.called is False

    def test_forks(self):
        """Show that a user can iterate over a gist's forks."""
        i = self.instance.forks()
        self.get_next(i)

        self.session.get.assert_called_once_with(
            url_for('forks'),
            params={'per_page': 100},
            headers={}
        )
<commit_msg>Add Gist#create_comment unit test<commit_after>"""Unit tests for the github3.gists module."""
import github3

from .helper import (create_example_data_helper, create_url_helper,
                     UnitHelper, UnitIteratorHelper)

gist_example_data = create_example_data_helper('gist_example_data')

url_for = create_url_helper(
    'https://api.github.com/gists/b4c7ac7be6e591d0d155'
)

class TestGist(UnitHelper):

    """Test regular Gist methods."""

    described_class = github3.gists.Gist
    example_data = gist_example_data()

    def test_create_comment(self):
        """Show that a user can create a comment."""
        self.instance.create_comment('some comment text')

        self.post_called_with(url_for('comments'),
                              data={'body': 'some comment text'})

class TestGistIterators(UnitIteratorHelper):

    """Test Gist methods that return Iterators."""

    described_class = github3.gists.Gist
    example_data = gist_example_data()

    def test_comments(self):
        """Show a user can iterate over the comments on a gist."""
        i = self.instance.comments()
        self.get_next(i)

        self.session.get.assert_called_once_with(
            url_for('comments'),
            params={'per_page': 100},
            headers={}
        )

    def test_commits(self):
        """Show a user can iterate over the commits on a gist."""
        i = self.instance.commits()
        self.get_next(i)

        self.session.get.assert_called_once_with(
            url_for('commits'),
            params={'per_page': 100},
            headers={}
        )

    def test_files(self):
        """Show that iterating over a gist's files does not make a request."""
        files = list(self.instance.files())
        assert len(files) > 0

        assert self.session.get.called is False

    def test_forks(self):
        """Show that a user can iterate over a gist's forks."""
        i = self.instance.forks()
        self.get_next(i)

        self.session.get.assert_called_once_with(
            url_for('forks'),
            params={'per_page': 100},
            headers={}
        )
<|endoftext|>


<commit_before># Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish, dis-
# tribute, sublicense, and/or sell copies of the Software, and to permit
# persons to whom the Software is furnished to do so, subject to the fol-
# lowing conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
import boto
from boto.utils import find_class
import datetime
from boto.sdb.db.key import Key

def object_lister(cls, query_lister, manager):
    for item in query_lister:
        if cls:
            yield cls(item.name)
        else:
            o = manager.get_object_from_id(item.name)
            if o:
                yield o

ISO8601 = '%Y-%m-%dT%H:%M:%SZ'

class SDBManager(object):

    def __init__(self, domain_name=None, aws_access_key_id=None, aws_secret_access_key=None, debug=0):
        self.domain_name = domain_name
        self.aws_access_key_id = aws_access_key_id
        self.aws_secret_access_key = aws_secret_access_key
        self.domain = None
        self.sdb = None
        self.s3 = None
        if not self.domain_name:
            self.domain_name = self.DefaultDomainName
            if self.domain_name:
                boto.log.info('No SimpleDB domain set, using default_domain: %s' % self.domain_name)
            else:
                boto.log.warning('No SimpleDB domain set, persistance is disabled')
        if self.domain_name:
            self.sdb = boto.connect_sdb(aws_access_key_id=self.aws_access_key_id,
                                        aws_secret_access_key=self.aws_secret_access_key,
                                        debug=debug)
            self.domain = self.sdb.lookup(self.domain_name)
            if not self.domain:
                self.domain = self.sdb.create_domain(self.domain_name)

    class Converter:
        """
        Responsible for converting base Python types to format compatible with underlying
        database.  For SimpleDB, that means everything needs to be converted to a string
        when stored in SimpleDB and from a string when retrieved.

        To convert a value, pass it to the encode or decode method.  The encode method
        will take a Python native value and convert to DB format.  The decode method will
        take a DB format value and convert it to Python native format.  To find the appropriate
        method to call, the generic encode/decode methods will look for the type-specific
        method by searching for a method called "encode_<type name>" or "decode_<type name>".
        """
        @classmethod
        def encode(cls, manager, prop, value):
            if hasattr(prop, 'reference_class'):
                return cls.encode_reference(manager, value)
            if isinstance(value, str) or isinstance(value, unicode):
                return value
            if isinstance(value, int) or isinstance(value, long):
                return cls.encode_int(manager, value)
            if isinstance(value, bool):
                return cls.encode_bool(manager, value)
            if isinstance(value, datetime.datetime):
                return cls.encode_datetime(manager, value)

        @classmethod
        def decode(cls, manager, prop, value):
            if isinstance(prop.data_type, str) or isinstance(prop.data_type, unicode):
                return value
            if isinstance(prop.data_type, int) or isinstance(prop.data_type, long):
                return cls.decode_int(manager, value)
            if isinstance(prop.data_type, bool):
                return cls.decode_bool(manager, value)
            if isinstance(prop.data_type, datetime.datetime):
                return cls.decode_datetime(manager, value)
            if isinstance(prop.data_type, Key):
                return cls.decode_reference(manager, value)

        @classmethod
        def encode_int(cls, manager, value):
            value = long(value)
            value += 9223372036854775808
            return '%020d' % value

        @classmethod
        def decode_int(cls, manager, value):
            value = long(value)
            value -= 9223372036854775808
            return value

        @classmethod
        def encode_bool(cls, manager, value):
            if value == True:
                return 'true'
            else:
                return 'false'

        @classmethod
        def decode_bool(cls, manager, value):
            if value.lower() == 'true':
                return True
            else:
                return False

        @classmethod
        def encode_datetime(cls, manager, value):
            return value.strftime(ISO8601)

        @classmethod
        def decode_datetime(cls, manager, value):
            try:
                return datetime.strptime(value, ISO8601)
            except:
                raise ValueError, 'Unable to convert %s to DateTime' % str_value

        @classmethod
        def encode_reference(cls, manager, value):
            if isinstance(value, str) or isinstance(value, unicode):
                return value
            if value == None:
                return ''
            else:
                return value.id

        @classmethod
        def decode_reference(cls, manager, value):
            if not value:
                return None
            try:
                return manager.get_object_from_id(value)
            except:
                raise ValueError, 'Unable to convert %s to Object' % str_value

    def get_s3_connection(self):
        if not self.s3:
            self.s3 = boto.connect_s3(self.aws_access_key_id, self.aws_secret_access_key)
        return self.s3

    def get_object(self, cls, id):
        a = self.domain.get_attributes(id, '__type__')
        if a.has_key('__type__'):
            return cls(id)
        else:
            raise SDBPersistenceError('%s object with id=%s does not exist' % (cls.__name__, id))

    def get_object_from_id(self, id):
        attrs = self.domain.get_attributes(id, ['__module__', '__type__', '__lineage__'])
        try:
            cls = find_class(attrs['__module__'], attrs['__type__'])
            return cls(id, manager=self)
        except ImportError:
            return None

    def query(self, cls, filters):
        if len(filters) > 4:
            raise SDBPersistenceError('Too many filters, max is 4')
        parts = ["['__type__'='%s'] union ['__lineage__'starts-with'%s']" % (cls.__name__, cls.get_lineage())]
        properties = cls.properties()
        for filter, value in filters:
            name, op = filter.strip().split()
            found = False
            for property in properties:
                if property.name == name:
                    found = True
                    value = self.encode_value(property, value)
                    parts.append("['%s' %s '%s']" % (name, op, value))
            if not found:
                raise SDBPersistenceError('%s is not a valid field' % key)
        query = ' intersection '.join(parts)
        rs = self.domain.query(query)
        return object_lister(cls, rs, self)

    def save_object(self, obj):
        attrs = {'__type__' : obj.__class__.__name__,
                 '__module__' : obj.__class__.__module__,
                 '__lineage__' : obj.get_lineage()}
        for property in obj.properties():
            if not property.__class__.__name__.startswith('_'):
                attrs[property.name] = property.get_value_for_datastore(obj)
        self.domain.put_attributes(obj.id, attrs, replace=True)

    def delete_object(self, obj):
        self.domain.delete_attributes(obj.id)

    def encode_value(self, prop, value):
        return self.Converter.encode(self, prop, value)

    def set_property(self, prop, obj, name, value):
        value = self.encode_value(self, prop, value)
        self.domain.put_attributes(obj.id, {name : value}, replace=True)

    def decode_value(self, prop, value):
        return self.Converter.decode(self, prop, value)

    def get_property(self, prop, obj, name):
        a = self.domain.get_attributes(obj.id, name)
        # try to get the attribute value from SDB
        if name in a:
            return self.decode_value(prop, a[name])
        raise AttributeError, '%s not found' % name

    def set_key_value(self, obj, name, value):
        self.domain.put_attributes(obj.id, {name : value}, replace=True)

    def get_key_value(self, obj, name):
        a = self.domain.get_attributes(obj.id, name)
        if a.has_key(name):
            return a[name]
        else:
            return None

    def get_raw_item(self, obj):
        return self.domain.get_item(obj.id)

<commit_msg>fixed a bug in set_property.<commit_after># Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish, dis-
# tribute, sublicense, and/or sell copies of the Software, and to permit
# persons to whom the Software is furnished to do so, subject to the fol-
# lowing conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
import boto
from boto.utils import find_class
import datetime
from boto.sdb.db.key import Key

def object_lister(cls, query_lister, manager):
    for item in query_lister:
        if cls:
            yield cls(item.name)
        else:
            o = manager.get_object_from_id(item.name)
            if o:
                yield o

ISO8601 = '%Y-%m-%dT%H:%M:%SZ'

class SDBManager(object):

    def __init__(self, domain_name=None, aws_access_key_id=None, aws_secret_access_key=None, debug=0):
        self.domain_name = domain_name
        self.aws_access_key_id = aws_access_key_id
        self.aws_secret_access_key = aws_secret_access_key
        self.domain = None
        self.sdb = None
        self.s3 = None
        if not self.domain_name:
            self.domain_name = self.DefaultDomainName
            if self.domain_name:
                boto.log.info('No SimpleDB domain set, using default_domain: %s' % self.domain_name)
            else:
                boto.log.warning('No SimpleDB domain set, persistance is disabled')
        if self.domain_name:
            self.sdb = boto.connect_sdb(aws_access_key_id=self.aws_access_key_id,
                                        aws_secret_access_key=self.aws_secret_access_key,
                                        debug=debug)
            self.domain = self.sdb.lookup(self.domain_name)
            if not self.domain:
                self.domain = self.sdb.create_domain(self.domain_name)

    class Converter:
        """
        Responsible for converting base Python types to format compatible with underlying
        database.  For SimpleDB, that means everything needs to be converted to a string
        when stored in SimpleDB and from a string when retrieved.

        To convert a value, pass it to the encode or decode method.  The encode method
        will take a Python native value and convert to DB format.  The decode method will
        take a DB format value and convert it to Python native format.  To find the appropriate
        method to call, the generic encode/decode methods will look for the type-specific
        method by searching for a method called "encode_<type name>" or "decode_<type name>".
        """
        @classmethod
        def encode(cls, manager, prop, value):
            if hasattr(prop, 'reference_class'):
                return cls.encode_reference(manager, value)
            if isinstance(value, str) or isinstance(value, unicode):
                return value
            if isinstance(value, int) or isinstance(value, long):
                return cls.encode_int(manager, value)
            if isinstance(value, bool):
                return cls.encode_bool(manager, value)
            if isinstance(value, datetime.datetime):
                return cls.encode_datetime(manager, value)

        @classmethod
        def decode(cls, manager, prop, value):
            if isinstance(prop.data_type, str) or isinstance(prop.data_type, unicode):
                return value
            if isinstance(prop.data_type, int) or isinstance(prop.data_type, long):
                return cls.decode_int(manager, value)
            if isinstance(prop.data_type, bool):
                return cls.decode_bool(manager, value)
            if isinstance(prop.data_type, datetime.datetime):
                return cls.decode_datetime(manager, value)
            if isinstance(prop.data_type, Key):
                return cls.decode_reference(manager, value)

        @classmethod
        def encode_int(cls, manager, value):
            value = long(value)
            value += 9223372036854775808
            return '%020d' % value

        @classmethod
        def decode_int(cls, manager, value):
            value = long(value)
            value -= 9223372036854775808
            return value

        @classmethod
        def encode_bool(cls, manager, value):
            if value == True:
                return 'true'
            else:
                return 'false'

        @classmethod
        def decode_bool(cls, manager, value):
            if value.lower() == 'true':
                return True
            else:
                return False

        @classmethod
        def encode_datetime(cls, manager, value):
            return value.strftime(ISO8601)

        @classmethod
        def decode_datetime(cls, manager, value):
            try:
                return datetime.strptime(value, ISO8601)
            except:
                raise ValueError, 'Unable to convert %s to DateTime' % str_value

        @classmethod
        def encode_reference(cls, manager, value):
            if isinstance(value, str) or isinstance(value, unicode):
                return value
            if value == None:
                return ''
            else:
                return value.id

        @classmethod
        def decode_reference(cls, manager, value):
            if not value:
                return None
            try:
                return manager.get_object_from_id(value)
            except:
                raise ValueError, 'Unable to convert %s to Object' % str_value

    def get_s3_connection(self):
        if not self.s3:
            self.s3 = boto.connect_s3(self.aws_access_key_id, self.aws_secret_access_key)
        return self.s3

    def get_object(self, cls, id):
        a = self.domain.get_attributes(id, '__type__')
        if a.has_key('__type__'):
            return cls(id)
        else:
            raise SDBPersistenceError('%s object with id=%s does not exist' % (cls.__name__, id))

    def get_object_from_id(self, id):
        attrs = self.domain.get_attributes(id, ['__module__', '__type__', '__lineage__'])
        try:
            cls = find_class(attrs['__module__'], attrs['__type__'])
            return cls(id, manager=self)
        except ImportError:
            return None

    def query(self, cls, filters):
        if len(filters) > 4:
            raise SDBPersistenceError('Too many filters, max is 4')
        parts = ["['__type__'='%s'] union ['__lineage__'starts-with'%s']" % (cls.__name__, cls.get_lineage())]
        properties = cls.properties()
        for filter, value in filters:
            name, op = filter.strip().split()
            found = False
            for property in properties:
                if property.name == name:
                    found = True
                    value = self.encode_value(property, value)
                    parts.append("['%s' %s '%s']" % (name, op, value))
            if not found:
                raise SDBPersistenceError('%s is not a valid field' % key)
        query = ' intersection '.join(parts)
        rs = self.domain.query(query)
        return object_lister(cls, rs, self)

    def save_object(self, obj):
        attrs = {'__type__' : obj.__class__.__name__,
                 '__module__' : obj.__class__.__module__,
                 '__lineage__' : obj.get_lineage()}
        for property in obj.properties():
            if not property.__class__.__name__.startswith('_'):
                attrs[property.name] = property.get_value_for_datastore(obj)
        self.domain.put_attributes(obj.id, attrs, replace=True)

    def delete_object(self, obj):
        self.domain.delete_attributes(obj.id)

    def encode_value(self, prop, value):
        return self.Converter.encode(self, prop, value)

    def set_property(self, prop, obj, name, value):
        value = self.encode_value(prop, value)
        self.domain.put_attributes(obj.id, {name : value}, replace=True)

    def decode_value(self, prop, value):
        return self.Converter.decode(self, prop, value)

    def get_property(self, prop, obj, name):
        a = self.domain.get_attributes(obj.id, name)
        # try to get the attribute value from SDB
        if name in a:
            return self.decode_value(prop, a[name])
        raise AttributeError, '%s not found' % name

    def set_key_value(self, obj, name, value):
        self.domain.put_attributes(obj.id, {name : value}, replace=True)

    def get_key_value(self, obj, name):
        a = self.domain.get_attributes(obj.id, name)
        if a.has_key(name):
            return a[name]
        else:
            return None

    def get_raw_item(self, obj):
        return self.domain.get_item(obj.id)

<|endoftext|>


<commit_before>from . import cfg
from . import gs
from . import kv
from . import vu
import sublime
import threading
import time

DOMAIN = 'gs.ev'

class Event(object):
	def __init__(self):
		self.lst = []
		self.lck = threading.Lock()
		self.post_add = None

	def __call__(self, *args, **kwargs):
		with self.lck:
			l = self.lst[:]

		for f in l:
			try:
				f(*args, **kwargs)
			except Exception:
				gs.print_traceback()

		return self

	def __iadd__(self, f):
		if not gs.callable(f):
			return

		with self.lck:
			self.lst.append(f)

		if self.post_add:
			try:
				self.post_add(self, f)
			except Exception:
				gs.print_traceback()

		return self

	def __isub__(self, f):
		with self.lck:
			self.lst.remove(f)

		return self

class Signal(Event):
	def __init__(self, delay_ms):
		Event.__init__(self)

		self.n = 0

		self.delay = max(10, delay_ms)
		self.deadline = 0
		self.running = False

	def __call__(self):
		with self.lck:
			self.deadline = (self.tms() + self.delay)

		self._sched()

	def tms(self):
		return int(time.time() * 1000)

	def _sched(self, ms=0):
		with self.lck:
			if not self.running:
				if ms <= 0:
					ms = self.delay

				self.running = True
				gs.do(DOMAIN, self._timedout, ms)

	def _timedout(self):
		with self.lck:
			self.running = False
			deadline = self.deadline

		ms = deadline - self.tms()
		if ms <= 0:
			Event.__call__(self)
		else:
			self._sched(ms)

kvs = kv.M()

def kvm(k):
	return kvs.get(k, lambda: (kv.M(), True))

def df_act_sig(view):
	def f():
		gs.do(DOMAIN, lambda: view_activated(view))
		gs.do(DOMAIN, lambda: lc(view))

	sig = Signal(1000)
	sig += f
	return (sig, True)

def sig_act(view):
	if ignore_view(view):
		return

	sig = kvm(view.id()).get('act-sig', lambda: df_act_sig(view))
	sig()

def df_mod_sig(view):
	def f():
		gs.do(DOMAIN, lambda: view_updated(view))

	sig = Signal(1000)
	sig += f
	return (sig, True)

def sig_mod(view):
	if ignore_view(view):
		return

	sig = kvm(view.id()).get('mod-sig', lambda: df_mod_sig(view))
	sig()

def lc(view):
	m = kvm(view.id())
	sel = gs.sel(view).begin()
	row, _ = view.rowcol(sel)
	if m.put('last-row', row) != row:
		gs.do(DOMAIN, lambda: line_changed(view))

def df_mov_sig(view):
	def f():
		gs.do(DOMAIN, lambda: cursor_moved(view))
		gs.do(DOMAIN, lambda: lc(view))

	sig = Signal(500)
	sig += f
	return (sig, True)

def sig_mov(view, reset_last_row=False):
	if ignore_view(view):
		return

	m = kvm(view.id())

	if reset_last_row:
		m.delete('last-row')

	sig = m.get('mov-sig', lambda: df_mov_sig(view))
	sig()

def ignore_view(view):
	return not vu.V(view).perma()

def df_sav_sig(view):
	def f():
		gs.do(DOMAIN, lambda: file_saved(view))
		gs.do(DOMAIN, lambda: file_sync(view))

	sig = Signal(100)
	sig += f
	return (sig, True)

def sig_sav(view):
	if ignore_view(view):
		return

	m = kvm(view.id())

	sig = m.get('sav-sig', lambda: df_sav_sig(view))
	sig()

def df_lod_sig(view):
	def f():
		gs.do(DOMAIN, lambda: file_loaded(view))
		gs.do(DOMAIN, lambda: file_sync(view))

	sig = Signal(1000)
	sig += f
	return (sig, True)

def sig_lod(view):
	if ignore_view(view):
		return

	m = kvm(view.id())

	sig = m.get('lod-sig', lambda: df_lod_sig(view))
	sig()

def sublime_event(k, view):
	if k in ('on_activated', 'on_post_save', 'on_load', 'on_close'):
		gs.do(DOMAIN, cfg.sync_vv(vu.active()))

	if not opts.margo_ready and k in ('on_load', 'on_activated'):
		return

	sig = ev_map.get(k)
	if sig:
		sig(view)

def margo_ready():
	opts.margo_ready = True

	for win in sublime.windows():
		for view in win.views():
			if not view.is_loading():
				sig_lod(view)
				# at this point ST might still be loading, so we don't know what the real active view
				# will be... but it should be safe to sig_act all loaded views (ST doesn't that anyway)
				sig_act(view)

	gs.ready = True
	ready()

opts = kv.O(
	margo_ready = False
)

debug = Event()
init = Event()
view_updated = Event()
view_activated = Event()
file_saved = Event()
file_loaded = Event()
file_sync = Event()
line_changed = Event()
cursor_moved = Event()
ready = Event()

ev_map = {
	'on_post_save': sig_sav,
	'on_activated': sig_act,
	'on_load': sig_lod,
	'on_modified': sig_mod,
	'on_selection_modified': sig_mov,
}

# todo: maybe impl view_closed... it's a little complex because closed views are treated as temp
# view and thus ignored...

<commit_msg>* add a fast view_updated event, for experimentation<commit_after>from . import cfg
from . import gs
from . import kv
from . import vu
import sublime
import threading
import time

DOMAIN = 'gs.ev'

class Event(object):
	def __init__(self):
		self.lst = []
		self.lck = threading.Lock()
		self.post_add = None

	def __call__(self, *args, **kwargs):
		with self.lck:
			l = self.lst[:]

		for f in l:
			try:
				f(*args, **kwargs)
			except Exception:
				gs.print_traceback()

		return self

	def __iadd__(self, f):
		if not gs.callable(f):
			return

		with self.lck:
			self.lst.append(f)

		if self.post_add:
			try:
				self.post_add(self, f)
			except Exception:
				gs.print_traceback()

		return self

	def __isub__(self, f):
		with self.lck:
			self.lst.remove(f)

		return self

class Signal(Event):
	def __init__(self, delay_ms):
		Event.__init__(self)

		self.n = 0

		self.delay = max(10, delay_ms)
		self.deadline = 0
		self.running = False

	def __call__(self):
		with self.lck:
			self.deadline = (self.tms() + self.delay)

		self._sched()

	def tms(self):
		return int(time.time() * 1000)

	def _sched(self, ms=0):
		with self.lck:
			if not self.running:
				if ms <= 0:
					ms = self.delay

				self.running = True
				gs.do(DOMAIN, self._timedout, ms)

	def _timedout(self):
		with self.lck:
			self.running = False
			deadline = self.deadline

		ms = deadline - self.tms()
		if ms <= 0:
			Event.__call__(self)
		else:
			self._sched(ms)

kvs = kv.M()

def kvm(k):
	return kvs.get(k, lambda: (kv.M(), True))

def df_act_sig(view):
	def f():
		gs.do(DOMAIN, lambda: view_activated(view))
		gs.do(DOMAIN, lambda: lc(view))

	sig = Signal(1000)
	sig += f
	return (sig, True)

def sig_act(view):
	if ignore_view(view):
		return

	sig = kvm(view.id()).get('act-sig', lambda: df_act_sig(view))
	sig()

def df_mod_sig(view):
	def f():
		gs.do(DOMAIN, lambda: view_updated(view))

	sig = Signal(1000)
	sig += f
	return (sig, True)

def df_mod_fast_sig(view):
	def f():
		gs.do(DOMAIN, lambda: view_updated_fast(view))

	sig = Signal(250)
	sig += f
	return (sig, True)

def sig_mod(view):
	if ignore_view(view):
		return

	sig = kvm(view.id()).get('mod-sig', lambda: df_mod_sig(view))
	sig()

	fast_sig = kvm(view.id()).get('mod-fast-sig', lambda: df_mod_fast_sig(view))
	fast_sig()

def lc(view):
	m = kvm(view.id())
	sel = gs.sel(view).begin()
	row, _ = view.rowcol(sel)
	if m.put('last-row', row) != row:
		gs.do(DOMAIN, lambda: line_changed(view))

def df_mov_sig(view):
	def f():
		gs.do(DOMAIN, lambda: cursor_moved(view))
		gs.do(DOMAIN, lambda: lc(view))

	sig = Signal(500)
	sig += f
	return (sig, True)

def sig_mov(view, reset_last_row=False):
	if ignore_view(view):
		return

	m = kvm(view.id())

	if reset_last_row:
		m.delete('last-row')

	sig = m.get('mov-sig', lambda: df_mov_sig(view))
	sig()

def ignore_view(view):
	return not vu.V(view).perma()

def df_sav_sig(view):
	def f():
		gs.do(DOMAIN, lambda: file_saved(view))
		gs.do(DOMAIN, lambda: file_sync(view))

	sig = Signal(100)
	sig += f
	return (sig, True)

def sig_sav(view):
	if ignore_view(view):
		return

	m = kvm(view.id())

	sig = m.get('sav-sig', lambda: df_sav_sig(view))
	sig()

def df_lod_sig(view):
	def f():
		gs.do(DOMAIN, lambda: file_loaded(view))
		gs.do(DOMAIN, lambda: file_sync(view))

	sig = Signal(1000)
	sig += f
	return (sig, True)

def sig_lod(view):
	if ignore_view(view):
		return

	m = kvm(view.id())

	sig = m.get('lod-sig', lambda: df_lod_sig(view))
	sig()

def sublime_event(k, view):
	if k in ('on_activated', 'on_post_save', 'on_load', 'on_close'):
		gs.do(DOMAIN, cfg.sync_vv(vu.active()))

	if not opts.margo_ready and k in ('on_load', 'on_activated'):
		return

	sig = ev_map.get(k)
	if sig:
		sig(view)

def margo_ready():
	opts.margo_ready = True

	for win in sublime.windows():
		for view in win.views():
			if not view.is_loading():
				sig_lod(view)
				# at this point ST might still be loading, so we don't know what the real active view
				# will be... but it should be safe to sig_act all loaded views (ST doesn't that anyway)
				sig_act(view)

	gs.ready = True
	ready()

opts = kv.O(
	margo_ready = False
)

debug = Event()
init = Event()
view_updated = Event()
view_updated_fast = Event()
view_activated = Event()
file_saved = Event()
file_loaded = Event()
file_sync = Event()
line_changed = Event()
cursor_moved = Event()
ready = Event()

ev_map = {
	'on_post_save': sig_sav,
	'on_activated': sig_act,
	'on_load': sig_lod,
	'on_modified': sig_mod,
	'on_selection_modified': sig_mov,
}

# todo: maybe impl view_closed... it's a little complex because closed views are treated as temp
# view and thus ignored...

<|endoftext|>


<commit_before>from __future__ import unicode_literals
from future.builtins import bytes, str
import sys

from django.conf import settings as django_settings
from django.utils.unittest import skipUnless

from mezzanine.conf import settings, registry, register_setting
from mezzanine.conf.models import Setting
from mezzanine.utils.tests import TestCase

class ConfTests(TestCase):

    @skipUnless(True, "Only run manually")
    def test_threading_race(self):
        import multiprocessing.pool
        import random
        from django.db import connections

        type_modifiers = {int: lambda s: s + 1,
                          float: lambda s: s + 1.0,
                          bool: lambda s: not s,
                          str: lambda s: s + u"test",
                          bytes: lambda s: s + b"test"}

        # Store a non-default value for every editable setting in the database
        editable_settings = {}
        for setting in registry.values():
            if setting["editable"]:
                modified = type_modifiers[setting["type"]](setting["default"])
                Setting.objects.create(name=setting["name"], value=modified)
                editable_settings[setting["name"]] = modified

        # Make our child threads use this thread's connections. Recent SQLite
        # do support access from multiple threads for in-memory databases, but
        # Django doesn't support it currently - so we have to resort to this
        # workaround, taken from Django's LiveServerTestCase.
        # See Django ticket #12118 for discussion.
        connections_override = {}
        for conn in connections.all():
            # If using in-memory sqlite databases, pass the connections to
            # the server thread.
            if (conn.vendor == 'sqlite'
                    and conn.settings_dict['NAME'] == ':memory:'):
                # Explicitly enable thread-shareability for this connection
                conn.allow_thread_sharing = True
                connections_override[conn.alias] = conn

        def initialise_thread():
            for alias, connection in connections_override.items():
                connections[alias] = connection

        thread_pool = multiprocessing.pool.ThreadPool(8, initialise_thread)

        def retrieve_setting(setting_name):
            settings.use_editable()
            return setting_name, getattr(settings, setting_name)

        def choose_random_setting(length=5000):
            choices = list(editable_settings)
            for _ in range(length):
                yield random.choice(choices)

        for result in thread_pool.imap_unordered(retrieve_setting,
                                                 choose_random_setting()):
            setting_name, retrieved_value = result
            if retrieved_value != editable_settings[setting_name]:
                self.fail("Setting race condition encountered")

    @skipUnless(sys.version_info[0] == 2,
                "Randomly fails or succeeds under Python 3 as noted in "
                "GH #858 - please fix.")
    def test_settings(self):
        """
        Test that an editable setting can be overridden with a DB
        value and that the data type is preserved when the value is
        returned back out of the DB. Also checks to ensure no
        unsupported types are defined for editable settings.
        """
        # Find an editable setting for each supported type.
        names_by_type = {}
        for setting in registry.values():
            if setting["editable"] and setting["type"] not in names_by_type:
                names_by_type[setting["type"]] = setting["name"]
        # Create a modified value for each setting and save it.
        values_by_name = {}
        for (setting_type, setting_name) in names_by_type.items():
            setting_value = registry[setting_name]["default"]
            if setting_type in (int, float):
                setting_value += 1
            elif setting_type is bool:
                setting_value = not setting_value
            elif setting_type is str:
                setting_value += u"test"
            elif setting_type is bytes:
                setting_value += b"test"
            else:
                setting = "%s: %s" % (setting_name, setting_type)
                self.fail("Unsupported setting type for %s" % setting)
            values_by_name[setting_name] = setting_value
            Setting.objects.create(name=setting_name, value=setting_value)
        # Load the settings and make sure the DB values have persisted.
        settings.use_editable()
        for (name, value) in values_by_name.items():
            self.assertEqual(getattr(settings, name), value)

    def test_editable_override(self):
        """
        Test that an editable setting is always overridden by a settings.py
        setting of the same name.
        """
        Setting.objects.all().delete()
        django_settings.FOO = "Set in settings.py"
        db_value = Setting(name="FOO", value="Set in database")
        db_value.save()
        settings.use_editable()
        first_value = settings.FOO
        settings.SITE_TITLE  # Triggers access?
        second_value = settings.FOO
        self.assertEqual(first_value, second_value)

    def test_bytes_conversion(self):
        register_setting(name="BYTES_TEST_SETTING", editable=True, default=b"")
        Setting.objects.create(name="BYTES_TEST_SETTING",
                               value="A unicode value")
        settings.use_editable()
        self.assertEqual(settings.BYTES_TEST_SETTING, b"A unicode value")
<commit_msg>Clear DB settings after test run<commit_after>from __future__ import unicode_literals
from future.builtins import bytes, str
import sys

from django.conf import settings as django_settings
from django.utils.unittest import skipUnless

from mezzanine.conf import settings, registry, register_setting
from mezzanine.conf.models import Setting
from mezzanine.utils.tests import TestCase

class ConfTests(TestCase):

    @skipUnless(True, "Only run manually")
    def test_threading_race(self):
        import multiprocessing.pool
        import random
        from django.db import connections

        type_modifiers = {int: lambda s: s + 1,
                          float: lambda s: s + 1.0,
                          bool: lambda s: not s,
                          str: lambda s: s + u"test",
                          bytes: lambda s: s + b"test"}

        # Store a non-default value for every editable setting in the database
        editable_settings = {}
        for setting in registry.values():
            if setting["editable"]:
                modified = type_modifiers[setting["type"]](setting["default"])
                Setting.objects.create(name=setting["name"], value=modified)
                editable_settings[setting["name"]] = modified

        # Make our child threads use this thread's connections. Recent SQLite
        # do support access from multiple threads for in-memory databases, but
        # Django doesn't support it currently - so we have to resort to this
        # workaround, taken from Django's LiveServerTestCase.
        # See Django ticket #12118 for discussion.
        connections_override = {}
        for conn in connections.all():
            # If using in-memory sqlite databases, pass the connections to
            # the server thread.
            if (conn.vendor == 'sqlite'
                    and conn.settings_dict['NAME'] == ':memory:'):
                # Explicitly enable thread-shareability for this connection
                conn.allow_thread_sharing = True
                connections_override[conn.alias] = conn

        def initialise_thread():
            for alias, connection in connections_override.items():
                connections[alias] = connection

        thread_pool = multiprocessing.pool.ThreadPool(8, initialise_thread)

        def retrieve_setting(setting_name):
            settings.use_editable()
            return setting_name, getattr(settings, setting_name)

        def choose_random_setting(length=5000):
            choices = list(editable_settings)
            for _ in range(length):
                yield random.choice(choices)

        for result in thread_pool.imap_unordered(retrieve_setting,
                                                 choose_random_setting()):
            setting_name, retrieved_value = result
            self.assertEqual(retrieved_value, editable_settings[setting_name])

        Setting.objects.all().delete()
        settings.use_editable()

    @skipUnless(sys.version_info[0] == 2,
                "Randomly fails or succeeds under Python 3 as noted in "
                "GH #858 - please fix.")
    def test_settings(self):
        """
        Test that an editable setting can be overridden with a DB
        value and that the data type is preserved when the value is
        returned back out of the DB. Also checks to ensure no
        unsupported types are defined for editable settings.
        """
        # Find an editable setting for each supported type.
        names_by_type = {}
        for setting in registry.values():
            if setting["editable"] and setting["type"] not in names_by_type:
                names_by_type[setting["type"]] = setting["name"]
        # Create a modified value for each setting and save it.
        values_by_name = {}
        for (setting_type, setting_name) in names_by_type.items():
            setting_value = registry[setting_name]["default"]
            if setting_type in (int, float):
                setting_value += 1
            elif setting_type is bool:
                setting_value = not setting_value
            elif setting_type is str:
                setting_value += u"test"
            elif setting_type is bytes:
                setting_value += b"test"
            else:
                setting = "%s: %s" % (setting_name, setting_type)
                self.fail("Unsupported setting type for %s" % setting)
            values_by_name[setting_name] = setting_value
            Setting.objects.create(name=setting_name, value=setting_value)
        # Load the settings and make sure the DB values have persisted.
        settings.use_editable()
        for (name, value) in values_by_name.items():
            self.assertEqual(getattr(settings, name), value)

    def test_editable_override(self):
        """
        Test that an editable setting is always overridden by a settings.py
        setting of the same name.
        """
        Setting.objects.all().delete()
        django_settings.FOO = "Set in settings.py"
        db_value = Setting(name="FOO", value="Set in database")
        db_value.save()
        settings.use_editable()
        first_value = settings.FOO
        settings.SITE_TITLE  # Triggers access?
        second_value = settings.FOO
        self.assertEqual(first_value, second_value)

    def test_bytes_conversion(self):
        register_setting(name="BYTES_TEST_SETTING", editable=True, default=b"")
        Setting.objects.create(name="BYTES_TEST_SETTING",
                               value="A unicode value")
        settings.use_editable()
        self.assertEqual(settings.BYTES_TEST_SETTING, b"A unicode value")
<|endoftext|>


<commit_before>from . import about
from . import ev
from . import gs
from collections import namedtuple
import os
import re
import string
import subprocess
import time

try:
	STARTUP_INFO = subprocess.STARTUPINFO()
	STARTUP_INFO.dwFlags |= subprocess.STARTF_USESHOWWINDOW
	STARTUP_INFO.wShowWindow = subprocess.SW_HIDE
except (AttributeError):
	STARTUP_INFO = None

Result = namedtuple('Result', 'out err ok exc')

class _command(object):
	def __init__(self):
		self.input = None
		self.stdin = subprocess.PIPE
		self.stdout = subprocess.PIPE
		self.stderr = subprocess.PIPE
		self.startupinfo = STARTUP_INFO
		self.wd = None
		self.env = {}

	def run(self):
		if self.wd:
			wd = self.wd
			try:
				os.makedirs(wd)
			except Exception:
				pass
		else:
			wd = None

		if self.input:
			input = gs.astr(self.input)
		else:
			input = None

		try:
			setsid = os.setsid
		except Exception:
			setsid = None

		out = ''
		err = ''
		exc = None

		nv = env(self.env)
		try:
			cmd_lst = self.cmd(nv)
			orig_cmd = cmd_lst[0]
			cmd_lst[0] = _which(orig_cmd, nv.get('PATH'))
			if not cmd_lst[0]:
				raise Exception('Cannot find command `%s`' % orig_cmd)

			ev.debug('sh.run', {
				'orig_cmd': orig_cmd,
				'cmd_lst': cmd_lst,
				'env': nv,
			})

			out, err = subprocess.Popen(
				cmd_lst,
				stdout=self.stdout,
				stderr=self.stderr,
				stdin=self.stdin,
				startupinfo=gs.STARTUP_INFO,
				shell=False,
				env=nv,
				cwd=wd,
				preexec_fn=setsid,
				bufsize=0
			).communicate(input=input)
		except Exception as e:
			exc = e

		return Result(
			out=gs.ustr(out),
			err=gs.ustr(err),
			ok=(not exc),
			exc=exc
		)

class ShellCommand(_command):
	def __init__(self, cmd_str):
		_command.__init__(self)
		self.cmd_str = gs.astr(cmd_str)

	def cmd(self, e):
		return _cmd(self.cmd_str, e)

class Command(_command):
	def __init__(self, cmd_lst):
		_command.__init__(self)
		self.cmd_lst = [gs.astr(s) for s in cmd_lst]

	def cmd(self, e):
		return self.cmd_lst

def shl(m={}):
	return _shl(env(m))

def _shl(e):
	l = gs.setting('shell', [])
	if not l:
		fn = e.get('SHELL') or e.get('COMSPEC')
		if fn:
			name, _ = os.path.splitext(os.path.basename(fn))
			f = globals().get('_shl_%s' % name)
			if f:
				l = f(fn)

	if not l:
		if gs.os_is_windows():
			l = _shl_cmd('cmd')
		else:
			l = _shl_sh('sh')

	return l

def _shl_cmd(fn):
	return [fn, '/C', '${CMD}']

def _shl_sh(fn):
	return [fn, '-c', '${CMD}']

def _shl_fish(fn):
	return [fn, '-l', '-c', '${CMD}']

def _shl_bash(fn):
	return [fn, '-l', '-c', '${CMD}']

def _shl_zsh(fn):
	return [fn, '-l', '-c', '${CMD}']

def cmd(cmd_str, m={}):
	return _cmd(cmd_str, env(m))

def _cmd(cmd_str, e):
	cmdm = {'CMD': cmd_str}
	cmdl = []
	for s in _shl(e):
		s = string.Template(s).safe_substitute(cmdm)
		s = gs.astr(s)
		if s:
			cmdl.append(s)

	return cmdl

def init():
	global _env_ext

	start = time.time()

	vars = [
		'PATH',
		'GOARCH',
		'GOBIN',
		'GOCHAR',
		'GOEXE',
		'GOGCCFLAGS',
		'GOHOSTARCH',
		'GOHOSTOS',
		'GOOS',
		'GOPATH',
		'GOROOT',
		'GOTOOLDIR',
		'CGO_ENABLED',
	]

	cmdl = []
	for k in vars:
		cmdl.append('[[[$'+k+']]'+k+'[[%'+k+'%]]]')
	cmd_str = 'echo "%s"' % ' '.join(cmdl)

	cr = ShellCommand(cmd_str).run()
	if cr.exc:
		_print('error loading env vars: %s' % cr.exc)

	out = cr.out + cr.err

	mats = re.findall(r'\[\[\[(.*?)\]\](%s)\[\[(.*?)\]\]\]' % '|'.join(vars), out)
	for m in mats:
		a, k, b = m
		v = ''
		if a != '$'+k:
			v = a
		elif b != '%'+k+'%':
			v = b

		if v:
			_env_ext[k] = v

	_print('load env vars: %0.3fs' % (time.time() - start))

def _print(s):
	print('GoSblime %s sh: %s' % (about.VERSION, s))

def getenv(name, default='', m={}):
	return env(m).get(name, default)

def env(m={}):
	"""
	Assemble environment information needed for correct operation. In particular,
	ensure that directories containing binaries are included in PATH.
	"""
	e = os.environ.copy()
	e.update(_env_ext)
	e.update(m)

	roots = gs.lst(e.get('GOPATH', '').split(os.pathsep), e.get('GOROOT', ''))
	lfn = gs.attr('last_active_go_fn', '')

	comps = lfn.split(os.sep)
	gs_gopath = []
	for i, s in enumerate(comps):
		if s.lower() == "src":
			p = os.sep.join(comps[:i])
			if p not in roots:
				gs_gopath.append(p)
	gs_gopath.reverse()
	e['GS_GOPATH'] = os.pathsep.join(gs_gopath)

	uenv = gs.setting('env', {})
	for k in uenv:
		try:
			uenv[k] = string.Template(uenv[k]).safe_substitute(e)
		except Exception as ex:
			gs.println('%s: Cannot expand env var `%s`: %s' % (NAME, k, ex))

	e.update(uenv)
	e.update(m)

	# For custom values of GOPATH, installed binaries via go install
	# will go into the "bin" dir of the corresponding GOPATH path.
	# Therefore, make sure these paths are included in PATH.

	add_path = [gs.home_dir_path('bin')]

	for s in gs.lst(e.get('GOROOT', ''), e.get('GOPATH', '').split(os.pathsep)):
		if s:
			s = os.path.join(s, 'bin')
			if s not in add_path:
				add_path.append(s)

	gobin = e.get('GOBIN', '')
	if gobin and gobin not in add_path:
		add_path.append(gobin)

	if gs.os_is_windows():
		l = [
			'~\\bin',
			'~\\go\\bin',
			'C:\\Go\\bin',
		]
	else:
		l = [
			'~/bin',
			'~/go/bin',
			'/usr/local/go/bin',
			'/usr/local/opt/go/bin',
			'/usr/local/bin',
			'/usr/bin',
		]

	for s in l:
		s = os.path.expanduser(s)
		if s not in add_path:
			add_path.append(s)

	psep = gs.setting('shell_path_sep') or os.pathsep

	for s in e.get('PATH', '').split(psep):
		if s and s not in add_path:
			add_path.append(s)

	e['PATH'] = psep.join(add_path)

	# Ensure no unicode objects leak through. The reason is twofold:
	# 	* On Windows, Python 2.6 (used by Sublime Text) subprocess.Popen
	# 	  can only take bytestrings as environment variables in the
	#	  "env"	parameter. Reference:
	# 	  https://github.com/DisposaBoy/GoSublime/issues/112
	# 	  http://stackoverflow.com/q/12253014/1670
	#   * Avoids issues with networking too.
	clean_env = {}
	for k, v in e.items():
		try:
			clean_env[gs.astr(k)] = gs.astr(v)
		except Exception as ex:
			gs.println('%s: Bad env: %s' % (NAME, ex))

	return clean_env

def which_ok(fn):
	try:
		return os.path.isfile(fn) and os.access(fn, os.X_OK)
	except Exception:
		return False

def which(cmd):
	return _which(cmd, getenv('PATH', ''))

def _which(cmd, env_path):
	if os.path.isabs(cmd):
		return cmd if which_ok(cmd) else ''

	# not supporting PATHEXT. period.
	if gs.os_is_windows():
		cmd = '%s.exe' % cmd

	psep = gs.setting('shell_path_sep') or os.pathsep

	seen = {}
	for p in env_path.split(psep):
		p = os.path.join(p, cmd)
		if p not in seen and which_ok(p):
			return p

		seen[p] = True

	return ''

def go(subcmd_str):
	cr = ShellCommand('go '+subcmd_str).run()
	out = cr.out.strip() + '\n' + cr.err.strip()
	return out.strip()

_env_ext = {}
init()
<commit_msg>* call the default shell as a login shell (it should be safe enough and ~/.profile should then be loaded)<commit_after>from . import about
from . import ev
from . import gs
from collections import namedtuple
import os
import re
import string
import subprocess
import time

try:
	STARTUP_INFO = subprocess.STARTUPINFO()
	STARTUP_INFO.dwFlags |= subprocess.STARTF_USESHOWWINDOW
	STARTUP_INFO.wShowWindow = subprocess.SW_HIDE
except (AttributeError):
	STARTUP_INFO = None

Result = namedtuple('Result', 'out err ok exc')

class _command(object):
	def __init__(self):
		self.input = None
		self.stdin = subprocess.PIPE
		self.stdout = subprocess.PIPE
		self.stderr = subprocess.PIPE
		self.startupinfo = STARTUP_INFO
		self.wd = None
		self.env = {}

	def run(self):
		if self.wd:
			wd = self.wd
			try:
				os.makedirs(wd)
			except Exception:
				pass
		else:
			wd = None

		if self.input:
			input = gs.astr(self.input)
		else:
			input = None

		try:
			setsid = os.setsid
		except Exception:
			setsid = None

		out = ''
		err = ''
		exc = None

		nv = env(self.env)
		try:
			cmd_lst = self.cmd(nv)
			orig_cmd = cmd_lst[0]
			cmd_lst[0] = _which(orig_cmd, nv.get('PATH'))
			if not cmd_lst[0]:
				raise Exception('Cannot find command `%s`' % orig_cmd)

			ev.debug('sh.run', {
				'orig_cmd': orig_cmd,
				'cmd_lst': cmd_lst,
				'env': nv,
			})

			out, err = subprocess.Popen(
				cmd_lst,
				stdout=self.stdout,
				stderr=self.stderr,
				stdin=self.stdin,
				startupinfo=gs.STARTUP_INFO,
				shell=False,
				env=nv,
				cwd=wd,
				preexec_fn=setsid,
				bufsize=0
			).communicate(input=input)
		except Exception as e:
			exc = e

		return Result(
			out=gs.ustr(out),
			err=gs.ustr(err),
			ok=(not exc),
			exc=exc
		)

class ShellCommand(_command):
	def __init__(self, cmd_str):
		_command.__init__(self)
		self.cmd_str = gs.astr(cmd_str)

	def cmd(self, e):
		return _cmd(self.cmd_str, e)

class Command(_command):
	def __init__(self, cmd_lst):
		_command.__init__(self)
		self.cmd_lst = [gs.astr(s) for s in cmd_lst]

	def cmd(self, e):
		return self.cmd_lst

def shl(m={}):
	return _shl(env(m))

def _shl(e):
	l = gs.setting('shell', [])
	if not l:
		fn = e.get('SHELL') or e.get('COMSPEC')
		if fn:
			name, _ = os.path.splitext(os.path.basename(fn))
			f = globals().get('_shl_%s' % name)
			if f:
				l = f(fn)

	if not l:
		if gs.os_is_windows():
			l = _shl_cmd('cmd')
		else:
			l = _shl_sh('sh')

	return l

def _shl_cmd(fn):
	return [fn, '/C', '${CMD}']

def _shl_sh(fn):
	return [fn, '-l', '-c', '${CMD}']

def _shl_fish(fn):
	return [fn, '-l', '-c', '${CMD}']

def _shl_bash(fn):
	return [fn, '-l', '-c', '${CMD}']

def _shl_zsh(fn):
	return [fn, '-l', '-c', '${CMD}']

def cmd(cmd_str, m={}):
	return _cmd(cmd_str, env(m))

def _cmd(cmd_str, e):
	cmdm = {'CMD': cmd_str}
	cmdl = []
	for s in _shl(e):
		s = string.Template(s).safe_substitute(cmdm)
		s = gs.astr(s)
		if s:
			cmdl.append(s)

	return cmdl

def init():
	global _env_ext

	start = time.time()

	vars = [
		'PATH',
		'GOARCH',
		'GOBIN',
		'GOCHAR',
		'GOEXE',
		'GOGCCFLAGS',
		'GOHOSTARCH',
		'GOHOSTOS',
		'GOOS',
		'GOPATH',
		'GOROOT',
		'GOTOOLDIR',
		'CGO_ENABLED',
	]

	cmdl = []
	for k in vars:
		cmdl.append('[[[$'+k+']]'+k+'[[%'+k+'%]]]')
	cmd_str = 'echo "%s"' % ' '.join(cmdl)

	cr = ShellCommand(cmd_str).run()
	if cr.exc:
		_print('error loading env vars: %s' % cr.exc)

	out = cr.out + cr.err

	mats = re.findall(r'\[\[\[(.*?)\]\](%s)\[\[(.*?)\]\]\]' % '|'.join(vars), out)
	for m in mats:
		a, k, b = m
		v = ''
		if a != '$'+k:
			v = a
		elif b != '%'+k+'%':
			v = b

		if v:
			_env_ext[k] = v

	_print('load env vars: %0.3fs' % (time.time() - start))

def _print(s):
	print('GoSblime %s sh: %s' % (about.VERSION, s))

def getenv(name, default='', m={}):
	return env(m).get(name, default)

def env(m={}):
	"""
	Assemble environment information needed for correct operation. In particular,
	ensure that directories containing binaries are included in PATH.
	"""
	e = os.environ.copy()
	e.update(_env_ext)
	e.update(m)

	roots = gs.lst(e.get('GOPATH', '').split(os.pathsep), e.get('GOROOT', ''))
	lfn = gs.attr('last_active_go_fn', '')

	comps = lfn.split(os.sep)
	gs_gopath = []
	for i, s in enumerate(comps):
		if s.lower() == "src":
			p = os.sep.join(comps[:i])
			if p not in roots:
				gs_gopath.append(p)
	gs_gopath.reverse()
	e['GS_GOPATH'] = os.pathsep.join(gs_gopath)

	uenv = gs.setting('env', {})
	for k in uenv:
		try:
			uenv[k] = string.Template(uenv[k]).safe_substitute(e)
		except Exception as ex:
			gs.println('%s: Cannot expand env var `%s`: %s' % (NAME, k, ex))

	e.update(uenv)
	e.update(m)

	# For custom values of GOPATH, installed binaries via go install
	# will go into the "bin" dir of the corresponding GOPATH path.
	# Therefore, make sure these paths are included in PATH.

	add_path = [gs.home_dir_path('bin')]

	for s in gs.lst(e.get('GOROOT', ''), e.get('GOPATH', '').split(os.pathsep)):
		if s:
			s = os.path.join(s, 'bin')
			if s not in add_path:
				add_path.append(s)

	gobin = e.get('GOBIN', '')
	if gobin and gobin not in add_path:
		add_path.append(gobin)

	if gs.os_is_windows():
		l = [
			'~\\bin',
			'~\\go\\bin',
			'C:\\Go\\bin',
		]
	else:
		l = [
			'~/bin',
			'~/go/bin',
			'/usr/local/go/bin',
			'/usr/local/opt/go/bin',
			'/usr/local/bin',
			'/usr/bin',
		]

	for s in l:
		s = os.path.expanduser(s)
		if s not in add_path:
			add_path.append(s)

	psep = gs.setting('shell_path_sep') or os.pathsep

	for s in e.get('PATH', '').split(psep):
		if s and s not in add_path:
			add_path.append(s)

	e['PATH'] = psep.join(add_path)

	# Ensure no unicode objects leak through. The reason is twofold:
	# 	* On Windows, Python 2.6 (used by Sublime Text) subprocess.Popen
	# 	  can only take bytestrings as environment variables in the
	#	  "env"	parameter. Reference:
	# 	  https://github.com/DisposaBoy/GoSublime/issues/112
	# 	  http://stackoverflow.com/q/12253014/1670
	#   * Avoids issues with networking too.
	clean_env = {}
	for k, v in e.items():
		try:
			clean_env[gs.astr(k)] = gs.astr(v)
		except Exception as ex:
			gs.println('%s: Bad env: %s' % (NAME, ex))

	return clean_env

def which_ok(fn):
	try:
		return os.path.isfile(fn) and os.access(fn, os.X_OK)
	except Exception:
		return False

def which(cmd):
	return _which(cmd, getenv('PATH', ''))

def _which(cmd, env_path):
	if os.path.isabs(cmd):
		return cmd if which_ok(cmd) else ''

	# not supporting PATHEXT. period.
	if gs.os_is_windows():
		cmd = '%s.exe' % cmd

	psep = gs.setting('shell_path_sep') or os.pathsep

	seen = {}
	for p in env_path.split(psep):
		p = os.path.join(p, cmd)
		if p not in seen and which_ok(p):
			return p

		seen[p] = True

	return ''

def go(subcmd_str):
	cr = ShellCommand('go '+subcmd_str).run()
	out = cr.out.strip() + '\n' + cr.err.strip()
	return out.strip()

_env_ext = {}
init()
<|endoftext|>