<commit_before># Copyright (C) 2011  Francois Marier <francois@libravatar.org>
# Copyright (C) 2010  Francois Marier <francois@libravatar.org>
#                     Jonathan Harker <jon@jon.geek.nz>
#                     Brett Wilkins <bushido.katana@gmail.com>
#
# This file is part of Libravatar
#
# Libravatar is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Libravatar is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Libravatar.  If not, see <http://www.gnu.org/licenses/>.

from gearman import libgearman
from hashlib import md5, sha1, sha256
import json
import mimetypes
from os import link, unlink, urandom, path
from urllib2 import urlopen

from django.db import models
from django.contrib.auth.models import User

from libravatar import settings
from libravatar.account.external_photos import identica_photo, gravatar_photo
from libravatar.public.views import resized_avatar

DEFAULT_IMAGE_FORMAT = 'jpg'
MAX_LENGTH_IPV6 = 45 # http://stackoverflow.com/questions/166132
MAX_LENGTH_URL = 2048 # http://stackoverflow.com/questions/754547

def delete_if_exists(filename):
    if path.isfile(filename):
        unlink(filename)

def password_reset_key(user):
    return sha256(user.username + user.password).hexdigest()

def mimetype_format(mime_type):
    if 'image/jpeg' == mime_type:
        return 'jpg'
    elif 'image/png' == mime_type:
        return 'png'
    return None

def uploaded_image_format(image):
    '''
    Take an UploadedFile and guess its type (png or jpg)
    '''
    # Check the mimetype sent by the browser
    image_format = mimetype_format(image.content_type)
    if image_format:
        return image_format

    # Fallback on the filename of the uploaded file
    (mime_type, unused) = mimetypes.guess_type(image.name)
    image_format = mimetype_format(mime_type)
    if image_format:
        return image_format

    print "WARN: cannot identify the remote image type: path=%s" % image.name
    return DEFAULT_IMAGE_FORMAT

def remote_image_format(image_url):
    '''
    Take an URL and extract the last part of it (without the
    query_string) and hope that it contains a file extension.
    '''
    (mime_type, unused) = mimetypes.guess_type(image_url)
    image_format = mimetype_format(mime_type)
    if image_format:
        return image_format

    print "WARN: cannot identify the remote image type: url=%s" % image_url
    return DEFAULT_IMAGE_FORMAT

def change_photo(photo, md5_hash, sha1_hash, sha256_hash):
    '''
    Change the photo that the given hashes point to by deleting/creating hard links.
    '''
    # TODO: use git-like hashed directories to avoid too many files in one directory
    md5_filename = settings.AVATAR_ROOT + md5_hash
    sha1_filename = settings.AVATAR_ROOT + sha1_hash
    sha256_filename = settings.AVATAR_ROOT + sha256_hash

    # Remove old image
    delete_if_exists(md5_filename)
    delete_if_exists(sha1_filename)
    delete_if_exists(sha256_filename)

    # Delete all resized images
    for size in xrange(settings.AVATAR_MIN_SIZE, settings.AVATAR_MAX_SIZE):
        size_dir = settings.AVATAR_ROOT + '/%s/' % size

        delete_if_exists(size_dir + md5_hash)
        delete_if_exists(size_dir + sha1_hash)
        delete_if_exists(size_dir + sha256_hash)

    if not photo:
        return

    source_filename = settings.USER_FILES_ROOT + photo.full_filename()
    if not path.isfile(source_filename):
        # cropped photo doesn't exist, don't change anything
        return

    link(source_filename, md5_filename)
    link(source_filename, sha1_filename)
    link(source_filename, sha256_filename)

    # Generate resized images for common sizes
    for size in settings.AVATAR_PREGENERATED_SIZES:
        (resized_filename, unused) = resized_avatar(md5_hash, size)

        # TODO: these should go once it's automatically done in image.py
        output_dir = settings.AVATAR_ROOT + '/%s/' % size
        link(resized_filename, output_dir + sha1_hash)
        link(resized_filename, output_dir + sha256_hash)

class PhotoManager(models.Manager):
    def delete_user_photos(self, user):
        for photo in self.filter(user=user):
            photo.delete() # deletes the photo on disk as well

class Photo(models.Model):
    user = models.ForeignKey(User, related_name='photos')
    ip_address = models.CharField(max_length=MAX_LENGTH_IPV6)
    filename = models.CharField(max_length=64) # sha256 hash is 64 characters
    format = models.CharField(max_length=3) # png or jpg
    add_date = models.DateTimeField(auto_now_add=True)
    objects = PhotoManager()

    def __unicode__(self):
        return settings.USER_FILES_URL + self.full_filename()

    def uploaded_pathname(self):
        return settings.UPLOADED_FILES_URL + self.full_filename()

    def upload_datetime(self):
        return self.add_date.strftime('%Y-%m-%d %H:%M:%S')

    def full_filename(self):
        return self.filename + '.' + self.format

    def save(self, image, force_insert=False, force_update=False):
        self.format = uploaded_image_format(image)
        self.filename = sha256(urandom(1024) + str(self.user.username)).hexdigest()
        super(Photo, self).save(force_insert, force_update)

        # Write file to disk
        dest_filename = settings.UPLOADED_FILES_ROOT + self.full_filename()
        with open(dest_filename, 'wb+') as destination:
            # FIXME: HACK: temporarily disabling chunks for now - need to be able to write from a non-filesystem file object or find some other way to handle writing to file from a buffer in memory.
            destination.write(image.read())

    def import_image(self, service_name, email_address):
        image_url = False

        if 'Identica' == service_name:
            identica = identica_photo(email_address)
            if identica:
                image_url = identica['image_url']
        elif 'Gravatar' == service_name:
            gravatar = gravatar_photo(email_address)
            if gravatar:
                image_url = gravatar['image_url']

        if not image_url:
            return False

        self.format = remote_image_format(image_url)
        self.filename = sha256(service_name + email_address).hexdigest()
        super(Photo, self).save()

        dest_filename = settings.UPLOADED_FILES_ROOT + self.full_filename()
        image = urlopen(image_url)

        # Write file to disk
        destination = open(dest_filename, 'wb+')
        destination.write(image.read())
        destination.close()
        self.crop()

        return True

    def delete(self):
        # Remove links to this photo
        for email in self.emails.all():
            email.set_photo(None)
        for openid in self.openids.all():
            openid.set_photo(None)

        # Queue a job for the photo deletion gearman worker
        gm_client = libgearman.Client()
        for server in settings.GEARMAN_SERVERS:
            gm_client.add_server(server)

        workload = {'filename' : self.full_filename()}
        gm_client.do_background('deletephoto', json.dumps(workload))

        super(Photo, self).delete()

    def crop(self, x=0, y=0, w=0, h=0):
        if path.isfile(settings.USER_FILES_ROOT + self.full_filename()):
            return # already done, skip

        if not path.isfile(settings.UPLOADED_FILES_ROOT + self.full_filename()):
            return # source image doesn't exist, can't crop it

        # Queue a job for the cropping/resizing gearman worker
        gm_client = libgearman.Client()
        for server in settings.GEARMAN_SERVERS:
            gm_client.add_server(server)

        workload = {'filename' : self.full_filename(),
                    'x' : x, 'y' : y, 'w' : w, 'h' : h}
        gm_client.do_background('cropresize', json.dumps(workload))

class ConfirmedEmail(models.Model):
    user = models.ForeignKey(User, related_name='confirmed_emails')
    ip_address = models.CharField(max_length=MAX_LENGTH_IPV6)
    email = models.EmailField(unique=True)
    photo = models.ForeignKey(Photo, related_name='emails', blank=True, null=True)
    add_date = models.DateTimeField(auto_now_add=True)

    def __unicode__(self):
        return self.email

    def photo_url(self):
        if self.photo:
            return self.photo
        return settings.MEDIA_URL + '/img/' + settings.DEFAULT_PHOTO

    def public_hash(self, algorithm):
        if 'md5' == algorithm:
            return md5(self.email.lower()).hexdigest()
        elif 'sha1' == algorithm:
            return sha1(self.email.lower()).hexdigest()
        else:
            return sha256(self.email.lower()).hexdigest()

    def set_photo(self, photo):
        self.photo = photo
        change_photo(photo, self.public_hash('md5'), self.public_hash('sha1'), self.public_hash('sha256'))
        self.save()

    def delete(self):
        self.set_photo(None)
        super(ConfirmedEmail, self).delete()

class UnconfirmedEmail(models.Model):
    user = models.ForeignKey(User, related_name='unconfirmed_emails')
    email = models.EmailField()
    verification_key = models.CharField(max_length=64)
    add_date = models.DateTimeField(auto_now_add=True)

    def __unicode__(self):
        return self.email + ' (unconfirmed)'

    def save(self, force_insert=False, force_update=False):
        salted_username = urandom(1024) + str(self.user.username)
        key = sha256(salted_username).hexdigest()
        self.verification_key = key

        super(UnconfirmedEmail, self).save(force_insert, force_update)

class LinkedOpenId(models.Model):
    user = models.ForeignKey(User, related_name='openids')
    ip_address = models.CharField(max_length=MAX_LENGTH_IPV6)
    openid = models.URLField(unique=True, verify_exists=False, max_length=MAX_LENGTH_URL)
    photo = models.ForeignKey(Photo, related_name='openids', blank=True, null=True)
    add_date = models.DateTimeField(auto_now_add=True)

    def __unicode__(self):
        return self.openid

    def photo_url(self):
        if self.photo:
            return self.photo
        return settings.MEDIA_URL + '/img/' + settings.DEFAULT_PHOTO

    def public_hash(self, algorithm):
        # TODO: lowercase the domain part of the OpenID
        if 'md5' == algorithm:
            return md5(self.openid).hexdigest()
        elif 'sha1' == algorithm:
            return sha1(self.openid).hexdigest()
        else:
            return sha256(self.openid).hexdigest()

    def set_photo(self, photo):
        self.photo = photo
        change_photo(photo, self.public_hash('md5'), self.public_hash('sha1'), self.public_hash('sha256'))
        self.save()

    def delete(self):
        self.set_photo(None)
        super(LinkedOpenId, self).delete()
<commit_msg>models: reorder methods to please django-lint<commit_after># Copyright (C) 2011  Francois Marier <francois@libravatar.org>
# Copyright (C) 2010  Francois Marier <francois@libravatar.org>
#                     Jonathan Harker <jon@jon.geek.nz>
#                     Brett Wilkins <bushido.katana@gmail.com>
#
# This file is part of Libravatar
#
# Libravatar is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Libravatar is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Libravatar.  If not, see <http://www.gnu.org/licenses/>.

from gearman import libgearman
from hashlib import md5, sha1, sha256
import json
import mimetypes
from os import link, unlink, urandom, path
from urllib2 import urlopen

from django.db import models
from django.contrib.auth.models import User

from libravatar import settings
from libravatar.account.external_photos import identica_photo, gravatar_photo
from libravatar.public.views import resized_avatar

DEFAULT_IMAGE_FORMAT = 'jpg'
MAX_LENGTH_IPV6 = 45 # http://stackoverflow.com/questions/166132
MAX_LENGTH_URL = 2048 # http://stackoverflow.com/questions/754547

def delete_if_exists(filename):
    if path.isfile(filename):
        unlink(filename)

def password_reset_key(user):
    return sha256(user.username + user.password).hexdigest()

def mimetype_format(mime_type):
    if 'image/jpeg' == mime_type:
        return 'jpg'
    elif 'image/png' == mime_type:
        return 'png'
    return None

def uploaded_image_format(image):
    '''
    Take an UploadedFile and guess its type (png or jpg)
    '''
    # Check the mimetype sent by the browser
    image_format = mimetype_format(image.content_type)
    if image_format:
        return image_format

    # Fallback on the filename of the uploaded file
    (mime_type, unused) = mimetypes.guess_type(image.name)
    image_format = mimetype_format(mime_type)
    if image_format:
        return image_format

    print "WARN: cannot identify the remote image type: path=%s" % image.name
    return DEFAULT_IMAGE_FORMAT

def remote_image_format(image_url):
    '''
    Take an URL and extract the last part of it (without the
    query_string) and hope that it contains a file extension.
    '''
    (mime_type, unused) = mimetypes.guess_type(image_url)
    image_format = mimetype_format(mime_type)
    if image_format:
        return image_format

    print "WARN: cannot identify the remote image type: url=%s" % image_url
    return DEFAULT_IMAGE_FORMAT

def change_photo(photo, md5_hash, sha1_hash, sha256_hash):
    '''
    Change the photo that the given hashes point to by deleting/creating hard links.
    '''
    # TODO: use git-like hashed directories to avoid too many files in one directory
    md5_filename = settings.AVATAR_ROOT + md5_hash
    sha1_filename = settings.AVATAR_ROOT + sha1_hash
    sha256_filename = settings.AVATAR_ROOT + sha256_hash

    # Remove old image
    delete_if_exists(md5_filename)
    delete_if_exists(sha1_filename)
    delete_if_exists(sha256_filename)

    # Delete all resized images
    for size in xrange(settings.AVATAR_MIN_SIZE, settings.AVATAR_MAX_SIZE):
        size_dir = settings.AVATAR_ROOT + '/%s/' % size

        delete_if_exists(size_dir + md5_hash)
        delete_if_exists(size_dir + sha1_hash)
        delete_if_exists(size_dir + sha256_hash)

    if not photo:
        return

    source_filename = settings.USER_FILES_ROOT + photo.full_filename()
    if not path.isfile(source_filename):
        # cropped photo doesn't exist, don't change anything
        return

    link(source_filename, md5_filename)
    link(source_filename, sha1_filename)
    link(source_filename, sha256_filename)

    # Generate resized images for common sizes
    for size in settings.AVATAR_PREGENERATED_SIZES:
        (resized_filename, unused) = resized_avatar(md5_hash, size)

        # TODO: these should go once it's automatically done in image.py
        output_dir = settings.AVATAR_ROOT + '/%s/' % size
        link(resized_filename, output_dir + sha1_hash)
        link(resized_filename, output_dir + sha256_hash)

class PhotoManager(models.Manager):
    def delete_user_photos(self, user):
        for photo in self.filter(user=user):
            photo.delete() # deletes the photo on disk as well

class Photo(models.Model):
    user = models.ForeignKey(User, related_name='photos')
    ip_address = models.CharField(max_length=MAX_LENGTH_IPV6)
    filename = models.CharField(max_length=64) # sha256 hash is 64 characters
    format = models.CharField(max_length=3) # png or jpg
    add_date = models.DateTimeField(auto_now_add=True)
    objects = PhotoManager()

    def __unicode__(self):
        return settings.USER_FILES_URL + self.full_filename()

    def save(self, image, force_insert=False, force_update=False):
        self.format = uploaded_image_format(image)
        self.filename = sha256(urandom(1024) + str(self.user.username)).hexdigest()
        super(Photo, self).save(force_insert, force_update)

        # Write file to disk
        dest_filename = settings.UPLOADED_FILES_ROOT + self.full_filename()
        with open(dest_filename, 'wb+') as destination:
            # FIXME: HACK: temporarily disabling chunks for now - need to be able to write from a non-filesystem file object or find some other way to handle writing to file from a buffer in memory.
            destination.write(image.read())

    def delete(self):
        # Remove links to this photo
        for email in self.emails.all():
            email.set_photo(None)
        for openid in self.openids.all():
            openid.set_photo(None)

        # Queue a job for the photo deletion gearman worker
        gm_client = libgearman.Client()
        for server in settings.GEARMAN_SERVERS:
            gm_client.add_server(server)

        workload = {'filename' : self.full_filename()}
        gm_client.do_background('deletephoto', json.dumps(workload))

        super(Photo, self).delete()

    def uploaded_pathname(self):
        return settings.UPLOADED_FILES_URL + self.full_filename()

    def upload_datetime(self):
        return self.add_date.strftime('%Y-%m-%d %H:%M:%S')

    def full_filename(self):
        return self.filename + '.' + self.format

    def import_image(self, service_name, email_address):
        image_url = False

        if 'Identica' == service_name:
            identica = identica_photo(email_address)
            if identica:
                image_url = identica['image_url']
        elif 'Gravatar' == service_name:
            gravatar = gravatar_photo(email_address)
            if gravatar:
                image_url = gravatar['image_url']

        if not image_url:
            return False

        self.format = remote_image_format(image_url)
        self.filename = sha256(service_name + email_address).hexdigest()
        super(Photo, self).save()

        dest_filename = settings.UPLOADED_FILES_ROOT + self.full_filename()
        image = urlopen(image_url)

        # Write file to disk
        destination = open(dest_filename, 'wb+')
        destination.write(image.read())
        destination.close()
        self.crop()

        return True

    def crop(self, x=0, y=0, w=0, h=0):
        if path.isfile(settings.USER_FILES_ROOT + self.full_filename()):
            return # already done, skip

        if not path.isfile(settings.UPLOADED_FILES_ROOT + self.full_filename()):
            return # source image doesn't exist, can't crop it

        # Queue a job for the cropping/resizing gearman worker
        gm_client = libgearman.Client()
        for server in settings.GEARMAN_SERVERS:
            gm_client.add_server(server)

        workload = {'filename' : self.full_filename(),
                    'x' : x, 'y' : y, 'w' : w, 'h' : h}
        gm_client.do_background('cropresize', json.dumps(workload))

class ConfirmedEmail(models.Model):
    user = models.ForeignKey(User, related_name='confirmed_emails')
    ip_address = models.CharField(max_length=MAX_LENGTH_IPV6)
    email = models.EmailField(unique=True)
    photo = models.ForeignKey(Photo, related_name='emails', blank=True, null=True)
    add_date = models.DateTimeField(auto_now_add=True)

    def __unicode__(self):
        return self.email

    def delete(self):
        self.set_photo(None)
        super(ConfirmedEmail, self).delete()

    def photo_url(self):
        if self.photo:
            return self.photo
        return settings.MEDIA_URL + '/img/' + settings.DEFAULT_PHOTO

    def public_hash(self, algorithm):
        if 'md5' == algorithm:
            return md5(self.email.lower()).hexdigest()
        elif 'sha1' == algorithm:
            return sha1(self.email.lower()).hexdigest()
        else:
            return sha256(self.email.lower()).hexdigest()

    def set_photo(self, photo):
        self.photo = photo
        change_photo(photo, self.public_hash('md5'), self.public_hash('sha1'), self.public_hash('sha256'))
        self.save()

class UnconfirmedEmail(models.Model):
    user = models.ForeignKey(User, related_name='unconfirmed_emails')
    email = models.EmailField()
    verification_key = models.CharField(max_length=64)
    add_date = models.DateTimeField(auto_now_add=True)

    def __unicode__(self):
        return self.email + ' (unconfirmed)'

    def save(self, force_insert=False, force_update=False):
        salted_username = urandom(1024) + str(self.user.username)
        key = sha256(salted_username).hexdigest()
        self.verification_key = key

        super(UnconfirmedEmail, self).save(force_insert, force_update)

class LinkedOpenId(models.Model):
    user = models.ForeignKey(User, related_name='openids')
    ip_address = models.CharField(max_length=MAX_LENGTH_IPV6)
    openid = models.URLField(unique=True, verify_exists=False, max_length=MAX_LENGTH_URL)
    photo = models.ForeignKey(Photo, related_name='openids', blank=True, null=True)
    add_date = models.DateTimeField(auto_now_add=True)

    def __unicode__(self):
        return self.openid

    def delete(self):
        self.set_photo(None)
        super(LinkedOpenId, self).delete()

    def photo_url(self):
        if self.photo:
            return self.photo
        return settings.MEDIA_URL + '/img/' + settings.DEFAULT_PHOTO

    def public_hash(self, algorithm):
        # TODO: lowercase the domain part of the OpenID
        if 'md5' == algorithm:
            return md5(self.openid).hexdigest()
        elif 'sha1' == algorithm:
            return sha1(self.openid).hexdigest()
        else:
            return sha256(self.openid).hexdigest()

    def set_photo(self, photo):
        self.photo = photo
        change_photo(photo, self.public_hash('md5'), self.public_hash('sha1'), self.public_hash('sha256'))
        self.save()
<|endoftext|>


<commit_before>#
# MLDBFB-345_improve_error_message_named_on_null.py
# Mich, 2016-02-01
# This file is part of MLDB. Copyright 2016 Datacratic. All rights reserved.
#
mldb = mldb_wrapper.wrap(mldb) # noqa

class ImproveErrorMessageNamedOnNullTest(MldbUnitTest): # noqa

    @classmethod
    def setUpClass(cls):
        ds = mldb.create_dataset({
            'id' : 'ds',
            'type' : 'sparse.mutable'
        })

        ds.record_row('row1', [['behA', 'a', 0]])
        ds.record_row('row2', [['behB', 'b', 0]])
        ds.commit()

    def test_working_case(self):
        mldb.post('/v1/procedures', {
            'type' : 'transform',
            'params' : {
                'inputData' :
                    # works because we only work on the row where behA != null
                    'SELECT * NAMED behA FROM ds WHERE behA IS NOT NULL',
                'outputDataset' : {
                    'id': 'out1'
                },
                'runOnCreation' : True
            }
        })

    def test_non_working_case(self):
        with self.assertRaises(mldb_wrapper.ResponseException) as exc: # noqa
            mldb.post('/v1/procedures', {
                'type' : 'transform',
                'params' : {
                    'inputData' : 'SELECT * NAMED behA FROM ds',
                    'outputDataset' : {
                        'id': 'out2'
                    },
                    'runOnCreation' : True
                }
            })
        # FIXME: It's ok to fail, but the error message should be helpful
        # currently returns "Can't convert from empty to string"
        mldb.log(exc.exception.response.json())

if __name__ == '__main__':
    mldb.run_tests()
<commit_msg>MLDBFB-345 Update test to expect proper err msg<commit_after>#
# MLDBFB-345_improve_error_message_named_on_null.py
# Mich, 2016-02-01
# This file is part of MLDB. Copyright 2016 Datacratic. All rights reserved.
#
import unittest

mldb = mldb_wrapper.wrap(mldb) # noqa

class ImproveErrorMessageNamedOnNullTest(MldbUnitTest): # noqa

    @classmethod
    def setUpClass(cls):
        ds = mldb.create_dataset({
            'id' : 'ds',
            'type' : 'sparse.mutable'
        })

        ds.record_row('row1', [['behA', 'a', 0]])
        ds.record_row('row2', [['behB', 'b', 0]])
        ds.commit()

    def test_working_case(self):
        # works because we only work on the row where behA != null
        mldb.query('SELECT * NAMED behA FROM ds WHERE behA IS NOT NULL')

    @unittest.expectedFailure
    def test_non_working_case(self):
        # FIXME: It's ok to fail, but the error message should be helpful
        # currently returns "Can't convert from empty to string"
        expect = "Can't create a row with a null or empty name."
        with self.assertMldbRaises(expected_regexp=expect):
            mldb.query('SELECT * NAMED behA FROM ds')

if __name__ == '__main__':
    mldb.run_tests()
<|endoftext|>


<commit_before>from rest_framework import serializers as ser
from modularodm import Q
from modularodm.exceptions import NoResultsFound, MultipleResultsFound
from api.base.serializers import (
    JSONAPISerializer, RelationshipField, IDField, JSONAPIListField, LinksField
)
from website.models import Node
from api.base.utils import absolute_reverse
from api.nodes.serializers import NodeTagField, NodeContributorsSerializer

class PreprintSerializer(JSONAPISerializer):

    filterable_fields = frozenset([
        'id',
        'title',
        'tags',
        'date_created',
        'date_modified',
        'authors',
        'preprint_subjects'
    ])

    title = ser.CharField(required=False)
    subjects = JSONAPIListField(required=False, source='preprint_subjects')
    date_created = ser.DateTimeField(read_only=True, source='preprint_created')
    date_modified = ser.DateTimeField(read_only=True)
    id = IDField(source='_id', required=False)
    abstract = ser.CharField(source='description', required=False)
    tags = JSONAPIListField(child=NodeTagField(), required=False)

    primary_file = RelationshipField(
        related_view='files:file-detail',
        related_view_kwargs={'file_id': '<preprint_file._id>'},
    )

    files = RelationshipField(
        related_view='nodes:node-providers',
        related_view_kwargs={'node_id': '<pk>'}
    )

    links = LinksField({'self': 'get_preprint_url', 'html': 'get_absolute_html_url'})

    authors = RelationshipField(
        related_view='preprints:preprint-authors',
        related_view_kwargs={'node_id': '<pk>'},
        related_meta={'count': 'get_contrib_count'},
    )

    class Meta:
        type_ = 'preprints'

    def get_preprint_url(self, obj):
        return absolute_reverse('preprints:preprint-detail', kwargs={'node_id': obj._id})

    def get_absolute_url(self, obj):
        return self.get_preprint_url(obj)

    def create(self, validated_data):
        node = validated_data.get('node')
        # TODO - get correct fields from validated_data
        pass

class PreprintDetailSerializer(PreprintSerializer):
    id = IDField(source='_id', required=True)

class PreprintAuthorSerializer(NodeContributorsSerializer):
    class Meta:
        type_ = 'authors'<commit_msg>Add custom PrimaryFileRelationshipField to allow for getting the correct object and returning correct internal value, use the field for the primary file along with read_only=False, add create method that calls proper methods to add attributes to node to create a preprint<commit_after>from rest_framework import serializers as ser
from modularodm import Q

from api.base.serializers import (
    JSONAPISerializer, IDField, JSONAPIListField, LinksField, RelationshipField
)
from api.base.utils import absolute_reverse, get_user_auth
from api.nodes.serializers import NodeTagField, NodeContributorsSerializer
from website.models import StoredFileNode

class PrimaryFileRelationshipField(RelationshipField):
    def get_object(self, file_id):
        return StoredFileNode.find_one(Q('_id', 'eq', file_id))

    def to_internal_value(self, data):
        file = self.get_object(data)
        return {'primary_file': file}

class PreprintSerializer(JSONAPISerializer):

    filterable_fields = frozenset([
        'id',
        'title',
        'tags',
        'date_created',
        'date_modified',
        'authors',
        'preprint_subjects'
    ])

    title = ser.CharField(required=False)
    subjects = JSONAPIListField(required=False, source='preprint_subjects')
    date_created = ser.DateTimeField(read_only=True, source='preprint_created')
    date_modified = ser.DateTimeField(read_only=True)
    id = IDField(source='_id', required=False)
    abstract = ser.CharField(source='description', required=False)
    tags = JSONAPIListField(child=NodeTagField(), required=False)

    primary_file = PrimaryFileRelationshipField(
        related_view='files:file-detail',
        related_view_kwargs={'file_id': '<preprint_file._id>'},
        lookup_url_kwarg='file_id',
        read_only=False
    )

    files = RelationshipField(
        related_view='nodes:node-providers',
        related_view_kwargs={'node_id': '<pk>'}
    )

    links = LinksField({'self': 'get_preprint_url', 'html': 'get_absolute_html_url'})

    authors = RelationshipField(
        related_view='preprints:preprint-authors',
        related_view_kwargs={'node_id': '<pk>'},
        related_meta={'count': 'get_contrib_count'},
    )

    class Meta:
        type_ = 'preprints'

    def get_preprint_url(self, obj):
        return absolute_reverse('preprints:preprint-detail', kwargs={'node_id': obj._id})

    def get_absolute_url(self, obj):
        return self.get_preprint_url(obj)

    def create(self, validated_data):
        node = validated_data.pop('node')
        auth = get_user_auth(self.context['request'])
        node.set_preprint_file(validated_data.pop('primary_file')._id, auth)
        if node._id != validated_data.pop('_id'):
            raise TypeError
        for key, value in validated_data.iteritems():
            setattr(node, key, value)
        node.save()
        return node

class PreprintDetailSerializer(PreprintSerializer):
    id = IDField(source='_id', required=True)

class PreprintAuthorSerializer(NodeContributorsSerializer):
    class Meta:
        type_ = 'authors'<|endoftext|>


<commit_before>#!/usr/users/4/stbrown/bin/python
# Copyright 2012 University of Pittsburgh
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License.  You may obtain a copy of
# the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
# License for the specific language governing permissions and limitations under
# the License.
from SimulatorService_services import runRequest
'''
Created on Nov 27, 2012

This is an example GenericEpidemicModelService implementation.

@author: John Levander
'''

from ZSI.wstools import logging
from SimulatorService_services_server import SimulatorService
from ZSI.ServiceContainer import AsServer
from ApolloFactory import *
from ApolloUtils import *
from SimulatorService_services import *
import os,sys
import shutil
#import fred
import random
import datetime
import paramiko
from fredUtils import FredSSHConn,infectiousPeriod,incubationPeriod

fredConn = FredSSHConn()

class FredWebService(SimulatorService):
        _wsdl = "".join(open("simulatorservice.wsdl").readlines())

        factory = ApolloFactory()
        utils = ApolloUtils()

        def soap_getSupportedLocations(self, ps, **kw):
            response = SimulatorService.soap_getSupportedLocations(self, ps, **kw)

            spl = self.factory.new_SupportedPopulationLocation()

            spl._populationLocation = "42003"
            spl._populationLocationDescription = "Allegheny County"

            response._supportedPopluationLocations.append(spl)
            return response

        #this method checks on the status of a job, indicated by the jobs runId
        def soap_getRunStatus(self, ps, **kw):
            response = SimulatorService.soap_getRunStatus(self, ps, **kw)
            #get the run id of the job
            request = ps.Parse(getRunStatusRequest.typecode)
            runId = request._runId

            status, message = fredConn._getStatus(runId)
            response._runStatus = self.factory.new_RunStatus()
            print "Fred status is:" + status
            response._runStatus._status = status.lower()
            print "Fred message is:" + message
            response._runStatus._message = message

            return response

        #this method runs an epidemic model
        def soap_run(self, ps, **kw):
	    fredConn._setup()
            response = SimulatorService.soap_run(self, ps, **kw)
            request = ps.Parse(runRequest.typecode)
            #extract the epidemic model input object from the request
            print "received run request"
            cfg = request._simulatorConfiguration

            #get the number of time steps in the simulation
            run_length = cfg._simulatorTimeSpecification._runLength

            #log the EpidemicModelInput request to a log file
            self.utils.log_simulatorConfiguration(cfg)

            ### STB CREATE FRED INPUT FILE
            ### do some error checking for this version of the model
            if cfg._simulatorTimeSpecification._timeStepUnit.lower() != 'days':
                    print ('This verison of FRED only supports time_step_unit of days')
                    print ('Apollo sent %s'%(cfg._simulatorTimeSpecification._timeStepUnit))
                    #### Put in Error Handling, have to ask John L.

            if cfg._simulatorTimeSpecification._timeStepValue != 1:
                    print ('This version of FRED only supports a time_step_value of 1')
                    print ('Apollo sent %d'%(cfg._simulatorTimeSpecification._timeStepValue))

	    #### Put in Error Handling, have to ask John L.

            #print ('WARNING: This service is currently ignoring infectious_period')
            #print ('WARNING: This service is currently ignoring latent_period')

            ### Make a random directory name so that multiple calls can be made
            randID = random.randint(0,100000)
            tempDirName = "fred.tmp.%d"%(randID)
            try:
                    os.mkdir(tempDirName)
            except:
                    raise RuntimeError("Cannot make temporary FRED directory %s"%(tempDirName))

            os.chdir(tempDirName)
            doVacc = False
            with open('fred_input.params','wb') as f:
                    f.write('days = %d\n'%cfg._simulatorTimeSpecification._runLength)
                    f.write('symp[0] = %g\n'%cfg._disease._asymptomaticInfectionFraction)
                    f.write('R0 = %g\n'%cfg._disease._reproductionNumber)
                    f.write('primary_cases_file[0] = fred_initial_population_0.txt\n')
                    f.write('residual_immunity_ages[0] = 2 0 100\n')
		    f.write('days_latent[0] = %s\n'\
			    %(incubationPeriod(float(cfg._disease._latentPeriod))))
		    f.write('days_incubating[0] = %s\n'\
			    %(incubationPeriod(float(cfg._disease._latentPeriod))))
		    f.write('days_infectious[0] = %s\n'\
			    %(infectiousPeriod(float(cfg._disease._infectiousPeriod))))
		    f.write('days_symptomatic[0] = %s\n'\
			    %(infectiousPeriod(float(cfg._disease._infectiousPeriod))))
                    totPop = self.utils.getTotalPopCount(cfg)
                    numImmune = self.utils.getPopCountGivenLocationAndDiseaseState(cfg, "ignoreed", "recovered")
                    percent_immune = float(numImmune)/float(totPop)
                    f.write('residual_immunity_values[0] = 1 %g\n'%(percent_immune))
		    f.write('track_infection_events = 1\n')
		    if sum(cfg._vaccinationControlMeasure._vaccineSupplySchedule) > 0.0:
                        doVacc = True
                        #print ('WARNING: This sevice is currently ignoring vaccine_cm_compliance due to someone fucking FRED up')
                        f.write("##### VACCINE PARAMETERS\n")
                        f.write("enable_behaviors = 1\n")
                        f.write("enable_vaccination = 1\n")
                        f.write("number_of_vaccines = 1\n")
			f.write("accept_vaccine_enabled = 1\n")
			f.write("accept_vaccine_strategy_distribution = 7 %g %g 0 0 0 0 0\n"\
				%(1.0-cfg._vaccinationControlMeasure._vaccineCmCompliance,\
				  cfg._vaccinationControlMeasure._vaccineCmCompliance))
                        f.write("### VACCINE 1\n")
                        f.write("vaccine_number_of_doses[0] = 1\n")
                        # Setting this to infinity for now
                        # need to implement production as a time map as well
                        f.write("vaccine_total_avail[0] = 300000000\n")
                        f.write("vaccine_additional_per_day[0] = 300000000\n")
                        f.write("vaccine_starting_day[0] = 0\n")
                        f.write("##### Vaccine 1 Dose 0\n")
                        f.write("vaccine_next_dosage_day[0][0] = 0\n")
                        f.write("vaccine_dose_efficacy_ages[0][0] = 2 0 100\n")
                        f.write("vaccine_dose_efficacy_values[0][0] = 1 %g\n"\
                                %cfg._vaccinationControlMeasure._vaccineEfficacy)
                        f.write("vaccine_dose_efficacy_delay_ages[0][0] = 2 0 100\n")
                        f.write("vaccine_dose_efficacy_delay_values[0][0] = 1 %d\n"\
                                %cfg._vaccinationControlMeasure._vaccineEfficacyDelay)
                        f.write("vaccination_capacity_file = fred-vaccination-schedule_0.txt\n")

                        with open('fred-vaccination-schedule_0.txt','wb') as g:
                            vaccineSupplySchedule = cfg._vaccinationControlMeasure._vaccineSupplySchedule
                            vaccineAdminSchedule = cfg._vaccinationControlMeasure._vaccinationAdminSchedule

                            for day in range(0,len(vaccineSupplySchedule)) :
                                ## Hack for now, because I don't have the resolution on production in FRED
                                g.write("%d %d\n"%(day,min(vaccineSupplySchedule[day],vaccineAdminSchedule[day])))

            with open('fred_initial_population_0.txt','wb') as f:
                    f.write('#line_format\n')
                    numInfectious = self.utils.getPopCountGivenLocationAndDiseaseState(cfg, "ignored", "exposed")
                    #f.write('0 0 %d\n'%(cfg._disease_dynamics._pop_count[2]))
                    f.write('0 0 %d\n'%(numInfectious))

		     ### if vaccination comes through, parse this
            with open('starttime','wb') as f:
		    f.write('Nothing')

	    idPrefix = cfg._simulatorIdentification._simulatorDeveloper +\
		       "_" + cfg._simulatorIdentification._simulatorName +\
		       "_" + cfg._simulatorIdentification._simulatorVersion + "_"

            fredConn._connect()

            ### Write the PBS submission Script
            with open('fred_submit.pbs','wb') as f:
                    f.write('#!/bin/csh\n')
		    if fredConn.machine == "blacklight.psc.xsede.org":
                        f.write('#PBS -l ncpus=16\n')
		    else:
			f.write('#PBS -l nodes=2:ppn=8\n')
                    f.write('#PBS -N fred.pbs.out\n')
                    f.write('#PBS -l walltime=30:00\n')
                    f.write('#PBS -j oe\n')
	     	    if fredConn.machine == "blacklight.psc.xsede.org":
                   	 f.write('#PBS -q debug\n')
                    f.write('\n')
		    if fredConn.machine == "blacklight.psc.xsede.org":
                        f.write('source /usr/share/modules/init/csh\n')
                        f.write('source /etc/csh.cshrc.psc\n')
                    f.write('module load fred\n')
		    f.write('module load python\n')
                    f.write('cd $PBS_O_WORKDIR\n')
		    f.write('echo `date` > starttime\n')
                    f.write('### Get the PBS ID\n')
                    f.write("set words = `echo $PBS_JOBID | sed 's/\./ /g'`\n")
                    f.write("set id = $words[1]\n")
                    f.write('fred_job -p fred_input.params -n 8 -t 2 -m 8 -k %s$id\n'%idPrefix)
		    f.write('touch .dbloading\n')
                    f.write('python $FRED_HOME/bin/fred_to_apollo_parallel.py -k %s$id\n'%(idPrefix))
                    f.write('rm -rf .dbloading\n')
                    #f.write('echo COMPLETED > job_status')

            ### for a direct Job.. write the csh script to run FRED
	    with open('fred_run.csh','wb') as f:
		f.write('#!/bin/csh\n')
		f.write('echo `date` > starttime\n')
		f.write('### Generate Timestamp for ID\n')
		f.write('fred_job -p fred_input.params -n 4 -t 2 -m 4 -k %s$id > out.run\n'%idPrefix)
		f.write('touch .dbloading\n')
		f.write('python $FRED_HOME/bin/fred_to_apollo_parallel.py -k %s$id >& out.db\n'%idPrefix)
		f.write('rm -rf .dbloading\n')

            os.chdir('../')
            #fredConn._setup("warhol.psc.edu",username="stbrown",privateKeyFile="./id_rsa")
            #fredConn._connect()
            #print "Created Connection"
            fredConn._mkdir(tempDirName)
            fredConn._sendFile(tempDirName+'/fred_submit.pbs')
	    fredConn._sendFile(tempDirName+'/fred_run.csh')
            fredConn._sendFile(tempDirName+'/fred_input.params')
            fredConn._sendFile(tempDirName+'/fred_initial_population_0.txt')
            fredConn._sendFile(tempDirName+'/starttime')
	    if doVacc:
		    fredConn._sendFile(tempDirName+"/fred-vaccination-schedule_0.txt")

	    ### we do not need temp directory anymore
	    shutil.rmtree(tempDirName)

            print "Running FRED"
            sys.stdout.flush()

            #returnVal = fredConn._submitPBSJob(randID)
	    returnVal = fredConn._submitJob(randID)
            print returnVal

	    ### parse the run ID
	    response._runId = idPrefix + str(returnVal)

            #ssh.close()
            #sftp.close()

            return response

#logger = logging.getLogger("");
#logger.setLevel(0)

#run a webserver on 8087
AsServer(port=8087, services=[FredWebService('fred'), ])

<commit_msg>Added new Fred parameters that now allow us to control seeding of both exposed and infectious people.  Thanks to Jay Depasse for implementing this is Fred so quickly.<commit_after>#!/usr/users/4/stbrown/bin/python
# Copyright 2012 University of Pittsburgh
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License.  You may obtain a copy of
# the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
# License for the specific language governing permissions and limitations under
# the License.
from SimulatorService_services import runRequest
'''
Created on Nov 27, 2012

This is an example GenericEpidemicModelService implementation.

@author: John Levander
'''

from ZSI.wstools import logging
from SimulatorService_services_server import SimulatorService
from ZSI.ServiceContainer import AsServer
from ApolloFactory import *
from ApolloUtils import *
from SimulatorService_services import *
import os,sys
import shutil
#import fred
import random
import datetime
import paramiko
from fredUtils import FredSSHConn,infectiousPeriod,incubationPeriod

fredConn = FredSSHConn()

class FredWebService(SimulatorService):
        _wsdl = "".join(open("simulatorservice.wsdl").readlines())

        factory = ApolloFactory()
        utils = ApolloUtils()

        def soap_getSupportedLocations(self, ps, **kw):
            response = SimulatorService.soap_getSupportedLocations(self, ps, **kw)

            spl = self.factory.new_SupportedPopulationLocation()

            spl._populationLocation = "42003"
            spl._populationLocationDescription = "Allegheny County"

            response._supportedPopluationLocations.append(spl)
            return response

        #this method checks on the status of a job, indicated by the jobs runId
        def soap_getRunStatus(self, ps, **kw):
            response = SimulatorService.soap_getRunStatus(self, ps, **kw)
            #get the run id of the job
            request = ps.Parse(getRunStatusRequest.typecode)
            runId = request._runId

            status, message = fredConn._getStatus(runId)
            response._runStatus = self.factory.new_RunStatus()
            print "Fred status is:" + status
            response._runStatus._status = status.lower()
            print "Fred message is:" + message
            response._runStatus._message = message

            return response

        #this method runs an epidemic model
        def soap_run(self, ps, **kw):
	    fredConn._setup()
            response = SimulatorService.soap_run(self, ps, **kw)
            request = ps.Parse(runRequest.typecode)
            #extract the epidemic model input object from the request
            print "received run request"
            cfg = request._simulatorConfiguration

            #get the number of time steps in the simulation
            run_length = cfg._simulatorTimeSpecification._runLength

            #log the EpidemicModelInput request to a log file
            self.utils.log_simulatorConfiguration(cfg)

            ### STB CREATE FRED INPUT FILE
            ### do some error checking for this version of the model
            if cfg._simulatorTimeSpecification._timeStepUnit.lower() != 'days':
                    print ('This verison of FRED only supports time_step_unit of days')
                    print ('Apollo sent %s'%(cfg._simulatorTimeSpecification._timeStepUnit))
                    #### Put in Error Handling, have to ask John L.

            if cfg._simulatorTimeSpecification._timeStepValue != 1:
                    print ('This version of FRED only supports a time_step_value of 1')
                    print ('Apollo sent %d'%(cfg._simulatorTimeSpecification._timeStepValue))

	    #### Put in Error Handling, have to ask John L.

            #print ('WARNING: This service is currently ignoring infectious_period')
            #print ('WARNING: This service is currently ignoring latent_period')

            ### Make a random directory name so that multiple calls can be made
            randID = random.randint(0,100000)
            tempDirName = "fred.tmp.%d"%(randID)
            try:
                    os.mkdir(tempDirName)
            except:
                    raise RuntimeError("Cannot make temporary FRED directory %s"%(tempDirName))

            os.chdir(tempDirName)
            doVacc = False
            with open('fred_input.params','wb') as f:
                    f.write('days = %d\n'%cfg._simulatorTimeSpecification._runLength)
                    f.write('symp[0] = %g\n'%cfg._disease._asymptomaticInfectionFraction)
                    f.write('R0 = %g\n'%cfg._disease._reproductionNumber)
                    f.write('primary_cases_file[0] = fred_initial_population_0.txt\n')
                    f.write('residual_immunity_ages[0] = 2 0 100\n')
		    f.write('days_latent[0] = %s\n'\
			    %(incubationPeriod(float(cfg._disease._latentPeriod))))
		    f.write('days_incubating[0] = %s\n'\
			    %(incubationPeriod(float(cfg._disease._latentPeriod))))
		    f.write('days_infectious[0] = %s\n'\
			    %(infectiousPeriod(float(cfg._disease._infectiousPeriod))))
		    f.write('days_symptomatic[0] = %s\n'\
			    %(infectiousPeriod(float(cfg._disease._infectiousPeriod))))
                    totPop = self.utils.getTotalPopCount(cfg)
                    numImmune = self.utils.getPopCountGivenLocationAndDiseaseState(cfg, "ignoreed", "recovered")
                    percent_immune = float(numImmune)/float(totPop)
                    f.write('residual_immunity_values[0] = 1 %g\n'%(percent_immune))
		    numExposed = float(self.utils.getPopCountGivenLocationAndDiseaseState(cfg,"ignoreed","exposed"))
		    numInfectious = float(self.utils.getPopCountGivenLocationAndDiseaseState(cfg,"ignoreed","infectious"))
		    fractionExposed = numExposed/(numExposed+numInfectious)
		    fractionInfectious = 1.0 - fractionExposed
		    f.write('advanced_seeding = exposed:%1.2f;infectious:%1.2f\n'%(fractionExposed,fractionInfectious))
		    f.write('track_infection_events = 1\n')
		    if sum(cfg._vaccinationControlMeasure._vaccineSupplySchedule) > 0.0:
                        doVacc = True
                        f.write("##### VACCINE PARAMETERS\n")
                        f.write("enable_behaviors = 1\n")
                        f.write("enable_vaccination = 1\n")
                        f.write("number_of_vaccines = 1\n")
			f.write("accept_vaccine_enabled = 1\n")
			f.write("accept_vaccine_strategy_distribution = 7 %g %g 0 0 0 0 0\n"\
				%(1.0-cfg._vaccinationControlMeasure._vaccineCmCompliance,\
				  cfg._vaccinationControlMeasure._vaccineCmCompliance))
                        f.write("### VACCINE 1\n")
                        f.write("vaccine_number_of_doses[0] = 1\n")
                        # Setting this to infinity for now
                        # need to implement production as a time map as well
                        f.write("vaccine_total_avail[0] = 300000000\n")
                        f.write("vaccine_additional_per_day[0] = 300000000\n")
                        f.write("vaccine_starting_day[0] = 0\n")
                        f.write("##### Vaccine 1 Dose 0\n")
                        f.write("vaccine_next_dosage_day[0][0] = 0\n")
                        f.write("vaccine_dose_efficacy_ages[0][0] = 2 0 100\n")
                        f.write("vaccine_dose_efficacy_values[0][0] = 1 %g\n"\
                                %cfg._vaccinationControlMeasure._vaccineEfficacy)
                        f.write("vaccine_dose_efficacy_delay_ages[0][0] = 2 0 100\n")
                        f.write("vaccine_dose_efficacy_delay_values[0][0] = 1 %d\n"\
                                %cfg._vaccinationControlMeasure._vaccineEfficacyDelay)
                        f.write("vaccination_capacity_file = fred-vaccination-schedule_0.txt\n")

                        with open('fred-vaccination-schedule_0.txt','wb') as g:
                            vaccineSupplySchedule = cfg._vaccinationControlMeasure._vaccineSupplySchedule
                            vaccineAdminSchedule = cfg._vaccinationControlMeasure._vaccinationAdminSchedule

                            for day in range(0,len(vaccineSupplySchedule)) :
                                ## Hack for now, because I don't have the resolution on production in FRED
                                g.write("%d %d\n"%(day,min(vaccineSupplySchedule[day],vaccineAdminSchedule[day])))

            with open('fred_initial_population_0.txt','wb') as f:
                    f.write('#line_format\n')
                    numInfectious = self.utils.getPopCountGivenLocationAndDiseaseState(cfg, "ignored", "exposed")
                    #f.write('0 0 %d\n'%(cfg._disease_dynamics._pop_count[2]))
                    f.write('0 0 %d\n'%(numInfectious))

		     ### if vaccination comes through, parse this
            with open('starttime','wb') as f:
		    f.write('Nothing')

	    idPrefix = cfg._simulatorIdentification._simulatorDeveloper +\
		       "_" + cfg._simulatorIdentification._simulatorName +\
		       "_" + cfg._simulatorIdentification._simulatorVersion + "_"

            fredConn._connect()

            ### Write the PBS submission Script
            with open('fred_submit.pbs','wb') as f:
                    f.write('#!/bin/csh\n')
		    if fredConn.machine == "blacklight.psc.xsede.org":
                        f.write('#PBS -l ncpus=16\n')
		    else:
			f.write('#PBS -l nodes=2:ppn=8\n')
                    f.write('#PBS -N fred.pbs.out\n')
                    f.write('#PBS -l walltime=30:00\n')
                    f.write('#PBS -j oe\n')
	     	    if fredConn.machine == "blacklight.psc.xsede.org":
                   	 f.write('#PBS -q debug\n')
                    f.write('\n')
		    if fredConn.machine == "blacklight.psc.xsede.org":
                        f.write('source /usr/share/modules/init/csh\n')
                        f.write('source /etc/csh.cshrc.psc\n')
                    f.write('module load fred\n')
		    f.write('module load python\n')
                    f.write('cd $PBS_O_WORKDIR\n')
		    f.write('echo `date` > starttime\n')
                    f.write('### Get the PBS ID\n')
                    f.write("set words = `echo $PBS_JOBID | sed 's/\./ /g'`\n")
                    f.write("set id = $words[1]\n")
                    f.write('fred_job -p fred_input.params -n 8 -t 2 -m 8 -k %s$id\n'%idPrefix)
		    f.write('touch .dbloading\n')
                    f.write('python $FRED_HOME/bin/fred_to_apollo_parallel.py -k %s$id\n'%(idPrefix))
                    f.write('rm -rf .dbloading\n')
                    #f.write('echo COMPLETED > job_status')

            ### for a direct Job.. write the csh script to run FRED
	    with open('fred_run.csh','wb') as f:
		f.write('#!/bin/csh\n')
		f.write('echo `date` > starttime\n')
		f.write('### Generate Timestamp for ID\n')
		f.write('fred_job -p fred_input.params -n 4 -t 2 -m 4 -k %s$id > out.run\n'%idPrefix)
		f.write('touch .dbloading\n')
		f.write('python $FRED_HOME/bin/fred_to_apollo_parallel.py -k %s$id >& out.db\n'%idPrefix)
		f.write('rm -rf .dbloading\n')

            os.chdir('../')
            #fredConn._setup("warhol.psc.edu",username="stbrown",privateKeyFile="./id_rsa")
            #fredConn._connect()
            #print "Created Connection"
            fredConn._mkdir(tempDirName)
            fredConn._sendFile(tempDirName+'/fred_submit.pbs')
	    fredConn._sendFile(tempDirName+'/fred_run.csh')
            fredConn._sendFile(tempDirName+'/fred_input.params')
            fredConn._sendFile(tempDirName+'/fred_initial_population_0.txt')
            fredConn._sendFile(tempDirName+'/starttime')
	    if doVacc:
		    fredConn._sendFile(tempDirName+"/fred-vaccination-schedule_0.txt")

	    ### we do not need temp directory anymore
	    shutil.rmtree(tempDirName)

            print "Running FRED"
            sys.stdout.flush()

            #returnVal = fredConn._submitPBSJob(randID)
	    returnVal = fredConn._submitJob(randID)
            print returnVal

	    ### parse the run ID
	    response._runId = idPrefix + str(returnVal)

            #ssh.close()
            #sftp.close()

            return response

#logger = logging.getLogger("");
#logger.setLevel(0)

#run a webserver on 8087
AsServer(port=8087, services=[FredWebService('fred'), ])

<|endoftext|>


<commit_before>from itertools import groupby
from collections import defaultdict
from xml.etree.cElementTree import Element

from casexml.apps.phone.fixtures import FixtureProvider
from corehq.apps.custom_data_fields.dbaccessors import get_by_domain_and_type
from corehq.apps.fixtures.utils import get_index_schema_node
from corehq.apps.locations.models import SQLLocation, LocationType, LocationFixtureConfiguration
from corehq import toggles

class LocationSet(object):
    """
    Very simple class for keeping track of a set of locations
    """

    def __init__(self, locations=None):
        self.by_id = {}
        self.root_locations = set()
        self.by_parent = defaultdict(set)
        if locations is not None:
            for loc in locations:
                self.add_location(loc)

    def add_location(self, location):
        self.by_id[location.location_id] = location
        parent = location.parent
        parent_id = parent.location_id if parent else None
        if parent_id is None:  # this is a root
            self.add_root(location)
        self.by_parent[parent_id].add(location)

    def add_root(self, location):
        self.root_locations.add(location)

    def __contains__(self, item):
        return item in self.by_id

def should_sync_locations(last_sync, locations_queryset, restore_user):
    """
    Determine if any locations (already filtered to be relevant
    to this user) require syncing.
    """
    if (
        not last_sync or
        not last_sync.date or
        restore_user.get_fixture_last_modified() >= last_sync.date
    ):
        return True

    return (
        locations_queryset.filter(last_modified__gte=last_sync.date).exists()
        or LocationType.objects.filter(domain=restore_user.domain,
                                       last_modified__gte=last_sync.date).exists()
    )

class LocationFixtureProvider(FixtureProvider):

    def __init__(self, id, serializer):
        self.id = id
        self.serializer = serializer

    def __call__(self, restore_state):
        """
        By default this will generate a fixture for the users
        location and it's "footprint", meaning the path
        to a root location through parent hierarchies.

        There is an admin feature flag that will make this generate
        a fixture with ALL locations for the domain.
        """
        restore_user = restore_state.restore_user

        if not self.serializer.should_sync(restore_user):
            return []

        # This just calls get_location_fixture_queryset but is memoized to the user
        locations_queryset = restore_user.get_locations_to_sync()
        if not should_sync_locations(restore_state.last_sync_log, locations_queryset, restore_user):
            return []

        data_fields = _get_location_data_fields(restore_user.domain)
        return self.serializer.get_xml_nodes(self.id, restore_user, locations_queryset, data_fields)

class HierarchicalLocationSerializer(object):

    def should_sync(self, restore_user):
        return should_sync_hierarchical_fixture(restore_user.project)

    def get_xml_nodes(self, fixture_id, restore_user, locations_queryset, data_fields):
        locations_db = LocationSet(locations_queryset)

        root_node = Element('fixture', {'id': fixture_id, 'user_id': restore_user.user_id})
        root_locations = locations_db.root_locations

        if root_locations:
            _append_children(root_node, locations_db, root_locations, data_fields)
        else:
            # There is a bug on mobile versions prior to 2.27 where
            # a parsing error will cause mobile to ignore the element
            # after this one if this element is empty.
            # So we have to add a dummy empty_element child to prevent
            # this element from being empty.
            root_node.append(Element("empty_element"))
        return [root_node]

class FlatLocationSerializer(object):

    def should_sync(self, restore_user):
        return should_sync_flat_fixture(restore_user.project)

    def get_xml_nodes(self, fixture_id, restore_user, locations_queryset, data_fields):

        all_types = LocationType.objects.filter(domain=restore_user.domain).values_list(
            'code', flat=True
        )
        location_type_attrs = ['{}_id'.format(t) for t in all_types if t is not None]
        attrs_to_index = ['@{}'.format(attr) for attr in location_type_attrs]
        attrs_to_index.extend(_get_indexed_field_name(field.slug) for field in data_fields
                              if field.index_in_fixture)
        attrs_to_index.extend(['@id', '@type', 'name'])

        return [get_index_schema_node(fixture_id, attrs_to_index),
                self._get_fixture_node(fixture_id, restore_user, locations_queryset,
                                       location_type_attrs, data_fields)]

    def _get_fixture_node(self, fixture_id, restore_user, locations_queryset,
                          location_type_attrs, data_fields):
        root_node = Element('fixture', {'id': fixture_id,
                                        'user_id': restore_user.user_id,
                                        'indexed': 'true'})
        outer_node = Element('locations')
        root_node.append(outer_node)
        all_locations = list(locations_queryset.order_by('site_code'))
        locations_by_id = {location.pk: location for location in all_locations}
        for location in all_locations:
            attrs = {
                'type': location.location_type.code,
                'id': location.location_id,
            }
            attrs.update({attr: '' for attr in location_type_attrs})
            attrs['{}_id'.format(location.location_type.code)] = location.location_id

            current_location = location
            while current_location.parent_id:
                try:
                    current_location = locations_by_id[current_location.parent_id]
                except KeyError:
                    current_location = current_location.parent

                    # For some reason this wasn't included in the locations we already fetched
                    from corehq.util.soft_assert import soft_assert
                    _soft_assert = soft_assert('{}@{}.com'.format('frener', 'dimagi'))
                    message = """
                        The flat location fixture didn't prefetch all parent locations:
                        {domain}: {location_id}
                    """.format(
                        domain=current_location.domain,
                        location_id=current_location.location_id,
                    )
                    _soft_assert(False, msg=message)

                attrs['{}_id'.format(current_location.location_type.code)] = current_location.location_id

            location_node = Element('location', attrs)
            _fill_in_location_element(location_node, location, data_fields)
            outer_node.append(location_node)

        return root_node

def should_sync_hierarchical_fixture(project):
    # Sync hierarchical fixture for domains with fixture toggle enabled for migration and
    # configuration set to use hierarchical fixture
    # Even if both fixtures are set up, this one takes priority for domains with toggle enabled
    return (
        project.uses_locations and
        toggles.HIERARCHICAL_LOCATION_FIXTURE.enabled(project.name) and
        LocationFixtureConfiguration.for_domain(project.name).sync_hierarchical_fixture
    )

def should_sync_flat_fixture(project):
    # Sync flat fixture for domains with conf for flat fixture enabled
    # This does not check for toggle for migration to allow domains those domains to migrate to flat fixture
    return (
        project.uses_locations and
        LocationFixtureConfiguration.for_domain(project.name).sync_flat_fixture
    )

location_fixture_generator = LocationFixtureProvider(
    id='commtrack:locations', serializer=HierarchicalLocationSerializer()
)
flat_location_fixture_generator = LocationFixtureProvider(
    id='locations', serializer=FlatLocationSerializer()
)

def get_location_fixture_queryset(user):
    if toggles.SYNC_ALL_LOCATIONS.enabled(user.domain):
        return SQLLocation.active_objects.filter(domain=user.domain).prefetch_related('location_type')

    all_locations = SQLLocation.objects.none()

    user_locations = user.get_sql_locations(user.domain).prefetch_related('location_type')

    all_locations |= _get_include_without_expanding_locations(user.domain, user_locations)

    for user_location in user_locations:
        location_type = user_location.location_type
        expand_from = location_type.expand_from or location_type
        expand_to_level = (SQLLocation.active_objects
                           .filter(domain__exact=user.domain, location_type=location_type.expand_to)
                           .values_list('level', flat=True)
                           .first())
        expand_from_locations = _get_expand_from_level(user.domain, user_location, expand_from)
        all_locations |= _get_children(expand_from_locations, expand_to_level)
        all_locations |= (SQLLocation.active_objects
                          .get_queryset_ancestors(expand_from_locations, include_self=True))

    return all_locations

def _get_expand_from_level(domain, user_location, expand_from):
    """From the users current location, return all locations of the highest
    level they want to start expanding from.
    """
    if user_location.location_type.expand_from_root:
        return SQLLocation.root_locations(domain=domain)
    else:
        ancestors = (
            user_location
            .get_ancestors(include_self=True)
            .filter(location_type=expand_from, is_archived=False)
            .prefetch_related('location_type')
        )
        return ancestors

def _get_children(expand_from_locations, expand_to_level):
    """From the topmost location, get all the children we want to sync
    """
    children = SQLLocation.active_objects.get_queryset_descendants(expand_from_locations).prefetch_related('location_type')
    if expand_to_level is not None:
        children = children.filter(level__lte=expand_to_level)
    return children

def _get_include_without_expanding_locations(domain, assigned_locations):
    """returns all locations set for inclusion along with their ancestors
    """
    # all loctypes to include, based on all assigned location types
    location_type_ids = {
        loc.location_type.include_without_expanding_id
        for loc in assigned_locations
        if loc.location_type.include_without_expanding_id is not None
    }
    # all levels to include, based on the above loctypes
    forced_levels = (SQLLocation.active_objects
                     .filter(domain__exact=domain,
                             location_type_id__in=location_type_ids)
                     .values_list('level', flat=True)
                     .order_by('level')
                     .distinct('level'))
    if forced_levels:
        return (SQLLocation.active_objects
                .filter(domain__exact=domain,
                        level__lte=max(forced_levels))
                .prefetch_related('location_type'))
    else:
        return SQLLocation.objects.none()

def _append_children(node, location_db, locations, data_fields):
    for type, locs in _group_by_type(locations):
        locs = sorted(locs, key=lambda loc: loc.name)
        node.append(_types_to_fixture(location_db, type, locs, data_fields))

def _group_by_type(locations):
    key = lambda loc: (loc.location_type.code, loc.location_type)
    for (code, type), locs in groupby(sorted(locations, key=key), key=key):
        yield type, list(locs)

def _types_to_fixture(location_db, type, locs, data_fields):
    type_node = Element('%ss' % type.code)  # hacky pluralization
    for loc in locs:
        type_node.append(_location_to_fixture(location_db, loc, type, data_fields))
    return type_node

def _get_metadata_node(location, data_fields):
    node = Element('location_data')
    # add default empty nodes for all known fields: http://manage.dimagi.com/default.asp?247786
    for field in data_fields:
        element = Element(field.slug)
        element.text = unicode(location.metadata.get(field.slug, ''))
        node.append(element)
    return node

def _location_to_fixture(location_db, location, type, data_fields):
    root = Element(type.code, {'id': location.location_id})
    _fill_in_location_element(root, location, data_fields)
    _append_children(root, location_db, location_db.by_parent[location.location_id], data_fields)
    return root

def _fill_in_location_element(xml_root, location, data_fields):
    fixture_fields = [
        'name',
        'site_code',
        'external_id',
        'latitude',
        'longitude',
        'location_type',
        'supply_point_id',
    ]
    for field in fixture_fields:
        field_node = Element(field)
        val = getattr(location, field)
        field_node.text = unicode(val if val is not None else '')
        xml_root.append(field_node)

    # in order to be indexed, custom data fields need to be top-level
    # so we stick them in there with the prefix data_
    for field in data_fields:
        if field.index_in_fixture:
            field_node = Element(_get_indexed_field_name(field.slug))
            val = location.metadata.get(field.slug)
            field_node.text = unicode(val if val is not None else '')
            xml_root.append(field_node)

    xml_root.append(_get_metadata_node(location, data_fields))

def _get_location_data_fields(domain):
    from corehq.apps.locations.views import LocationFieldsView
    fields_definition = get_by_domain_and_type(domain, LocationFieldsView.field_type)
    if fields_definition:
        return fields_definition.fields
    else:
        return []

def _get_indexed_field_name(slug):
    return "data_{}".format(slug)
<commit_msg>Eliminate leading whitespace in message<commit_after>from itertools import groupby
from collections import defaultdict
from xml.etree.cElementTree import Element

from casexml.apps.phone.fixtures import FixtureProvider
from corehq.apps.custom_data_fields.dbaccessors import get_by_domain_and_type
from corehq.apps.fixtures.utils import get_index_schema_node
from corehq.apps.locations.models import SQLLocation, LocationType, LocationFixtureConfiguration
from corehq import toggles

class LocationSet(object):
    """
    Very simple class for keeping track of a set of locations
    """

    def __init__(self, locations=None):
        self.by_id = {}
        self.root_locations = set()
        self.by_parent = defaultdict(set)
        if locations is not None:
            for loc in locations:
                self.add_location(loc)

    def add_location(self, location):
        self.by_id[location.location_id] = location
        parent = location.parent
        parent_id = parent.location_id if parent else None
        if parent_id is None:  # this is a root
            self.add_root(location)
        self.by_parent[parent_id].add(location)

    def add_root(self, location):
        self.root_locations.add(location)

    def __contains__(self, item):
        return item in self.by_id

def should_sync_locations(last_sync, locations_queryset, restore_user):
    """
    Determine if any locations (already filtered to be relevant
    to this user) require syncing.
    """
    if (
        not last_sync or
        not last_sync.date or
        restore_user.get_fixture_last_modified() >= last_sync.date
    ):
        return True

    return (
        locations_queryset.filter(last_modified__gte=last_sync.date).exists()
        or LocationType.objects.filter(domain=restore_user.domain,
                                       last_modified__gte=last_sync.date).exists()
    )

class LocationFixtureProvider(FixtureProvider):

    def __init__(self, id, serializer):
        self.id = id
        self.serializer = serializer

    def __call__(self, restore_state):
        """
        By default this will generate a fixture for the users
        location and it's "footprint", meaning the path
        to a root location through parent hierarchies.

        There is an admin feature flag that will make this generate
        a fixture with ALL locations for the domain.
        """
        restore_user = restore_state.restore_user

        if not self.serializer.should_sync(restore_user):
            return []

        # This just calls get_location_fixture_queryset but is memoized to the user
        locations_queryset = restore_user.get_locations_to_sync()
        if not should_sync_locations(restore_state.last_sync_log, locations_queryset, restore_user):
            return []

        data_fields = _get_location_data_fields(restore_user.domain)
        return self.serializer.get_xml_nodes(self.id, restore_user, locations_queryset, data_fields)

class HierarchicalLocationSerializer(object):

    def should_sync(self, restore_user):
        return should_sync_hierarchical_fixture(restore_user.project)

    def get_xml_nodes(self, fixture_id, restore_user, locations_queryset, data_fields):
        locations_db = LocationSet(locations_queryset)

        root_node = Element('fixture', {'id': fixture_id, 'user_id': restore_user.user_id})
        root_locations = locations_db.root_locations

        if root_locations:
            _append_children(root_node, locations_db, root_locations, data_fields)
        else:
            # There is a bug on mobile versions prior to 2.27 where
            # a parsing error will cause mobile to ignore the element
            # after this one if this element is empty.
            # So we have to add a dummy empty_element child to prevent
            # this element from being empty.
            root_node.append(Element("empty_element"))
        return [root_node]

class FlatLocationSerializer(object):

    def should_sync(self, restore_user):
        return should_sync_flat_fixture(restore_user.project)

    def get_xml_nodes(self, fixture_id, restore_user, locations_queryset, data_fields):

        all_types = LocationType.objects.filter(domain=restore_user.domain).values_list(
            'code', flat=True
        )
        location_type_attrs = ['{}_id'.format(t) for t in all_types if t is not None]
        attrs_to_index = ['@{}'.format(attr) for attr in location_type_attrs]
        attrs_to_index.extend(_get_indexed_field_name(field.slug) for field in data_fields
                              if field.index_in_fixture)
        attrs_to_index.extend(['@id', '@type', 'name'])

        return [get_index_schema_node(fixture_id, attrs_to_index),
                self._get_fixture_node(fixture_id, restore_user, locations_queryset,
                                       location_type_attrs, data_fields)]

    def _get_fixture_node(self, fixture_id, restore_user, locations_queryset,
                          location_type_attrs, data_fields):
        root_node = Element('fixture', {'id': fixture_id,
                                        'user_id': restore_user.user_id,
                                        'indexed': 'true'})
        outer_node = Element('locations')
        root_node.append(outer_node)
        all_locations = list(locations_queryset.order_by('site_code'))
        locations_by_id = {location.pk: location for location in all_locations}
        for location in all_locations:
            attrs = {
                'type': location.location_type.code,
                'id': location.location_id,
            }
            attrs.update({attr: '' for attr in location_type_attrs})
            attrs['{}_id'.format(location.location_type.code)] = location.location_id

            current_location = location
            while current_location.parent_id:
                try:
                    current_location = locations_by_id[current_location.parent_id]
                except KeyError:
                    current_location = current_location.parent

                    # For some reason this wasn't included in the locations we already fetched
                    from corehq.util.soft_assert import soft_assert
                    _soft_assert = soft_assert('{}@{}.com'.format('frener', 'dimagi'))
                    message = (
                        "The flat location fixture didn't prefetch all parent "
                        "locations: {domain}: {location_id}"
                    ).format(
                        domain=current_location.domain,
                        location_id=current_location.location_id,
                    )
                    _soft_assert(False, msg=message)

                attrs['{}_id'.format(current_location.location_type.code)] = current_location.location_id

            location_node = Element('location', attrs)
            _fill_in_location_element(location_node, location, data_fields)
            outer_node.append(location_node)

        return root_node

def should_sync_hierarchical_fixture(project):
    # Sync hierarchical fixture for domains with fixture toggle enabled for migration and
    # configuration set to use hierarchical fixture
    # Even if both fixtures are set up, this one takes priority for domains with toggle enabled
    return (
        project.uses_locations and
        toggles.HIERARCHICAL_LOCATION_FIXTURE.enabled(project.name) and
        LocationFixtureConfiguration.for_domain(project.name).sync_hierarchical_fixture
    )

def should_sync_flat_fixture(project):
    # Sync flat fixture for domains with conf for flat fixture enabled
    # This does not check for toggle for migration to allow domains those domains to migrate to flat fixture
    return (
        project.uses_locations and
        LocationFixtureConfiguration.for_domain(project.name).sync_flat_fixture
    )

location_fixture_generator = LocationFixtureProvider(
    id='commtrack:locations', serializer=HierarchicalLocationSerializer()
)
flat_location_fixture_generator = LocationFixtureProvider(
    id='locations', serializer=FlatLocationSerializer()
)

def get_location_fixture_queryset(user):
    if toggles.SYNC_ALL_LOCATIONS.enabled(user.domain):
        return SQLLocation.active_objects.filter(domain=user.domain).prefetch_related('location_type')

    all_locations = SQLLocation.objects.none()

    user_locations = user.get_sql_locations(user.domain).prefetch_related('location_type')

    all_locations |= _get_include_without_expanding_locations(user.domain, user_locations)

    for user_location in user_locations:
        location_type = user_location.location_type
        expand_from = location_type.expand_from or location_type
        expand_to_level = (SQLLocation.active_objects
                           .filter(domain__exact=user.domain, location_type=location_type.expand_to)
                           .values_list('level', flat=True)
                           .first())
        expand_from_locations = _get_expand_from_level(user.domain, user_location, expand_from)
        all_locations |= _get_children(expand_from_locations, expand_to_level)
        all_locations |= (SQLLocation.active_objects
                          .get_queryset_ancestors(expand_from_locations, include_self=True))

    return all_locations

def _get_expand_from_level(domain, user_location, expand_from):
    """From the users current location, return all locations of the highest
    level they want to start expanding from.
    """
    if user_location.location_type.expand_from_root:
        return SQLLocation.root_locations(domain=domain)
    else:
        ancestors = (
            user_location
            .get_ancestors(include_self=True)
            .filter(location_type=expand_from, is_archived=False)
            .prefetch_related('location_type')
        )
        return ancestors

def _get_children(expand_from_locations, expand_to_level):
    """From the topmost location, get all the children we want to sync
    """
    children = SQLLocation.active_objects.get_queryset_descendants(expand_from_locations).prefetch_related('location_type')
    if expand_to_level is not None:
        children = children.filter(level__lte=expand_to_level)
    return children

def _get_include_without_expanding_locations(domain, assigned_locations):
    """returns all locations set for inclusion along with their ancestors
    """
    # all loctypes to include, based on all assigned location types
    location_type_ids = {
        loc.location_type.include_without_expanding_id
        for loc in assigned_locations
        if loc.location_type.include_without_expanding_id is not None
    }
    # all levels to include, based on the above loctypes
    forced_levels = (SQLLocation.active_objects
                     .filter(domain__exact=domain,
                             location_type_id__in=location_type_ids)
                     .values_list('level', flat=True)
                     .order_by('level')
                     .distinct('level'))
    if forced_levels:
        return (SQLLocation.active_objects
                .filter(domain__exact=domain,
                        level__lte=max(forced_levels))
                .prefetch_related('location_type'))
    else:
        return SQLLocation.objects.none()

def _append_children(node, location_db, locations, data_fields):
    for type, locs in _group_by_type(locations):
        locs = sorted(locs, key=lambda loc: loc.name)
        node.append(_types_to_fixture(location_db, type, locs, data_fields))

def _group_by_type(locations):
    key = lambda loc: (loc.location_type.code, loc.location_type)
    for (code, type), locs in groupby(sorted(locations, key=key), key=key):
        yield type, list(locs)

def _types_to_fixture(location_db, type, locs, data_fields):
    type_node = Element('%ss' % type.code)  # hacky pluralization
    for loc in locs:
        type_node.append(_location_to_fixture(location_db, loc, type, data_fields))
    return type_node

def _get_metadata_node(location, data_fields):
    node = Element('location_data')
    # add default empty nodes for all known fields: http://manage.dimagi.com/default.asp?247786
    for field in data_fields:
        element = Element(field.slug)
        element.text = unicode(location.metadata.get(field.slug, ''))
        node.append(element)
    return node

def _location_to_fixture(location_db, location, type, data_fields):
    root = Element(type.code, {'id': location.location_id})
    _fill_in_location_element(root, location, data_fields)
    _append_children(root, location_db, location_db.by_parent[location.location_id], data_fields)
    return root

def _fill_in_location_element(xml_root, location, data_fields):
    fixture_fields = [
        'name',
        'site_code',
        'external_id',
        'latitude',
        'longitude',
        'location_type',
        'supply_point_id',
    ]
    for field in fixture_fields:
        field_node = Element(field)
        val = getattr(location, field)
        field_node.text = unicode(val if val is not None else '')
        xml_root.append(field_node)

    # in order to be indexed, custom data fields need to be top-level
    # so we stick them in there with the prefix data_
    for field in data_fields:
        if field.index_in_fixture:
            field_node = Element(_get_indexed_field_name(field.slug))
            val = location.metadata.get(field.slug)
            field_node.text = unicode(val if val is not None else '')
            xml_root.append(field_node)

    xml_root.append(_get_metadata_node(location, data_fields))

def _get_location_data_fields(domain):
    from corehq.apps.locations.views import LocationFieldsView
    fields_definition = get_by_domain_and_type(domain, LocationFieldsView.field_type)
    if fields_definition:
        return fields_definition.fields
    else:
        return []

def _get_indexed_field_name(slug):
    return "data_{}".format(slug)
<|endoftext|>


<commit_before>import sys
import random
import copy
from PIL import Image, ImageTk
from io import BytesIO
import Tkinter as tk

class Action(object):
  def __init__(self, size, param):
    self.size = size
    self.param = param
  def mod(self, image):
    return image
  def mutate(self):
    pass
  def update(self, param):
    self.param = param

class ActionDoNothing(Action):
  def __init__(self, size, param):
    Action.__init__(self, size, param)

class ActionDelete(Action):
  def __init__(self, size, param):
    Action.__init__(self, size, param)
    self.begin = random.randint(0, self.size-1)
    self.end = random.randint(self.begin, min(self.begin+self.param, self.size-1))
  def mod(self, image):
    newimage = list(image)
    del newimage[self.begin:self.end]
    return newimage
  def mutate(self):
    self.begin = random.randint(0, self.size-1)
    self.end = random.randint(self.begin, min(self.begin+self.param, self.size-1))

class ActionAdd(Action):
  def __init__(self, size, param):
    Action.__init__(self, size, param)
    self.begin = random.randint(0, self.size-1)
    self.chunk = [chr(random.randint(0, 255)) for i in range(random.randint(1, self.param))]
  def mod(self, image):
    return image[:self.begin] + self.chunk + image[self.begin:]
  def mutate(self):
    self.begin = random.randint(0, self.size-1)
    self.chunk = [chr(random.randint(0, 255)) for i in range(random.randint(1, self.param))]

class ActionMove(Action):
  def __init__(self, size, param):
    Action.__init__(self, size, param)
    self.begin = random.randint(0, self.size-1)
    self.end = random.randint(self.begin, min(self.begin+self.param, self.size-1))
    self.insert = random.randint(0, self.size-(self.end-self.begin)-1)
  def mod(self, image):
    newimage = list(image)
    del newimage[self.begin:self.end]
    return newimage[:self.insert] + image[self.begin:self.end] + newimage[self.insert:]
  def mutate(self):
    self.begin = random.randint(0, self.size-1)
    self.end = random.randint(self.begin, min(self.begin+self.param, self.size-1))
    self.insert = random.randint(0, self.size-(self.end-self.begin)-1)

class Genome(object):
  def __init__(self, filename, length, param):
    self.im_data = list(open(filename, "rb").read())
    self.size = len(self.im_data)
    self.param = param
    self.genome = [random.choice([ActionDoNothing(self.size, self.param), ActionDelete(self.size, self.param), ActionAdd(self.size, self.param), ActionMove(self.size, self.param)]) for i in range(length)]
    while not self.test():
      self.genome = [random.choice([ActionDoNothing(self.size, self.param), ActionDelete(self.size, self.param), ActionAdd(self.size, self.param), ActionMove(self.size, self.param)]) for i in range(length)]
  def test(self):
    im = list(self.im_data)
    for action in self.genome:
      im = action.mod(im)
    try:
      Image.open(BytesIO("".join(im))).load()
    except:
      return False
    return True
  def update(self, param):
    self.param = param
    for action in self.genome:
      action.update(param)
    while not self.test():
      self.genome = [random.choice([ActionDoNothing(self.size, self.param), ActionDelete(self.size, self.param), ActionAdd(self.size, self.param), ActionMove(self.size, self.param)]) for i in range(length)]
  def resize(self, length):
    if length < len(self.genome):
      while length != len(self.genome):
        pos = random.randint(0, len(self.genome)-1)
        del self.genome[pos]
    elif length > len(self.genome):
      splice = random.randint(0, len(self.genome))
      self.genome.insert(splice, random.choice([ActionDoNothing(self.size, self.param), ActionDelete(self.size, self.param), ActionAdd(self.size, self.param), ActionMove(self.size, self.param)]))
      while not self.test():
        self.genome[splice] = random.choice([ActionDoNothing(self.size, self.param), ActionDelete(self.size, self.param), ActionAdd(self.size, self.param), ActionMove(self.size, self.param)])
  def mutate(self):
    old_genome = copy.deepcopy(self.genome)
    for action in self.genome:
      if random.random() < 0.1:
        action.mutate()
    while not self.test():
      self.genome = copy.deepcopy(old_genome)
      for action in self.genome:
        if random.random() < 0.1:
          action.mutate()
  def cross(self, other):
    new_genome = copy.deepcopy(other)
    splice = random.randint(0, len(self.genome))
    for i in range(len(self.genome)):
      if i <= splice:
        new_genome.genome[i] = copy.deepcopy(self.genome[i])
      else:
        new_genome.genome[i] = copy.deepcopy(other.genome[i])
    return new_genome
  def get_orig(self):
    return Image.open(BytesIO("".join(self.im_data)))
  def get_orig_tk(self):
    return ImageTk.PhotoImage(self.get_orig())
  def get_mod(self):
    im = list(self.im_data)
    for action in self.genome:
      im = action.mod(im)
    return Image.open(BytesIO("".join(im)))
  def get_mod_tk(self):
    return ImageTk.PhotoImage(self.get_mod())
  def get_mod_thumb(self, width):
    im = self.get_mod()
    height = int(float(im.size[1])/float(im.size[0]) * width)
    im = im.resize((width, height))
    return im
  def get_mod_thumb_tk(self, width):
    return ImageTk.PhotoImage(self.get_mod_thumb(width))

class Gui(object):
  def __init__(self, root, filename):
    self.filename = filename
    self.num_genomes = 12
    # generate initial genomes
    self.genomes = [Genome(self.filename, 0, 0) for i in range(self.num_genomes)]
    # GUI
    self.root = root
    # image
    self.im_vars = [tk.IntVar() for i in range(self.num_genomes)]
    self.im_labels = [tk.Checkbutton(self.root, text="genome "+str(i), variable=self.im_vars[i], indicatoron=False) for i in range(self.num_genomes)]
    for i in range(3):
      for j in range(4):
        self.im_labels[i*4+j].grid(row=i, column=j)
    self.old_im_labels = [self.im_labels[i] for i in range(self.num_genomes)]
    # slider
    self.length_scale = tk.Scale(self.root, from_=0, to=10, label="genome length", orient=tk.HORIZONTAL, length=200)
    self.length_scale.grid(row=3, column=0)
    self.param_scale = tk.Scale(self.root, from_=1, to=1000, label="param", orient=tk.HORIZONTAL, length=200)
    self.param_scale.grid(row=3, column=1)
    # buttons
    self.gen_button = tk.Button(self.root, text="gen", command=self.show_genomes)
    self.gen_button.grid(row=3, column=2)
    self.exit_button = tk.Button(self.root, text="Exit", command=self.root.quit)
    self.exit_button.grid(row=3, column=3)
  def evolve(self):
    good_genomes = [self.genomes[i] for i in range(self.num_genomes) if self.im_vars[i].get()]
    if len(good_genomes) == 0:
      for i in range(self.num_genomes):
        self.genomes[i] = Genome(self.filename, self.length_scale.get(), self.param_scale.get())
    else:
      for i in range(self.num_genomes):
        self.genomes[i] = random.choice(good_genomes).cross(random.choice(good_genomes))
        self.genomes[i].mutate()
  def show_genomes(self):
    for genome in self.genomes:
      genome.update(self.param_scale.get())
    if len(self.genomes[0].genome) != self.length_scale.get():
      for genome in self.genomes:
        genome.resize(self.length_scale.get())
    self.evolve()
    for var in self.im_vars:
      var.set(0)
    for i in range(3):
      for j in range(4):
        tkim = self.genomes[i*4+j].get_mod_thumb_tk(300)
        self.im_labels[i*4+j] = tk.Checkbutton(self.root, image=tkim, variable=self.im_vars[i*4+j], indicatoron=False, bd=10, selectcolor="green")
        self.im_labels[i*4+j].grid(row=i, column=j)
        self.im_labels[i*4+j].image = tkim
        self.old_im_labels[i*4+j].destroy()
        self.old_im_labels[i*4+j] = self.im_labels[i*4+j]

if __name__ == "__main__":
  if len(sys.argv) != 2:
    print "USAGE: " + sys.argv[0] + " image"
    sys.exit()
  root = tk.Tk()
  gui = Gui(root, sys.argv[1])
  root.mainloop()
  root.destroy()
<commit_msg>added mutation_prob scale, scales now have standard values<commit_after>import sys
import random
import copy
from PIL import Image, ImageTk
from io import BytesIO
import Tkinter as tk

class Action(object):
  def __init__(self, size, param):
    self.size = size
    self.param = param
  def mod(self, image):
    return image
  def mutate(self):
    pass
  def update(self, param):
    self.param = param

class ActionDoNothing(Action):
  def __init__(self, size, param):
    Action.__init__(self, size, param)

class ActionDelete(Action):
  def __init__(self, size, param):
    Action.__init__(self, size, param)
    self.begin = random.randint(0, self.size-1)
    self.end = random.randint(self.begin, min(self.begin+self.param, self.size-1))
  def mod(self, image):
    newimage = list(image)
    del newimage[self.begin:self.end]
    return newimage
  def mutate(self):
    self.begin = random.randint(0, self.size-1)
    self.end = random.randint(self.begin, min(self.begin+self.param, self.size-1))

class ActionAdd(Action):
  def __init__(self, size, param):
    Action.__init__(self, size, param)
    self.begin = random.randint(0, self.size-1)
    self.chunk = [chr(random.randint(0, 255)) for i in range(random.randint(1, self.param))]
  def mod(self, image):
    return image[:self.begin] + self.chunk + image[self.begin:]
  def mutate(self):
    self.begin = random.randint(0, self.size-1)
    self.chunk = [chr(random.randint(0, 255)) for i in range(random.randint(1, self.param))]

class ActionMove(Action):
  def __init__(self, size, param):
    Action.__init__(self, size, param)
    self.begin = random.randint(0, self.size-1)
    self.end = random.randint(self.begin, min(self.begin+self.param, self.size-1))
    self.insert = random.randint(0, self.size-(self.end-self.begin)-1)
  def mod(self, image):
    newimage = list(image)
    del newimage[self.begin:self.end]
    return newimage[:self.insert] + image[self.begin:self.end] + newimage[self.insert:]
  def mutate(self):
    self.begin = random.randint(0, self.size-1)
    self.end = random.randint(self.begin, min(self.begin+self.param, self.size-1))
    self.insert = random.randint(0, self.size-(self.end-self.begin)-1)

class Genome(object):
  def __init__(self, filename, length, param, mutation_prob):
    self.im_data = list(open(filename, "rb").read())
    self.size = len(self.im_data)
    self.param = param
    self.mutation_prob = mutation_prob
    self.genome = [random.choice([ActionDoNothing(self.size, self.param), ActionDelete(self.size, self.param), ActionAdd(self.size, self.param), ActionMove(self.size, self.param)]) for i in range(length)]
    while not self.test():
      self.genome = [random.choice([ActionDoNothing(self.size, self.param), ActionDelete(self.size, self.param), ActionAdd(self.size, self.param), ActionMove(self.size, self.param)]) for i in range(length)]
  def test(self):
    im = list(self.im_data)
    for action in self.genome:
      im = action.mod(im)
    try:
      Image.open(BytesIO("".join(im))).load()
    except:
      return False
    return True
  def update(self, param, mutation_prob):
    self.param = param
    self.mutation_prob = mutation_prob
    for action in self.genome:
      action.update(param)
    while not self.test():
      self.genome = [random.choice([ActionDoNothing(self.size, self.param), ActionDelete(self.size, self.param), ActionAdd(self.size, self.param), ActionMove(self.size, self.param)]) for i in range(length)]
  def resize(self, length):
    if length < len(self.genome):
      while length != len(self.genome):
        pos = random.randint(0, len(self.genome)-1)
        del self.genome[pos]
    elif length > len(self.genome):
      splice = random.randint(0, len(self.genome))
      self.genome.insert(splice, random.choice([ActionDoNothing(self.size, self.param), ActionDelete(self.size, self.param), ActionAdd(self.size, self.param), ActionMove(self.size, self.param)]))
      while not self.test():
        self.genome[splice] = random.choice([ActionDoNothing(self.size, self.param), ActionDelete(self.size, self.param), ActionAdd(self.size, self.param), ActionMove(self.size, self.param)])
  def mutate(self):
    old_genome = copy.deepcopy(self.genome)
    for action in self.genome:
      if random.random() < self.mutation_prob:
        action.mutate()
    while not self.test():
      self.genome = copy.deepcopy(old_genome)
      for action in self.genome:
        if random.random() < self.mutation_prob:
          action.mutate()
  def cross(self, other):
    new_genome = copy.deepcopy(other)
    splice = random.randint(0, len(self.genome))
    for i in range(len(self.genome)):
      if i <= splice:
        new_genome.genome[i] = copy.deepcopy(self.genome[i])
      else:
        new_genome.genome[i] = copy.deepcopy(other.genome[i])
    return new_genome
  def get_orig(self):
    return Image.open(BytesIO("".join(self.im_data)))
  def get_orig_tk(self):
    return ImageTk.PhotoImage(self.get_orig())
  def get_mod(self):
    im = list(self.im_data)
    for action in self.genome:
      im = action.mod(im)
    return Image.open(BytesIO("".join(im)))
  def get_mod_tk(self):
    return ImageTk.PhotoImage(self.get_mod())
  def get_mod_thumb(self, width):
    im = self.get_mod()
    height = int(float(im.size[1])/float(im.size[0]) * width)
    im = im.resize((width, height))
    return im
  def get_mod_thumb_tk(self, width):
    return ImageTk.PhotoImage(self.get_mod_thumb(width))

class Gui(object):
  def __init__(self, root, filename):
    self.filename = filename
    self.num_genomes = 12
    # generate initial genomes
    self.genomes = [Genome(self.filename, 0, 0, 0) for i in range(self.num_genomes)]
    # GUI
    self.root = root
    # image
    self.im_vars = [tk.IntVar() for i in range(self.num_genomes)]
    self.im_labels = []
    for i in range(3):
      for j in range(4):
        tkim = self.genomes[i*4+j].get_mod_thumb_tk(300)
        self.im_labels.append(tk.Checkbutton(self.root, image=tkim, variable=self.im_vars[i*4+j], indicatoron=False, bd=10, selectcolor="green"))
        self.im_labels[i*4+j].grid(row=i, column=j)
        self.im_labels[i*4+j].image = tkim
    self.old_im_labels = [self.im_labels[i] for i in range(self.num_genomes)]
    # slider
    self.length_scale = tk.Scale(self.root, from_=0, to=10, label="genome length", orient=tk.HORIZONTAL, length=200)
    self.length_scale.grid(row=3, column=0)
    self.length_scale.set(4)
    self.param_scale = tk.Scale(self.root, from_=1, to=1000, label="param", orient=tk.HORIZONTAL, length=200)
    self.param_scale.grid(row=3, column=1)
    self.param_scale.set(70)
    self.mutation_scale = tk.Scale(self.root, from_=0, to=100, label="mutation probability", orient=tk.HORIZONTAL, length=200)
    self.mutation_scale.grid(row=3, column=2)
    self.mutation_scale.set(15)
    # buttons
    self.gen_button = tk.Button(self.root, text="gen", command=self.show_genomes)
    self.gen_button.grid(row=4, column=2)
    self.exit_button = tk.Button(self.root, text="Exit", command=self.root.quit)
    self.exit_button.grid(row=4, column=3)
  def evolve(self):
    good_genomes = [self.genomes[i] for i in range(self.num_genomes) if self.im_vars[i].get()]
    if len(good_genomes) == 0:
      for i in range(self.num_genomes):
        self.genomes[i] = Genome(self.filename, self.length_scale.get(), self.param_scale.get(), float(self.mutation_scale.get())/100.0)
    else:
      for i in range(self.num_genomes):
        self.genomes[i] = random.choice(good_genomes).cross(random.choice(good_genomes))
        self.genomes[i].mutate()
  def show_genomes(self):
    for genome in self.genomes:
      genome.update(self.param_scale.get(), float(self.mutation_scale.get())/100.0)
    if len(self.genomes[0].genome) != self.length_scale.get():
      for genome in self.genomes:
        genome.resize(self.length_scale.get())
    self.evolve()
    for var in self.im_vars:
      var.set(0)
    for i in range(3):
      for j in range(4):
        tkim = self.genomes[i*4+j].get_mod_thumb_tk(300)
        self.im_labels[i*4+j] = tk.Checkbutton(self.root, image=tkim, variable=self.im_vars[i*4+j], indicatoron=False, bd=10, selectcolor="green")
        self.im_labels[i*4+j].grid(row=i, column=j)
        self.im_labels[i*4+j].image = tkim
        self.old_im_labels[i*4+j].destroy()
        self.old_im_labels[i*4+j] = self.im_labels[i*4+j]

if __name__ == "__main__":
  if len(sys.argv) != 2:
    print "USAGE: " + sys.argv[0] + " image"
    sys.exit()
  root = tk.Tk()
  gui = Gui(root, sys.argv[1])
  root.mainloop()
  root.destroy()
<|endoftext|>