<commit_before>import requests
from requests_oauthlib import OAuth1Session
import flask
import json

BASE_RESPONSE = {
    "version": "1",
    "response": {}
}

with open(".scorekeeper_keys") as f:
    keys = json.load(f)

TWITTER_APP_KEY = keys["api_key"]
TWITTER_APP_KEY_SECRET = keys["api_secret"]
TWITTER_ACCESS_TOKEN = keys["access_token"]
TWITTER_ACCESS_TOKEN_SECRET = keys["token_secret"]

twitter = OAuth1Session(
    TWITTER_APP_KEY,
    client_secret=TWITTER_APP_KEY_SECRET,
    resource_owner_key=TWITTER_ACCESS_TOKEN,
    resource_owner_secret=TWITTER_ACCESS_TOKEN_SECRET
)

api = flask.Flask(__name__)

def get_points():
    points = {}
    try:
        with open('.points') as p:
            points = json.load(p)
    except FileNotFoundError:
        with open('.points', 'w') as p:
            json.dump({}, p)
    return points

def add_point(person):
    points = get_points()
    person = person.lower()
    if person in points:
        points[person] += 1
    else:
        points[person] = 1

    with open('.points', 'w') as p:
        json.dump(points, p)

def points_for(person):
    return get_points().get(person.lower(), 0)

def do_tweet(person, reason):
    if reason:
        if reason.lower().startswith("for "):
            reason = reason[4:]
    r = twitter.post("https://api.twitter.com/1.1/statuses/update.json",
                      data={"status": "Crazy point to {}: {}".format(person.title(), reason)})
    print("Status:", r.text)
    print("Person", person, "\nReason", reason)

@api.route('/', methods=['POST', 'GET'])
def alexa():
    data = flask.request.data
    if data:
        data = json.loads(data.decode('UTF-8'))
    else:
        raise ValueError("Invalid arguments; no data")

    with open('/tmp/alexa_test', 'a+') as f:
        print("Data  : " + json.dumps(data, indent=2, separators=(',', ': ')) + '\n')

    request = data["request"]
    req_type = request["type"]
    session_id = data["session"]["sessionId"]

    attrs = {"person": None, "reason": None}
    attrs.update(data["session"].get("attributes", {}))

    if req_type == "LaunchRequest":
        descs = ["{} has {} point{}".format(k, v, '' if v == 1 else 's') for (k, v) in
                 sorted(get_points().items(), key=lambda n:n[1], reverse=True)]
        pretty_scores = "There are no scores yet."
        if len(descs) == 1:
            pretty_scores = ';'.join(descs)
        elif len(descs) > 1:
            pretty_scores = ';'.join(descs[:-1]) + '; and ' + descs[-1]
        resp = dict(BASE_RESPONSE)
        resp["response"] = {
            "outputSpeech": {
                "type": "PlainText",
                "text": "Here is the Hackafe score breakdown. {}".format(pretty_scores)
            },
            "shouldEndSession": True
        }
        return flask.jsonify(resp)
    elif req_type == "SessionEndedRequest":
        resp = dict(BASE_RESPONSE)
        resp["response"] = {
            "shouldEndSession": True
        }
        return flask.jsonify(resp)
    elif req_type == "IntentRequest":
        # TODO validate timestamp
        if "intent" in request:
            intent = request["intent"]
            name = intent['name']
            slots = intent['slots']

            if name == "Point" or name == "ConfirmPerson":
                for slot in slots.values():
                    slot_name = slot["name"]
                    slot_val = slot.get("value", None)

                    if slot_name == "Person":
                        attrs["person"] = slot_val
                    elif slot_name == "Reason":
                        attrs["reason"] = slot_val

                if attrs["reason"] and attrs["person"]:
                    resp = dict(BASE_RESPONSE)
                    resp["response"] = {
                        "outputSpeech": {
                            "type": "PlainText",
                            "text": "Should I give {} a point for {}?".format(
                                attrs["person"], attrs["reason"])
                        },
                        "shouldEndSession": False
                    }
                    resp["sessionAttributes"] = attrs
                    return flask.jsonify(resp)
                elif not attrs["person"]:
                    print("Didn't get a name, reprompting")
                    resp = dict(BASE_RESPONSE)
                    resp["response"] = {
                            "outputSpeech": {
                                "type": "PlainText",
                                "text": "Who should I give a point to?"
                            },
                        "shouldEndSession": False
                    }
                    resp["sessionAttributes"] = attrs
                    return flask.jsonify(resp)
                elif not attrs["reason"]:
                    print("Prompting for a reason")
                    resp = dict(BASE_RESPONSE)
                    resp["response"] = {
                        "outputSpeech": {
                            "type": "PlainText",
                            "text": "What's the reason?"
                        },
                        "shouldEndSession": False
                    }
                    resp["sessionAttributes"] = attrs
                    return flask.jsonify(resp)
            elif name == "GiveReason":
                for slot in slots.values():
                    slot_name = slot["name"]
                    slot_val = slot.get("value", None)

                    if slot_name == "Reason":
                        attrs["reason"] = slot_val
                if attrs["reason"] and attrs["person"]:
                    resp = dict(BASE_RESPONSE)
                    resp["response"] = {
                        "outputSpeech": {
                            "type": "PlainText",
                            "text": "Should I give {} a point for {}?".format(
                                attrs["person"], attrs["reason"])
                        },
                        "shouldEndSession": False
                    }
                    resp["sessionAttributes"] = attrs
                    return flask.jsonify(resp)
                elif not attrs["person"]:
                    print("Didn't get a name, reprompting")
                    resp = dict(BASE_RESPONSE)
                    resp["response"] = {
                            "outputSpeech": {
                                "type": "PlainText",
                                "text": "Who should I give a point to?"
                            },
                        "shouldEndSession": False
                    }
                    resp["sessionAttributes"] = attrs
                    return flask.jsonify(resp)
                elif not attrs["reason"]:
                    print("Prompting for a reason")
                    resp = dict(BASE_RESPONSE)
                    resp["response"] = {
                        "outputSpeech": {
                            "type": "PlainText",
                            "text": "What's the reason?"
                        },
                        "shouldEndSession": False
                    }
                    resp["sessionAttributes"] = attrs
                    return flask.jsonify(resp)
            elif name == "ConfirmPoint":
                for slot in slots.values():
                    slot_name = slot["name"]
                    slot_val = slot.get("value", None)
                    confirmed = False

                    if slot_name == "Confirmed":
                        confirmed = (slot_val.lower() == "yes")

                    if confirmed:
                        add_point(attrs["person"])
                        num_points = points_for(attrs["person"])
                        resp = dict(BASE_RESPONSE)
                        resp["response"] = {
                            "outputSpeech": {
                                "type": "PlainText",
                                "text": "{} now has {} point{}!".format(attrs["person"], num_points, "" if num_points == 1 else "s")
                            },
                        "shouldEndSession": True
                        }
                        do_tweet(attrs["person"],
                                 attrs["reason"])
                        resp["sessionAttributes"] = attrs
                        return flask.jsonify(resp)
                    else:
                        resp = dict(BASE_RESPONSE)
                        resp["response"] = {
                            "shouldEndSession": True
                        }
                        resp["sessionAttributes"] = attrs
                        return flask.jsonify(resp)

    return {'args': repr(args), 'kwargs': repr(kwargs)}

get_points()
api.run('0.0.0.0', port=80, debug=True)
<commit_msg>Add sign messages<commit_after>import threading
import requests
from requests_oauthlib import OAuth1Session
import flask
import json
import sign

BASE_RESPONSE = {
    "version": "1",
    "response": {}
}

with open(".scorekeeper_keys") as f:
    keys = json.load(f)

TWITTER_APP_KEY = keys["api_key"]
TWITTER_APP_KEY_SECRET = keys["api_secret"]
TWITTER_ACCESS_TOKEN = keys["access_token"]
TWITTER_ACCESS_TOKEN_SECRET = keys["token_secret"]

twitter = OAuth1Session(
    TWITTER_APP_KEY,
    client_secret=TWITTER_APP_KEY_SECRET,
    resource_owner_key=TWITTER_ACCESS_TOKEN,
    resource_owner_secret=TWITTER_ACCESS_TOKEN_SECRET
)

api = flask.Flask(__name__)

SIGN = sign.Sign("192.168.4.105", 8800)

def get_points():
    points = {}
    try:
        with open('.points') as p:
            points = json.load(p)
    except FileNotFoundError:
        with open('.points', 'w') as p:
            json.dump({}, p)
    return points

def add_point(person):
    points = get_points()
    person = person.lower()
    if person in points:
        points[person] += 1
    else:
        points[person] = 1

    with open('.points', 'w') as p:
        json.dump(points, p)

def points_for(person):
    return get_points().get(person.lower(), 0)

def do_tweet(person, reason):
    if reason:
        if reason.lower().startswith("for "):
            reason = reason[4:]
    r = twitter.post("https://api.twitter.com/1.1/statuses/update.json",
                      data={"status": "Crazy point to {}: {}".format(person.title(), reason)})
    print("Status:", r.text)
    print("Person", person, "\nReason", reason)

def _display_thread():
    texts = ["{}: {}".format(*i) for i in
             sorted(get_points().items(), key=lambda n:n[1], reverse=True)]
    total_lifetime = len(texts) * 2

    for text in texts:
        SIGN.add_message(texts, priority=2, lifetime=total_lifetime)

def do_display():
    threading.Thread(target=_display_thread).start()

@api.route('/', methods=['POST', 'GET'])
def alexa():
    data = flask.request.data
    if data:
        data = json.loads(data.decode('UTF-8'))
    else:
        raise ValueError("Invalid arguments; no data")

    with open('/tmp/alexa_test', 'a+') as f:
        print("Data  : " + json.dumps(data, indent=2, separators=(',', ': ')) + '\n')

    request = data["request"]
    req_type = request["type"]
    session_id = data["session"]["sessionId"]

    attrs = {"person": None, "reason": None}
    attrs.update(data["session"].get("attributes", {}))

    if req_type == "LaunchRequest":
        descs = ["{} has {} point{}".format(k, v, '' if v == 1 else 's') for (k, v) in
                 sorted(get_points().items(), key=lambda n:n[1], reverse=True)]
        pretty_scores = "There are no scores yet."
        if len(descs) == 1:
            pretty_scores = ';'.join(descs)
        elif len(descs) > 1:
            pretty_scores = ';'.join(descs[:-1]) + '; and ' + descs[-1]
        resp = dict(BASE_RESPONSE)
        resp["response"] = {
            "outputSpeech": {
                "type": "PlainText",
                "text": "Here is the Hackafe score breakdown. {}".format(pretty_scores)
            },
            "shouldEndSession": True
        }
        do_display()
        return flask.jsonify(resp)
    elif req_type == "SessionEndedRequest":
        resp = dict(BASE_RESPONSE)
        resp["response"] = {
            "shouldEndSession": True
        }
        return flask.jsonify(resp)
    elif req_type == "IntentRequest":
        # TODO validate timestamp
        if "intent" in request:
            intent = request["intent"]
            name = intent['name']
            slots = intent['slots']

            if name == "Point" or name == "ConfirmPerson":
                for slot in slots.values():
                    slot_name = slot["name"]
                    slot_val = slot.get("value", None)

                    if slot_name == "Person":
                        attrs["person"] = slot_val
                    elif slot_name == "Reason":
                        attrs["reason"] = slot_val

                if attrs["reason"] and attrs["person"]:
                    resp = dict(BASE_RESPONSE)
                    resp["response"] = {
                        "outputSpeech": {
                            "type": "PlainText",
                            "text": "Should I give {} a point for {}?".format(
                                attrs["person"], attrs["reason"])
                        },
                        "shouldEndSession": False
                    }
                    resp["sessionAttributes"] = attrs
                    return flask.jsonify(resp)
                elif not attrs["person"]:
                    print("Didn't get a name, reprompting")
                    resp = dict(BASE_RESPONSE)
                    resp["response"] = {
                            "outputSpeech": {
                                "type": "PlainText",
                                "text": "Who should I give a point to?"
                            },
                        "shouldEndSession": False
                    }
                    resp["sessionAttributes"] = attrs
                    return flask.jsonify(resp)
                elif not attrs["reason"]:
                    print("Prompting for a reason")
                    resp = dict(BASE_RESPONSE)
                    resp["response"] = {
                        "outputSpeech": {
                            "type": "PlainText",
                            "text": "What's the reason?"
                        },
                        "shouldEndSession": False
                    }
                    resp["sessionAttributes"] = attrs
                    return flask.jsonify(resp)
            elif name == "GiveReason":
                for slot in slots.values():
                    slot_name = slot["name"]
                    slot_val = slot.get("value", None)

                    if slot_name == "Reason":
                        attrs["reason"] = slot_val
                if attrs["reason"] and attrs["person"]:
                    resp = dict(BASE_RESPONSE)
                    resp["response"] = {
                        "outputSpeech": {
                            "type": "PlainText",
                            "text": "Should I give {} a point for {}?".format(
                                attrs["person"], attrs["reason"])
                        },
                        "shouldEndSession": False
                    }
                    resp["sessionAttributes"] = attrs
                    return flask.jsonify(resp)
                elif not attrs["person"]:
                    print("Didn't get a name, reprompting")
                    resp = dict(BASE_RESPONSE)
                    resp["response"] = {
                            "outputSpeech": {
                                "type": "PlainText",
                                "text": "Who should I give a point to?"
                            },
                        "shouldEndSession": False
                    }
                    resp["sessionAttributes"] = attrs
                    return flask.jsonify(resp)
                elif not attrs["reason"]:
                    print("Prompting for a reason")
                    resp = dict(BASE_RESPONSE)
                    resp["response"] = {
                        "outputSpeech": {
                            "type": "PlainText",
                            "text": "What's the reason?"
                        },
                        "shouldEndSession": False
                    }
                    resp["sessionAttributes"] = attrs
                    return flask.jsonify(resp)
            elif name == "ConfirmPoint":
                for slot in slots.values():
                    slot_name = slot["name"]
                    slot_val = slot.get("value", None)
                    confirmed = False

                    if slot_name == "Confirmed":
                        confirmed = (slot_val.lower() == "yes")

                    if confirmed:
                        add_point(attrs["person"])
                        num_points = points_for(attrs["person"])
                        resp = dict(BASE_RESPONSE)
                        resp["response"] = {
                            "outputSpeech": {
                                "type": "PlainText",
                                "text": "{} now has {} point{}!".format(attrs["person"], num_points, "" if num_points == 1 else "s")
                            },
                        "shouldEndSession": True
                        }
                        do_tweet(attrs["person"],
                                 attrs["reason"])
                        resp["sessionAttributes"] = attrs
                        return flask.jsonify(resp)
                    else:
                        resp = dict(BASE_RESPONSE)
                        resp["response"] = {
                            "shouldEndSession": True
                        }
                        resp["sessionAttributes"] = attrs
                        return flask.jsonify(resp)

    return {'args': repr(args), 'kwargs': repr(kwargs)}

get_points()
api.run('0.0.0.0', port=80, debug=True)
<|endoftext|>


<commit_before>from collections import defaultdict

from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.db.models.fields.related import ManyToManyRel
from django.db.models.query_utils import QueryWrapper

from taggit.forms import TagField
from taggit.models import Tag, TaggedItem
from taggit.utils import require_instance_manager

class TaggableRel(ManyToManyRel):
    def __init__(self):
        self.to = TaggedItem
        self.related_name = None
        self.limit_choices_to = {}
        self.symmetrical = True
        self.multiple = True
        self.through = None

class TaggableManager(object):
    def __init__(self):
        self.rel = TaggableRel()
        self.editable = True
        self.unique = False
        self.creates_table = False
        self.db_column = None
        self.choices = None
        self.creation_counter = models.Field.creation_counter
        models.Field.creation_counter += 1

    def __get__(self, instance, type):
        manager = _TaggableManager()
        manager.model = type
        if instance is None:
            manager.object_id = None
        else:
            manager.object_id = instance.pk
        return manager

    def contribute_to_class(self, cls, name):
        self.name = self.column = name
        self.model = cls
        cls._meta.add_field(self)
        setattr(cls, name, self)

    def save_form_data(self, instance, value):
        getattr(instance, self.name).set(*value)

    def get_db_prep_lookup(self, lookup_type, value):
        if lookup_type !=  "in":
            raise ValueError("You can't do lookups other than \"in\" on Tags")
        if all(isinstance(v, Tag) for v in value):
            qs = TaggedItem.objects.filter(tag__in=value)
        elif all(isinstance(v, basestring) for v in value):
            qs = TaggedItem.objects.filter(tag__name__in=value)
        elif all(isinstance(v, (int, long)) for v in value):
            # This one is really ackward, just don't do it.  The ORM does it
            # for deletes, but no one else gets to.
            return value
        else:
            # Fucking flip-floppers.
            raise ValueError("You can't combine Tag objects and strings.  '%s' was provided." % value)
        sql, params = qs.values_list("pk", flat=True).query.as_sql()
        return QueryWrapper(("(%s)" % sql), params)

    def formfield(self, form_class=TagField, **kwargs):
        defaults = {
            "label": "Tags",
            "help_text": "A comma seperated list of tags."
        }
        defaults.update(kwargs)
        return form_class(**kwargs)

    def value_from_object(self, instance):
        return ", ".join(map(unicode, getattr(instance, self.name).all()))

    def related_query_name(self):
        return None

    def m2m_reverse_name(self):
        return "id"

    def m2m_column_name(self):
        return "object_id"

    def db_type(self):
        return None

    def m2m_db_table(self):
        return self.rel.to._meta.db_table

    def extra_filters(self, pieces, pos, negate):
        if negate:
            return []
        prefix = "__".join(pieces[:pos+1])
        return [("%s__content_type" % prefix, ContentType.objects.get_for_model(self.model))]

class _TaggableManager(models.Manager):
    def get_query_set(self):
        ct = ContentType.objects.get_for_model(self.model)
        if self.object_id is not None:
            return Tag.objects.filter(items__object_id=self.object_id,
                items__content_type=ct)
        else:
            return Tag.objects.filter(items__content_type=ct).distinct()

    @require_instance_manager
    def add(self, *tags):
        for tag in tags:
            if not isinstance(tag, Tag):
                tag, _ = Tag.objects.get_or_create(name=tag)
            TaggedItem.objects.create(object_id=self.object_id,
                content_type=ContentType.objects.get_for_model(self.model), tag=tag)

    @require_instance_manager
    def set(self, *tags):
        self.clear()
        self.add(*tags)

    @require_instance_manager
    def remove(self, *tags):
        TaggedItem.objects.filter(object_id=self.object_id,
            content_type=ContentType.objects.get_for_model(self.model)).filter(
            tag__name__in=tags).delete()

    @require_instance_manager
    def clear(self):
        TaggedItem.objects.filter(object_id=self.object_id,
            content_type=ContentType.objects.get_for_model(self.model)).delete()

    def most_common(self):
        return self.get_query_set().annotate(
            num_times=models.Count('items')
        ).order_by('-num_times')

    @require_instance_manager
    def similar_objects(self):
        qs = TaggedItem.objects.values('object_id', 'content_type')
        qs = qs.annotate(n=models.Count('pk'))
        qs = qs.exclude(
            object_id=self.object_id,
            content_type=ContentType.objects.get_for_model(self.model)
        )
        qs = qs.filter(tag__in=self.all())
        qs = qs.order_by('-n')

        preload = defaultdict(set)
        for result in qs:
            preload[result["content_type"]].add(result["object_id"])

        items = {}
        for ct, obj_ids in preload.iteritems():
            ct = ContentType.objects.get_for_id(ct)
            items[ct.pk] = dict((o.pk, o) for o in
                ct.model_class()._default_manager.filter(pk__in=obj_ids)
            )

        results = []
        for result in qs:
            obj = items[result["content_type"]][result["object_id"]]
            obj.similar_tags = result["n"]
            results.append(obj)
        return results
<commit_msg>Update to work with django 1.2<commit_after>from collections import defaultdict

import django
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.db.models.fields.related import ManyToManyRel
from django.db.models.query_utils import QueryWrapper

from taggit.forms import TagField
from taggit.models import Tag, TaggedItem
from taggit.utils import require_instance_manager

class TaggableRel(ManyToManyRel):
    def __init__(self):
        self.to = TaggedItem
        self.related_name = None
        self.limit_choices_to = {}
        self.symmetrical = True
        self.multiple = True
        self.through = None

class TaggableManager(object):
    def __init__(self):
        self.rel = TaggableRel()
        self.editable = True
        self.unique = False
        self.creates_table = False
        self.db_column = None
        self.choices = None
        self.creation_counter = models.Field.creation_counter
        models.Field.creation_counter += 1

    def __get__(self, instance, type):
        manager = _TaggableManager()
        manager.model = type
        if instance is None:
            manager.object_id = None
        else:
            manager.object_id = instance.pk
        return manager

    def contribute_to_class(self, cls, name):
        self.name = self.column = name
        self.model = cls
        cls._meta.add_field(self)
        setattr(cls, name, self)

    def save_form_data(self, instance, value):
        getattr(instance, self.name).set(*value)

    def get_prep_lookup(self, lookup_type, value):
        if lookup_type !=  "in":
            raise ValueError("You can't do lookups other than \"in\" on Tags")
        if all(isinstance(v, Tag) for v in value):
            qs = TaggedItem.objects.filter(tag__in=value)
        elif all(isinstance(v, basestring) for v in value):
            qs = TaggedItem.objects.filter(tag__name__in=value)
        elif all(isinstance(v, (int, long)) for v in value):
            # This one is really ackward, just don't do it.  The ORM does it
            # for deletes, but no one else gets to.
            return value
        else:
            # Fucking flip-floppers.
            raise ValueError("You can't combine Tag objects and strings.  '%s' was provided." % value)
        sql, params = qs.values_list("pk", flat=True).query.as_sql()
        return QueryWrapper(("(%s)" % sql), params)

    if django.VERSION < (1, 2):
        get_db_prep_lookup = get_prep_lookup

    def formfield(self, form_class=TagField, **kwargs):
        defaults = {
            "label": "Tags",
            "help_text": "A comma seperated list of tags."
        }
        defaults.update(kwargs)
        return form_class(**kwargs)

    def value_from_object(self, instance):
        return ", ".join(map(unicode, getattr(instance, self.name).all()))

    def related_query_name(self):
        return None

    def m2m_reverse_name(self):
        return "id"

    def m2m_column_name(self):
        return "object_id"

    def db_type(self):
        return None

    def m2m_db_table(self):
        return self.rel.to._meta.db_table

    def extra_filters(self, pieces, pos, negate):
        if negate:
            return []
        prefix = "__".join(pieces[:pos+1])
        return [("%s__content_type" % prefix, ContentType.objects.get_for_model(self.model))]

class _TaggableManager(models.Manager):
    def get_query_set(self):
        ct = ContentType.objects.get_for_model(self.model)
        if self.object_id is not None:
            return Tag.objects.filter(items__object_id=self.object_id,
                items__content_type=ct)
        else:
            return Tag.objects.filter(items__content_type=ct).distinct()

    @require_instance_manager
    def add(self, *tags):
        for tag in tags:
            if not isinstance(tag, Tag):
                tag, _ = Tag.objects.get_or_create(name=tag)
            TaggedItem.objects.create(object_id=self.object_id,
                content_type=ContentType.objects.get_for_model(self.model), tag=tag)

    @require_instance_manager
    def set(self, *tags):
        self.clear()
        self.add(*tags)

    @require_instance_manager
    def remove(self, *tags):
        TaggedItem.objects.filter(object_id=self.object_id,
            content_type=ContentType.objects.get_for_model(self.model)).filter(
            tag__name__in=tags).delete()

    @require_instance_manager
    def clear(self):
        TaggedItem.objects.filter(object_id=self.object_id,
            content_type=ContentType.objects.get_for_model(self.model)).delete()

    def most_common(self):
        return self.get_query_set().annotate(
            num_times=models.Count('items')
        ).order_by('-num_times')

    @require_instance_manager
    def similar_objects(self):
        qs = TaggedItem.objects.values('object_id', 'content_type')
        qs = qs.annotate(n=models.Count('pk'))
        qs = qs.exclude(
            object_id=self.object_id,
            content_type=ContentType.objects.get_for_model(self.model)
        )
        qs = qs.filter(tag__in=self.all())
        qs = qs.order_by('-n')

        preload = defaultdict(set)
        for result in qs:
            preload[result["content_type"]].add(result["object_id"])

        items = {}
        for ct, obj_ids in preload.iteritems():
            ct = ContentType.objects.get_for_id(ct)
            items[ct.pk] = dict((o.pk, o) for o in
                ct.model_class()._default_manager.filter(pk__in=obj_ids)
            )

        results = []
        for result in qs:
            obj = items[result["content_type"]][result["object_id"]]
            obj.similar_tags = result["n"]
            results.append(obj)
        return results
<|endoftext|>


<commit_before>from __future__ import print_function
import matplotlib.pyplot as plt
import numpy as np
from scipy import misc
import STR2CTC
import os

def read_image_list(pathToList):
    """Reads a .txt file containing paths to the images
    Args:
       image_list_file: a .txt file with one /path/to/image per line
       label: optionally, if set label will be pasted after each line
    Returns:
       List with all filenames in file image_list_file
    """
    f = open(pathToList, 'r')
    filenames = []
    for line in f:
        filenames.append(line[:-1])
    f.close()
    return filenames

def get_batch_labels(bList, cm):
    labels = []
    for path in bList:
        labelFile = path[:] + ".txt"
        tmp = open(labelFile, 'r')
        str = tmp.readline();
        labels.append(str)
        #print(str)
        if tmp is not None:
            tmp.close()
    idx, val, shape = STR2CTC.target_string_list_to_ctc_tensor_repr(labels, cm)
    return idx, val, shape

def get_batch_imgs(bList, imgW, mvn):
    imgs = []
    seqL = []

    for path in bList:
        aImg = misc.imread(path)
        width = aImg.shape[1]
        hei = aImg.shape[0]
        aSeqL = min(width, imgW)
        aSeqL = max(aSeqL, imgW/2)
        seqL.append(aSeqL)
       # aImg = aImg.astype('float32')
        aImg = aImg / 255.0
        if mvn:
            std = np.std(aImg)
            mean = np.mean(aImg)
            tImg = (aImg - mean) / std
            aImg = tImg
        if width < imgW:
            padW = imgW - width
            npad = ((0,0), (0,padW))
            tImg = np.pad(aImg, npad, mode='constant', constant_values=0)
            aImg = tImg
        if width > imgW:
            tImg = aImg[:,:imgW]
            aImg = tImg
        #plt.imshow(aImg, cmap=plt.cm.gray)
        #plt.show()
        imgs.append(aImg)
        bSize = len(bList)
        imgBatched = np.zeros((bSize, hei, imgW, 1), dtype='float32')
        # batch the image list
        for idx, img in enumerate(imgs):
            imgBatched[idx,:,:,0] = img
    return imgBatched, seqL

def get_list_vals(bList, cm, imgW, mvn=False):
    tgtIdx, tgtVal, tgtShape = get_batch_labels(bList, cm)
    inpBatch, inpSeqL = get_batch_imgs(bList, imgW, mvn)
    return inpBatch, inpSeqL, tgtIdx, tgtVal, tgtShape

if __name__ == '__main__':
    os.chdir("..")
    list = read_image_list('./resources/lp_only_train.lst')
    imgBatches, seqL = get_list_vals(list, STR2CTC.get_charmap_lp(), 100)
    #print(seqL)
    print(imgBatches.shape)
    print(imgBatches.dtype)
    plt.imshow(imgBatches[129], cmap=plt.cm.gray)
    plt.show()
<commit_msg>changed folder sturcture: models/<type_of_model>/<model_name><commit_after>from __future__ import print_function
import matplotlib.pyplot as plt
import numpy as np
from scipy import misc
import STR2CTC
import os

def read_image_list(pathToList):
    """Reads a .txt file containing paths to the images
    Args:
       image_list_file: a .txt file with one /path/to/image per line
       label: optionally, if set label will be pasted after each line
    Returns:
       List with all filenames in file image_list_file
    """
    f = open(pathToList, 'r')
    filenames = []
    for line in f:
        if(line[-1]=='\n'):
            filenames.append(line[:-1])
        else:
            filenames.append(line)
    f.close()
    return filenames

def get_batch_labels(bList, cm):
    labels = []
    for path in bList:
        labelFile = path[:] + ".txt"
        tmp = open(labelFile, 'r')
        str = tmp.readline();
        labels.append(str)
        #print(str)
        if tmp is not None:
            tmp.close()
    idx, val, shape = STR2CTC.target_string_list_to_ctc_tensor_repr(labels, cm)
    return idx, val, shape

def get_batch_imgs(bList, imgW, mvn):
    imgs = []
    seqL = []

    for path in bList:
        aImg = misc.imread(path)
        width = aImg.shape[1]
        hei = aImg.shape[0]
        aSeqL = min(width, imgW)
        aSeqL = max(aSeqL, imgW/2)
        seqL.append(aSeqL)
       # aImg = aImg.astype('float32')
        aImg = aImg / 255.0
        if mvn:
            std = np.std(aImg)
            mean = np.mean(aImg)
            tImg = (aImg - mean) / std
            aImg = tImg
        if width < imgW:
            padW = imgW - width
            npad = ((0,0), (0,padW))
            tImg = np.pad(aImg, npad, mode='constant', constant_values=0)
            aImg = tImg
        if width > imgW:
            tImg = aImg[:,:imgW]
            aImg = tImg
        #plt.imshow(aImg, cmap=plt.cm.gray)
        #plt.show()
        imgs.append(aImg)
        bSize = len(bList)
        imgBatched = np.zeros((bSize, hei, imgW, 1), dtype='float32')
        # batch the image list
        for idx, img in enumerate(imgs):
            imgBatched[idx,:,:,0] = img
    return imgBatched, seqL

def get_list_vals(bList, cm, imgW, mvn=False):
    tgtIdx, tgtVal, tgtShape = get_batch_labels(bList, cm)
    inpBatch, inpSeqL = get_batch_imgs(bList, imgW, mvn)
    return inpBatch, inpSeqL, tgtIdx, tgtVal, tgtShape

if __name__ == '__main__':
    os.chdir("..")
    list = read_image_list('./resources/lp_only_train.lst')
    imgBatches, seqL = get_list_vals(list, STR2CTC.get_charmap_lp(), 100)
    #print(seqL)
    print(imgBatches.shape)
    print(imgBatches.dtype)
    plt.imshow(imgBatches[129], cmap=plt.cm.gray)
    plt.show()
<|endoftext|>


<commit_before>def export_mtz(integrated_data, experiment_list, hklout):
  '''Export data from integrated_data corresponding to experiment_list to an
  MTZ file hklout.'''

  # for the moment assume (and assert) that we will convert data from exactly
  # one lattice...

  # FIXME test to see if the model is scan varying, and if it is (which it can be)
  # write out a different U matrix for each batch... This should be easy. N.B.
  # should use U matrix from middle of batch (probably)

  # FIXME allow for more than one experiment in here: this is fine just add
  # multiple MTZ data sets (DIALS1...DIALSN) and multiple batch headers: one
  # range of batches for each experiment

  assert(len(experiment_list) == 1)
  assert(min(integrated_data['id']) == max(integrated_data['id']) == 0)

  # strip out negative variance reflections: these should not really be there
  # FIXME Doing select on summation results. Should do on profile result if
  # present?
  selection = integrated_data['intensity.sum.variance'] < 0
  if selection.count(True) > 0:
    integrated_data.del_selected(selection)
    print 'Removing %d reflections with negative variance' % selection.count(True)

  # FIXME TODO for more than one experiment into an MTZ file:
  #
  # - add an epoch (or recover an epoch) from the scan and add this as an extra
  #   column to the MTZ file for scaling, so we know that the two lattices were
  #   integrated at the same time
  # - decide a sensible BATCH increment to apply to the BATCH value between
  #   experiments and add this
  #
  # At the moment this is probably enough to be working on.

  experiment = experiment_list[0]

  # also only work with one panel(for the moment)

  assert(len(experiment.detector) == 1)

  axis = experiment.goniometer.get_rotation_axis()
  s0 = experiment.beam.get_s0()
  wavelength = experiment.beam.get_wavelength()

  from scitbx import matrix

  panel = experiment.detector[0]
  origin = matrix.col(panel.get_origin())
  fast = matrix.col(panel.get_fast_axis())
  slow = matrix.col(panel.get_slow_axis())

  pixel_size = panel.get_pixel_size()

  fast *= pixel_size[0]
  slow *= pixel_size[1]

  U = experiment.crystal.get_U()
  unit_cell = experiment.crystal.get_unit_cell()
  from iotbx import mtz

  from scitbx.array_family import flex
  from math import floor, sin, sqrt

  m = mtz.object()
  m.set_title('from dials.export_mtz')
  m.set_space_group_info(experiment.crystal.get_space_group().info())

  image_range = experiment.scan.get_image_range()

  for b in range(image_range[0], image_range[1] + 1):
    o = m.add_batch().set_num(b).set_nbsetid(1).set_ncryst(1)
    o.set_time1(0.0).set_time2(0.0).set_title('Batch %d' % b)
    o.set_ndet(1).set_theta(flex.float((0.0, 0.0))).set_lbmflg(0)
    o.set_alambd(wavelength).set_delamb(0.0).set_delcor(0.0)
    o.set_divhd(0.0).set_divvd(0.0)

    # FIXME hard-coded assumption on indealized beam vector below... this may be
    # broken when we come to process data from a non-imgCIF frame
    o.set_so(flex.float(s0)).set_source(flex.float((0, 0, -1)))

    # these are probably 0, 1 respectively, also flags for how many are set, sd
    o.set_bbfac(0.0).set_bscale(1.0)
    o.set_sdbfac(0.0).set_sdbscale(0.0).set_nbscal(0)

    # unit cell (this is fine) and the what-was-refined-flags FIXME hardcoded

    # take time-varying parameters from the *end of the frame* unlikely to
    # be much different at the end - however only exist if time-varying refinement
    # was used
    if experiment.crystal.num_scan_points:
      _unit_cell = experiment.crystal.get_unit_cell_at_scan_point(b)
      _U = experiment.crystal.get_U_at_scan_point(b)
    else:
      _unit_cell = unit_cell
      _U = U

    # FIXME need to get what was refined and what was constrained from the
    # crystal model
    o.set_cell(flex.float(_unit_cell.parameters()))
    o.set_lbcell(flex.int((-1, -1, -1, -1, -1, -1)))
    o.set_umat(flex.float(_U.transpose().elems))

    # get the mosaic spread though today it may not actually be set
    mosaic = experiment.crystal.get_mosaicity()
    o.set_crydat(flex.float([mosaic, 0.0, 0.0, 0.0, 0.0, 0.0,
                             0.0, 0.0, 0.0, 0.0, 0.0, 0.0]))

    o.set_lcrflg(0)
    o.set_datum(flex.float((0.0, 0.0, 0.0)))

    # detector size, distance
    o.set_detlm(flex.float([0.0, panel.get_image_size()[0],
                            0.0, panel.get_image_size()[1],
                            0, 0, 0, 0]))
    o.set_dx(flex.float([panel.get_distance(), 0.0]))

    # goniometer axes and names, and scan axis number, and number of axes, missets
    o.set_e1(flex.float(axis))
    o.set_e2(flex.float((0.0, 0.0, 0.0)))
    o.set_e3(flex.float((0.0, 0.0, 0.0)))
    o.set_gonlab(flex.std_string(('AXIS', '', '')))
    o.set_jsaxs(1)
    o.set_ngonax(1)
    o.set_phixyz(flex.float((0.0, 0.0, 0.0, 0.0, 0.0, 0.0)))

    # scan ranges, axis
    phi_start, phi_range = experiment.scan.get_image_oscillation(b)
    o.set_phistt(phi_start)
    o.set_phirange(phi_range)
    o.set_phiend(phi_start + phi_range)
    o.set_scanax(flex.float(axis))

    # number of misorientation angles
    o.set_misflg(0)

    # crystal axis closest to rotation axis (why do I want this?)
    o.set_jumpax(0)

    # type of data - 1; 2D, 2; 3D, 3; Laue
    o.set_ldtype(2)

  # now create the actual data structures - first keep a track of the columns
  # H K L M/ISYM BATCH I SIGI IPR SIGIPR FRACTIONCALC XDET YDET ROT WIDTH
  # LP MPART FLAG BGPKRATIOS

  from cctbx.array_family import flex as cflex
  from cctbx.miller import map_to_asu_isym

  # gather the required information for the reflection file

  x_px, y_px, z_px = zip(*integrated_data['xyzcal.px'])

  xdet = flex.double(x_px)
  ydet = flex.double(y_px)
  zdet = flex.double(z_px)

  # compute ROT values
  rot = flex.double([experiment.scan.get_angle_from_image_index(z) for z in zdet])

  # compute BATCH values
  batch = flex.floor(zdet).iround() + 1

  # compute M/ISYM
  mi = integrated_data['miller_index']
  h, k, l = zip(*integrated_data['miller_index'])
  isym = flex.int(len(integrated_data['miller_index']))
  misym = flex.int(len(isym))
  map_to_asu_isym(experiment.crystal.get_space_group().type(), True,
                  integrated_data['miller_index'], isym)

  for j, i in enumerate(isym):
    if i < 0:
      mis = (-i + 1) * 2
    else:
      mis = (i + 1) * 2 - 1
    misym[j] = mis

  # we're working with full reflections so...
  fractioncalc = flex.double(len(isym), 1.0)

  # now go for it and make an MTZ file...

  x = m.add_crystal('XTAL', 'DIALS', unit_cell.parameters())
  d = x.add_dataset('FROMDIALS', wavelength)

  # now add column information...

  type_table = {'IPR': 'J', 'BGPKRATIOS': 'R', 'WIDTH': 'R', 'I': 'J',
                'H': 'H', 'K': 'H', 'MPART': 'I', 'L': 'H', 'BATCH': 'B',
                'M_ISYM': 'Y', 'SIGI': 'Q', 'FLAG': 'I', 'XDET': 'R', 'LP': 'R',
                'YDET': 'R', 'SIGIPR': 'Q', 'FRACTIONCALC': 'R', 'ROT': 'R'}

  h, k, l = zip(*integrated_data['miller_index'])
  h, k, l = flex.int(h), flex.int(k), flex.int(l)

  m.adjust_column_array_sizes(len(h))
  m.set_n_reflections(len(h))

  # assign H, K, L
  d.add_column('H', type_table['H']).set_values(h.as_double().as_float())
  d.add_column('K', type_table['K']).set_values(k.as_double().as_float())
  d.add_column('L', type_table['L']).set_values(l.as_double().as_float())

  d.add_column('M_ISYM', type_table['M_ISYM']).set_values(
    misym.as_double().as_float())
  d.add_column('BATCH', type_table['BATCH']).set_values(
    batch.as_double().as_float())

  lp = integrated_data['lp']
  if 'intensity.prf.value' in integrated_data:
    I = integrated_data['intensity.prf.value'] * lp
    V = integrated_data['intensity.prf.variance'] * lp
  else:
    I = integrated_data['intensity.sum.value'] * lp
    V = integrated_data['intensity.sum.variance'] * lp

  d.add_column('I', type_table['I']).set_values(I.as_float())

  # Trap negative variances

  assert ((V < 0).count(True) == 0)

  d.add_column('SIGI', type_table['SIGI']).set_values(flex.sqrt(V).as_float())

  d.add_column('FRACTIONCALC', type_table['FRACTIONCALC']).set_values(
    fractioncalc.as_float())

  d.add_column('XDET', type_table['XDET']).set_values(xdet.as_float())
  d.add_column('YDET', type_table['YDET']).set_values(ydet.as_float())
  d.add_column('ROT', type_table['ROT']).set_values(rot.as_float())

  m.write(hklout)

  return m
<commit_msg>Working in m.replace_original_index_miller_indices; intermediate commit with all the original code in but commented out...<commit_after>def export_mtz(integrated_data, experiment_list, hklout):
  '''Export data from integrated_data corresponding to experiment_list to an
  MTZ file hklout.'''

  # for the moment assume (and assert) that we will convert data from exactly
  # one lattice...

  # FIXME test to see if the model is scan varying, and if it is (which it can be)
  # write out a different U matrix for each batch... This should be easy. N.B.
  # should use U matrix from middle of batch (probably)

  # FIXME allow for more than one experiment in here: this is fine just add
  # multiple MTZ data sets (DIALS1...DIALSN) and multiple batch headers: one
  # range of batches for each experiment

  assert(len(experiment_list) == 1)
  assert(min(integrated_data['id']) == max(integrated_data['id']) == 0)

  # strip out negative variance reflections: these should not really be there
  # FIXME Doing select on summation results. Should do on profile result if
  # present?
  selection = integrated_data['intensity.sum.variance'] < 0
  if selection.count(True) > 0:
    integrated_data.del_selected(selection)
    print 'Removing %d reflections with negative variance' % selection.count(True)

  # FIXME TODO for more than one experiment into an MTZ file:
  #
  # - add an epoch (or recover an epoch) from the scan and add this as an extra
  #   column to the MTZ file for scaling, so we know that the two lattices were
  #   integrated at the same time
  # - decide a sensible BATCH increment to apply to the BATCH value between
  #   experiments and add this
  #
  # At the moment this is probably enough to be working on.

  experiment = experiment_list[0]

  # also only work with one panel(for the moment)

  assert(len(experiment.detector) == 1)

  axis = experiment.goniometer.get_rotation_axis()
  s0 = experiment.beam.get_s0()
  wavelength = experiment.beam.get_wavelength()

  from scitbx import matrix

  panel = experiment.detector[0]
  origin = matrix.col(panel.get_origin())
  fast = matrix.col(panel.get_fast_axis())
  slow = matrix.col(panel.get_slow_axis())

  pixel_size = panel.get_pixel_size()

  fast *= pixel_size[0]
  slow *= pixel_size[1]

  U = experiment.crystal.get_U()
  unit_cell = experiment.crystal.get_unit_cell()
  from iotbx import mtz

  from scitbx.array_family import flex
  from math import floor, sin, sqrt

  m = mtz.object()
  m.set_title('from dials.export_mtz')
  m.set_space_group_info(experiment.crystal.get_space_group().info())

  image_range = experiment.scan.get_image_range()

  for b in range(image_range[0], image_range[1] + 1):
    o = m.add_batch().set_num(b).set_nbsetid(1).set_ncryst(1)
    o.set_time1(0.0).set_time2(0.0).set_title('Batch %d' % b)
    o.set_ndet(1).set_theta(flex.float((0.0, 0.0))).set_lbmflg(0)
    o.set_alambd(wavelength).set_delamb(0.0).set_delcor(0.0)
    o.set_divhd(0.0).set_divvd(0.0)

    # FIXME hard-coded assumption on indealized beam vector below... this may be
    # broken when we come to process data from a non-imgCIF frame
    o.set_so(flex.float(s0)).set_source(flex.float((0, 0, -1)))

    # these are probably 0, 1 respectively, also flags for how many are set, sd
    o.set_bbfac(0.0).set_bscale(1.0)
    o.set_sdbfac(0.0).set_sdbscale(0.0).set_nbscal(0)

    # unit cell (this is fine) and the what-was-refined-flags FIXME hardcoded

    # take time-varying parameters from the *end of the frame* unlikely to
    # be much different at the end - however only exist if time-varying refinement
    # was used
    if experiment.crystal.num_scan_points:
      _unit_cell = experiment.crystal.get_unit_cell_at_scan_point(b)
      _U = experiment.crystal.get_U_at_scan_point(b)
    else:
      _unit_cell = unit_cell
      _U = U

    # FIXME need to get what was refined and what was constrained from the
    # crystal model
    o.set_cell(flex.float(_unit_cell.parameters()))
    o.set_lbcell(flex.int((-1, -1, -1, -1, -1, -1)))
    o.set_umat(flex.float(_U.transpose().elems))

    # get the mosaic spread though today it may not actually be set
    mosaic = experiment.crystal.get_mosaicity()
    o.set_crydat(flex.float([mosaic, 0.0, 0.0, 0.0, 0.0, 0.0,
                             0.0, 0.0, 0.0, 0.0, 0.0, 0.0]))

    o.set_lcrflg(0)
    o.set_datum(flex.float((0.0, 0.0, 0.0)))

    # detector size, distance
    o.set_detlm(flex.float([0.0, panel.get_image_size()[0],
                            0.0, panel.get_image_size()[1],
                            0, 0, 0, 0]))
    o.set_dx(flex.float([panel.get_distance(), 0.0]))

    # goniometer axes and names, and scan axis number, and number of axes, missets
    o.set_e1(flex.float(axis))
    o.set_e2(flex.float((0.0, 0.0, 0.0)))
    o.set_e3(flex.float((0.0, 0.0, 0.0)))
    o.set_gonlab(flex.std_string(('AXIS', '', '')))
    o.set_jsaxs(1)
    o.set_ngonax(1)
    o.set_phixyz(flex.float((0.0, 0.0, 0.0, 0.0, 0.0, 0.0)))

    # scan ranges, axis
    phi_start, phi_range = experiment.scan.get_image_oscillation(b)
    o.set_phistt(phi_start)
    o.set_phirange(phi_range)
    o.set_phiend(phi_start + phi_range)
    o.set_scanax(flex.float(axis))

    # number of misorientation angles
    o.set_misflg(0)

    # crystal axis closest to rotation axis (why do I want this?)
    o.set_jumpax(0)

    # type of data - 1; 2D, 2; 3D, 3; Laue
    o.set_ldtype(2)

  # now create the actual data structures - first keep a track of the columns
  # H K L M/ISYM BATCH I SIGI IPR SIGIPR FRACTIONCALC XDET YDET ROT WIDTH
  # LP MPART FLAG BGPKRATIOS

  from cctbx.array_family import flex as cflex
  from cctbx.miller import map_to_asu_isym

  # gather the required information for the reflection file

  x_px, y_px, z_px = zip(*integrated_data['xyzcal.px'])

  xdet = flex.double(x_px)
  ydet = flex.double(y_px)
  zdet = flex.double(z_px)

  # compute ROT values
  rot = flex.double([experiment.scan.get_angle_from_image_index(z) for z in zdet])

  # compute BATCH values
  batch = flex.floor(zdet).iround() + 1

  # compute M/ISYM
  # mi = integrated_data['miller_index']
  # h, k, l = zip(*integrated_data['miller_index'])
  # isym = flex.int(len(integrated_data['miller_index']))
  # misym = flex.int(len(isym))
  # map_to_asu_isym(experiment.crystal.get_space_group().type(), True,
  #                 integrated_data['miller_index'], isym)

  # for j, i in enumerate(isym):
  #   if i < 0:
  #     mis = (-i + 1) * 2
  #   else:
  #     mis = (i + 1) * 2 - 1
  #   misym[j] = mis

  # we're working with full reflections so...
  fractioncalc = flex.double(len(rot), 1.0)

  # now go for it and make an MTZ file...

  x = m.add_crystal('XTAL', 'DIALS', unit_cell.parameters())
  d = x.add_dataset('FROMDIALS', wavelength)

  # now add column information...

  type_table = {'IPR': 'J', 'BGPKRATIOS': 'R', 'WIDTH': 'R', 'I': 'J',
                'H': 'H', 'K': 'H', 'MPART': 'I', 'L': 'H', 'BATCH': 'B',
                'M_ISYM': 'Y', 'SIGI': 'Q', 'FLAG': 'I', 'XDET': 'R', 'LP': 'R',
                'YDET': 'R', 'SIGIPR': 'Q', 'FRACTIONCALC': 'R', 'ROT': 'R'}

  # OK these are probably the original indices not the reduced indices that the
  # MTZ file should have - now use Richard's new method to add the H, K, L
  # columns and M_ISYM N.B. now no longer mapping Miller indices to ASU above
  # so these HKL are currently nonsense values...
  # h, k, l = zip(*integrated_data['miller_index'])
  # h, k, l = flex.int(h), flex.int(k), flex.int(l)

  nref = len(integrated_data['miller_index'])
  m.adjust_column_array_sizes(nref)
  m.set_n_reflections(nref)

  # assign H, K, L space
  # d.add_column('H', type_table['H']).set_values(h.as_double().as_float())
  # d.add_column('K', type_table['K']).set_values(k.as_double().as_float())
  # d.add_column('L', type_table['L']).set_values(l.as_double().as_float())
  for column in 'H', 'K', 'L':
    d.add_column(column, type_table[column]).set_values(
      flex.double(nref, 0.0).as_float())

  # make M_ISYM space
  d.add_column('M_ISYM', type_table['M_ISYM']).set_values(
    flex.double(nref, 0.0).as_float())

  m.replace_original_index_miller_indices(integrated_data['miller_index'])

  #d.add_column('M_ISYM', type_table['M_ISYM']).set_values(
  #  misym.as_double().as_float())
  d.add_column('BATCH', type_table['BATCH']).set_values(
    batch.as_double().as_float())

  lp = integrated_data['lp']
  if 'intensity.prf.value' in integrated_data:
    I = integrated_data['intensity.prf.value'] * lp
    V = integrated_data['intensity.prf.variance'] * lp
  else:
    I = integrated_data['intensity.sum.value'] * lp
    V = integrated_data['intensity.sum.variance'] * lp

  d.add_column('I', type_table['I']).set_values(I.as_float())

  # Trap negative variances

  assert ((V < 0).count(True) == 0)

  d.add_column('SIGI', type_table['SIGI']).set_values(flex.sqrt(V).as_float())

  d.add_column('FRACTIONCALC', type_table['FRACTIONCALC']).set_values(
    fractioncalc.as_float())

  d.add_column('XDET', type_table['XDET']).set_values(xdet.as_float())
  d.add_column('YDET', type_table['YDET']).set_values(ydet.as_float())
  d.add_column('ROT', type_table['ROT']).set_values(rot.as_float())

  m.write(hklout)

  return m
<|endoftext|>


<commit_before># Copyright (c) 2013 - Rackspace Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.

"""
    Usage: python usage_seed.py [period_length] [sql_connection]
    python usage_seed hour mysql://user:password@nova-db.example.com/nova?charset=utf8

    The idea behind usage seeding is to take the current state of all
    active instances on active compute hosts and insert that data into
    Stacktach's usage tables. This script should be run against the
    nova database in each cell which has active compute nodes. The
    reason for that is because the global cell does not have information
    on active compute hosts.
"""

import __builtin__
setattr(__builtin__, '_', lambda x: x)
import datetime
import os
import sys

from oslo.config import cfg
CONF = cfg.CONF
CONF.config_file = "/etc/nova/nova.conf"

if __name__ == '__main__':
    if len(sys.argv) != 3:
        print "Proper Usage: usage_seed.py [period_length] [sql_connection]"
        sys.exit(1)
    CONF.sql_connection = sys.argv[2]

from nova.compute import task_states
from nova.context import RequestContext
from nova.db import api as novadb
from nova.db.sqlalchemy import api
from nova.db.sqlalchemy import models as novamodels

POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
                                                os.pardir, os.pardir))
if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'stacktach')):
    sys.path.insert(0, POSSIBLE_TOPDIR)

from stacktach import datetime_to_decimal as dt
from stacktach import models

# start yanked from reports/nova_usage_audit.py
def get_previous_period(time, period_length):
    if period_length == 'day':
        last_period = time - datetime.timedelta(days=1)
        start = datetime.datetime(year=last_period.year,
                                  month=last_period.month,
                                  day=last_period.day)
        end = datetime.datetime(year=time.year,
                                month=time.month,
                                day=time.day)
        return start, end
    elif period_length == 'hour':
        last_period = time - datetime.timedelta(hours=1)
        start = datetime.datetime(year=last_period.year,
                                  month=last_period.month,
                                  day=last_period.day,
                                  hour=last_period.hour)
        end = datetime.datetime(year=time.year,
                                month=time.month,
                                day=time.day,
                                hour=time.hour)
        return start, end
# end yanked from reports/nova_usage_audit.py

def _usage_for_instance(instance, task=None):
    usage = {
        'instance': instance['uuid'],
        'tenant': instance['project_id'],
        'instance_type_id': instance.get('instance_type_id'),
    }

    launched_at = instance.get('launched_at')
    if launched_at is not None:
        usage['launched_at'] = dt.dt_to_decimal(launched_at)

    if task is not None:
        usage['task'] = task

    return usage

def _delete_for_instance(instance):
    delete = {
        'instance': instance['uuid'],
        'deleted_at': dt.dt_to_decimal(instance.get('terminated_at')),
    }

    launched_at = instance.get('launched_at')
    if launched_at is not None:
        delete['launched_at'] = dt.dt_to_decimal(launched_at)
    return delete

def get_active_instances(period_length):
    context = RequestContext('1', '1', is_admin=True)
    start, end = get_previous_period(datetime.datetime.utcnow(), period_length)
    session = api.get_session()
    computes = novadb.service_get_all_by_topic(context, 'compute')
    active_instances = []
    yesterday = datetime.datetime.utcnow() - datetime.timedelta(days=1)
    for compute in computes:
        if compute.updated_at > yesterday:
            query = session.query(novamodels.Instance)

            active_filter = api.or_(novamodels.Instance.terminated_at == None,
                                    novamodels.Instance.terminated_at > start)
            query = query.filter(active_filter)
            query = query.filter_by(host=compute.host)

            for instance in query.all():
                active_instances.append(instance)
    return active_instances

def get_action_for_instance(context, instance_uuid, action_name):
    actions = novadb.actions_get(context, instance_uuid)
    for action in actions:
        if action['action'] == action_name:
            return action

rebuild_tasks = [task_states.REBUILDING,
                 task_states.REBUILD_BLOCK_DEVICE_MAPPING,
                 task_states.REBUILD_SPAWNING]

resize_tasks = [task_states.RESIZE_PREP,
                task_states.RESIZE_MIGRATING,
                task_states.RESIZE_MIGRATED,
                task_states.RESIZE_FINISH]

resize_revert_tasks = [task_states.RESIZE_REVERTING]

rescue_tasks = [task_states.RESCUING]

in_flight_tasks = (rebuild_tasks + resize_tasks +
                   resize_revert_tasks + rescue_tasks)

def seed(period_length):
    usages = []
    building_usages = []
    in_flight_usages = []
    deletes = []

    context = RequestContext(1, 1, is_admin=True)

    print "Selecting all active instances"
    active_instances = get_active_instances(period_length)
    print "Selected all active instances"

    print "Populating active usages, preparing for in-flight"
    for instance in active_instances:
        vm_state = instance['vm_state']
        task_state = instance['task_state']

        if vm_state == 'building':
            building_usages.append(_usage_for_instance(instance))
            if instance['deleted'] != 0:
                # Just in case...
                deletes.append(_delete_for_instance(instance))
        else:
            if task_state in in_flight_tasks:
                in_flight_usages.append(_usage_for_instance(instance,
                                                            task=task_state))
                if instance['deleted'] != 0:
                    # Just in case...
                    deletes.append(_delete_for_instance(instance))
            else:
                usages.append(_usage_for_instance(instance))
                if instance['deleted'] != 0:
                    deletes.append(_delete_for_instance(instance))
    print "Populated active instances, processing building"
    for usage in building_usages:
        action = get_action_for_instance(context, usage['instance'], 'create')
        if action is not None:
            usage['request_id'] = action['request_id']

    print "Populated building, processing in-flight"
    for usage in in_flight_usages:
        instance = usage['instance']
        action = None
        if usage['task'] in rebuild_tasks:
            action = get_action_for_instance(context, instance, 'rebuild')
        elif usage['task'] in resize_tasks:
            action = get_action_for_instance(context, instance, 'resize')
        elif usage['task'] in resize_revert_tasks:
            action = get_action_for_instance(context, instance, 'resizeRevert')
        elif usage['task'] in rescue_tasks:
            action = get_action_for_instance(context, instance, 'rescue')

        if action is not None:
            usage['request_id'] = action['request_id']
        del usage['task']

    print "Done cataloging usage"

    print "Saving active instances"
    active_InstanceUsages = map(lambda x: models.InstanceUsage(**x),
                                usages)
    models.InstanceUsage.objects.bulk_create(active_InstanceUsages,
                                             batch_size=100)

    print "Saving building instances"
    building_InstanceUsages = map(lambda x: models.InstanceUsage(**x),
                                  building_usages)
    models.InstanceUsage.objects.bulk_create(building_InstanceUsages,
                                             batch_size=100)

    print "Saving in-flight instances"
    in_flight_InstanceUsages = map(lambda x: models.InstanceUsage(**x),
                                   in_flight_usages)
    models.InstanceUsage.objects.bulk_create(in_flight_InstanceUsages,
                                             batch_size=100)

    print "Saving deletes"
    all_InstanceDeletes = map(lambda x: models.InstanceDeletes(**x),
                              deletes)
    models.InstanceDeletes.objects.bulk_create(all_InstanceDeletes,
                                               batch_size=100)

    return (len(usages), len(building_usages),
            len(in_flight_usages), len(deletes))

if __name__ == '__main__':
    msg = ("Seeded system with: \n"
           "%s Active Instances \n"
           "%s Building Instances \n"
           "%s In Flight Instances \n"
           "%s Deleted Instances \n")
    print msg % seed(sys.argv[1])

<commit_msg>Only seed deleted instance when deleted today.<commit_after># Copyright (c) 2013 - Rackspace Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.

"""
    Usage: python usage_seed.py [period_length] [sql_connection]
    python usage_seed hour mysql://user:password@nova-db.example.com/nova?charset=utf8

    The idea behind usage seeding is to take the current state of all
    active instances on active compute hosts and insert that data into
    Stacktach's usage tables. This script should be run against the
    nova database in each cell which has active compute nodes. The
    reason for that is because the global cell does not have information
    on active compute hosts.
"""

import __builtin__
setattr(__builtin__, '_', lambda x: x)
import datetime
import os
import sys

from oslo.config import cfg
CONF = cfg.CONF
CONF.config_file = "/etc/nova/nova.conf"

if __name__ == '__main__':
    if len(sys.argv) != 3:
        print "Proper Usage: usage_seed.py [period_length] [sql_connection]"
        sys.exit(1)
    CONF.sql_connection = sys.argv[2]

from nova.compute import task_states
from nova.context import RequestContext
from nova.db import api as novadb
from nova.db.sqlalchemy import api
from nova.db.sqlalchemy import models as novamodels

POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
                                                os.pardir, os.pardir))
if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'stacktach')):
    sys.path.insert(0, POSSIBLE_TOPDIR)

from stacktach import datetime_to_decimal as dt
from stacktach import models

# start yanked from reports/nova_usage_audit.py
def get_previous_period(time, period_length):
    if period_length == 'day':
        last_period = time - datetime.timedelta(days=1)
        start = datetime.datetime(year=last_period.year,
                                  month=last_period.month,
                                  day=last_period.day)
        end = datetime.datetime(year=time.year,
                                month=time.month,
                                day=time.day)
        return start, end
    elif period_length == 'hour':
        last_period = time - datetime.timedelta(hours=1)
        start = datetime.datetime(year=last_period.year,
                                  month=last_period.month,
                                  day=last_period.day,
                                  hour=last_period.hour)
        end = datetime.datetime(year=time.year,
                                month=time.month,
                                day=time.day,
                                hour=time.hour)
        return start, end
# end yanked from reports/nova_usage_audit.py

def _usage_for_instance(instance, task=None):
    usage = {
        'instance': instance['uuid'],
        'tenant': instance['project_id'],
        'instance_type_id': instance.get('instance_type_id'),
    }

    launched_at = instance.get('launched_at')
    if launched_at is not None:
        usage['launched_at'] = dt.dt_to_decimal(launched_at)

    if task is not None:
        usage['task'] = task

    return usage

def _delete_for_instance(instance):
    delete = {
        'instance': instance['uuid'],
        'deleted_at': dt.dt_to_decimal(instance.get('terminated_at')),
    }

    launched_at = instance.get('launched_at')
    if launched_at is not None:
        delete['launched_at'] = dt.dt_to_decimal(launched_at)
    return delete

def get_active_instances(period_length):
    context = RequestContext('1', '1', is_admin=True)
    start, end = get_previous_period(datetime.datetime.utcnow(), period_length)
    session = api.get_session()
    computes = novadb.service_get_all_by_topic(context, 'compute')
    active_instances = []
    yesterday = datetime.datetime.utcnow() - datetime.timedelta(days=1)
    for compute in computes:
        if compute.updated_at > yesterday:
            query = session.query(novamodels.Instance)

            active_filter = api.or_(novamodels.Instance.terminated_at == None,
                                    novamodels.Instance.terminated_at > start)
            query = query.filter(active_filter)
            query = query.filter_by(host=compute.host)

            for instance in query.all():
                active_instances.append(instance)
    return active_instances

def get_action_for_instance(context, instance_uuid, action_name):
    actions = novadb.actions_get(context, instance_uuid)
    for action in actions:
        if action['action'] == action_name:
            return action

rebuild_tasks = [task_states.REBUILDING,
                 task_states.REBUILD_BLOCK_DEVICE_MAPPING,
                 task_states.REBUILD_SPAWNING]

resize_tasks = [task_states.RESIZE_PREP,
                task_states.RESIZE_MIGRATING,
                task_states.RESIZE_MIGRATED,
                task_states.RESIZE_FINISH]

resize_revert_tasks = [task_states.RESIZE_REVERTING]

rescue_tasks = [task_states.RESCUING]

in_flight_tasks = (rebuild_tasks + resize_tasks +
                   resize_revert_tasks + rescue_tasks)

def seed(period_length):
    usages = []
    building_usages = []
    in_flight_usages = []
    deletes = []

    start, end = get_previous_period(datetime.datetime.utcnow(), period_length)

    context = RequestContext(1, 1, is_admin=True)

    print "Selecting all active instances"
    active_instances = get_active_instances(period_length)
    print "Selected all active instances"

    print "Populating active usages, preparing for in-flight"
    for instance in active_instances:
        vm_state = instance['vm_state']
        task_state = instance['task_state']

        if vm_state == 'building':
            if instance['deleted'] != 0 and instance['deleted_at'] >= start:
                building_usages.append(_usage_for_instance(instance))
                deletes.append(_delete_for_instance(instance))
            elif instance['deleted'] == 0:
                building_usages.append(_usage_for_instance(instance))
        else:
            if task_state in in_flight_tasks:
                if (instance['deleted'] != 0 and
                        instance['deleted_at'] >= start):
                    # Just in case...
                    deletes.append(_delete_for_instance(instance))
                    in_flight_usages.append(_usage_for_instance(instance,
                                            task=task_state))
                elif instance['deleted'] == 0:
                    in_flight_usages.append(_usage_for_instance(instance,
                                            task=task_state))
            else:
                if (instance['deleted'] != 0 and
                        instance['deleted_at'] >= start):
                    deletes.append(_delete_for_instance(instance))
                    usages.append(_usage_for_instance(instance))
                elif instance['deleted'] == 0:
                    usages.append(_usage_for_instance(instance))

    print "Populated active instances, processing building"
    for usage in building_usages:
        action = get_action_for_instance(context, usage['instance'], 'create')
        if action is not None:
            usage['request_id'] = action['request_id']

    print "Populated building, processing in-flight"
    for usage in in_flight_usages:
        instance = usage['instance']
        action = None
        if usage['task'] in rebuild_tasks:
            action = get_action_for_instance(context, instance, 'rebuild')
        elif usage['task'] in resize_tasks:
            action = get_action_for_instance(context, instance, 'resize')
        elif usage['task'] in resize_revert_tasks:
            action = get_action_for_instance(context, instance, 'resizeRevert')
        elif usage['task'] in rescue_tasks:
            action = get_action_for_instance(context, instance, 'rescue')

        if action is not None:
            usage['request_id'] = action['request_id']
        del usage['task']

    print "Done cataloging usage"

    print "Saving active instances"
    active_InstanceUsages = map(lambda x: models.InstanceUsage(**x),
                                usages)
    models.InstanceUsage.objects.bulk_create(active_InstanceUsages,
                                             batch_size=100)

    print "Saving building instances"
    building_InstanceUsages = map(lambda x: models.InstanceUsage(**x),
                                  building_usages)
    models.InstanceUsage.objects.bulk_create(building_InstanceUsages,
                                             batch_size=100)

    print "Saving in-flight instances"
    in_flight_InstanceUsages = map(lambda x: models.InstanceUsage(**x),
                                   in_flight_usages)
    models.InstanceUsage.objects.bulk_create(in_flight_InstanceUsages,
                                             batch_size=100)

    print "Saving deletes"
    all_InstanceDeletes = map(lambda x: models.InstanceDeletes(**x),
                              deletes)
    models.InstanceDeletes.objects.bulk_create(all_InstanceDeletes,
                                               batch_size=100)

    return (len(usages), len(building_usages),
            len(in_flight_usages), len(deletes))

if __name__ == '__main__':
    msg = ("Seeded system with: \n"
           "%s Active Instances \n"
           "%s Building Instances \n"
           "%s In Flight Instances \n"
           "%s Deleted Instances \n")
    print msg % seed(sys.argv[1])

<|endoftext|>


<commit_before>import subprocess
from collections import defaultdict
from datetime import datetime

class Scheduler(object):
    """The scheduler can run instances of smif as a subprocess
    and can provide information whether the modelrun is running,
    is done or has failed.
    """
    def __init__(self):
        self._status = defaultdict(lambda: 'unstarted')
        self._process = {}
        self._output = defaultdict(str)
        self._err = {}
        self.lock = False

    def add(self, model_run_name, args):
        """Add a model_run to the Modelrun scheduler.

        Parameters
        ----------
        model_run_name: str
            Name of the modelrun
        args: dict
            Arguments for the command-line interface

        Exception
        ---------
        Exception
            When the modelrun was already started

        Notes
        -----
        There is no queuing mechanism implemented, each `add`
        will directly start a subprocess. This means that it
        is possible to run multiple modelruns concurrently.
        This may cause conflicts, it depends on the
        implementation whether a certain sector model / wrapper
        touches the filesystem or other shared resources.
        """
        if self._status[model_run_name] is not 'running':

            smif_call = (
                'exec smif ' +
                '-'*(int(args['verbosity']) > 0) + 'v'*int(args['verbosity']) +
                ' run' + ' ' + model_run_name + ' ' +
                '-d' + ' ' + args['directory'] + ' ' +
                '-w'*args['warm_start'] + ' '*args['warm_start'] +
                '-i' + ' ' + args['output_format']
            )

            self._process[model_run_name] = subprocess.Popen(
                smif_call,
                shell=True,
                stdout=subprocess.PIPE, stderr=subprocess.STDOUT
            )
            self._output[model_run_name] = "\x1b[1;34mModelrun \x1b \x1b[0m" + \
                                           model_run_name + "\n"
            self._output[model_run_name] += "\x1b[1;34mTime \x1b \x1b " \
                                            "\x1b \x1b \x1b \x1b[0m" + \
                                            datetime.now().strftime('%Y-%m-%d %H:%M:%S') + "\n"
            self._output[model_run_name] += "\x1b[1;34mPID" + " \x1b"*7 + "[0m" + \
                                            str(self._process[model_run_name].pid) + "\n"
            self._output[model_run_name] += "\x1b[1;34mCommand" + " \x1b"*3 + "[0m" + \
                                            smif_call + "\n"

            self._output[model_run_name] += "-" * 100 + "\n"
            self._status[model_run_name] = 'running'
        else:
            raise Exception('Model is already running.')

    def kill(self, model_run_name):
        """ Kill a Modelrun that is already running

        Parameters
        ----------
        model_run_name: str
            Name of the modelrun
        """
        if self._status[model_run_name] == 'running':
            self._process[model_run_name].kill()
            self._status[model_run_name] = 'stopped'

    def get_status(self, model_run_name):
        """Get the status from the Modelrun scheduler.

        Parameters
        ----------
        model_run_name: str
            Name of the modelrun

        Returns
        -------
        dict: A message containing the status, command-line
        output and error that can be directly sent back over
        the http api.

        Notes
        -----
        Possible status:

        unstarted:
            Model run was not started
        queing:
            Model run is waiting to be executed
        running:
            Model run is running
        stopped:
            Model run was stopped (killed) by user
        done:
            Model run was completed succesfully
        failed:
            Model run completed running with an exit code
        """
        if self._status[model_run_name] == 'running':
            if self.lock is False:
                self.lock = True
                for line in iter(self._process[model_run_name].stdout.readline, b''):
                    self._output[model_run_name] += line.decode()
                    self._process[model_run_name].stdout.flush()
                self.lock = False

            if self._process[model_run_name].poll() == 0:
                self._status[model_run_name] = 'done'
            elif self._process[model_run_name].poll() == 1:
                self._status[model_run_name] = 'failed'

        return {
            'status': self._status[model_run_name],
            'output': self._output[model_run_name]
        }
<commit_msg>Remove exec to allow running subprocesses under windows<commit_after>import subprocess
from collections import defaultdict
from datetime import datetime

class Scheduler(object):
    """The scheduler can run instances of smif as a subprocess
    and can provide information whether the modelrun is running,
    is done or has failed.
    """
    def __init__(self):
        self._status = defaultdict(lambda: 'unstarted')
        self._process = {}
        self._output = defaultdict(str)
        self._err = {}
        self.lock = False

    def add(self, model_run_name, args):
        """Add a model_run to the Modelrun scheduler.

        Parameters
        ----------
        model_run_name: str
            Name of the modelrun
        args: dict
            Arguments for the command-line interface

        Exception
        ---------
        Exception
            When the modelrun was already started

        Notes
        -----
        There is no queuing mechanism implemented, each `add`
        will directly start a subprocess. This means that it
        is possible to run multiple modelruns concurrently.
        This may cause conflicts, it depends on the
        implementation whether a certain sector model / wrapper
        touches the filesystem or other shared resources.
        """
        if self._status[model_run_name] is not 'running':

            smif_call = (
                'smif ' +
                '-'*(int(args['verbosity']) > 0) + 'v'*int(args['verbosity']) +
                ' run' + ' ' + model_run_name + ' ' +
                '-d' + ' ' + args['directory'] + ' ' +
                '-w'*args['warm_start'] + ' '*args['warm_start'] +
                '-i' + ' ' + args['output_format']
            )

            self._process[model_run_name] = subprocess.Popen(
                smif_call,
                shell=True,
                stdout=subprocess.PIPE, stderr=subprocess.STDOUT
            )
            self._output[model_run_name] = "\x1b[1;34mModelrun \x1b \x1b[0m" + \
                                           model_run_name + "\n"
            self._output[model_run_name] += "\x1b[1;34mTime \x1b \x1b " \
                                            "\x1b \x1b \x1b \x1b[0m" + \
                                            datetime.now().strftime('%Y-%m-%d %H:%M:%S') + "\n"
            self._output[model_run_name] += "\x1b[1;34mPID" + " \x1b"*7 + "[0m" + \
                                            str(self._process[model_run_name].pid) + "\n"
            self._output[model_run_name] += "\x1b[1;34mCommand" + " \x1b"*3 + "[0m" + \
                                            smif_call + "\n"

            self._output[model_run_name] += "-" * 100 + "\n"
            self._status[model_run_name] = 'running'
        else:
            raise Exception('Model is already running.')

    def kill(self, model_run_name):
        """ Kill a Modelrun that is already running

        Parameters
        ----------
        model_run_name: str
            Name of the modelrun
        """
        if self._status[model_run_name] == 'running':
            self._process[model_run_name].kill()
            self._status[model_run_name] = 'stopped'

    def get_status(self, model_run_name):
        """Get the status from the Modelrun scheduler.

        Parameters
        ----------
        model_run_name: str
            Name of the modelrun

        Returns
        -------
        dict: A message containing the status, command-line
        output and error that can be directly sent back over
        the http api.

        Notes
        -----
        Possible status:

        unstarted:
            Model run was not started
        queing:
            Model run is waiting to be executed
        running:
            Model run is running
        stopped:
            Model run was stopped (killed) by user
        done:
            Model run was completed succesfully
        failed:
            Model run completed running with an exit code
        """
        if self._status[model_run_name] == 'running':
            if self.lock is False:
                self.lock = True
                for line in iter(self._process[model_run_name].stdout.readline, b''):
                    self._output[model_run_name] += line.decode()
                    self._process[model_run_name].stdout.flush()
                self.lock = False

            if self._process[model_run_name].poll() == 0:
                self._status[model_run_name] = 'done'
            elif self._process[model_run_name].poll() == 1:
                self._status[model_run_name] = 'failed'

        return {
            'status': self._status[model_run_name],
            'output': self._output[model_run_name]
        }
<|endoftext|>


<commit_before>import bs
import bsInternal
import threading
import json
import urllib2
import weakref
import os
import os.path

branch = "master"
url = "https://rawgit.com/Mrmaxmeier/BombSquad-Community-Mod-Manager/" + branch + "/index.json"
modPath = bs.getEnvironment()['userScriptsDirectory'] + "/"
mod = "modManager"

class SimpleGetThread(threading.Thread):
    def __init__(self, url, callback=None):
        threading.Thread.__init__(self)
        self._url = url.encode("ascii")  # embedded python2.7 has weird encoding issues
        self._callback = callback or (lambda d: None)
        self._context = bs.Context('current')
        # save and restore the context we were created from
        activity = bs.getActivity(exceptionOnNone=False)
        self._activity = weakref.ref(activity) if activity is not None else None

    def _runCallback(self, arg):
        # if we were created in an activity context and that activity has since died, do nothing
        # (hmm should we be using a context-call instead of doing this manually?)
        if self._activity is not None and (self._activity() is None or self._activity().isFinalized()):
            return
        # (technically we could do the same check for session contexts, but not gonna worry about it for now)
        with self._context:
            self._callback(arg)

    def run(self):
        try:
            bsInternal._setThreadName("SimpleGetThread")
            response = urllib2.urlopen(self._url)
            bs.callInGameThread(bs.Call(self._runCallback, response.read()))
        except:
            bs.printException()
            bs.callInGameThread(bs.Call(self._runCallback, None))

installed = []
installing = []

def check_finished():
    if any([m not in installed for m in installing]):
        return
    bs.screenMessage("installed everything.")
    if os.path.isfile(modPath + __name__ + ".pyc"):
        os.remove(modPath + __name__ + ".pyc")
    if os.path.isfile(modPath + __name__ + ".py"):
        os.remove(modPath + __name__ + ".py")
        bs.screenMessage("deleted self")
    bs.screenMessage("activating modManager")
    __import__(mod)

def install(data, mod):
    installing.append(mod)
    bs.screenMessage("installing " + str(mod))
    print("installing", mod)
    for dep in data[mod].get("requires", []):
        install(data, dep)
    url = data[mod]["url"]
    filename = data[mod]["filename"]

    def f(data):
        print("writing", filename)
        with open(modPath + filename, "w") as f:
            f.write(data)
        installed.append(mod)
        check_finished()

    SimpleGetThread(url, f).start()

def onIndex(data):
    if not data:
        bs.screenMessage("network error :(")
        return
    data = json.loads(data)
    install(data["mods"], mod)

SimpleGetThread(url, onIndex).start()
<commit_msg>try multiple urls before bailing out<commit_after>import bs
import bsInternal
import threading
import json
import urllib2
import weakref
import os
import os.path

modPath = bs.getEnvironment()['userScriptsDirectory'] + "/"
branch = "master"
mod = "modManager"
user_repo = "Mrmaxmeier/BombSquad-Community-Mod-Manager"

def index_url():
    yield "https://raw.githubusercontent.com/{}/{}/index.json".format(user_repo, branch)
    yield "https://rawgit.com/{}/{}/index.json".format(user_repo, branch)
    yield "http://raw.githack.com/{}/{}/index.json".format(user_repo, branch)
    yield "http://rawgit.com/{}/{}/index.json".format(user_repo, branch)

def mod_url(data):
    if "url" in data:
        yield data["url"]
    commit_hexsha = data["commit_hexsha"]
    filename = data["filename"]
    yield "https://cdn.rawgit.com/{}/{}/mods/{}".format(user_repo, commit_hexsha, filename)
    yield "http://rawcdn.githack.com/{}/{}/mods/{}".format(user_repo, commit_hexsha, filename)

def try_fetch_cb(generator, callback):
    def f(data):
        if data:
            callback(data)
        else:
            try:
                SimpleGetThread(next(generator), f).start()
            except StopIteration:
                callback(None)
    SimpleGetThread(next(generator), f).start()

class SimpleGetThread(threading.Thread):
    def __init__(self, url, callback=None):
        threading.Thread.__init__(self)
        self._url = url.encode("ascii")  # embedded python2.7 has weird encoding issues
        self._callback = callback or (lambda d: None)
        self._context = bs.Context('current')
        # save and restore the context we were created from
        activity = bs.getActivity(exceptionOnNone=False)
        self._activity = weakref.ref(activity) if activity is not None else None

    def _runCallback(self, arg):
        # if we were created in an activity context and that activity has since died, do nothing
        # (hmm should we be using a context-call instead of doing this manually?)
        if self._activity is not None and (self._activity() is None or self._activity().isFinalized()):
            return
        # (technically we could do the same check for session contexts, but not gonna worry about it for now)
        with self._context:
            self._callback(arg)

    def run(self):
        try:
            bsInternal._setThreadName("SimpleGetThread")
            response = urllib2.urlopen(self._url)
            bs.callInGameThread(bs.Call(self._runCallback, response.read()))
        except:
            bs.printException()
            bs.callInGameThread(bs.Call(self._runCallback, None))

installed = []
installing = []

def check_finished():
    if any([m not in installed for m in installing]):
        return
    bs.screenMessage("installed everything.")
    if os.path.isfile(modPath + __name__ + ".pyc"):
        os.remove(modPath + __name__ + ".pyc")
    if os.path.isfile(modPath + __name__ + ".py"):
        os.remove(modPath + __name__ + ".py")
        bs.screenMessage("deleted self")
    bs.screenMessage("activating modManager")
    __import__(mod)

def install(data, mod):
    installing.append(mod)
    bs.screenMessage("installing " + str(mod))
    print("installing", mod)
    for dep in data[mod].get("requires", []):
        install(data, dep)
    filename = data[mod]["filename"]

    def f(data):
        if not data:
            bs.screenMessage("failed to download mod '{}'".format(filename))
        print("writing", filename)
        with open(modPath + filename, "w") as f:
            f.write(data)
        installed.append(mod)
        check_finished()

    try_fetch_cb(mod_url(data[mod]), f)

def onIndex(data):
    if not data:
        bs.screenMessage("network error :(")
        return
    data = json.loads(data)
    install(data["mods"], mod)

try_fetch_cb(index_url(), onIndex)
<|endoftext|>