<commit_before>#!/usr/bin/env python

'''
Medium export image fill 1.00
by Marcin Wichary (aresluna.org)

Site: https://github.com/mwichary/medium-export-image-fill

This is free and unencumbered software released into the public domain.

Anyone is free to copy, modify, publish, use, compile, sell, or
distribute this software, either in source code form or as a compiled
binary, for any purpose, commercial or non-commercial, and by any
means.

For more information, please refer to <http://unlicense.org/>
'''

# Imports
# ---------------------------------

import argparse
import json
import os
import re
import stat
import string
import sys
import time
import urllib
from shutil import copyfile
# The location of urlretrieve changed modules in Python 3
if (sys.version_info > (3, 0)):
    from urllib.request import urlretrieve
else:
    from urllib import urlretrieve

# Main entry point
# ---------------------------------

# Introduce yourself

print("Medium export image fill 1.00")
print("by Marcin Wichary (aresluna.org) and others")
print("use --help to see options")
print("")

# Prepare variables etc.

image_count_global = 0
if not os.path.isdir("images"):
  os.mkdir("images")
if not os.path.isdir("original_articles"):
  os.mkdir("original_articles")

# We need to impersonate a browser for downloads from CloudFlare to succeed
urllib.URLopener.version = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36 SE 2.X MetaSr 1.0'

# Find all the articles to be processed

articles = []
for filename in os.listdir("."):
  if filename.endswith(".html"):
    articles.append(filename)

print("To process: %i article(s)..." % (len(articles)))
print("(You can cancel any time. Next time you run, the script should resume at the last point.)")
print("")

# Loop 1: Go through all the articles
# -----------------------------------

for article_count, article_filename in enumerate(articles):
  try:
    # Display progress
    count_string = '[%i/%i] ' % (article_count + 1, len(articles))
    print '%s%s...' % (count_string, article_filename)

    # Make a copy of the original file, just in case (only if it doesn't exist before)
    backup_filename = "original_articles/" + article_filename
    if not os.path.isfile(backup_filename):
      copyfile(article_filename, backup_filename)

    # Open file, find all images
    with open(article_filename, 'r') as file:
      article_contents = file.read()

    images = re.findall(r'()', article_contents)

    # Loop 2: Go through all the images in an article
    # -----------------------------------------------

    for image_count, image in enumerate(images):

      # Create an article-specific directory to download images into
      directory_name = 'images/' + os.path.splitext(article_filename)[0]
      if (image_count == 0) and not os.path.isdir(directory_name):
        os.mkdir(directory_name)

      image_server_url = image[2]

      # Check the URL. If it starts with https://, it means we have to download the image.
      # If it doesn't, it means we already downloaded the image and rewrote the URL to point
      # to a local file
      skip_download = False
      if image_server_url[:8] != 'https://':
        skip_download = True

      # Get the image id, used to determine local filename
      image_id = re.findall(r'\/(([^\/]*?)\.([a-z]+))$', image_server_url)[0][0]

      # Update the user
      progress_string = "[%i/%i] %s %s..." %\
          (image_count + 1, len(images), "Skipping" if skip_download else "Downloading", image_id)
      progress_string = progress_string.rjust(len(progress_string) + len(count_string))
      sys.stdout.write("\r" + progress_string)
      sys.stdout.write("\033[K") # Clear the end of the line
      sys.stdout.flush()

      if not skip_download:
        # Rewrite the URL to get the maximum quality image from the server
        image_server_high_quality_url = re.sub(r'\/max\/[0-9]+\/', '/', image_server_url, 1)

        image_local_filename = directory_name + '/' + image_id

        # Download the file (try a few times if need be)
        downloaded = False
        download_tries = 3
        while not downloaded:
          try:
            urlretrieve(image_server_high_quality_url, image_local_filename)
          except:
            download_tries = download_tries - 1
            if download_tries == 0:
              print("")
              print("Failed to download %s after 3 tries." % better_url)
              print("Please try again later?")
              sys.exit()
            time.sleep(5) # Wait 5 seconds before retrying
          else:
            downloaded = True

        image_count_global = image_count_global + 1

        # Rewrite the URL to point to a local file, and re-save the article
        article_contents = re.sub(r'()'.format(re.escape(image_server_url)),
            r''.format(image_local_filename),
            article_contents, 1)
        with open(article_filename, 'w') as file:
          file.write(article_contents)

    # Remove the last downloaded file from the screen
    sys.stdout.write("\r")
    sys.stdout.write("\033[K") # Clear the end of the line
    sys.stdout.flush()

  except KeyboardInterrupt:
    print("")
    print("Interrupted! Come back any time.")
    sys.exit()

# End loop 1 (all the months)
print("")
print("Done!")
print("%i images downloaded in total." % image_count_global)
print("")
<commit_msg>Don’t use images/ directory.<commit_after>#!/usr/bin/env python

'''
Medium export image fill 1.00
by Marcin Wichary (aresluna.org)

Site: https://github.com/mwichary/medium-export-image-fill

This is free and unencumbered software released into the public domain.

Anyone is free to copy, modify, publish, use, compile, sell, or
distribute this software, either in source code form or as a compiled
binary, for any purpose, commercial or non-commercial, and by any
means.

For more information, please refer to <http://unlicense.org/>
'''

# Imports
# ---------------------------------

import argparse
import json
import os
import re
import stat
import string
import sys
import time
import urllib
from shutil import copyfile
# The location of urlretrieve changed modules in Python 3
if (sys.version_info > (3, 0)):
    from urllib.request import urlretrieve
else:
    from urllib import urlretrieve

# Main entry point
# ---------------------------------

# Introduce yourself

print("Medium export image fill 1.00")
print("by Marcin Wichary (aresluna.org) and others")
print("use --help to see options")
print("")

# Prepare variables etc.

image_count_global = 0
if not os.path.isdir("original_articles"):
  os.mkdir("original_articles")

# We need to impersonate a browser for downloads from CloudFlare to succeed
urllib.URLopener.version = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36 SE 2.X MetaSr 1.0'

# Find all the articles to be processed

articles = []
for filename in os.listdir("."):
  if filename.endswith(".html"):
    articles.append(filename)

print("To process: %i article(s)..." % (len(articles)))
print("(You can cancel any time. Next time you run, the script should resume at the last point.)")
print("")

# Loop 1: Go through all the articles
# -----------------------------------

for article_count, article_filename in enumerate(articles):
  try:
    # Display progress
    count_string = '[%i/%i] ' % (article_count + 1, len(articles))
    print '%s%s...' % (count_string, article_filename)

    # Make a copy of the original file, just in case (only if it doesn't exist before)
    backup_filename = "original_articles/" + article_filename
    if not os.path.isfile(backup_filename):
      copyfile(article_filename, backup_filename)

    # Open file, find all images
    with open(article_filename, 'r') as file:
      article_contents = file.read()

    images = re.findall(r'()', article_contents)

    # Loop 2: Go through all the images in an article
    # -----------------------------------------------

    for image_count, image in enumerate(images):

      # Create an article-specific directory to download images into
      directory_name = os.path.splitext(article_filename)[0]
      if (image_count == 0) and not os.path.isdir(directory_name):
        os.mkdir(directory_name)

      image_server_url = image[2]

      # Check the URL. If it starts with https://, it means we have to download the image.
      # If it doesn't, it means we already downloaded the image and rewrote the URL to point
      # to a local file
      skip_download = False
      if image_server_url[:8] != 'https://':
        skip_download = True

      # Get the image id, used to determine local filename
      image_id = re.findall(r'\/(([^\/]*?)\.([a-z]+))$', image_server_url)[0][0]

      # Update the user
      progress_string = "[%i/%i] %s %s..." %\
          (image_count + 1, len(images), "Skipping" if skip_download else "Downloading", image_id)
      progress_string = progress_string.rjust(len(progress_string) + len(count_string))
      sys.stdout.write("\r" + progress_string)
      sys.stdout.write("\033[K") # Clear the end of the line
      sys.stdout.flush()

      if not skip_download:
        # Rewrite the URL to get the maximum quality image from the server
        image_server_high_quality_url = re.sub(r'\/max\/[0-9]+\/', '/', image_server_url, 1)

        image_local_filename = directory_name + '/' + image_id

        # Download the file (try a few times if need be)
        downloaded = False
        download_tries = 3
        while not downloaded:
          try:
            urlretrieve(image_server_high_quality_url, image_local_filename)
          except:
            download_tries = download_tries - 1
            if download_tries == 0:
              print("")
              print("Failed to download %s after 3 tries." % better_url)
              print("Please try again later?")
              sys.exit()
            time.sleep(5) # Wait 5 seconds before retrying
          else:
            downloaded = True

        image_count_global = image_count_global + 1

        # Rewrite the URL to point to a local file, and re-save the article
        article_contents = re.sub(r'()'.format(re.escape(image_server_url)),
            r''.format(image_local_filename),
            article_contents, 1)
        with open(article_filename, 'w') as file:
          file.write(article_contents)

    # Remove the last downloaded file from the screen
    sys.stdout.write("\r")
    sys.stdout.write("\033[K") # Clear the end of the line
    sys.stdout.flush()

  except KeyboardInterrupt:
    print("")
    print("Interrupted! Come back any time.")
    sys.exit()

# End loop 1 (all the months)
print("")
print("Done!")
print("%i images downloaded in total." % image_count_global)
print("")
<|endoftext|>


<commit_before>import six

from six.moves.urllib.parse import urlparse

from dache.backends.base import CacheKeyWarning  # noqa
from dache.utils.module_loading import import_string

__version__ = '0.0.2'

__all__ = ('register_backend', 'Cache', 'CacheKeyWarning')

_BACKENDS = {
    'file': 'dache.backends.filebased.FileBasedCache',
    'leveldb': 'dache.backends.leveldb.LevelDBCache',
    'locmem': 'dache.backends.locmem.LocMemCache',
    'memcached': 'dache.backends.memcached.MemcachedCache',
    'pylibmc': 'dache.backends.memcached.PyLibMCCache',
    'redis': 'dache.backends.redis.RedisCache',
}

def register_backend(url_scheme, backend_class):
    """Register a cache backend."""
    _BACKENDS[url_scheme] = backend_class

class Cache(object):

    def __init__(self, url, **options):
        # Create cache backend
        result = urlparse(url)
        backend_class = _BACKENDS[result.scheme]
        if isinstance(backend_class, six.string_types):
            backend_class = import_string(backend_class)

        self._backend = backend_class(result, **options)

        public_methods = ('add', 'get', 'set', 'delete', 'get_many', 'has_key',
                          'incr', 'decr', 'set_many', 'delete_many', 'clear',
                          'validate_key', 'incr_version', 'decr_version',
                          'close')
        for method in public_methods:
            setattr(self, method, getattr(self._backend, method))

    def __contains__(self, item):
        return item in self._backend
<commit_msg>Bump version to 0.0.3<commit_after>import six

from six.moves.urllib.parse import urlparse

from dache.backends.base import CacheKeyWarning  # noqa
from dache.utils.module_loading import import_string

__version__ = '0.0.3'

__all__ = ('register_backend', 'Cache', 'CacheKeyWarning')

_BACKENDS = {
    'file': 'dache.backends.filebased.FileBasedCache',
    'leveldb': 'dache.backends.leveldb.LevelDBCache',
    'locmem': 'dache.backends.locmem.LocMemCache',
    'memcached': 'dache.backends.memcached.MemcachedCache',
    'pylibmc': 'dache.backends.memcached.PyLibMCCache',
    'redis': 'dache.backends.redis.RedisCache',
}

def register_backend(url_scheme, backend_class):
    """Register a cache backend."""
    _BACKENDS[url_scheme] = backend_class

class Cache(object):

    def __init__(self, url, **options):
        # Create cache backend
        result = urlparse(url)
        backend_class = _BACKENDS[result.scheme]
        if isinstance(backend_class, six.string_types):
            backend_class = import_string(backend_class)

        self._backend = backend_class(result, **options)

        public_methods = ('add', 'get', 'set', 'delete', 'get_many', 'has_key',
                          'incr', 'decr', 'set_many', 'delete_many', 'clear',
                          'validate_key', 'incr_version', 'decr_version',
                          'close')
        for method in public_methods:
            setattr(self, method, getattr(self._backend, method))

    def __contains__(self, item):
        return item in self._backend
<|endoftext|>


<commit_before>import rospy
from specification import Specification
from rated_statistics import RatedStatisticsContainer
from arni_core.helper import *
import arni_msgs
from arni_msgs.msg import RatedStatistics, RatedStatisticsEntity

class SpecificationHandler:
    """
    Loads the specifications from the parameter server, stores and compares them to the actual metadata.
    """

    __namespace = '/arni/specifications'

    def __load_specifications(self):
        """
        Loads Specifications from the configurations and stores them in the
        internal storage.
        """
        try:
            params = rospy.get_param(self.__namespace)
            if isinstance(params, dict):
                specifications = []
                for x in params.values():
                    for y in x:
                        specifications.append(y)
            else:
                specifications = params
            for o in specifications:
                for seuid in o.keys():
                    if SEUID().is_valid(seuid):
                        spec = Specification()
                        spec.seuid = seuid
                        for k in o[seuid].keys():
                            spec.add_tuple((k, o[seuid][k]))
                        self.__specifications[seuid] = spec
                    else:
                        rospy.logdebug("[SpecificationHandler][__load_specifications] %s is not a valid seuid." % seuid)
        except KeyError:
            pass
        rospy.loginfo("[SpecificationHandler] Loaded %s parameters." % str(len(self.__specifications.keys())))

    def loaded_specifications(self):
        """
        Returns a list containing all seuids of loaded specifications.

        :return: A list of strings.
        """
        return self.__specifications.keys()

    def get(self, identifier):
        """
        Returns the Specification object from the internal storage.

        :param identifier: The seuid describing the desired Specification object.
        :type identifier: str
        :return: The Specification object with the given identifier, None if it was not found.
        """
        if identifier in self.__specifications.keys():
            return self.__specifications[identifier]
        if identifier[0] == "c":
            if SEUID(identifier).topic in self.__specifications.keys():
                return self.__specifications[SEUID(identifier).topic]
        return None

    def compare(self, data, identifier, specification=None):
        """
        Compares a Message object with a Specification object regarding all available matching fields.

        :param data: The actual data.
        :type data: object.
        :param identifier: The identifier of the metadata package.
        :type identifier: str
        :param specification: The Specification object, alternatively a string identifying it.
        :type specification: Specification or str.
        :returns: A RatedStatisticsContainer object representing the result.
        """
        if identifier is None:
            rospy.logdebug("[SpecificationHandler][compare] No identifier given.")
            return None
        if not SEUID().is_valid(identifier):
            rospy.logdebug("[SpecificationHandler][compare] Given identifier is invalid.")
            return None
        if data is None:
            rospy.logdebug("[SpecificationHandler][compare] No data given.")
            return None
        result = RatedStatisticsContainer(identifier)
        if identifier[0] == "n":
            result.host = data.host
        if specification is None:
            # if identifier in self.__specifications.keys():
            #     specification = self.__specifications[identifier]
            # else:
            #     if identifier[0] == "c":
            #         if SEUID(identifier).topic in self.__specifications.keys():
            #             specification = self.__specifications[SEUID(identifier).topic]
            specification = self.get(identifier)
        # if specification is None:
        # rospy.logdebug("[SpecificationHandler][compare] No Specification available for %s" % identifier)
        window_len = data.window_stop - data.window_start
        if window_len.to_sec() == 0:
            window_len = rospy.Duration(1)
        fields = dir(data)
        # exclude = ("window_start", "window_stop")
        # for x in exclude:
        # if x in fields:
        # fields.remove(x)
        if identifier[0] == "c":
            fields.append("bandwidth")
            fields.append("frequency")
        for field in fields:
            value = None
            if field[0] == "_" or "serialize" in field:
                continue
            current_obj = {}
            if field == "bandwidth":
                value = data.traffic / window_len.to_sec()
            elif field == "frequency":
                if hasattr(data, "delivered_msgs"):
                    value = data.delivered_msgs / window_len.to_sec()
            else:
                value = getattr(data, field)
            if value is not None:
                limits = self.__get_limits(specification, field)
                if isinstance(value, (list, tuple)):
                    current_obj["state"] = []
                    current_obj["actual"] = []
                    current_obj["expected"] = []
                    for i, v in enumerate(value):
                        limits = self.__get_limits(specification, field, i)
                        current_obj["actual"].append(v)
                        current_obj["state"].append(self.__compare(v, limits))
                        current_obj["expected"].append(limits)
                else:
                    status = self.__compare(value, limits)
                    current_obj["state"] = status
                    current_obj["actual"] = value
                    current_obj["expected"] = limits
                result.add_value(field, current_obj["actual"], current_obj["expected"], current_obj["state"])
        result.add_value("alive", ["True"], ["True"], [2])
        return result

    def compare_topic(self, data=None):
        """
        Compares Messages about one topic

        :param data: List of Statistics messages
        :return: list of RatedStatistics messages
        """
        if not data:
            data = []
        by_topic = {}
        result = []
        delivered_msgs_set = False
        for message in data:
            seuid = SEUID(message)
            if not seuid.get_seuid("topic") in by_topic.keys():
                by_topic[seuid.get_seuid("topic")] = {
                    "window_min": rospy.Time.now(),
                    "window_max": rospy.Time(0),
                    "delivered_msgs": 0,
                    "dropped_msgs": 0,
                    "frequency": {},
                    "traffic": 0,
                    "stamp_age_mean": [],
                    "stamp_age_stddev": [],
                    "stamp_age_max": rospy.Duration(0),
                    "packages": 0
                }
            window_len = message.window_stop - message.window_start
            if window_len.to_sec() == 0:
                window_len = rospy.Duration(1)
            # scale = window_len.to_sec() / window_len_max.to_sec()
            scale = 1  # placeholder
            by_topic[seuid.get_seuid("topic")]["window_min"] = min(message.window_start,
                                                                   by_topic[seuid.get_seuid("topic")]["window_min"])
            by_topic[seuid.get_seuid("topic")]["window_max"] = max(message.window_stop,
                                                                   by_topic[seuid.get_seuid("topic")]["window_max"])
            if hasattr(message, "delivered_msgs"):
                delivered_msgs_set = True
                by_topic[seuid.get_seuid("topic")]["delivered_msgs"] += message.delivered_msgs * scale
                if seuid.get_seuid("publisher") not in by_topic[seuid.get_seuid("topic")]["frequency"].keys():
                    by_topic[seuid.get_seuid("topic")]["frequency"][seuid.get_seuid("publisher")] = {
                        "value": 0,
                        "subscriber": seuid.get_seuid("subscriber")
                    }
                if seuid.get_seuid("subscriber") == \
                        by_topic[seuid.get_seuid("topic")]["frequency"][seuid.get_seuid("publisher")]["subscriber"]:
                    by_topic[seuid.get_seuid("topic")]["frequency"][seuid.get_seuid("publisher")]["value"] += \
                        message.delivered_msgs / window_len.to_sec() * scale
            by_topic[seuid.get_seuid("topic")]["dropped_msgs"] += message.dropped_msgs * scale
            by_topic[seuid.get_seuid("topic")]["traffic"] += message.traffic * scale
            by_topic[seuid.get_seuid("topic")]["stamp_age_max"] = max(message.stamp_age_max,
                                                                      by_topic[seuid.get_seuid("topic")][
                                                                          "stamp_age_max"])
            by_topic[seuid.get_seuid("topic")]["stamp_age_mean"].append(message.stamp_age_mean * scale)
            by_topic[seuid.get_seuid("topic")]["packages"] += 1
        for topic, data in by_topic.iteritems():
            specification = self.get(topic)
            r = RatedStatistics()
            r.window_start = data["window_min"]
            r.window_stop = data["window_max"]
            window_len = data["window_max"] - data["window_min"]
            r.seuid = topic
            fields = ["dropped_msgs", "traffic", "traffic_per_second", "stamp_age_max", "stamp_age_mean", "packages",
                      "packages_per_second"]
            fields.remove("traffic")
            if delivered_msgs_set:
                fields.append("delivered_msgs")
                fields.append("frequency")
            alt_names = {"traffic_per_second": "bandwidth"}
            for f in fields:
                re = RatedStatisticsEntity()
                re.statistic_type = f
                if f in alt_names.keys():
                    re.statistic_type = alt_names[f]
                if f == "frequency":
                    value = 0
                    for pub, freq in data["frequency"].iteritems():
                        value += freq["value"]
                elif f[-10:] == "per_second":
                    value = data[f[0:-11]] / window_len.to_sec()
                elif f == "stamp_age_mean":
                    x = data[f]
                    value = sum(y.to_sec() for y in x) / len(x)
                else:
                    value = data[f]
                re.actual_value.append(str(value))
                limits = self.__get_limits(specification, re.statistic_type)
                re.expected_value.append(str(limits))
                re.state = [self.__compare(value, limits)]
                r.rated_statistics_entity.append(re)
            re = RatedStatisticsEntity()
            re.statistic_type = "alive"
            re.expected_value = ["True"]
            re.actual_value = ["True"]
            re.state = [2]
            r.rated_statistics_entity.append(re)
            result.append(r)
        return result

    def __get_limits(self, specification, field, offset=0):
        if specification is None or field is None:
            return None
        key = "%s_%s_%s" % (specification.seuid, field, str(offset))
        if key in self.__limit_cache.keys():
            return self.__limit_cache[key]
        try:
            specs = specification.get(field)[1]
            if isinstance(specs, list) and len(specs) > 0 and isinstance(specs[0], list):
                if len(specs) > offset:
                    specs = specs[offset]
                else:
                    return None
            limits = specs[0:2]
            if len(specs) > 2 and specs[2][0].lower() == "r":
                if limits[1] > 1:
                    limits[1] -= 1
                m = limits[0]
                r = limits[1]
                limits[0] = m - m * r
                limits[1] = m + m * r
        except TypeError:
            limits = None
        except AttributeError:
            limits = None
        self.__limit_cache[key] = limits
        return limits

    def __compare(self, value, reference):
        if not isinstance(reference, list) or len(reference) < 2 or \
                not isinstance(reference[0], (int, long, float, complex)) or \
                not isinstance(reference[1], (int, long, float, complex)):
            return 2
        reference.sort()
        # r = (reference[0] - reference[1]) / 2
        # m = reference[0] + r
        if reference[0] > value:
            state = 1
        elif reference[1] < value:
            state = 0
        else:
            state = 3
        return state

    def reload_specifications(self, msg=None):
        """
        Reloads all specifications loaded into the namespace /arni/specifications
        """
        self.__limit_cache = {}
        self.__specifications = {}
        self.__load_specifications()
        return []

    def __init__(self):
        """
        Initiates the SpecificationHandler kicking off the loading of available specifications.
        """
        self.__limit_cache = {}
        self.__specifications = {}
        self.reload_specifications()<commit_msg>timewindow length fix<commit_after>import rospy
from specification import Specification
from rated_statistics import RatedStatisticsContainer
from arni_core.helper import *
import arni_msgs
from arni_msgs.msg import RatedStatistics, RatedStatisticsEntity

class SpecificationHandler:
    """
    Loads the specifications from the parameter server, stores and compares them to the actual metadata.
    """

    __namespace = '/arni/specifications'

    def __load_specifications(self):
        """
        Loads Specifications from the configurations and stores them in the
        internal storage.
        """
        try:
            params = rospy.get_param(self.__namespace)
            if isinstance(params, dict):
                specifications = []
                for x in params.values():
                    for y in x:
                        specifications.append(y)
            else:
                specifications = params
            for o in specifications:
                for seuid in o.keys():
                    if SEUID().is_valid(seuid):
                        spec = Specification()
                        spec.seuid = seuid
                        for k in o[seuid].keys():
                            spec.add_tuple((k, o[seuid][k]))
                        self.__specifications[seuid] = spec
                    else:
                        rospy.logdebug("[SpecificationHandler][__load_specifications] %s is not a valid seuid." % seuid)
        except KeyError:
            pass
        rospy.loginfo("[SpecificationHandler] Loaded %s parameters." % str(len(self.__specifications.keys())))

    def loaded_specifications(self):
        """
        Returns a list containing all seuids of loaded specifications.

        :return: A list of strings.
        """
        return self.__specifications.keys()

    def get(self, identifier):
        """
        Returns the Specification object from the internal storage.

        :param identifier: The seuid describing the desired Specification object.
        :type identifier: str
        :return: The Specification object with the given identifier, None if it was not found.
        """
        if identifier in self.__specifications.keys():
            return self.__specifications[identifier]
        if identifier[0] == "c":
            if SEUID(identifier).topic in self.__specifications.keys():
                return self.__specifications[SEUID(identifier).topic]
        return None

    def compare(self, data, identifier, specification=None):
        """
        Compares a Message object with a Specification object regarding all available matching fields.

        :param data: The actual data.
        :type data: object.
        :param identifier: The identifier of the metadata package.
        :type identifier: str
        :param specification: The Specification object, alternatively a string identifying it.
        :type specification: Specification or str.
        :returns: A RatedStatisticsContainer object representing the result.
        """
        if identifier is None:
            rospy.logdebug("[SpecificationHandler][compare] No identifier given.")
            return None
        if not SEUID().is_valid(identifier):
            rospy.logdebug("[SpecificationHandler][compare] Given identifier is invalid.")
            return None
        if data is None:
            rospy.logdebug("[SpecificationHandler][compare] No data given.")
            return None
        result = RatedStatisticsContainer(identifier)
        if identifier[0] == "n":
            result.host = data.host
        if specification is None:
            # if identifier in self.__specifications.keys():
            #     specification = self.__specifications[identifier]
            # else:
            #     if identifier[0] == "c":
            #         if SEUID(identifier).topic in self.__specifications.keys():
            #             specification = self.__specifications[SEUID(identifier).topic]
            specification = self.get(identifier)
        # if specification is None:
        # rospy.logdebug("[SpecificationHandler][compare] No Specification available for %s" % identifier)
        window_len = data.window_stop - data.window_start
        if window_len.to_sec() == 0:
            window_len = rospy.Duration(1)
        fields = dir(data)
        # exclude = ("window_start", "window_stop")
        # for x in exclude:
        # if x in fields:
        # fields.remove(x)
        if identifier[0] == "c":
            fields.append("bandwidth")
            fields.append("frequency")
        for field in fields:
            value = None
            if field[0] == "_" or "serialize" in field:
                continue
            current_obj = {}
            if field == "bandwidth":
                value = data.traffic / window_len.to_sec()
            elif field == "frequency":
                if hasattr(data, "delivered_msgs"):
                    value = data.delivered_msgs / window_len.to_sec()
            else:
                value = getattr(data, field)
            if value is not None:
                limits = self.__get_limits(specification, field)
                if isinstance(value, (list, tuple)):
                    current_obj["state"] = []
                    current_obj["actual"] = []
                    current_obj["expected"] = []
                    for i, v in enumerate(value):
                        limits = self.__get_limits(specification, field, i)
                        current_obj["actual"].append(v)
                        current_obj["state"].append(self.__compare(v, limits))
                        current_obj["expected"].append(limits)
                else:
                    status = self.__compare(value, limits)
                    current_obj["state"] = status
                    current_obj["actual"] = value
                    current_obj["expected"] = limits
                result.add_value(field, current_obj["actual"], current_obj["expected"], current_obj["state"])
        result.add_value("alive", ["True"], ["True"], [2])
        return result

    def compare_topic(self, data=None):
        """
        Compares Messages about one topic

        :param data: List of Statistics messages
        :return: list of RatedStatistics messages
        """
        if not data:
            data = []
        by_topic = {}
        result = []
        delivered_msgs_set = False
        for message in data:
            seuid = SEUID(message)
            if not seuid.get_seuid("topic") in by_topic.keys():
                by_topic[seuid.get_seuid("topic")] = {
                    "window_min": rospy.Time.now(),
                    "window_max": rospy.Time(0),
                    "delivered_msgs": 0,
                    "dropped_msgs": 0,
                    "frequency": {},
                    "traffic": 0,
                    "stamp_age_mean": [],
                    "stamp_age_stddev": [],
                    "stamp_age_max": rospy.Duration(0),
                    "packages": 0
                }
            window_len = message.window_stop - message.window_start
            if window_len.to_sec() == 0:
                window_len = rospy.Duration(1)
            # scale = window_len.to_sec() / window_len_max.to_sec()
            scale = 1  # placeholder
            by_topic[seuid.get_seuid("topic")]["window_min"] = min(message.window_start,
                                                                   by_topic[seuid.get_seuid("topic")]["window_min"])
            by_topic[seuid.get_seuid("topic")]["window_max"] = max(message.window_stop,
                                                                   by_topic[seuid.get_seuid("topic")]["window_max"])
            if hasattr(message, "delivered_msgs"):
                delivered_msgs_set = True
                by_topic[seuid.get_seuid("topic")]["delivered_msgs"] += message.delivered_msgs * scale
                if seuid.get_seuid("publisher") not in by_topic[seuid.get_seuid("topic")]["frequency"].keys():
                    by_topic[seuid.get_seuid("topic")]["frequency"][seuid.get_seuid("publisher")] = {
                        "value": 0,
                        "subscriber": seuid.get_seuid("subscriber")
                    }
                if seuid.get_seuid("subscriber") == \
                        by_topic[seuid.get_seuid("topic")]["frequency"][seuid.get_seuid("publisher")]["subscriber"]:
                    by_topic[seuid.get_seuid("topic")]["frequency"][seuid.get_seuid("publisher")]["value"] += \
                        message.delivered_msgs / window_len.to_sec() * scale
            by_topic[seuid.get_seuid("topic")]["dropped_msgs"] += message.dropped_msgs * scale
            by_topic[seuid.get_seuid("topic")]["traffic"] += message.traffic * scale
            by_topic[seuid.get_seuid("topic")]["stamp_age_max"] = max(message.stamp_age_max,
                                                                      by_topic[seuid.get_seuid("topic")][
                                                                          "stamp_age_max"])
            by_topic[seuid.get_seuid("topic")]["stamp_age_mean"].append(message.stamp_age_mean * scale)
            by_topic[seuid.get_seuid("topic")]["packages"] += 1
        for topic, data in by_topic.iteritems():
            specification = self.get(topic)
            r = RatedStatistics()
            r.window_start = data["window_min"]
            r.window_stop = data["window_max"]
            window_len = data["window_max"] - data["window_min"]
            if window_len.to_sec() == 0:
                window_len = rospy.Duration(1)
            r.seuid = topic
            fields = ["dropped_msgs", "traffic", "traffic_per_second", "stamp_age_max", "stamp_age_mean", "packages",
                      "packages_per_second"]
            fields.remove("traffic")
            if delivered_msgs_set:
                fields.append("delivered_msgs")
                fields.append("frequency")
            alt_names = {"traffic_per_second": "bandwidth"}
            for f in fields:
                re = RatedStatisticsEntity()
                re.statistic_type = f
                if f in alt_names.keys():
                    re.statistic_type = alt_names[f]
                if f == "frequency":
                    value = 0
                    for pub, freq in data["frequency"].iteritems():
                        value += freq["value"]
                elif f[-10:] == "per_second":
                    value = data[f[0:-11]] / window_len.to_sec()
                elif f == "stamp_age_mean":
                    x = data[f]
                    value = sum(y.to_sec() for y in x) / len(x)
                else:
                    value = data[f]
                re.actual_value.append(str(value))
                limits = self.__get_limits(specification, re.statistic_type)
                re.expected_value.append(str(limits))
                re.state = [self.__compare(value, limits)]
                r.rated_statistics_entity.append(re)
            re = RatedStatisticsEntity()
            re.statistic_type = "alive"
            re.expected_value = ["True"]
            re.actual_value = ["True"]
            re.state = [2]
            r.rated_statistics_entity.append(re)
            result.append(r)
        return result

    def __get_limits(self, specification, field, offset=0):
        if specification is None or field is None:
            return None
        key = "%s_%s_%s" % (specification.seuid, field, str(offset))
        if key in self.__limit_cache.keys():
            return self.__limit_cache[key]
        try:
            specs = specification.get(field)[1]
            if isinstance(specs, list) and len(specs) > 0 and isinstance(specs[0], list):
                if len(specs) > offset:
                    specs = specs[offset]
                else:
                    return None
            limits = specs[0:2]
            if len(specs) > 2 and specs[2][0].lower() == "r":
                if limits[1] > 1:
                    limits[1] -= 1
                m = limits[0]
                r = limits[1]
                limits[0] = m - m * r
                limits[1] = m + m * r
        except TypeError:
            limits = None
        except AttributeError:
            limits = None
        self.__limit_cache[key] = limits
        return limits

    def __compare(self, value, reference):
        if not isinstance(reference, list) or len(reference) < 2 or \
                not isinstance(reference[0], (int, long, float, complex)) or \
                not isinstance(reference[1], (int, long, float, complex)):
            return 2
        reference.sort()
        # r = (reference[0] - reference[1]) / 2
        # m = reference[0] + r
        if reference[0] > value:
            state = 1
        elif reference[1] < value:
            state = 0
        else:
            state = 3
        return state

    def reload_specifications(self, msg=None):
        """
        Reloads all specifications loaded into the namespace /arni/specifications
        """
        self.__limit_cache = {}
        self.__specifications = {}
        self.__load_specifications()
        return []

    def __init__(self):
        """
        Initiates the SpecificationHandler kicking off the loading of available specifications.
        """
        self.__limit_cache = {}
        self.__specifications = {}
        self.reload_specifications()<|endoftext|>


<commit_before>from collections import defaultdict
import numpy as np
import matplotlib.pyplot as plt

from dark import ncbidb

def basePlotter(blastHits, title):
    """
    Plot the reads and the subject, so that bases in the reads which are
    different from the subject are shown. Else a '.' is shown.
    like so:
    subject_gi  ATGCGTACGTACGACACC
    read_1         A......TTC..T

    @param blastHits: A L{dark.blast.BlastHits} instance.
    @param title: A C{str} sequence title that was hit by BLAST. We plot the
        reads that hit this title.
    """
    result = []
    params = blastHits.plotParams
    assert params is not None, ('Oops, it looks like you forgot to run '
                                'computePlotInfo.')

    sequence = ncbidb.getSequence(title, blastHits.records.blastDb)
    subject = sequence.seq
    gi = title.split('|')[1]
    sub = '%s \t \t \t %s' % (gi, subject)
    result.append(sub)

    plotInfo = blastHits.titles[title]['plotInfo']
    items = plotInfo['items']
    count = 0
    for item in items:
        count += 1
        hsp = item['hsp']
        queryTitle = blastHits.fasta[item['readNum']].id
        # If the product of the subject and query frame values is +ve,
        # then they're either both +ve or both -ve, so we just use the
        # query as is. Otherwise, we need to reverse complement it.
        if item['frame']['subject'] * item['frame']['query'] > 0:
            query = blastHits.fasta[item['readNum']].seq
            reverse = False
        else:
            # One of the subject or query has negative sense.
            #print 'reversing query'
            query = blastHits.fasta[
                item['readNum']].reverse_complement().seq
            reverse = True
        query = query.upper()
        queryStart = hsp['queryStart']
        subjectStart = hsp['subjectStart']
        queryEnd = hsp['queryEnd']
        subjectEnd = hsp['subjectEnd']

        # Before comparing the read to the subject, make a string of the
        # same length as the subject, which contains the read and
        # has ' ' at the offsets.
        # 3 parts need to be taken into account:
        # 1) the left offset (if the query doesn't stick out to the left)
        # 2) the query. if the frame is -1, it has to be reversed.
        # The query consists of 3 parts: left, middle (control for gaps)
        # 3) the right offset

        # do part 1) and 2).
        if queryStart < 0:
            # The query is sticking out to the left
            leftQuery = ''
            if subjectStart == 0:
                # The match starts at the first base of the subject
                middleLeftQuery = ''
                pass
            else:
                # The match starts into the subject.
                # Determine the length of the not matching query
                # part to the left
                leftOffset = -1 * queryStart
                rightOffset = subjectStart + leftOffset
                middleLeftQuery = query[leftOffset:rightOffset]
        else:
            # The query is not sticking out to the left
            # make the left offset
            leftQuery = queryStart * ' '

            leftQueryOffset = subjectStart - queryStart
            middleLeftQuery = query[:leftQueryOffset]

        # do part 3)
        # disregard gaps in subject while adding
        matchQuery = item['origHsp'].query
        matchSubject = item['origHsp'].sbjct
        index = 0
        mid = ''
        for item in range(len(matchQuery)):
            if matchSubject[index] != ' ':
                mid = mid + matchQuery[index]
                index += 1
            else:
                index += 1
        # if the query has been reversed, turn it around
        if reverse:
            rev = ''
            toReverse = mid
            reverseDict = {' ': ' ', '-': '-', 'A': 'T', 'T': 'A',
                           'C': 'G', 'G': 'C', '.': '.', 'N': 'N'}
            for item in toReverse:
                newItem = reverseDict[item]
                rev = rev + newItem
            mid = rev[::-1]

        middleQuery = middleLeftQuery + mid

        # add right not-matching part of the query
        rightQueryOffset = queryEnd - subjectEnd
        rightQuery = query[-rightQueryOffset:]
        middleQuery += rightQuery

        read = leftQuery + middleQuery

        # do part 3)
        sbLen = len(subject)
        offset = sbLen - len(read)
        # if the read is sticking out to the right
        # chop it off
        if offset < 0:
            read = read[:offset]
        # if it's not sticking out, fill the space with ' '
        elif offset > 0:
            addOn = offset * ' '
            read = read + addOn

        # compare the subject and the read, make a string
        # called 'comparison', which contains a '.' if the bases
        # are equal and the letter of the read if they are not.
        index = 0
        comparison = ''
        for base in range(len(read)):
            subjectBase = subject[index]
            readBase = read[index]
            if readBase == ' ':
                comparison += ' '
            elif readBase == subjectBase:
                comparison += '.'
            elif readBase != subjectBase:
                comparison += readBase
            index += 1
        que = '%s \t %s' % (queryTitle, comparison)
        result.append(que)

        # sanity checks
        assert (len(comparison) == len(subject)), (
            '%d "!=" %d' % (len(comparison), len(subject)))

        index = 0
        if comparison[index] == ' ':
            index += 1
        else:
            start = index - 1
            assert (start == queryStart or start == -1), (start, queryStart)

    return result

def getAPOBECFrequencies(dotAlignment, orig, new, pattern):
    """
    Gets mutation frequencies if they are in a certain pattern.

    @param dotAlignment: result from calling basePlotter
    @param orig: A C{str}, naming the original base
    @param new: A C{str}, what orig was mutated to
    @param pattern: A C{str}m which pattern we're looking for
        (must be one of 'cPattern', 'tPattern')
    """
    cPattern = ['ACA', 'ACC', 'ACG', 'ACT', 'CCA', 'CCC', 'CCG', 'CCT',
                'GCA', 'GCC', 'GCG', 'GCT', 'TCA', 'TCC', 'TCG', 'TCT']
    tPattern = ['ATA', 'ATC', 'ATG', 'ATT', 'CTA', 'CTC', 'CTG', 'CTT',
                'GTA', 'GTC', 'GTG', 'GTT', 'TTA', 'TTC', 'TTG', 'TTT']
    # choose the right pattern
    if pattern == 'cPattern':
        patterns = cPattern
        middleBase = 'C'
    else:
        patterns = tPattern
        middleBase = 'T'
    # generate the freqs dict with the right pattern
    freqs = {}
    for pattern in patterns:
        freqs[pattern] = 0
    # get the subject sequence from dotAlignment
    subject = dotAlignment[0].split('\t')[3]
    # exclude the subject from the dotAlignment, so just the queries
    # are left over
    queries = dotAlignment[1:]
    for item in queries:
        query = item.split('\t')[1]
        index = 0
        for queryBase in query:
            qBase = query[index]
            sBase = subject[index]
            if qBase == new and sBase == orig:
                try:
                    plusSb = subject[index + 1]
                    minusSb = subject[index - 1]
                except IndexError:
                    plusSb = 'end'
                motif = '%s%s%s' % (minusSb, middleBase, plusSb)
                if motif in freqs.keys():
                    freqs[motif] += 1
            index += 1

    return freqs

def getCompleteFreqs(blastHits):
    """
    Make a dictionary which collects all mutation frequencies from
    all reads.
    Calls basePlotter to get dotAlignment, which is passed to
    getAPOBECFrequencies with the respective parameter, to collect
    the frequencies.

    @param blastHits: A L{dark.blast.BlastHits} instance.
    """
    allFreqs = {}
    for title in blastHits.titles:
        allFreqs[title] = {
            'C>A': {},
            'C>G': {},
            'C>T': {},
            'T>A': {},
            'T>C': {},
            'T>G': {},
        }
        basesPlotted = basePlotter(blastHits, title)
        for mutation in allFreqs[title]:
            orig = mutation[0]
            new = mutation[2]
            if orig == 'C':
                pattern = 'cPattern'
            else:
                pattern = 'tPattern'
            freqs = getAPOBECFrequencies(basesPlotted, orig, new, pattern)
            allFreqs[title][mutation] = freqs
        allFreqs[title]['numberOfReads'] = len(blastHits.titles[title]['plotInfo']['items'])
        allFreqs[title]['bitScoreMax'] = blastHits.titles[title]['plotInfo']['bitScoreMax']
    return allFreqs

def makeFrequencyGraph(allFreqs, title, substitution, pattern,
                       color='blue', createFigure=True, showFigure=True,
                       readsAx=False):
    """
    For a title, make a graph showing the frequencies.

    @param allFreqs: result from getCompleteFreqs
    @param title: A C{str}, title of virus of which frequencies should be
        plotted.
    @param substitution: A C{str}, which substitution should be plotted;
        must be one of 'C>A', 'C>G', 'C>T', 'T>A', 'T>C', 'T>G'.
    @param pattern: A C{str}, which pattern we're looking for ( must be
        one of 'cPattern', 'tPattern')
    @param color: A C{str}, color of bars.
    """
    cPattern = ['ACA', 'ACC', 'ACG', 'ACT', 'CCA', 'CCC', 'CCG', 'CCT',
                'GCA', 'GCC', 'GCG', 'GCT', 'TCA', 'TCC', 'TCG', 'TCT']
    tPattern = ['ATA', 'ATC', 'ATG', 'ATT', 'CTA', 'CTC', 'CTG', 'CTT',
                'GTA', 'GTC', 'GTG', 'GTT', 'TTA', 'TTC', 'TTG', 'TTT']

    # choose the right pattern
    if pattern == 'cPattern':
        patterns = cPattern
    else:
        patterns = tPattern

    fig = plt.figure(figsize=(10, 10))
    ax = readsAx or fig.add_subplot(111)
    # how many bars
    N = 16
    ind = np.arange(N)
    width = 0.4
    # make a list in the right order, so that it can be plotted easily
    divisor = allFreqs[title]['numberOfReads']
    toPlot = allFreqs[title][substitution]
    index = 0
    data = []
    for item in patterns:
        newData = toPlot[patterns[index]]/divisor
        #newData = toPlot[patterns[index]]
        data.append(newData)
        index += 1
    # create the bars
    rects1 = ax.bar(ind, data, width, color=color)
    maxY = np.max(data) + 5
    # axes and labels
    if createFigure:
        title = title.split('|')[4][:50]
        ax.set_title('%s \n %s' % (title, substitution), fontsize=20)
        ax.set_ylim(0, maxY)
        ax.set_ylabel('Absolute Number of Mutations', fontsize=16)
        ax.set_xticks(ind+width)
        xTickNames = ax.set_xticklabels(patterns, rotation=45, fontsize=8)
    if not createFigure:
        ax.set_xticks(ind+width)
        xTickNames = ax.set_xticklabels(patterns, rotation=45, fontsize=0)
    if createFigure:
        if showFigure:
            plt.show()
    return maxY

def makeFrequencyPanel(allFreqs, patientName):
    """
    For a title, make a graph showing the frequencies.

    @param allFreqs: result from getCompleteFreqs
    @param patientName: A C{str}, title for the panel
    """
    titles = sorted(
        allFreqs.iterkeys(),
        key=lambda title: (allFreqs[title]['bitScoreMax'], title))

    origMaxY = 0
    cols = 6
    rows = len(allFreqs)
    figure, ax = plt.subplots(rows, cols, squeeze=False)
    substitutions = ['C>A', 'C>G', 'C>T', 'T>A', 'T>C', 'T>G']
    colors = ['blue', 'black', 'red', 'yellow', 'green', 'orange']

    for i, title in enumerate(titles):
        index = 0
        while index < 6:
            for subst in allFreqs[str(title)]:
                substitution = substitutions[index]
                print i, index, title, 'substitution', substitutions[index]
                if substitution[0] == 'C':
                    pattern = 'cPattern'
                else:
                    pattern = 'tPattern'
                maxY = makeFrequencyGraph(allFreqs, title, substitution,
                                          pattern, color=colors[index],
                                          createFigure=False, showFigure=False,
                                          readsAx=ax[i][index])
                if maxY > origMaxY:
                    origMaxY = maxY

            # add title for individual plot.
            # if used for other viruses, this will have to be adapted.
            if index == 0:
                gi = title.split('|')[1]
                titles = title.split(' ')
                try:
                    typeIndex = titles.index('type')
                except ValueError:
                    typeIndex = -1
                if typeIndex > -1:
                    typeNumber = titles[typeIndex+1]
                else:
                    typeNumber = 'gi: %s' % gi

                ax[i][index].set_ylabel(('Type %s \n maxBitScore: %s' % (typeNumber, allFreqs[title]['bitScoreMax'])),
                                        fontsize=10)
            # add xAxis tick labels
            if i == 0:
                ax[i][index].set_title(substitution, fontsize=13)
            if i == len(allFreqs)-1 or i == (len(allFreqs)-1)/2:
                if index < 3:
                    pat = ['ACA', 'ACC', 'ACG', 'ACT', 'CCA', 'CCC', 'CCG',
                           'CCT', 'GCA', 'GCC', 'GCG', 'GCT', 'TCA', 'TCC',
                           'TCG', 'TCT']
                else:
                    pat = ['ATA', 'ATC', 'ATG', 'ATT', 'CTA', 'CTC', 'CTG',
                           'CTT', 'GTA', 'GTC', 'GTG', 'GTT', 'TTA', 'TTC',
                           'TTG', 'TTT']
                ax[i][index].set_xticklabels(pat, rotation=45, fontsize=8)
            index += 1

    # make Y-axis equal
    for i, title in enumerate(allFreqs):
        index = 0
        while index < 6:
            a = ax[i][index]
            a.set_ylim([0, origMaxY])
            index += 1
    # add title of whole panel
    figure.suptitle('Mutation Signatures in %s' % patientName, fontsize=20)
    figure.set_size_inches(5 * cols, 3 * rows, forward=True)
    figure.show()

    return allFreqs
<commit_msg>correnctions<commit_after>from collections import defaultdict
import numpy as np
import matplotlib.pyplot as plt

from dark import ncbidb

def basePlotter(blastHits, title):
    """
    Plot the reads and the subject, so that bases in the reads which are
    different from the subject are shown. Else a '.' is shown.
    like so:
    subject_gi  ATGCGTACGTACGACACC
    read_1         A......TTC..T

    @param blastHits: A L{dark.blast.BlastHits} instance.
    @param title: A C{str} sequence title that was hit by BLAST. We plot the
        reads that hit this title.
    """
    result = []
    params = blastHits.plotParams
    assert params is not None, ('Oops, it looks like you forgot to run '
                                'computePlotInfo.')

    sequence = ncbidb.getSequence(title, blastHits.records.blastDb)
    subject = sequence.seq
    gi = title.split('|')[1]
    sub = '%s\t \t \t%s' % (gi, subject)
    result.append(sub)

    plotInfo = blastHits.titles[title]['plotInfo']
    assert plotInfo is not None, ('Oops, it looks like you forgot to run '
                                'computePlotInfo.')

    items = plotInfo['items']
    count = 0
    for item in items:
        count += 1
        hsp = item['hsp']
        queryTitle = blastHits.fasta[item['readNum']].id
        # If the product of the subject and query frame values is +ve,
        # then they're either both +ve or both -ve, so we just use the
        # query as is. Otherwise, we need to reverse complement it.
        if item['frame']['subject'] * item['frame']['query'] > 0:
            query = blastHits.fasta[item['readNum']].seq
            reverse = False
        else:
            # One of the subject or query has negative sense.
            #print 'reversing query'
            query = blastHits.fasta[
                item['readNum']].reverse_complement().seq
            reverse = True
        query = query.upper()
        queryStart = hsp['queryStart']
        subjectStart = hsp['subjectStart']
        queryEnd = hsp['queryEnd']
        subjectEnd = hsp['subjectEnd']

        # Before comparing the read to the subject, make a string of the
        # same length as the subject, which contains the read and
        # has ' ' where the read does not match.
        # 3 parts need to be taken into account:
        # 1) the left offset (if the query doesn't stick out to the left)
        # 2) the query. if the frame is -1, it has to be reversed.
        # The query consists of 3 parts: left, middle (control for gaps)
        # 3) the right offset

        # do part 1) and 2).
        if queryStart < 0:
            # The query is sticking out to the left
            leftQuery = ''
            if subjectStart == 0:
                # The match starts at the first base of the subject
                middleLeftQuery = ''
            else:
                # The match starts into the subject.
                # Determine the length of the not matching query
                # part to the left
                leftOffset = -1 * queryStart
                rightOffset = subjectStart + leftOffset
                middleLeftQuery = query[leftOffset:rightOffset]
        else:
            # The query is not sticking out to the left
            # make the left offset
            leftQuery = queryStart * ' '

            leftQueryOffset = subjectStart - queryStart
            middleLeftQuery = query[:leftQueryOffset]

        # do part 3)
        # disregard gaps in subject while adding
        matchQuery = item['origHsp'].query
        matchSubject = item['origHsp'].sbjct
        index = 0
        mid = ''
        for item in range(len(matchQuery)):
            if matchSubject[index] != ' ':
                mid += matchQuery[index]
            index += 1
        # if the query has been reversed, turn the matched part around
        if reverse:
            rev = ''
            toReverse = mid
            reverseDict = {' ': ' ', '-': '-', 'A': 'T', 'T': 'A',
                           'C': 'G', 'G': 'C', '.': '.', 'N': 'N'}
            for item in toReverse:
                newItem = reverseDict[item]
                rev += newItem
            mid = rev[::-1]

        middleQuery = middleLeftQuery + mid

        # add right not-matching part of the query
        rightQueryOffset = queryEnd - subjectEnd
        rightQuery = query[-rightQueryOffset:]
        middleQuery += rightQuery

        read = leftQuery + middleQuery

        # do part 3)
        offset = len(subject) - len(read)
        # if the read is sticking out to the right
        # chop it off
        if offset < 0:
            read = read[:offset]
        # if it's not sticking out, fill the space with ' '
        elif offset > 0:
            read += offset * ' '

        # compare the subject and the read, make a string
        # called 'comparison', which contains a '.' if the bases
        # are equal and the letter of the read if they are not.
        comparison = ''
        for readBase, subjectBase in zip(read, subject):
            if readBase == ' ':
                comparison += ' '
            elif readBase == subjectBase:
                comparison += '.'
            elif readBase != subjectBase:
                comparison += readBase
            index += 1
        que = '%s \t %s' % (queryTitle, comparison)
        result.append(que)

        # sanity checks
        assert (len(comparison) == len(subject)), (
            '%d != %d' % (len(comparison), len(subject)))

        index = 0
        if comparison[index] == ' ':
            index += 1
        else:
            start = index - 1
            assert (start == queryStart or start == -1),
                   ('%s != %s or %s != -1' % (start, queryStart, start)

    return result

def getAPOBECFrequencies(dotAlignment, orig, new, pattern):
    """
    Gets mutation frequencies if they are in a certain pattern.

    @param dotAlignment: result from calling basePlotter
    @param orig: A C{str}, naming the original base
    @param new: A C{str}, what orig was mutated to
    @param pattern: A C{str}m which pattern we're looking for
        (must be one of 'cPattern', 'tPattern')
    """
    cPattern = ['ACA', 'ACC', 'ACG', 'ACT', 'CCA', 'CCC', 'CCG', 'CCT',
                'GCA', 'GCC', 'GCG', 'GCT', 'TCA', 'TCC', 'TCG', 'TCT']
    tPattern = ['ATA', 'ATC', 'ATG', 'ATT', 'CTA', 'CTC', 'CTG', 'CTT',
                'GTA', 'GTC', 'GTG', 'GTT', 'TTA', 'TTC', 'TTG', 'TTT']
    # choose the right pattern
    if pattern == 'cPattern':
        patterns = cPattern
        middleBase = 'C'
    else:
        patterns = tPattern
        middleBase = 'T'
    # generate the freqs dict with the right pattern
    freqs = defaultdict(int)
    for pattern in patterns:
        freqs[pattern] = 0
    # get the subject sequence from dotAlignment
    subject = dotAlignment[0].split('\t')[3]
    # exclude the subject from the dotAlignment, so just the queries
    # are left over
    queries = dotAlignment[1:]
    for item in queries:
        query = item.split('\t')[1]
        index = 0
        for queryBase in query:
            qBase = query[index]
            sBase = subject[index]
            if qBase == new and sBase == orig:
                try:
                    plusSb = subject[index + 1]
                    minusSb = subject[index - 1]
                except IndexError:
                    plusSb = 'end'
                motif = '%s%s%s' % (minusSb, middleBase, plusSb)
                if motif in freqs:
                    freqs[motif] += 1
            index += 1

    return freqs

def getCompleteFreqs(blastHits):
    """
    Make a dictionary which collects all mutation frequencies from
    all reads.
    Calls basePlotter to get dotAlignment, which is passed to
    getAPOBECFrequencies with the respective parameter, to collect
    the frequencies.

    @param blastHits: A L{dark.blast.BlastHits} instance.
    """
    allFreqs = {}
    for title in blastHits.titles:
        allFreqs[title] = {
            'C>A': {},
            'C>G': {},
            'C>T': {},
            'T>A': {},
            'T>C': {},
            'T>G': {},
        }
        basesPlotted = basePlotter(blastHits, title)
        for mutation in allFreqs[title]:
            orig = mutation[0]
            new = mutation[2]
            if orig == 'C':
                pattern = 'cPattern'
            else:
                pattern = 'tPattern'
            freqs = getAPOBECFrequencies(basesPlotted, orig, new, pattern)
            allFreqs[title][mutation] = freqs
        allFreqs[title]['numberOfReads'] = len(blastHits.titles[title]['plotInfo']['items'])
        allFreqs[title]['bitScoreMax'] = blastHits.titles[title]['plotInfo']['bitScoreMax']
    return allFreqs

def makeFrequencyGraph(allFreqs, title, substitution, pattern,
                       color='blue', createFigure=True, showFigure=True,
                       readsAx=False):
    """
    For a title, make a graph showing the frequencies.

    @param allFreqs: result from getCompleteFreqs
    @param title: A C{str}, title of virus of which frequencies should be
        plotted.
    @param substitution: A C{str}, which substitution should be plotted;
        must be one of 'C>A', 'C>G', 'C>T', 'T>A', 'T>C', 'T>G'.
    @param pattern: A C{str}, which pattern we're looking for ( must be
        one of 'cPattern', 'tPattern')
    @param color: A C{str}, color of bars.
    @param createFigure: If C{True}, create a figure.
    @param showFigure: If C{True}, show the created figure.
    @param readsAx: If not None, use this as the subplot for displaying reads.
    """
    cPattern = ['ACA', 'ACC', 'ACG', 'ACT', 'CCA', 'CCC', 'CCG', 'CCT',
                'GCA', 'GCC', 'GCG', 'GCT', 'TCA', 'TCC', 'TCG', 'TCT']
    tPattern = ['ATA', 'ATC', 'ATG', 'ATT', 'CTA', 'CTC', 'CTG', 'CTT',
                'GTA', 'GTC', 'GTG', 'GTT', 'TTA', 'TTC', 'TTG', 'TTT']

    # choose the right pattern
    if pattern == 'cPattern':
        patterns = cPattern
    else:
        patterns = tPattern

    fig = plt.figure(figsize=(10, 10))
    ax = readsAx or fig.add_subplot(111)
    # how many bars
    N = 16
    ind = np.arange(N)
    width = 0.4
    # make a list in the right order, so that it can be plotted easily
    divisor = allFreqs[title]['numberOfReads']
    toPlot = allFreqs[title][substitution]
    index = 0
    data = []
    for item in patterns:
        newData = toPlot[patterns[index]]/divisor
        data.append(newData)
        index += 1
    # create the bars
    rects1 = ax.bar(ind, data, width, color=color)
    maxY = np.max(data) + 5
    # axes and labels
    if createFigure:
        title = title.split('|')[4][:50]
        ax.set_title('%s \n %s' % (title, substitution), fontsize=20)
        ax.set_ylim(0, maxY)
        ax.set_ylabel('Absolute Number of Mutations', fontsize=16)
        ax.set_xticks(ind+width)
        xTickNames = ax.set_xticklabels(patterns, rotation=45, fontsize=8)
    if createFigure == False:
        ax.set_xticks(ind+width)
        xTickNames = ax.set_xticklabels(patterns, rotation=45, fontsize=0)
    else:
        if showFigure:
            plt.show()
    return maxY

def makeFrequencyPanel(allFreqs, patientName):
    """
    For a title, make a graph showing the frequencies.

    @param allFreqs: result from getCompleteFreqs
    @param patientName: A C{str}, title for the panel
    """
    titles = sorted(
        allFreqs.iterkeys(),
        key=lambda title: (allFreqs[title]['bitScoreMax'], title))

    origMaxY = 0
    cols = 6
    rows = len(allFreqs)
    figure, ax = plt.subplots(rows, cols, squeeze=False)
    substitutions = ['C>A', 'C>G', 'C>T', 'T>A', 'T>C', 'T>G']
    colors = ['blue', 'black', 'red', 'yellow', 'green', 'orange']

    for i, title in enumerate(titles):
        for index in range(6):
            for subst in allFreqs[str(title)]:
                substitution = substitutions[index]
                print i, index, title, 'substitution', substitutions[index]
                if substitution[0] == 'C':
                    pattern = 'cPattern'
                else:
                    pattern = 'tPattern'
                maxY = makeFrequencyGraph(allFreqs, title, substitution,
                                          pattern, color=colors[index],
                                          createFigure=False, showFigure=False,
                                          readsAx=ax[i][index])
                if maxY > origMaxY:
                    origMaxY = maxY

            # add title for individual plot.
            # if used for other viruses, this will have to be adapted.
            if index == 0:
                gi = title.split('|')[1]
                titles = title.split(' ')
                try:
                    typeIndex = titles.index('type')
                except ValueError:
                    typeNumber = 'gi: %s' % gi
                else:
                    typeNumber = titles[typeIndex + 1]

                ax[i][index].set_ylabel(('Type %s \n maxBitScore: %s' %
                    (typeNumber, allFreqs[title]['bitScoreMax'])), fontsize=10)
            # add xAxis tick labels
            if i == 0:
                ax[i][index].set_title(substitution, fontsize=13)
            if i == len(allFreqs)-1 or i == (len(allFreqs)-1)/2:
                if index < 3:
                    pat = ['ACA', 'ACC', 'ACG', 'ACT', 'CCA', 'CCC', 'CCG',
                           'CCT', 'GCA', 'GCC', 'GCG', 'GCT', 'TCA', 'TCC',
                           'TCG', 'TCT']
                else:
                    pat = ['ATA', 'ATC', 'ATG', 'ATT', 'CTA', 'CTC', 'CTG',
                           'CTT', 'GTA', 'GTC', 'GTG', 'GTT', 'TTA', 'TTC',
                           'TTG', 'TTT']
                ax[i][index].set_xticklabels(pat, rotation=45, fontsize=8)

    # make Y-axis equal
    for i, title in enumerate(allFreqs):
        for index in range(6)
            a = ax[i][index]
            a.set_ylim([0, origMaxY])
    # add title of whole panel
    figure.suptitle('Mutation Signatures in %s' % patientName, fontsize=20)
    figure.set_size_inches(5 * cols, 3 * rows, forward=True)
    figure.show()

    return allFreqs
<|endoftext|>


<commit_before>from numpy.linalg import inv,cholesky

class BaseMetricLearner(object):
  def __init__(self):
    raise NotImplementedError('BaseMetricLearner should not be instantiated')

  def metric(self):
    """Computes the Mahalanobis matrix from the transformation matrix.

    .. math:: M = L^{\\top} L

    Returns
    -------
    M : (d x d) matrix
    """
    L = self.transformer()
    return L.T.dot(L)

  def transformer(self):
    """Computes the transformation matrix from the Mahalanobis matrix.

    L = inv(cholesky(M))

    Returns
    -------
    L : (d x d) matrix
    """
    return inv(cholesky(self.metric()))

  def transform(self, X=None):
    """Applies the metric transformation.

    Parameters
    ----------
    X : (n x d) matrix, optional
        Data to transform. If not supplied, the training data will be used.

    Returns
    -------
    transformed : (n x d) matrix
        Input data transformed to the metric space by :math:`XL^{\\top}`
    """
    if X is None:
      X = self.X
    L = self.transformer()
    return X.dot(L.T)

  def get_params(self, deep=False):
    """Get parameters for this metric learner.

    Parameters
    ----------
    deep: boolean, optional
        @WARNING doesn't do anything, only exists because scikit-learn has this on BaseEstimator.

    Returns
    -------
    params : mapping of string to any
        Parameter names mapped to their values.
    """
    return self.params

  def set_params(self, **kwarg):
    """Set the parameters of this metric learner.

    Overwrites any default parameters or parameters specified in constructor.

    Returns
    -------
    self
    """
    self.params = {**self.params, **kwarg}
<commit_msg>set_params rewritten to be compatible with python 2<commit_after>from numpy.linalg import inv,cholesky

class BaseMetricLearner(object):
  def __init__(self):
    raise NotImplementedError('BaseMetricLearner should not be instantiated')

  def metric(self):
    """Computes the Mahalanobis matrix from the transformation matrix.

    .. math:: M = L^{\\top} L

    Returns
    -------
    M : (d x d) matrix
    """
    L = self.transformer()
    return L.T.dot(L)

  def transformer(self):
    """Computes the transformation matrix from the Mahalanobis matrix.

    L = inv(cholesky(M))

    Returns
    -------
    L : (d x d) matrix
    """
    return inv(cholesky(self.metric()))

  def transform(self, X=None):
    """Applies the metric transformation.

    Parameters
    ----------
    X : (n x d) matrix, optional
        Data to transform. If not supplied, the training data will be used.

    Returns
    -------
    transformed : (n x d) matrix
        Input data transformed to the metric space by :math:`XL^{\\top}`
    """
    if X is None:
      X = self.X
    L = self.transformer()
    return X.dot(L.T)

  def get_params(self, deep=False):
    """Get parameters for this metric learner.

    Parameters
    ----------
    deep: boolean, optional
        @WARNING doesn't do anything, only exists because scikit-learn has this on BaseEstimator.

    Returns
    -------
    params : mapping of string to any
        Parameter names mapped to their values.
    """
    return self.params

  def set_params(self, **kwarg):
    """Set the parameters of this metric learner.

    Overwrites any default parameters or parameters specified in constructor.

    Returns
    -------
    self
    """
    self.params.update(kwarg)
<|endoftext|>


<commit_before>
import os, subprocess, select, fcntl, signal

from zmq.eventloop.ioloop import PeriodicCallback
from cc.handler import CCHandler

import skytools

__all__ = ['JobMgr']

CC_HANDLER = 'JobMgr'

def set_nonblocking(fd, onoff):
    """Toggle the O_NONBLOCK flag.
    If onoff==None then return current setting.
    """
    flags = fcntl.fcntl(fd, fcntl.F_GETFL)
    if onoff is None:
        return (flags & os.O_NONBLOCK) > 0
    if onoff:
        flags |= os.O_NONBLOCK
    else:
        flags &= ~os.O_NONBLOCK
    fcntl.fcntl(fd, fcntl.F_SETFL, flags)

def set_cloexec(fd, onoff):
    """Toggle the FD_CLOEXEC flag.
    If onoff==None then return current setting.
    """
    flags = fcntl.fcntl(fd, fcntl.F_GETFD)
    if onoff is None:
        return (flags & fcntl.FD_CLOEXEC) > 0
    if onoff:
        flags |= fcntl.FD_CLOEXEC
    else:
        flags &= ~fcntl.FD_CLOEXEC
    fcntl.fcntl(fd, fcntl.F_SETFL, flags)

#
# JobMgr
#

class JobState:
    def __init__(self, jname, jcf, log, cc_url, ioloop):
        self.jname = jname
        self.jcf = jcf
        self.proc = None
        self.log = log
        self.cc_url = cc_url
        self.timer = None
        self.ioloop = ioloop
        pidfiledir = self.jcf.getfile('pidfiledir', '~/pid')
        self.pidfile = "%s/%s.pid" % (pidfiledir, self.jname)
        self.cfdict = {
                'job_name': self.jname,
                'pidfile': self.pidfile,
        }
        for o in self.jcf.options():
            self.cfdict[o] = self.jcf.get(o)

    def handle_timer(self):
        if self.proc:
            self.log.info('JobState.handle_timer')
            data = self.proc.stdout.read()
            if data:
                self.log.info('handle_timer: stdout=%s', repr(data.strip()))
            if self.proc.poll() is not None:
                x = self.proc.wait()
                self.log.info('handle_timer: proc exited with %s', repr(x))
                self.proc = None
        else:
            # daemonization successful?
            live = skytools.signal_pidfile(self.pidfile, 0)
            if live:
                self.log.debug('handle_timer: %s is alive', self.jname)
            else:
                self.log.info('handle_timer: %s is dead', self.jname)

    def start(self):
        # unsure about the best way to specify target
        mod = self.jcf.get('module', '')
        script = self.jcf.get('module', '')
        cls = self.jcf.get('class', '')
        args = ['-d', '--cc', self.cc_url, '--ccdaemon', self.jname]
        if mod:
            cmd = ['python', '-m', mod] + args
        elif script:
            cmd = [script] + args
        else:
            raise skytools.UsageError('dunno how to launch class')

        self.log.info('Launching %s: %s', self.jname, cmd)
        self.proc = subprocess.Popen(cmd, close_fds=True,
                                stdin = open(os.devnull, 'rb'),
                                stdout = subprocess.PIPE,
                                stderr = subprocess.STDOUT)

        set_nonblocking(self.proc.stdout, True)

        self.timer = PeriodicCallback(self.handle_timer, 2*1000, self.ioloop)
        self.timer.start()

    def stop(self):
        try:
            self.log.info('Killing %s', self.jname)
            skytools.signal_pidfile(self.pidfile, signal.SIGINT)
        except:
            self.log.exception('signal_pidfile failed')

class JobMgr(CCHandler):
    """Provide config to local daemons / tasks."""

    def __init__(self, hname, hcf, ccscript):
        super(JobMgr, self).__init__(hname, hcf, ccscript)

        self.local_url = ccscript.local_url

        self.jobs = {}
        for dname in self.cf.getlist('daemons'):
            self.add_job(dname)

    def add_job(self, jname):
        jcf = skytools.Config(jname, self.cf.filename, ignore_defs = True)
        jstate = JobState(jname, jcf, self.log, self.local_url, self.ioloop)
        self.jobs[jname] = jstate
        jstate.start()

    def handle_msg(self, cmsg):
        """Got message from client, send to remote CC"""

        self.log.info('JobMgr req: %s', cmsg)
        data = cmsg.get_payload()

        res = {'req': data['req']}
        if data['req'] == 'job.config':
            # send config
            job = self.jobs[data['job_name']]
            res['config'] = job.cfdict
        else:
            res['msg'] = 'Unsupported req'
        ans = cmsg.make_reply(res)
        self.cclocal.send_cmsg(ans)
        self.log.info('JobMgr answer: %s', ans)

    def stop(self):
        for j in self.jobs.values():
            j.stop()

<commit_msg>jobmgr: fix pidfiledir<commit_after>
import os, subprocess, select, fcntl, signal

from zmq.eventloop.ioloop import PeriodicCallback
from cc.handler import CCHandler
from cc.util import set_nonblocking

import skytools

__all__ = ['JobMgr']

CC_HANDLER = 'JobMgr'

#
# JobMgr
#

class JobState:
    def __init__(self, jname, jcf, log, cc_url, ioloop, pidfiledir):
        self.jname = jname
        self.jcf = jcf
        self.proc = None
        self.log = log
        self.cc_url = cc_url
        self.timer = None
        self.ioloop = ioloop
        self.pidfile = "%s/%s.pid" % (pidfiledir, self.jname)
        self.cfdict = {
                'job_name': self.jname,
                'pidfile': self.pidfile,
        }
        for o in self.jcf.options():
            self.cfdict[o] = self.jcf.get(o)

    def handle_timer(self):
        if self.proc:
            self.log.info('JobState.handle_timer')
            data = self.proc.stdout.read()
            if data:
                self.log.info('handle_timer: stdout=%s', repr(data.strip()))
            if self.proc.poll() is not None:
                x = self.proc.wait()
                self.log.info('handle_timer: proc exited with %s', repr(x))
                self.proc = None
        else:
            # daemonization successful?
            live = skytools.signal_pidfile(self.pidfile, 0)
            if live:
                self.log.debug('handle_timer: %s is alive', self.jname)
            else:
                self.log.info('handle_timer: %s is dead', self.jname)

    def start(self):
        # unsure about the best way to specify target
        mod = self.jcf.get('module', '')
        script = self.jcf.get('module', '')
        cls = self.jcf.get('class', '')
        args = ['-d', '--cc', self.cc_url, '--ccdaemon', self.jname]
        if mod:
            cmd = ['python', '-m', mod] + args
        elif script:
            cmd = [script] + args
        else:
            raise skytools.UsageError('dunno how to launch class')

        self.log.info('Launching %s: %s', self.jname, cmd)
        self.proc = subprocess.Popen(cmd, close_fds=True,
                                stdin = open(os.devnull, 'rb'),
                                stdout = subprocess.PIPE,
                                stderr = subprocess.STDOUT)

        set_nonblocking(self.proc.stdout, True)

        self.timer = PeriodicCallback(self.handle_timer, 2*1000, self.ioloop)
        self.timer.start()

    def stop(self):
        try:
            self.log.info('Killing %s', self.jname)
            skytools.signal_pidfile(self.pidfile, signal.SIGINT)
        except:
            self.log.exception('signal_pidfile failed')

class JobMgr(CCHandler):
    """Provide config to local daemons / tasks."""

    def __init__(self, hname, hcf, ccscript):
        super(JobMgr, self).__init__(hname, hcf, ccscript)

        self.local_url = ccscript.local_url
        self.pidfiledir = hcf.getfile('pidfiledir', '~/pid')

        self.jobs = {}
        for dname in self.cf.getlist('daemons'):
            self.add_job(dname)

    def add_job(self, jname):
        jcf = skytools.Config(jname, self.cf.filename, ignore_defs = True)
        jstate = JobState(jname, jcf, self.log, self.local_url, self.ioloop, self.pidfiledir)
        self.jobs[jname] = jstate
        jstate.start()

    def handle_msg(self, cmsg):
        """Got message from client, send to remote CC"""

        self.log.info('JobMgr req: %s', cmsg)
        data = cmsg.get_payload()

        res = {'req': data['req']}
        if data['req'] == 'job.config':
            # send config
            job = self.jobs[data['job_name']]
            res['config'] = job.cfdict
        else:
            res['msg'] = 'Unsupported req'
        ans = cmsg.make_reply(res)
        self.cclocal.send_cmsg(ans)
        self.log.info('JobMgr answer: %s', ans)

    def stop(self):
        for j in self.jobs.values():
            j.stop()

<|endoftext|>


<commit_before>from datetime import datetime
import random

from django.db import models
from django.contrib.auth.models import User
from django.db.models.signals import post_save, pre_save
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist

from cc.general.models import VarCharField, EmailField
from cc.geo.models import Location
import cc.ripple.api as ripple
from cc.general.util import cache_on_object
from cc.general.mail import send_mail, email_str

CODE_LENGTH = 20
CODE_CHARS = '1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'

class Profile(models.Model):
    user = models.OneToOneField(User, related_name='profile')
    name = VarCharField(blank=True)
    location = models.ForeignKey(Location, null=True, blank=True)
    photo = models.ImageField(
        upload_to='user/%Y/%m', max_length=256, blank=True)
    description = models.TextField(blank=True)

    created = models.DateTimeField(auto_now_add=True)
    updated = models.DateTimeField(auto_now_add=True)

    trusted_profiles = models.ManyToManyField(
        'Profile', symmetrical=False, related_name='trusting_profiles',
        blank=True)

    # TODO: Should a profile always trust itself?

    FEED_TEMPLATE = 'profile_feed_item.html'

    def __unicode__(self):
        return self.name or self.username

    @models.permalink
    def get_absolute_url(self):
        return 'profile', (self.username,)

    def set_updated(self):
        """
        Call this every time a user-facing change is made to profile that
        should cause it to be bumped to the top of the feed.
        Must still save the profile after to have any effect.
        """
        self.updated = datetime.now()

    @property
    def username(self):
        # Username is stored in django User model.
        return self.user.username

    @property
    def email(self):
        return self.settings.email

    @property
    def endorsement_limited(self):
        return self.settings.endorsement_limited

    def endorsement_for(self, recipient):
        "Returns this profile's endorsement of recipient, or None."
        try:
            return self.endorsements_made.get(recipient=recipient)
        except ObjectDoesNotExist:
            return None

    @property
    def date(self):
        return self.updated

    # TODO: Consider putting profile at top of feed whenever they come
    # back online?

    @property
    def text(self):
        return self.description

    @property
    def feed_public(self):
        return True

    @property
    def feed_recipient(self):
        return None

    @property
    def feed_poster(self):
        return self

    def get_search_text(self):
        return [(self.name, 'A'),
                (self.username, 'A'),
                (self.description, 'B'),
               ]

    @property
    @cache_on_object
    def endorsement_count(self):
        return self.endorsements_received.all().count()

    @property
    @cache_on_object
    def endorsement_sum(self):
        return self.endorsements_received.all().aggregate(
            models.Sum('weight')).get('weight__sum') or 0

    @property
    @cache_on_object
    def endorsements_made_sum(self):
        return (
            (self.endorsements_made.all().aggregate(
                models.Sum('weight')).get('weight__sum') or 0) +
            (self.invitations_sent.all().aggregate(
                models.Sum('endorsement_weight')).get(
                'endorsement_weight__sum') or 0))

    @property
    @cache_on_object
    def endorsements_remaining(self):
        return max(((self.endorsement_count) * settings.ENDORSEMENT_BONUS -
                    self.endorsements_made_sum), 0)

    @property
    def can_endorse(self):
        return not self.endorsement_limited or self.endorsements_remaining > 0

    def reputation(self, asker):
        """"
        Returns max flow of endorsements from asker to self across endorsement
        network.
        """
        return ripple.credit_reputation(self, asker)

    def overall_balance(self):
        return ripple.overall_balance(self)

    def trusted_balance(self):
        "Overall balance minus acknowledgements received beyond trusted limits."
        return ripple.trusted_balance(self)

    def trusts(self, profile):
        return self.trusted_profiles.filter(pk=profile.id).count() > 0

    def account(self, profile):
        return ripple.get_account(self, profile)

    def email_str(self):
        """
        Returns '"Name" <email>' suitable for email headers.
        """
        return email_str(self.name, self.email)

    @classmethod
    def get_by_id(cls, id):
        return cls.objects.get(pk=id)

    @classmethod
    def post_save(cls, sender, instance, created, **kwargs):
        # Create Settings for this profile if it is new.
        if created:
            Settings.objects.create(profile=instance)

post_save.connect(Profile.post_save, sender=Profile,
                  dispatch_uid='profile.models')

class Settings(models.Model):
    "Profile settings."
    profile = models.OneToOneField(Profile, related_name='settings')
    email = EmailField(blank=True)
    endorsement_limited = models.BooleanField(default=True)
    send_notifications = models.BooleanField(
        default=True, help_text=(
            "Receive email whenever someone endorses or acknowledges you."))

    # Sticky form settings.
    feed_radius = models.IntegerField(null=True, blank=True)
    feed_trusted = models.BooleanField(default=False)

    def __unicode__(self):
        return u"Settings for %s" % self.profile

class Invitation(models.Model):
    from_profile = models.ForeignKey(Profile, related_name='invitations_sent')
    to_email = EmailField("Friend's email")
    endorsement_weight = models.PositiveIntegerField("Hearts", help_text=(
            "Each heart represents an hour of value you'd provide "
            "in exchange for acknowledgements."))
    endorsement_text = models.TextField("Testimonial", blank=True)
    message = models.TextField(
        "Private message", blank=True,
        help_text="Sent with the invitation email only. Not public.")

    date = models.DateTimeField(auto_now_add=True)
    code = VarCharField(unique=True)

    # TODO: Some help text in fields above.

    # TODO: Cron job to delete old invitations?

    def __unicode__(self):
        return u"%s invites %s" % (self.from_profile, self.to_email)

    @models.permalink
    def get_absolute_url(self):
        return 'invitation', (self.code,)

    def set_code(self):
        self.code = ''.join(
            (random.choice(CODE_CHARS) for i in xrange(CODE_LENGTH)))

    def send(self):
        send_mail("%s Has Invited You To Villages.cc" % self.from_profile,
                  self.from_profile, self.to_email, 'invitation_email.txt',
                  {'invitation': self})
        self.date = datetime.now()
        self.save()

    @classmethod
    def pre_save(cls, sender, instance, **kwargs):
        if not instance.code:
            instance.set_code()

# Fill in code before saving.
pre_save.connect(Invitation.pre_save, sender=Invitation,
                 dispatch_uid='profile.models')
<commit_msg>Give 5 hearts to give out before any endorsement.<commit_after>from datetime import datetime
import random

from django.db import models
from django.contrib.auth.models import User
from django.db.models.signals import post_save, pre_save
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist

from cc.general.models import VarCharField, EmailField
from cc.geo.models import Location
import cc.ripple.api as ripple
from cc.general.util import cache_on_object
from cc.general.mail import send_mail, email_str

CODE_LENGTH = 20
CODE_CHARS = '1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'

class Profile(models.Model):
    user = models.OneToOneField(User, related_name='profile')
    name = VarCharField(blank=True)
    location = models.ForeignKey(Location, null=True, blank=True)
    photo = models.ImageField(
        upload_to='user/%Y/%m', max_length=256, blank=True)
    description = models.TextField(blank=True)

    created = models.DateTimeField(auto_now_add=True)
    updated = models.DateTimeField(auto_now_add=True)

    trusted_profiles = models.ManyToManyField(
        'Profile', symmetrical=False, related_name='trusting_profiles',
        blank=True)

    # TODO: Should a profile always trust itself?

    FEED_TEMPLATE = 'profile_feed_item.html'

    def __unicode__(self):
        return self.name or self.username

    @models.permalink
    def get_absolute_url(self):
        return 'profile', (self.username,)

    def set_updated(self):
        """
        Call this every time a user-facing change is made to profile that
        should cause it to be bumped to the top of the feed.
        Must still save the profile after to have any effect.
        """
        self.updated = datetime.now()

    @property
    def username(self):
        # Username is stored in django User model.
        return self.user.username

    @property
    def email(self):
        return self.settings.email

    @property
    def endorsement_limited(self):
        return self.settings.endorsement_limited

    def endorsement_for(self, recipient):
        "Returns this profile's endorsement of recipient, or None."
        try:
            return self.endorsements_made.get(recipient=recipient)
        except ObjectDoesNotExist:
            return None

    @property
    def date(self):
        return self.updated

    # TODO: Consider putting profile at top of feed whenever they come
    # back online?

    @property
    def text(self):
        return self.description

    @property
    def feed_public(self):
        return True

    @property
    def feed_recipient(self):
        return None

    @property
    def feed_poster(self):
        return self

    def get_search_text(self):
        return [(self.name, 'A'),
                (self.username, 'A'),
                (self.description, 'B'),
               ]

    @property
    @cache_on_object
    def endorsement_count(self):
        return self.endorsements_received.all().count()

    @property
    @cache_on_object
    def endorsement_sum(self):
        return self.endorsements_received.all().aggregate(
            models.Sum('weight')).get('weight__sum') or 0

    @property
    @cache_on_object
    def endorsements_made_sum(self):
        return (
            (self.endorsements_made.all().aggregate(
                models.Sum('weight')).get('weight__sum') or 0) +
            (self.invitations_sent.all().aggregate(
                models.Sum('endorsement_weight')).get(
                'endorsement_weight__sum') or 0))

    @property
    @cache_on_object
    def endorsements_remaining(self):
        return max(((self.endorsement_count + 1) * settings.ENDORSEMENT_BONUS -
                    self.endorsements_made_sum), 0)

    @property
    def can_endorse(self):
        return not self.endorsement_limited or self.endorsements_remaining > 0

    def reputation(self, asker):
        """"
        Returns max flow of endorsements from asker to self across endorsement
        network.
        """
        return ripple.credit_reputation(self, asker)

    def overall_balance(self):
        return ripple.overall_balance(self)

    def trusted_balance(self):
        "Overall balance minus acknowledgements received beyond trusted limits."
        return ripple.trusted_balance(self)

    def trusts(self, profile):
        return self.trusted_profiles.filter(pk=profile.id).count() > 0

    def account(self, profile):
        return ripple.get_account(self, profile)

    def email_str(self):
        """
        Returns '"Name" <email>' suitable for email headers.
        """
        return email_str(self.name, self.email)

    @classmethod
    def get_by_id(cls, id):
        return cls.objects.get(pk=id)

    @classmethod
    def post_save(cls, sender, instance, created, **kwargs):
        # Create Settings for this profile if it is new.
        if created:
            Settings.objects.create(profile=instance)

post_save.connect(Profile.post_save, sender=Profile,
                  dispatch_uid='profile.models')

class Settings(models.Model):
    "Profile settings."
    profile = models.OneToOneField(Profile, related_name='settings')
    email = EmailField(blank=True)
    endorsement_limited = models.BooleanField(default=True)
    send_notifications = models.BooleanField(
        default=True, help_text=(
            "Receive email whenever someone endorses or acknowledges you."))

    # Sticky form settings.
    feed_radius = models.IntegerField(null=True, blank=True)
    feed_trusted = models.BooleanField(default=False)

    def __unicode__(self):
        return u"Settings for %s" % self.profile

class Invitation(models.Model):
    from_profile = models.ForeignKey(Profile, related_name='invitations_sent')
    to_email = EmailField("Friend's email")
    endorsement_weight = models.PositiveIntegerField("Hearts", help_text=(
            "Each heart represents an hour of value you'd provide "
            "in exchange for acknowledgements."))
    endorsement_text = models.TextField("Testimonial", blank=True)
    message = models.TextField(
        "Private message", blank=True,
        help_text="Sent with the invitation email only. Not public.")

    date = models.DateTimeField(auto_now_add=True)
    code = VarCharField(unique=True)

    # TODO: Some help text in fields above.

    # TODO: Cron job to delete old invitations?

    def __unicode__(self):
        return u"%s invites %s" % (self.from_profile, self.to_email)

    @models.permalink
    def get_absolute_url(self):
        return 'invitation', (self.code,)

    def set_code(self):
        self.code = ''.join(
            (random.choice(CODE_CHARS) for i in xrange(CODE_LENGTH)))

    def send(self):
        send_mail("%s Has Invited You To Villages.cc" % self.from_profile,
                  self.from_profile, self.to_email, 'invitation_email.txt',
                  {'invitation': self})
        self.date = datetime.now()
        self.save()

    @classmethod
    def pre_save(cls, sender, instance, **kwargs):
        if not instance.code:
            instance.set_code()

# Fill in code before saving.
pre_save.connect(Invitation.pre_save, sender=Invitation,
                 dispatch_uid='profile.models')
<|endoftext|>


<commit_before>from flask.ext import restful
from . import database
from flask import abort, send_file
from werkzeug import secure_filename
import os

class DownloadInstance(restful.Resource):
    def get(self, id):
        document = database.get_document(id)
        if document is None:
            abort(404)
        return send_file(document['path'])

<commit_msg>Make sure we give people a download with the actual name of the file<commit_after>from flask.ext import restful
from . import database
from flask import abort, send_file
import mimetypes

class DownloadInstance(restful.Resource):
    def get(self, id):
        document = database.get_document(id)
        if document is None:
            abort(404)
        mt = mimetypes.guess_type(document['path'])[0]
        return send_file(document['path'],
                         as_attachment=True,
                         attachment_filename=document['name'],
                         mimetype=mt)
<|endoftext|>