<commit_before>#!/usr/bin/env python
"""An object that models the current state of the MCP.

2013-08-23 ROwen    Added apogeeGangLabelDict in lieu of figure out how to access labelHelp from an opscore Key
"""
__all__ = ["Model"]

import urlparse
import opscore.actor.model as actorModel

_theModel = None

def Model():
    global _theModel
    if not _theModel:
        _theModel = _Model()
    return _theModel

class _Model (actorModel.Model):
    def __init__(self):
        actorModel.Model.__init__(self, "mcp")
        self.apogeeGangLabelDict = {
             "0": "Unknown",
             "1": "Disconnected",
             "2": "At Cart",
             "4": "Podium: ?",
            "12": "Podium: Dense",
            "20": "Podium: Sparse",
            "36": "Podium: 1M",
        }
<commit_msg>Removed unused import<commit_after>#!/usr/bin/env python
"""An object that models the current state of the MCP.

2013-08-23 ROwen    Added apogeeGangLabelDict in lieu of figure out how to access labelHelp from an opscore Key
2014-12-18 ROwen    Reemoved unused import.
"""
__all__ = ["Model"]

import opscore.actor.model as actorModel

_theModel = None

def Model():
    global _theModel
    if not _theModel:
        _theModel = _Model()
    return _theModel

class _Model (actorModel.Model):
    def __init__(self):
        actorModel.Model.__init__(self, "mcp")
        self.apogeeGangLabelDict = {
             "0": "Unknown",
             "1": "Disconnected",
             "2": "At Cart",
             "4": "Podium: ?",
            "12": "Podium: Dense",
            "20": "Podium: Sparse",
            "36": "Podium: 1M",
        }
<|endoftext|>


<commit_before>'''Report plugin that plots some graphs'''

from Tank.Plugins.Aggregator import AggregateResultListener, AggregatorPlugin
from Tank.Plugins.Monitoring import MonitoringPlugin
from Tank.MonCollector.collector import MonitoringDataListener, MonitoringDataDecoder
from tankcore import AbstractPlugin
import datetime
import time
import string
import json
import os
from collections import defaultdict

class ReportPlugin(AbstractPlugin, AggregateResultListener, MonitoringDataListener):
    '''Graphite data uploader'''

    SECTION = 'report'

    @staticmethod
    def get_key():
        return __file__

    def __init__(self, core):
        AbstractPlugin.__init__(self, core)
        self.decoder = MonitoringDataDecoder()
        self.mon_data = {}
        def create_storage():
            return {
                'avg': defaultdict(list),
                'quantiles': defaultdict(list),
                'threads': {
                    'active_threads': []
                },
                'rps': {
                    'RPS': []
                },
            }
        self.overall = create_storage()
        self.cases = defaultdict(create_storage)

    def monitoring_data(self, data_string):
        self.log.debug("Mon report data: %s", data_string)
        for line in data_string.splitlines():
            if not line.strip():
                continue

            def append_data(host, ts, data):
                if host not in self.mon_data:
                    self.mon_data[host] = {}
                host_data = self.mon_data[host]
                for key, value in data.iteritems():
                    try:
                        value = float(value)
                        if '_' in key:
                            group, key = key.split('_', 1)
                        else:
                            group = key
                        if group not in host_data:
                            host_data[group] = {}
                        group_data = host_data[group]
                        if key not in group_data:
                            group_data[key] = []
                        group_data[key].append((int(ts), value))
                    except ValueError:
                        pass
            host, data, _, ts = self.decoder.decode_line(line)
            append_data(host, ts, data)

    def get_available_options(self):
        return []

    def start_test(self):
        start_time = datetime.datetime.now()
        self.start_time = start_time.strftime("%H:%M%%20%Y%m%d")

    def end_test(self, retcode):
        end_time = datetime.datetime.now() + datetime.timedelta(minutes = 1)
        self.end_time = end_time.strftime("%H:%M%%20%Y%m%d")

    def configure(self):
        '''Read configuration'''
        self.show_graph = self.get_option("show_graph", "")
        default_template = "/report.tpl"
        self.template = self.get_option("template", os.path.dirname(__file__) + default_template)
        aggregator = self.core.get_plugin_of_type(AggregatorPlugin)
        aggregator.add_result_listener(self)
        try:
            self.mon = self.core.get_plugin_of_type(MonitoringPlugin)
            if self.mon.monitoring:
                self.mon.monitoring.add_listener(self)
        except KeyError:
            self.log.warning("No monitoring module, monitroing report disabled")

    def aggregate_second(self, data):
        """
        @data: SecondAggregateData
        """
        ts = int(time.mktime(data.time.timetuple()))
        def add_aggreagted_second(data_item, storage):
            data_dict = data_item.__dict__
            avg = storage['avg']
            for key in ["avg_connect_time", "avg_send_time", "avg_latency", "avg_receive_time"]:
                avg[key].append((ts, data_dict.get(key, None)))
            quantiles = storage['quantiles']
            for key, value in data_item.quantiles.iteritems():
                quantiles[key].append((ts, value))
        add_aggreagted_second(data.overall, self.overall)
        for case, case_data in data.cases.iteritems():
            add_aggreagted_second(case_data, self.cases[case])

    def post_process(self, retcode):
        results = {
            'overall': self.overall,
            'cases': self.cases,
            'monitoring': self.mon_data,
        }
        template = open(self.template, 'r').read()
        report_html = self.core.mkstemp(".html", "report_")
        self.core.add_artifact_file(report_html)
        with open(report_html, 'w') as report_html_file:
            report_html_file.write(
                string.Template(template).safe_substitute(
                    metrics=json.dumps(results),
                )
            )

<commit_msg>additional metrics<commit_after>'''Report plugin that plots some graphs'''

from Tank.Plugins.Aggregator import AggregateResultListener, AggregatorPlugin
from Tank.Plugins.Monitoring import MonitoringPlugin
from Tank.MonCollector.collector import MonitoringDataListener, MonitoringDataDecoder
from tankcore import AbstractPlugin
import datetime
import time
import string
import json
import os
from collections import defaultdict

class ReportPlugin(AbstractPlugin, AggregateResultListener, MonitoringDataListener):
    '''Graphite data uploader'''

    SECTION = 'report'

    @staticmethod
    def get_key():
        return __file__

    def __init__(self, core):
        AbstractPlugin.__init__(self, core)
        self.decoder = MonitoringDataDecoder()
        self.mon_data = {}
        def create_storage():
            return {
                'avg': defaultdict(list),
                'quantiles': defaultdict(list),
                'threads': {
                    'active_threads': []
                },
                'rps': {
                    'RPS': []
                },
                'http_codes': defaultdict(list),
                'net_codes': defaultdict(list),
            }
        self.overall = create_storage()
        self.cases = defaultdict(create_storage)

    def monitoring_data(self, data_string):
        self.log.debug("Mon report data: %s", data_string)
        for line in data_string.splitlines():
            if not line.strip():
                continue

            def append_data(host, ts, data):
                if host not in self.mon_data:
                    self.mon_data[host] = {}
                host_data = self.mon_data[host]
                for key, value in data.iteritems():
                    try:
                        value = float(value)
                        if '_' in key:
                            group, key = key.split('_', 1)
                        else:
                            group = key
                        if group not in host_data:
                            host_data[group] = {}
                        group_data = host_data[group]
                        if key not in group_data:
                            group_data[key] = []
                        group_data[key].append((int(ts), value))
                    except ValueError:
                        pass
            host, data, _, ts = self.decoder.decode_line(line)
            append_data(host, ts, data)

    def get_available_options(self):
        return []

    def start_test(self):
        start_time = datetime.datetime.now()
        self.start_time = start_time.strftime("%H:%M%%20%Y%m%d")

    def end_test(self, retcode):
        end_time = datetime.datetime.now() + datetime.timedelta(minutes = 1)
        self.end_time = end_time.strftime("%H:%M%%20%Y%m%d")

    def configure(self):
        '''Read configuration'''
        self.show_graph = self.get_option("show_graph", "")
        default_template = "/report.tpl"
        self.template = self.get_option("template", os.path.dirname(__file__) + default_template)
        aggregator = self.core.get_plugin_of_type(AggregatorPlugin)
        aggregator.add_result_listener(self)
        try:
            self.mon = self.core.get_plugin_of_type(MonitoringPlugin)
            if self.mon.monitoring:
                self.mon.monitoring.add_listener(self)
        except KeyError:
            self.log.warning("No monitoring module, monitroing report disabled")

    def aggregate_second(self, data):
        """
        @data: SecondAggregateData
        """
        ts = int(time.mktime(data.time.timetuple()))
        def add_aggreagted_second(data_item, storage):
            data_dict = data_item.__dict__
            avg = storage['avg']
            for key in ["avg_connect_time", "avg_send_time", "avg_latency", "avg_receive_time"]:
                avg[key].append((ts, data_dict.get(key, None)))
            quantiles = storage['quantiles']
            for key, value in data_item.quantiles.iteritems():
                quantiles[key].append((ts, value))
            storage['threads']['active_threads'].append((ts, data_item.active_threads))
            storage['rps']['RPS'].append((ts, data_item.RPS))
            http_codes = storage['http_codes']
            for key, value in data_item.http_codes.iteritems():
                http_codes[key].append((ts, value))
            net_codes = storage['net_codes']
            for key, value in data_item.net_codes.iteritems():
                net_codes[key].append((ts, value))

        add_aggreagted_second(data.overall, self.overall)
        for case, case_data in data.cases.iteritems():
            add_aggreagted_second(case_data, self.cases[case])

    def post_process(self, retcode):
        results = {
            'overall': self.overall,
            'cases': self.cases,
            'monitoring': self.mon_data,
        }
        template = open(self.template, 'r').read()
        report_html = self.core.mkstemp(".html", "report_")
        self.core.add_artifact_file(report_html)
        with open(report_html, 'w') as report_html_file:
            report_html_file.write(
                string.Template(template).safe_substitute(
                    metrics=json.dumps(results),
                )
            )

<|endoftext|>


<commit_before>"""
django-helpdesk - A Django powered ticket tracker for small enterprise.

(c) Copyright 2008 Jutda. All Rights Reserved. See LICENSE for details.

lib.py - Common functions (eg multipart e-mail)
"""

chart_colours = ('80C65A', '990066', 'FF9900', '3399CC', 'BBCCED', '3399CC', 'FFCC33')

try:
    from base64 import urlsafe_b64encode as b64encode
except ImportError:
    from base64 import encodestring as b64encode
try:
    from base64 import urlsafe_b64decode as b64decode
except ImportError:
    from base64 import decodestring as b64decode

import logging
logger = logging.getLogger('helpdesk')

from django.utils.encoding import smart_str

def send_templated_mail(template_name, email_context, recipients, sender=None, bcc=None, fail_silently=False, files=None):
    """
    send_templated_mail() is a warpper around Django's e-mail routines that
    allows us to easily send multipart (text/plain & text/html) e-mails using
    templates that are stored in the database. This lets the admin provide
    both a text and a HTML template for each message.

    template_name is the slug of the template to use for this message (see
        models.EmailTemplate)

    email_context is a dictionary to be used when rendering the template

    recipients can be either a string, eg 'a@b.com', or a list of strings.

    sender should contain a string, eg 'My Site <me@z.com>'. If you leave it
        blank, it'll use settings.DEFAULT_FROM_EMAIL as a fallback.

    bcc is an optional list of addresses that will receive this message as a
        blind carbon copy.

    fail_silently is passed to Django's mail routine. Set to 'True' to ignore
        any errors at send time.

    files can be a list of file paths to be attached, or it can be left blank.
        eg ('/tmp/file1.txt', '/tmp/image.png')

    """
    from django.conf import settings
    from django.core.mail import EmailMultiAlternatives
    from django.template import loader, Context

    from helpdesk.models import EmailTemplate
    import os

    context = Context(email_context)

    if hasattr(context['queue'], 'locale'):
        locale = getattr(context['queue'], 'locale', '')
    else:
        locale = context['queue'].get('locale', 'en')
    if not locale:
        locale = 'en'

    t = None
    try:
        t = EmailTemplate.objects.get(template_name__iexact=template_name, locale=locale)
    except EmailTemplate.DoesNotExist:
        pass

    if not t:
        try:
            t = EmailTemplate.objects.get(template_name__iexact=template_name, locale__isnull=True)
        except EmailTemplate.DoesNotExist:
            logger.warning('template "%s" does not exist, no mail sent' %
			   template_name)
            return # just ignore if template doesn't exist

    if not sender:
        sender = settings.DEFAULT_FROM_EMAIL

    footer_file = os.path.join('helpdesk', locale, 'email_text_footer.txt')

    text_part = loader.get_template_from_string(
        "%s{%% include '%s' %%}" % (t.plain_text, footer_file)
        ).render(context)

    email_html_base_file = os.path.join('helpdesk', locale, 'email_html_base.html')

    ''' keep new lines in html emails '''
    from django.utils.safestring import mark_safe

    if context.has_key('comment'):
        html_txt = context['comment']
        html_txt = html_txt.replace('\r\n', '
')
        context['comment'] = mark_safe(html_txt)

    html_part = loader.get_template_from_string(
        "{%% extends '%s' %%}{%% block title %%}%s{%% endblock %%}{%% block content %%}%s{%% endblock %%}" % (email_html_base_file, t.heading, t.html)
        ).render(context)

    subject_part = loader.get_template_from_string(
        "{{ ticket.ticket }} {{ ticket.title|safe }} %s" % t.subject
        ).render(context)

    if isinstance(recipients,(str,unicode)):
        if recipients.find(','):
            recipients = recipients.split(',')
    elif type(recipients) != list:
        recipients = [recipients,]

    msg = EmailMultiAlternatives(   subject_part,
                                    text_part,
                                    sender,
                                    recipients,
                                    bcc=bcc)
    msg.attach_alternative(html_part, "text/html")

    if files:
        if type(files) != list:
            files = [files,]

        for file in files:
            msg.attach_file(file)

    return msg.send(fail_silently)

def query_to_dict(results, descriptions):
    """
    Replacement method for cursor.dictfetchall() as that method no longer
    exists in psycopg2, and I'm guessing in other backends too.

    Converts the results of a raw SQL query into a list of dictionaries, suitable
    for use in templates etc.
    """

    output = []
    for data in results:
        row = {}
        i = 0
        for column in descriptions:
            row[column[0]] = data[i]
            i += 1

        output.append(row)
    return output

def apply_query(queryset, params):
    """
    Apply a dict-based set of filters & parameters to a queryset.

    queryset is a Django queryset, eg MyModel.objects.all() or
             MyModel.objects.filter(user=request.user)

    params is a dictionary that contains the following:
        filtering: A dict of Django ORM filters, eg:
            {'user__id__in': [1, 3, 103], 'title__contains': 'foo'}
        other_filter: Another filter of some type, most likely a
            set of Q() objects.
        sorting: The name of the column to sort by
    """
    for key in params['filtering'].keys():
        filter = {key: params['filtering'][key]}
        queryset = queryset.filter(**filter)

    if params.get('other_filter', None):
        # eg a Q() set
        queryset = queryset.filter(params['other_filter'])

    sorting = params.get('sorting', None)
    if sorting:
        sortreverse = params.get('sortreverse', None)
        if sortreverse:
            sorting = "-%s" % sorting
        queryset = queryset.order_by(sorting)

    return queryset

def safe_template_context(ticket):
    """
    Return a dictionary that can be used as a template context to render
    comments and other details with ticket or queue parameters. Note that
    we don't just provide the Ticket & Queue objects to the template as
    they could reveal confidential information. Just imagine these two options:
        * {{ ticket.queue.email_box_password }}
        * {{ ticket.assigned_to.password }}

    Ouch!

    The downside to this is that if we make changes to the model, we will also
    have to update this code. Perhaps we can find a better way in the future.
    """

    context = {
        'queue': {},
        'ticket': {},
        }
    queue = ticket.queue

    for field in (  'title', 'slug', 'email_address', 'from_address', 'locale'):
        attr = getattr(queue, field, None)
        if callable(attr):
            context['queue'][field] = attr()
        else:
            context['queue'][field] = attr

    for field in (  'title', 'created', 'modified', 'submitter_email',
                    'status', 'get_status_display', 'on_hold', 'description',
                    'resolution', 'priority', 'get_priority_display',
                    'last_escalation', 'ticket', 'ticket_for_url',
                    'get_status', 'ticket_url', 'staff_url', '_get_assigned_to'
                 ):
        attr = getattr(ticket, field, None)
        if callable(attr):
            context['ticket'][field] = '%s' % attr()
        else:
            context['ticket'][field] = attr

    context['ticket']['queue'] = context['queue']
    context['ticket']['assigned_to'] = context['ticket']['_get_assigned_to']

    return context

def text_is_spam(text, request):
    # Based on a blog post by 'sciyoshi':
    # http://sciyoshi.com/blog/2008/aug/27/using-akismet-djangos-new-comments-framework/
    # This will return 'True' is the given text is deemed to be spam, or
    # False if it is not spam. If it cannot be checked for some reason, we
    # assume it isn't spam.
    from django.contrib.sites.models import Site
    from django.conf import settings
    try:
        from helpdesk.akismet import Akismet
    except:
        return False
    try:
        site = Site.objects.get_current()
    except:
        site = Site(domain='configure-django-sites.com')

    ak = Akismet(
        blog_url='http://%s/' % site.domain,
        agent='django-helpdesk',
    )

    if hasattr(settings, 'TYPEPAD_ANTISPAM_API_KEY'):
        ak.setAPIKey(key = settings.TYPEPAD_ANTISPAM_API_KEY)
        ak.baseurl = 'api.antispam.typepad.com/1.1/'
    elif hasattr(settings, 'AKISMET_API_KEY'):
        ak.setAPIKey(key = settings.AKISMET_API_KEY)
    else:
        return False

    if ak.verify_key():
        ak_data = {
            'user_ip': request.META.get('REMOTE_ADDR', '127.0.0.1'),
            'user_agent': request.META.get('HTTP_USER_AGENT', ''),
            'referrer': request.META.get('HTTP_REFERER', ''),
            'comment_type': 'comment',
            'comment_author': '',
        }

        return ak.comment_check(smart_str(text), data=ak_data)

    return False
<commit_msg>Removed logging<commit_after>"""
django-helpdesk - A Django powered ticket tracker for small enterprise.

(c) Copyright 2008 Jutda. All Rights Reserved. See LICENSE for details.

lib.py - Common functions (eg multipart e-mail)
"""

chart_colours = ('80C65A', '990066', 'FF9900', '3399CC', 'BBCCED', '3399CC', 'FFCC33')

try:
    from base64 import urlsafe_b64encode as b64encode
except ImportError:
    from base64 import encodestring as b64encode
try:
    from base64 import urlsafe_b64decode as b64decode
except ImportError:
    from base64 import decodestring as b64decode

from django.utils.encoding import smart_str

def send_templated_mail(template_name, email_context, recipients, sender=None, bcc=None, fail_silently=False, files=None):
    """
    send_templated_mail() is a warpper around Django's e-mail routines that
    allows us to easily send multipart (text/plain & text/html) e-mails using
    templates that are stored in the database. This lets the admin provide
    both a text and a HTML template for each message.

    template_name is the slug of the template to use for this message (see
        models.EmailTemplate)

    email_context is a dictionary to be used when rendering the template

    recipients can be either a string, eg 'a@b.com', or a list of strings.

    sender should contain a string, eg 'My Site <me@z.com>'. If you leave it
        blank, it'll use settings.DEFAULT_FROM_EMAIL as a fallback.

    bcc is an optional list of addresses that will receive this message as a
        blind carbon copy.

    fail_silently is passed to Django's mail routine. Set to 'True' to ignore
        any errors at send time.

    files can be a list of file paths to be attached, or it can be left blank.
        eg ('/tmp/file1.txt', '/tmp/image.png')

    """
    from django.conf import settings
    from django.core.mail import EmailMultiAlternatives
    from django.template import loader, Context

    from helpdesk.models import EmailTemplate
    import os

    context = Context(email_context)

    if hasattr(context['queue'], 'locale'):
        locale = getattr(context['queue'], 'locale', '')
    else:
        locale = context['queue'].get('locale', 'en')
    if not locale:
        locale = 'en'

    t = None
    try:
        t = EmailTemplate.objects.get(template_name__iexact=template_name, locale=locale)
    except EmailTemplate.DoesNotExist:
        pass

    if not t:
        try:
            t = EmailTemplate.objects.get(template_name__iexact=template_name, locale__isnull=True)
        except EmailTemplate.DoesNotExist:
            return # just ignore if template doesn't exist

    if not sender:
        sender = settings.DEFAULT_FROM_EMAIL

    footer_file = os.path.join('helpdesk', locale, 'email_text_footer.txt')

    text_part = loader.get_template_from_string(
        "%s{%% include '%s' %%}" % (t.plain_text, footer_file)
        ).render(context)

    email_html_base_file = os.path.join('helpdesk', locale, 'email_html_base.html')

    ''' keep new lines in html emails '''
    from django.utils.safestring import mark_safe

    if context.has_key('comment'):
        html_txt = context['comment']
        html_txt = html_txt.replace('\r\n', '
')
        context['comment'] = mark_safe(html_txt)

    html_part = loader.get_template_from_string(
        "{%% extends '%s' %%}{%% block title %%}%s{%% endblock %%}{%% block content %%}%s{%% endblock %%}" % (email_html_base_file, t.heading, t.html)
        ).render(context)

    subject_part = loader.get_template_from_string(
        "{{ ticket.ticket }} {{ ticket.title|safe }} %s" % t.subject
        ).render(context)

    if isinstance(recipients,(str,unicode)):
        if recipients.find(','):
            recipients = recipients.split(',')
    elif type(recipients) != list:
        recipients = [recipients,]

    msg = EmailMultiAlternatives(   subject_part,
                                    text_part,
                                    sender,
                                    recipients,
                                    bcc=bcc)
    msg.attach_alternative(html_part, "text/html")

    if files:
        if type(files) != list:
            files = [files,]

        for file in files:
            msg.attach_file(file)

    return msg.send(fail_silently)

def query_to_dict(results, descriptions):
    """
    Replacement method for cursor.dictfetchall() as that method no longer
    exists in psycopg2, and I'm guessing in other backends too.

    Converts the results of a raw SQL query into a list of dictionaries, suitable
    for use in templates etc.
    """

    output = []
    for data in results:
        row = {}
        i = 0
        for column in descriptions:
            row[column[0]] = data[i]
            i += 1

        output.append(row)
    return output

def apply_query(queryset, params):
    """
    Apply a dict-based set of filters & parameters to a queryset.

    queryset is a Django queryset, eg MyModel.objects.all() or
             MyModel.objects.filter(user=request.user)

    params is a dictionary that contains the following:
        filtering: A dict of Django ORM filters, eg:
            {'user__id__in': [1, 3, 103], 'title__contains': 'foo'}
        other_filter: Another filter of some type, most likely a
            set of Q() objects.
        sorting: The name of the column to sort by
    """
    for key in params['filtering'].keys():
        filter = {key: params['filtering'][key]}
        queryset = queryset.filter(**filter)

    if params.get('other_filter', None):
        # eg a Q() set
        queryset = queryset.filter(params['other_filter'])

    sorting = params.get('sorting', None)
    if sorting:
        sortreverse = params.get('sortreverse', None)
        if sortreverse:
            sorting = "-%s" % sorting
        queryset = queryset.order_by(sorting)

    return queryset

def safe_template_context(ticket):
    """
    Return a dictionary that can be used as a template context to render
    comments and other details with ticket or queue parameters. Note that
    we don't just provide the Ticket & Queue objects to the template as
    they could reveal confidential information. Just imagine these two options:
        * {{ ticket.queue.email_box_password }}
        * {{ ticket.assigned_to.password }}

    Ouch!

    The downside to this is that if we make changes to the model, we will also
    have to update this code. Perhaps we can find a better way in the future.
    """

    context = {
        'queue': {},
        'ticket': {},
        }
    queue = ticket.queue

    for field in (  'title', 'slug', 'email_address', 'from_address', 'locale'):
        attr = getattr(queue, field, None)
        if callable(attr):
            context['queue'][field] = attr()
        else:
            context['queue'][field] = attr

    for field in (  'title', 'created', 'modified', 'submitter_email',
                    'status', 'get_status_display', 'on_hold', 'description',
                    'resolution', 'priority', 'get_priority_display',
                    'last_escalation', 'ticket', 'ticket_for_url',
                    'get_status', 'ticket_url', 'staff_url', '_get_assigned_to'
                 ):
        attr = getattr(ticket, field, None)
        if callable(attr):
            context['ticket'][field] = '%s' % attr()
        else:
            context['ticket'][field] = attr

    context['ticket']['queue'] = context['queue']
    context['ticket']['assigned_to'] = context['ticket']['_get_assigned_to']

    return context

def text_is_spam(text, request):
    # Based on a blog post by 'sciyoshi':
    # http://sciyoshi.com/blog/2008/aug/27/using-akismet-djangos-new-comments-framework/
    # This will return 'True' is the given text is deemed to be spam, or
    # False if it is not spam. If it cannot be checked for some reason, we
    # assume it isn't spam.
    from django.contrib.sites.models import Site
    from django.conf import settings
    try:
        from helpdesk.akismet import Akismet
    except:
        return False
    try:
        site = Site.objects.get_current()
    except:
        site = Site(domain='configure-django-sites.com')

    ak = Akismet(
        blog_url='http://%s/' % site.domain,
        agent='django-helpdesk',
    )

    if hasattr(settings, 'TYPEPAD_ANTISPAM_API_KEY'):
        ak.setAPIKey(key = settings.TYPEPAD_ANTISPAM_API_KEY)
        ak.baseurl = 'api.antispam.typepad.com/1.1/'
    elif hasattr(settings, 'AKISMET_API_KEY'):
        ak.setAPIKey(key = settings.AKISMET_API_KEY)
    else:
        return False

    if ak.verify_key():
        ak_data = {
            'user_ip': request.META.get('REMOTE_ADDR', '127.0.0.1'),
            'user_agent': request.META.get('HTTP_USER_AGENT', ''),
            'referrer': request.META.get('HTTP_REFERER', ''),
            'comment_type': 'comment',
            'comment_author': '',
        }

        return ak.comment_check(smart_str(text), data=ak_data)

    return False
<|endoftext|>


<commit_before>"""
django-helpdesk - A Django powered ticket tracker for small enterprise.

(c) Copyright 2008 Jutda. All Rights Reserved. See LICENSE for details.

lib.py - Common functions (eg multipart e-mail)
"""

import logging
import mimetypes
import os
from smtplib import SMTPException

try:
    # Python 2 support
    from base64 import urlsafe_b64encode as b64encode
except ImportError:
    # Python 3 support
    from base64 import encodebytes as b64encode
try:
    # Python 2 support
    from base64 import urlsafe_b64decode as b64decode
except ImportError:
    # Python 3 support
    from base64 import decodebytes as b64decode

from django.conf import settings
from django.db.models import Q
from django.utils import six
from django.utils.encoding import smart_text
from django.utils.safestring import mark_safe

from helpdesk.models import Attachment, EmailTemplate

logger = logging.getLogger('helpdesk')

def send_templated_mail(template_name,
                        context,
                        recipients,
                        sender=None,
                        bcc=None,
                        fail_silently=False,
                        files=None):
    """
    send_templated_mail() is a wrapper around Django's e-mail routines that
    allows us to easily send multipart (text/plain & text/html) e-mails using
    templates that are stored in the database. This lets the admin provide
    both a text and a HTML template for each message.

    template_name is the slug of the template to use for this message (see
        models.EmailTemplate)

    context is a dictionary to be used when rendering the template

    recipients can be either a string, eg 'a@b.com', or a list of strings.

    sender should contain a string, eg 'My Site <me@z.com>'. If you leave it
        blank, it'll use settings.DEFAULT_FROM_EMAIL as a fallback.

    bcc is an optional list of addresses that will receive this message as a
        blind carbon copy.

    fail_silently is passed to Django's mail routine. Set to 'True' to ignore
        any errors at send time.

    files can be a list of tuples. Each tuple should be a filename to attach,
        along with the File objects to be read. files can be blank.

    """
    from django.core.mail import EmailMultiAlternatives
    from django.template import engines
    from_string = engines['django'].from_string

    from helpdesk.models import EmailTemplate
    from helpdesk.settings import HELPDESK_EMAIL_SUBJECT_TEMPLATE, \
        HELPDESK_EMAIL_FALLBACK_LOCALE

    locale = context['queue'].get('locale') or HELPDESK_EMAIL_FALLBACK_LOCALE

    try:
        t = EmailTemplate.objects.get(template_name__iexact=template_name, locale=locale)
    except EmailTemplate.DoesNotExist:
        try:
            t = EmailTemplate.objects.get(template_name__iexact=template_name, locale__isnull=True)
        except EmailTemplate.DoesNotExist:
            logger.warning('template "%s" does not exist, no mail sent', template_name)
            return  # just ignore if template doesn't exist

    subject_part = from_string(
        HELPDESK_EMAIL_SUBJECT_TEMPLATE % {
            "subject": t.subject
        }).render(context).replace('\n', '').replace('\r', '')

    footer_file = os.path.join('helpdesk', locale, 'email_text_footer.txt')

    text_part = from_string(
        "%s{%% include '%s' %%}" % (t.plain_text, footer_file)
    ).render(context)

    email_html_base_file = os.path.join('helpdesk', locale, 'email_html_base.html')
    # keep new lines in html emails
    if 'comment' in context:
        context['comment'] = mark_safe(context['comment'].replace('\r\n', '
'))

    html_part = from_string(
        "{%% extends '%s' %%}{%% block title %%}"
        "%s"
        "{%% endblock %%}{%% block content %%}%s{%% endblock %%}" %
        (email_html_base_file, t.heading, t.html)
    ).render(context)

    if isinstance(recipients, str):
        if recipients.find(','):
            recipients = recipients.split(',')
    elif type(recipients) != list:
        recipients = [recipients]

    msg = EmailMultiAlternatives(subject_part, text_part,
                                 sender or settings.DEFAULT_FROM_EMAIL,
                                 recipients, bcc=bcc)
    msg.attach_alternative(html_part, "text/html")

    if files:
        for filename, filefield in files:
            mime = mimetypes.guess_type(filename)
            if mime[0] is not None and mime[0] == "text/plain":
                with open(filefield.path, 'r') as attachedfile:
                    content = attachedfile.read()
                    msg.attach(filename, content)
            else:
                if six.PY3:
                    msg.attach_file(filefield.path)
                else:
                    with open(filefield.path, 'rb') as attachedfile:
                        content = attachedfile.read()
                        msg.attach(filename, content)

    logger.debug('Sending email to: {!r}'.format(recipients))

    try:
        return msg.send()
    except SMTPException:
        logger.exception('SMTPException raised while sending email to {}'.format(recipients))
        if not fail_silently:
            raise e
        return 0

def query_to_dict(results, descriptions):
    """
    Replacement method for cursor.dictfetchall() as that method no longer
    exists in psycopg2, and I'm guessing in other backends too.

    Converts the results of a raw SQL query into a list of dictionaries, suitable
    for use in templates etc.
    """

    output = []
    for data in results:
        row = {}
        i = 0
        for column in descriptions:
            row[column[0]] = data[i]
            i += 1

        output.append(row)
    return output

def apply_query(queryset, params):
    """
    Apply a dict-based set of filters & parameters to a queryset.

    queryset is a Django queryset, eg MyModel.objects.all() or
             MyModel.objects.filter(user=request.user)

    params is a dictionary that contains the following:
        filtering: A dict of Django ORM filters, eg:
            {'user__id__in': [1, 3, 103], 'title__contains': 'foo'}

        search_string: A freetext search string

        sorting: The name of the column to sort by
    """
    for key in params['filtering'].keys():
        filter = {key: params['filtering'][key]}
        queryset = queryset.filter(**filter)

    search = params.get('search_string', None)
    if search:
        qset = (
            Q(title__icontains=search) |
            Q(description__icontains=search) |
            Q(resolution__icontains=search) |
            Q(submitter_email__icontains=search)
        )

        queryset = queryset.filter(qset)

    sorting = params.get('sorting', None)
    if sorting:
        sortreverse = params.get('sortreverse', None)
        if sortreverse:
            sorting = "-%s" % sorting
        queryset = queryset.order_by(sorting)

    return queryset

def ticket_template_context(ticket):
    context = {}

    for field in ('title', 'created', 'modified', 'submitter_email',
                  'status', 'get_status_display', 'on_hold', 'description',
                  'resolution', 'priority', 'get_priority_display',
                  'last_escalation', 'ticket', 'ticket_for_url',
                  'get_status', 'ticket_url', 'staff_url', '_get_assigned_to'
                  ):
        attr = getattr(ticket, field, None)
        if callable(attr):
            context[field] = '%s' % attr()
        else:
            context[field] = attr
    context['assigned_to'] = context['_get_assigned_to']

    return context

def queue_template_context(queue):
    context = {}

    for field in ('title', 'slug', 'email_address', 'from_address', 'locale'):
        attr = getattr(queue, field, None)
        if callable(attr):
            context[field] = attr()
        else:
            context[field] = attr

    return context

def safe_template_context(ticket):
    """
    Return a dictionary that can be used as a template context to render
    comments and other details with ticket or queue parameters. Note that
    we don't just provide the Ticket & Queue objects to the template as
    they could reveal confidential information. Just imagine these two options:
        * {{ ticket.queue.email_box_password }}
        * {{ ticket.assigned_to.password }}

    Ouch!

    The downside to this is that if we make changes to the model, we will also
    have to update this code. Perhaps we can find a better way in the future.
    """

    context = {
        'queue': queue_template_context(ticket.queue),
        'ticket': ticket_template_context(ticket),
    }
    context['ticket']['queue'] = context['queue']

    return context

def text_is_spam(text, request):
    # Based on a blog post by 'sciyoshi':
    # http://sciyoshi.com/blog/2008/aug/27/using-akismet-djangos-new-comments-framework/
    # This will return 'True' is the given text is deemed to be spam, or
    # False if it is not spam. If it cannot be checked for some reason, we
    # assume it isn't spam.
    from django.contrib.sites.models import Site
    from django.core.exceptions import ImproperlyConfigured
    try:
        from helpdesk.akismet import Akismet
    except ImportError:
        return False
    try:
        site = Site.objects.get_current()
    except ImproperlyConfigured:
        site = Site(domain='configure-django-sites.com')

    ak = Akismet(
        blog_url='http://%s/' % site.domain,
        agent='django-helpdesk',
    )

    if hasattr(settings, 'TYPEPAD_ANTISPAM_API_KEY'):
        ak.setAPIKey(key=settings.TYPEPAD_ANTISPAM_API_KEY)
        ak.baseurl = 'api.antispam.typepad.com/1.1/'
    elif hasattr(settings, 'AKISMET_API_KEY'):
        ak.setAPIKey(key=settings.AKISMET_API_KEY)
    else:
        return False

    if ak.verify_key():
        ak_data = {
            'user_ip': request.META.get('REMOTE_ADDR', '127.0.0.1'),
            'user_agent': request.META.get('HTTP_USER_AGENT', ''),
            'referrer': request.META.get('HTTP_REFERER', ''),
            'comment_type': 'comment',
            'comment_author': '',
        }

        return ak.comment_check(smart_text(text), data=ak_data)

    return False

def process_attachments(followup, attached_files):
    max_email_attachment_size = getattr(settings, 'MAX_EMAIL_ATTACHMENT_SIZE', 512000)
    attachments = []

    for attached in attached_files:
        if attached.size:
            filename = smart_text(attached.name)
            att = Attachment(
                followup=followup,
                file=attached,
                filename=filename,
                mime_type=attached.content_type or
                mimetypes.guess_type(filename, strict=False)[0] or
                'application/octet-stream',
                size=attached.size,
            )
            att.save()

            if attached.size < max_email_attachment_size:
                # Only files smaller than 512kb (or as defined in
                # settings.MAX_EMAIL_ATTACHMENT_SIZE) are sent via email.
                attachments.append([filename, att.file])

    return attachments
<commit_msg>Fix broken try-except statement in lib, for #577<commit_after>"""
django-helpdesk - A Django powered ticket tracker for small enterprise.

(c) Copyright 2008 Jutda. All Rights Reserved. See LICENSE for details.

lib.py - Common functions (eg multipart e-mail)
"""

import logging
import mimetypes
import os
from smtplib import SMTPException

try:
    # Python 2 support
    from base64 import urlsafe_b64encode as b64encode
except ImportError:
    # Python 3 support
    from base64 import encodebytes as b64encode
try:
    # Python 2 support
    from base64 import urlsafe_b64decode as b64decode
except ImportError:
    # Python 3 support
    from base64 import decodebytes as b64decode

from django.conf import settings
from django.db.models import Q
from django.utils import six
from django.utils.encoding import smart_text
from django.utils.safestring import mark_safe

from helpdesk.models import Attachment, EmailTemplate

logger = logging.getLogger('helpdesk')

def send_templated_mail(template_name,
                        context,
                        recipients,
                        sender=None,
                        bcc=None,
                        fail_silently=False,
                        files=None):
    """
    send_templated_mail() is a wrapper around Django's e-mail routines that
    allows us to easily send multipart (text/plain & text/html) e-mails using
    templates that are stored in the database. This lets the admin provide
    both a text and a HTML template for each message.

    template_name is the slug of the template to use for this message (see
        models.EmailTemplate)

    context is a dictionary to be used when rendering the template

    recipients can be either a string, eg 'a@b.com', or a list of strings.

    sender should contain a string, eg 'My Site <me@z.com>'. If you leave it
        blank, it'll use settings.DEFAULT_FROM_EMAIL as a fallback.

    bcc is an optional list of addresses that will receive this message as a
        blind carbon copy.

    fail_silently is passed to Django's mail routine. Set to 'True' to ignore
        any errors at send time.

    files can be a list of tuples. Each tuple should be a filename to attach,
        along with the File objects to be read. files can be blank.

    """
    from django.core.mail import EmailMultiAlternatives
    from django.template import engines
    from_string = engines['django'].from_string

    from helpdesk.models import EmailTemplate
    from helpdesk.settings import HELPDESK_EMAIL_SUBJECT_TEMPLATE, \
        HELPDESK_EMAIL_FALLBACK_LOCALE

    locale = context['queue'].get('locale') or HELPDESK_EMAIL_FALLBACK_LOCALE

    try:
        t = EmailTemplate.objects.get(template_name__iexact=template_name, locale=locale)
    except EmailTemplate.DoesNotExist:
        try:
            t = EmailTemplate.objects.get(template_name__iexact=template_name, locale__isnull=True)
        except EmailTemplate.DoesNotExist:
            logger.warning('template "%s" does not exist, no mail sent', template_name)
            return  # just ignore if template doesn't exist

    subject_part = from_string(
        HELPDESK_EMAIL_SUBJECT_TEMPLATE % {
            "subject": t.subject
        }).render(context).replace('\n', '').replace('\r', '')

    footer_file = os.path.join('helpdesk', locale, 'email_text_footer.txt')

    text_part = from_string(
        "%s{%% include '%s' %%}" % (t.plain_text, footer_file)
    ).render(context)

    email_html_base_file = os.path.join('helpdesk', locale, 'email_html_base.html')
    # keep new lines in html emails
    if 'comment' in context:
        context['comment'] = mark_safe(context['comment'].replace('\r\n', '
'))

    html_part = from_string(
        "{%% extends '%s' %%}{%% block title %%}"
        "%s"
        "{%% endblock %%}{%% block content %%}%s{%% endblock %%}" %
        (email_html_base_file, t.heading, t.html)
    ).render(context)

    if isinstance(recipients, str):
        if recipients.find(','):
            recipients = recipients.split(',')
    elif type(recipients) != list:
        recipients = [recipients]

    msg = EmailMultiAlternatives(subject_part, text_part,
                                 sender or settings.DEFAULT_FROM_EMAIL,
                                 recipients, bcc=bcc)
    msg.attach_alternative(html_part, "text/html")

    if files:
        for filename, filefield in files:
            mime = mimetypes.guess_type(filename)
            if mime[0] is not None and mime[0] == "text/plain":
                with open(filefield.path, 'r') as attachedfile:
                    content = attachedfile.read()
                    msg.attach(filename, content)
            else:
                if six.PY3:
                    msg.attach_file(filefield.path)
                else:
                    with open(filefield.path, 'rb') as attachedfile:
                        content = attachedfile.read()
                        msg.attach(filename, content)

    logger.debug('Sending email to: {!r}'.format(recipients))

    try:
        return msg.send()
    except SMTPException as e:
        logger.exception('SMTPException raised while sending email to {}'.format(recipients))
        if not fail_silently:
            raise e
        return 0

def query_to_dict(results, descriptions):
    """
    Replacement method for cursor.dictfetchall() as that method no longer
    exists in psycopg2, and I'm guessing in other backends too.

    Converts the results of a raw SQL query into a list of dictionaries, suitable
    for use in templates etc.
    """

    output = []
    for data in results:
        row = {}
        i = 0
        for column in descriptions:
            row[column[0]] = data[i]
            i += 1

        output.append(row)
    return output

def apply_query(queryset, params):
    """
    Apply a dict-based set of filters & parameters to a queryset.

    queryset is a Django queryset, eg MyModel.objects.all() or
             MyModel.objects.filter(user=request.user)

    params is a dictionary that contains the following:
        filtering: A dict of Django ORM filters, eg:
            {'user__id__in': [1, 3, 103], 'title__contains': 'foo'}

        search_string: A freetext search string

        sorting: The name of the column to sort by
    """
    for key in params['filtering'].keys():
        filter = {key: params['filtering'][key]}
        queryset = queryset.filter(**filter)

    search = params.get('search_string', None)
    if search:
        qset = (
            Q(title__icontains=search) |
            Q(description__icontains=search) |
            Q(resolution__icontains=search) |
            Q(submitter_email__icontains=search)
        )

        queryset = queryset.filter(qset)

    sorting = params.get('sorting', None)
    if sorting:
        sortreverse = params.get('sortreverse', None)
        if sortreverse:
            sorting = "-%s" % sorting
        queryset = queryset.order_by(sorting)

    return queryset

def ticket_template_context(ticket):
    context = {}

    for field in ('title', 'created', 'modified', 'submitter_email',
                  'status', 'get_status_display', 'on_hold', 'description',
                  'resolution', 'priority', 'get_priority_display',
                  'last_escalation', 'ticket', 'ticket_for_url',
                  'get_status', 'ticket_url', 'staff_url', '_get_assigned_to'
                  ):
        attr = getattr(ticket, field, None)
        if callable(attr):
            context[field] = '%s' % attr()
        else:
            context[field] = attr
    context['assigned_to'] = context['_get_assigned_to']

    return context

def queue_template_context(queue):
    context = {}

    for field in ('title', 'slug', 'email_address', 'from_address', 'locale'):
        attr = getattr(queue, field, None)
        if callable(attr):
            context[field] = attr()
        else:
            context[field] = attr

    return context

def safe_template_context(ticket):
    """
    Return a dictionary that can be used as a template context to render
    comments and other details with ticket or queue parameters. Note that
    we don't just provide the Ticket & Queue objects to the template as
    they could reveal confidential information. Just imagine these two options:
        * {{ ticket.queue.email_box_password }}
        * {{ ticket.assigned_to.password }}

    Ouch!

    The downside to this is that if we make changes to the model, we will also
    have to update this code. Perhaps we can find a better way in the future.
    """

    context = {
        'queue': queue_template_context(ticket.queue),
        'ticket': ticket_template_context(ticket),
    }
    context['ticket']['queue'] = context['queue']

    return context

def text_is_spam(text, request):
    # Based on a blog post by 'sciyoshi':
    # http://sciyoshi.com/blog/2008/aug/27/using-akismet-djangos-new-comments-framework/
    # This will return 'True' is the given text is deemed to be spam, or
    # False if it is not spam. If it cannot be checked for some reason, we
    # assume it isn't spam.
    from django.contrib.sites.models import Site
    from django.core.exceptions import ImproperlyConfigured
    try:
        from helpdesk.akismet import Akismet
    except ImportError:
        return False
    try:
        site = Site.objects.get_current()
    except ImproperlyConfigured:
        site = Site(domain='configure-django-sites.com')

    ak = Akismet(
        blog_url='http://%s/' % site.domain,
        agent='django-helpdesk',
    )

    if hasattr(settings, 'TYPEPAD_ANTISPAM_API_KEY'):
        ak.setAPIKey(key=settings.TYPEPAD_ANTISPAM_API_KEY)
        ak.baseurl = 'api.antispam.typepad.com/1.1/'
    elif hasattr(settings, 'AKISMET_API_KEY'):
        ak.setAPIKey(key=settings.AKISMET_API_KEY)
    else:
        return False

    if ak.verify_key():
        ak_data = {
            'user_ip': request.META.get('REMOTE_ADDR', '127.0.0.1'),
            'user_agent': request.META.get('HTTP_USER_AGENT', ''),
            'referrer': request.META.get('HTTP_REFERER', ''),
            'comment_type': 'comment',
            'comment_author': '',
        }

        return ak.comment_check(smart_text(text), data=ak_data)

    return False

def process_attachments(followup, attached_files):
    max_email_attachment_size = getattr(settings, 'MAX_EMAIL_ATTACHMENT_SIZE', 512000)
    attachments = []

    for attached in attached_files:
        if attached.size:
            filename = smart_text(attached.name)
            att = Attachment(
                followup=followup,
                file=attached,
                filename=filename,
                mime_type=attached.content_type or
                mimetypes.guess_type(filename, strict=False)[0] or
                'application/octet-stream',
                size=attached.size,
            )
            att.save()

            if attached.size < max_email_attachment_size:
                # Only files smaller than 512kb (or as defined in
                # settings.MAX_EMAIL_ATTACHMENT_SIZE) are sent via email.
                attachments.append([filename, att.file])

    return attachments
<|endoftext|>


<commit_before>import time
import os

dir_path = dir_path = os.path.dirname(os.path.abspath(__file__))

start_num = 0
if os.path.isfile(dir_path + '/../pickles/start_num.txt'):
    with open(dir_path + '/../pickles/start_num.txt', 'r') as f:
        l = f.readline()
        start_num = int(l)

i = start_num
with open(dir_path + '/../pickles/numbers.csv', 'a') as f:
	stime = time.clock()
	while time.clock() - stime < 2:
		f.write(str(i) + ',')
		i += 1

with open(dir_path + '/../pickles/start_num.txt', 'w') as f:
	f.write(str(i))<commit_msg>Delete test.py<commit_after><|endoftext|>


<commit_before>"""Models for the ``document_library`` app."""
from django.conf import settings
from django.db import models
from django.utils.translation import get_language
from django.utils.translation import ugettext_lazy as _

from filer.fields.file import FilerFileField
from simple_translation.utils import get_translation_queryset

class Document(models.Model):
    """
    A document consists of a title and description and a number of filer-files.

    See ``DocumentTitle`` for the translateable fields of this model.

    :creation_date: DateTime when this document was created.
    :user: Optional FK to the User who created this document.
    :position: If you want to order the documents other than by creation date,
      enter numbers for positioning here.

    """
    creation_date = models.DateTimeField(
        auto_now_add=True,
        verbose_name=_('Creation date'),
    )

    user = models.ForeignKey(
        'auth.User',
        verbose_name=_('User'),
        null=True, blank=True,
    )

    position = models.PositiveIntegerField(
        verbose_name=_('Position'),
        null=True, blank=True,
    )

    is_published = models.BooleanField(
        default=False,
        verbose_name=_('Is published'),
    )

    class Meta:
        ordering = ('position', '-creation_date', )

    def __unicode__(self):
        return self.get_title()

    def get_title(self):
        lang = get_language()
        return get_translation_queryset(self).filter(language=lang)[0].title

class DocumentTitle(models.Model):
    """
    The translateable fields of the ``Document`` model.

    :title: The title of the document.
    :description: A short description of the document.
    :filer_file: FK to the File of the document version for this language.

    """
    title = models.CharField(
        max_length=512,
        verbose_name=_('Title'),
    )

    description = models.TextField(
        verbose_name=_('Description'),
        blank=True,
    )

    filer_file = FilerFileField(
        verbose_name=_('File'),
        null=True, blank=True,
    )

    # Needed by simple-translation
    document = models.ForeignKey(Document)
    language = models.CharField(max_length=5, choices=settings.LANGUAGES)
<commit_msg>Improved docstring.<commit_after>"""Models for the ``document_library`` app."""
from django.conf import settings
from django.db import models
from django.utils.translation import get_language
from django.utils.translation import ugettext_lazy as _

from filer.fields.file import FilerFileField
from simple_translation.utils import get_translation_queryset

class Document(models.Model):
    """
    A document consists of a title and description and a number of filer-files.

    See ``DocumentTitle`` for the translateable fields of this model.

    :creation_date: DateTime when this document was created.
    :user: Optional FK to the User who created this document.
    :position: If you want to order the documents other than by creation date,
      enter numbers for positioning here.
    :is_published: If ``False`` the object will be excluded from the library
      views.

    """
    creation_date = models.DateTimeField(
        auto_now_add=True,
        verbose_name=_('Creation date'),
    )

    user = models.ForeignKey(
        'auth.User',
        verbose_name=_('User'),
        null=True, blank=True,
    )

    position = models.PositiveIntegerField(
        verbose_name=_('Position'),
        null=True, blank=True,
    )

    is_published = models.BooleanField(
        default=False,
        verbose_name=_('Is published'),
    )

    class Meta:
        ordering = ('position', '-creation_date', )

    def __unicode__(self):
        return self.get_title()

    def get_title(self):
        lang = get_language()
        return get_translation_queryset(self).filter(language=lang)[0].title

class DocumentTitle(models.Model):
    """
    The translateable fields of the ``Document`` model.

    :title: The title of the document.
    :description: A short description of the document.
    :filer_file: FK to the File of the document version for this language.

    """
    title = models.CharField(
        max_length=512,
        verbose_name=_('Title'),
    )

    description = models.TextField(
        verbose_name=_('Description'),
        blank=True,
    )

    filer_file = FilerFileField(
        verbose_name=_('File'),
        null=True, blank=True,
    )

    # Needed by simple-translation
    document = models.ForeignKey(Document)
    language = models.CharField(max_length=5, choices=settings.LANGUAGES)
<|endoftext|>


<commit_before>from time import sleep
from datetime import datetime

import urllib
import tg
from pymongo import bson
from pylons import c

from ming import schema
from ming.orm import MappedClass, session
from ming.orm import FieldProperty, ForeignIdProperty, RelationProperty

from pyforge.model import Artifact, VersionedArtifact, Snapshot, Message, project_orm_session, Project
from pyforge.model import File, User, Feed, Thread, Post, Notification
from pyforge.lib import helpers as h

common_suffix = tg.config.get('forgemail.domain', '.sourceforge.net')

class Globals(MappedClass):

    class __mongometa__:
        name = 'globals'
        session = project_orm_session

    type_s = 'Globals'
    _id = FieldProperty(schema.ObjectId)
    app_config_id = ForeignIdProperty('AppConfig', if_missing=lambda:c.app.config._id)
    last_ticket_num = FieldProperty(int)
    status_names = FieldProperty(str)
    milestone_names = FieldProperty(str, if_missing='')
    custom_fields = FieldProperty([{str:None}])

    @classmethod
    def next_ticket_num(cls):
        g = cls.query.find_and_modify(
            query=dict(app_config_id=c.app.config._id),
            update={'$inc': { 'last_ticket_num': 1}},
            new=True)
        return g.last_ticket_num+1

    @classmethod
    def for_current_tracker(cls):
        return cls.query.get(app_config_id=c.app.config._id)

    def sortable_custom_fields_shown_in_search(self):
        return [dict(sortable_name='%s_s' % field.name,
                     name=field.name,
                     label=field.label)
                for field in self.custom_fields
                if field.show_in_search]

class TicketHistory(Snapshot):

    class __mongometa__:
        name = 'ticket_history'

    def original(self):
        return Ticket.query.get(_id=self.artifact_id)

    def shorthand_id(self):
        return '%s#%s' % (self.original().shorthand_id(), self.version)

    def url(self):
        return self.original().url() + '?version=%d' % self.version

    @property
    def assigned_to(self):
        if self.data.assigned_to_id is None: return None
        return User.query.get(_id=self.data.assigned_to_id)

    def index(self):
        result = Snapshot.index(self)
        result.update(
            title_s='Version %d of %s' % (
                self.version,self.original().summary),
            type_s='Ticket Snapshot',
            text=self.data.summary)
        return result

class Bin(Artifact):
    class __mongometa__:
        name = 'bin'

    type_s = 'Bin'
    _id = FieldProperty(schema.ObjectId)
    summary = FieldProperty(str)
    terms = FieldProperty(str, if_missing='')

    def url(self):
        return self.app_config.url() + 'search/?q=' + urllib.quote_plus(str(self.terms))

    def shorthand_id(self):
        return str(self.summary)

    def index(self):
        result = Artifact.index(self)
        result.update(
            type_s=self.type_s,
            summary_t=self.summary,
            terms_s=self.terms)
        return result

class Ticket(VersionedArtifact):
    class __mongometa__:
        name = 'ticket'
        history_class = TicketHistory

    type_s = 'Ticket'
    _id = FieldProperty(schema.ObjectId)
    created_date = FieldProperty(datetime, if_missing=datetime.utcnow)

    super_id = FieldProperty(schema.ObjectId, if_missing=None)
    sub_ids = FieldProperty([schema.ObjectId], if_missing=None)
    ticket_num = FieldProperty(int)
    summary = FieldProperty(str)
    description = FieldProperty(str, if_missing='')
    reported_by_id = ForeignIdProperty(User, if_missing=lambda:c.user._id)
    assigned_to_id = ForeignIdProperty(User, if_missing=None)
    milestone = FieldProperty(str, if_missing='')
    status = FieldProperty(str, if_missing='')
    custom_fields = FieldProperty({str:None})

    reported_by = RelationProperty(User, via='reported_by_id')

    def index(self):
        result = VersionedArtifact.index(self)
        result.update(
            title_s='Ticket %s' % self.ticket_num,
            version_i=self.version,
            type_s=self.type_s,
            ticket_num_i=self.ticket_num,
            summary_t=self.summary,
            milestone_s=self.milestone,
            status_s=self.status,
            text=self.description)
        for k,v in self.custom_fields.iteritems():
            result[k + '_s'] = str(v)
        if self.reported_by:
            result['reported_by_s'] = self.reported_by.username
        if self.assigned_to:
            result['assigned_to_s'] = self.assigned_to.username
        return result

    @property
    def assigned_to(self):
        if self.assigned_to_id is None: return None
        return User.query.get(_id=self.assigned_to_id)

    @property
    def reported_by_username(self):
        if self.reported_by:
            return self.reported_by.username
        return 'nobody'

    @property
    def assigned_to_username(self):
        if self.assigned_to:
            return self.assigned_to.username
        return 'nobody'

    @property
    def email_address(self):
        domain = '.'.join(reversed(self.app.url[1:-1].split('/'))).replace('_', '-')
        return '%s@%s%s' % (self.ticket_num, domain, common_suffix)

    def commit(self):
        VersionedArtifact.commit(self)
        if self.version > 1:
            hist = TicketHistory.query.get(artifact_id=self._id, version=self.version-1)
            old = hist.data
            changes = ['Ticket %s has been modified: %s' % (self.ticket_num, self.summary)]
            fields = [
                ('Summary', old.summary, self.summary),
                ('Status', old.status, self.status) ]
            for key in self.custom_fields:
                fields.append((key, old.custom_fields.get(key, ''), self.custom_fields[key]))
            for title, o, n in fields:
                if o != n:
                    changes.append('%s updated: %r => %r' % (
                            title, o, n))
            o = hist.assigned_to
            n = self.assigned_to
            if o != n:
                changes.append('Owner updated: %r => %r' % (
                        o and o.username, n and n.username))
                self.subscribe(user=n)
            if old.description != self.description:
                changes.append('Description updated:')
                changes.append(h.diff_text(old.description, self.description))
            description = '
'.join(changes)
            subject = 'Ticket %s modified: %s' % (self.ticket_num, self.summary)
        else:
            self.subscribe()
            if self.assigned_to_id:
                self.subscribe(user=User.query.get(_id=self.assigned_to_id))
            description = 'Ticket %s created: %s' % (
                self.ticket_num, self.summary)
            subject = 'Ticket %s created' % self.ticket_num
            Thread(discussion_id=self.app_config.discussion_id,
                   artifact_reference=self.dump_ref(),
                   subject='#%s discussion' % self.ticket_num)
        Feed.post(self, description)
        Notification.post(artifact=self, topic='metadata', text=description, subject=subject)

    def url(self):
        return self.app_config.url() + str(self.ticket_num) + '/'

    def shorthand_id(self):
        return '#' + str(self.ticket_num)

    def assigned_to_name(self):
        who = self.assigned_to
        if who in (None, User.anonymous()): return 'nobody'
        return who.display_name

    @property
    def attachments(self):
        return Attachment.by_metadata(ticket_id=self._id,type='attachment')

    def set_as_subticket_of(self, new_super_id):
        # For this to be generally useful we would have to check first that
        # new_super_id is not a sub_id (recursively) of self

        if self.super_id == new_super_id:
            return

        if self.super_id is not None:
            old_super = Ticket.query.get(_id=self.super_id, app_config_id=c.app.config._id)
            old_super.sub_ids = [id for id in old_super.sub_ids if id != self._id]
            old_super.dirty_sums(dirty_self=True)

        self.super_id = new_super_id

        if new_super_id is not None:
            new_super = Ticket.query.get(_id=new_super_id, app_config_id=c.app.config._id)
            if new_super.sub_ids is None:
                new_super.sub_ids = []
            if self._id not in new_super.sub_ids:
                new_super.sub_ids.append(self._id)
            new_super.dirty_sums(dirty_self=True)

    def recalculate_sums(self, super_sums=None):
        """Calculate custom fields of type 'sum' (if any) by recursing into subtickets (if any)."""
        if super_sums is None:
            super_sums = {}
            globals = Globals.query.get(app_config_id=c.app.config._id)
            for k in [cf.name for cf in globals.custom_fields or [] if cf.type=='sum']:
                super_sums[k] = float(0)

        # if there are no custom fields of type 'sum', we're done
        if not super_sums:
            return

        # if this ticket has no subtickets, use its field values directly
        if not self.sub_ids:
            for k in super_sums:
                try:
                    v = float(self.custom_fields.get(k, 0))
                except (TypeError, ValueError):
                    v = 0
                super_sums[k] += v

        # else recurse into subtickets
        else:
            sub_sums = {}
            for k in super_sums:
                sub_sums[k] = float(0)
            for id in self.sub_ids:
                subticket = Ticket.query.get(_id=id, app_config_id=c.app.config._id)
                subticket.recalculate_sums(sub_sums)
            for k, v in sub_sums.iteritems():
                self.custom_fields[k] = v
                super_sums[k] += v

    def dirty_sums(self, dirty_self=False):
        """From a changed ticket, climb the superticket chain to call recalculate_sums at the root."""
        root = self if dirty_self else None
        next_id = self.super_id
        while next_id is not None:
            root = Ticket.query.get(_id=next_id, app_config_id=c.app.config._id)
            next_id = root.super_id
        if root is not None:
            root.recalculate_sums()

    def update(self,ticket_form, tracker_globals):
        tags = (ticket_form.pop('tags', None) or '').split(',')
        if tags == ['']:
            tags = []
        labels = (ticket_form.pop('labels', None) or '').split(',')
        if labels == ['']:
            labels = []
        self.labels = labels
        h.tag_artifact(self, c.user, tags)
        custom_sums = set()
        other_custom_fields = set()
        for cf in tracker_globals.custom_fields or []:
            (custom_sums if cf.type=='sum' else other_custom_fields).add(cf.name)
            if cf.type == 'boolean' and 'custom_fields.'+cf.name not in ticket_form:
                self.custom_fields[cf.name] = 'False'
        for k, v in ticket_form.iteritems():
            if 'custom_fields.' in k:
                k = k.split('custom_fields.')[1]
            if k in custom_sums:
                # sums must be coerced to numeric type
                try:
                    self.custom_fields[k] = float(v)
                except (TypeError, ValueError):
                    self.custom_fields[k] = 0
            elif k in other_custom_fields:
                # strings are good enough for any other custom fields
                self.custom_fields[k] = v
            elif k == 'assigned_to':
                if v: self.assigned_to_id = bson.ObjectId(v)
            elif k != 'super_id':
                # if it's not a custom field, set it right on the ticket (but don't overwrite super_id)
                setattr(self, k, v)
        self.commit()
        # flush so we can participate in a subticket search (if any)
        session(self).flush()
        super_id = ticket_form.get('super_id')
        if super_id:
            self.set_as_subticket_of(bson.ObjectId(super_id))

    def __json__(self):
        return dict(
            _id=str(self._id),
            created_date=self.created_date,
            super_id=self.super_id,
            sub_ids=self.sub_ids,
            ticket_num=self.ticket_num,
            summary=self.summary,
            description=self.description,
            reported_by=self.reported_by_username,
            assigned_to=self.assigned_to_username,
            reported_by_id=self.reported_by_id and str(self.reported_by_id) or None,
            assigned_to_id=self.assigned_to_id and str(self.assigned_to_id) or None,
            milestone=self.milestone,
            status=self.status,
            custom_fields=self.custom_fields)

class Attachment(File):
    class __mongometa__:
        name = 'attachment.files'
        indexes = [
            'metadata.filename',
            'metadata.ticket_id' ]

    # Override the metadata schema here
    metadata=FieldProperty(dict(
            ticket_id=schema.ObjectId,
            app_config_id=schema.ObjectId,
            type=str,
            filename=str))

    @property
    def ticket(self):
        return Ticket.query.get(_id=self.metadata.ticket_id)

    def url(self):
        return self.ticket.url() + 'attachment/' + self.filename

MappedClass.compile_all()
<commit_msg>Fix for broken tracker with unicode chars in saved search name<commit_after>from time import sleep
from datetime import datetime

import urllib
import tg
from pymongo import bson
from pylons import c

from ming import schema
from ming.orm import MappedClass, session
from ming.orm import FieldProperty, ForeignIdProperty, RelationProperty

from pyforge.model import Artifact, VersionedArtifact, Snapshot, Message, project_orm_session, Project
from pyforge.model import File, User, Feed, Thread, Post, Notification
from pyforge.lib import helpers as h

common_suffix = tg.config.get('forgemail.domain', '.sourceforge.net')

class Globals(MappedClass):

    class __mongometa__:
        name = 'globals'
        session = project_orm_session

    type_s = 'Globals'
    _id = FieldProperty(schema.ObjectId)
    app_config_id = ForeignIdProperty('AppConfig', if_missing=lambda:c.app.config._id)
    last_ticket_num = FieldProperty(int)
    status_names = FieldProperty(str)
    milestone_names = FieldProperty(str, if_missing='')
    custom_fields = FieldProperty([{str:None}])

    @classmethod
    def next_ticket_num(cls):
        g = cls.query.find_and_modify(
            query=dict(app_config_id=c.app.config._id),
            update={'$inc': { 'last_ticket_num': 1}},
            new=True)
        return g.last_ticket_num+1

    @classmethod
    def for_current_tracker(cls):
        return cls.query.get(app_config_id=c.app.config._id)

    def sortable_custom_fields_shown_in_search(self):
        return [dict(sortable_name='%s_s' % field.name,
                     name=field.name,
                     label=field.label)
                for field in self.custom_fields
                if field.show_in_search]

class TicketHistory(Snapshot):

    class __mongometa__:
        name = 'ticket_history'

    def original(self):
        return Ticket.query.get(_id=self.artifact_id)

    def shorthand_id(self):
        return '%s#%s' % (self.original().shorthand_id(), self.version)

    def url(self):
        return self.original().url() + '?version=%d' % self.version

    @property
    def assigned_to(self):
        if self.data.assigned_to_id is None: return None
        return User.query.get(_id=self.data.assigned_to_id)

    def index(self):
        result = Snapshot.index(self)
        result.update(
            title_s='Version %d of %s' % (
                self.version,self.original().summary),
            type_s='Ticket Snapshot',
            text=self.data.summary)
        return result

class Bin(Artifact):
    class __mongometa__:
        name = 'bin'

    type_s = 'Bin'
    _id = FieldProperty(schema.ObjectId)
    summary = FieldProperty(str, required=True)
    terms = FieldProperty(str, if_missing='')

    def url(self):
        return self.app_config.url() + 'search/?q=' + urllib.quote_plus(str(self.terms))

    def shorthand_id(self):
        return self.summary

    def index(self):
        result = Artifact.index(self)
        result.update(
            type_s=self.type_s,
            summary_t=self.summary,
            terms_s=self.terms)
        return result

class Ticket(VersionedArtifact):
    class __mongometa__:
        name = 'ticket'
        history_class = TicketHistory

    type_s = 'Ticket'
    _id = FieldProperty(schema.ObjectId)
    created_date = FieldProperty(datetime, if_missing=datetime.utcnow)

    super_id = FieldProperty(schema.ObjectId, if_missing=None)
    sub_ids = FieldProperty([schema.ObjectId], if_missing=None)
    ticket_num = FieldProperty(int)
    summary = FieldProperty(str)
    description = FieldProperty(str, if_missing='')
    reported_by_id = ForeignIdProperty(User, if_missing=lambda:c.user._id)
    assigned_to_id = ForeignIdProperty(User, if_missing=None)
    milestone = FieldProperty(str, if_missing='')
    status = FieldProperty(str, if_missing='')
    custom_fields = FieldProperty({str:None})

    reported_by = RelationProperty(User, via='reported_by_id')

    def index(self):
        result = VersionedArtifact.index(self)
        result.update(
            title_s='Ticket %s' % self.ticket_num,
            version_i=self.version,
            type_s=self.type_s,
            ticket_num_i=self.ticket_num,
            summary_t=self.summary,
            milestone_s=self.milestone,
            status_s=self.status,
            text=self.description)
        for k,v in self.custom_fields.iteritems():
            result[k + '_s'] = str(v)
        if self.reported_by:
            result['reported_by_s'] = self.reported_by.username
        if self.assigned_to:
            result['assigned_to_s'] = self.assigned_to.username
        return result

    @property
    def assigned_to(self):
        if self.assigned_to_id is None: return None
        return User.query.get(_id=self.assigned_to_id)

    @property
    def reported_by_username(self):
        if self.reported_by:
            return self.reported_by.username
        return 'nobody'

    @property
    def assigned_to_username(self):
        if self.assigned_to:
            return self.assigned_to.username
        return 'nobody'

    @property
    def email_address(self):
        domain = '.'.join(reversed(self.app.url[1:-1].split('/'))).replace('_', '-')
        return '%s@%s%s' % (self.ticket_num, domain, common_suffix)

    def commit(self):
        VersionedArtifact.commit(self)
        if self.version > 1:
            hist = TicketHistory.query.get(artifact_id=self._id, version=self.version-1)
            old = hist.data
            changes = ['Ticket %s has been modified: %s' % (self.ticket_num, self.summary)]
            fields = [
                ('Summary', old.summary, self.summary),
                ('Status', old.status, self.status) ]
            for key in self.custom_fields:
                fields.append((key, old.custom_fields.get(key, ''), self.custom_fields[key]))
            for title, o, n in fields:
                if o != n:
                    changes.append('%s updated: %r => %r' % (
                            title, o, n))
            o = hist.assigned_to
            n = self.assigned_to
            if o != n:
                changes.append('Owner updated: %r => %r' % (
                        o and o.username, n and n.username))
                self.subscribe(user=n)
            if old.description != self.description:
                changes.append('Description updated:')
                changes.append(h.diff_text(old.description, self.description))
            description = '
'.join(changes)
            subject = 'Ticket %s modified: %s' % (self.ticket_num, self.summary)
        else:
            self.subscribe()
            if self.assigned_to_id:
                self.subscribe(user=User.query.get(_id=self.assigned_to_id))
            description = 'Ticket %s created: %s' % (
                self.ticket_num, self.summary)
            subject = 'Ticket %s created' % self.ticket_num
            Thread(discussion_id=self.app_config.discussion_id,
                   artifact_reference=self.dump_ref(),
                   subject='#%s discussion' % self.ticket_num)
        Feed.post(self, description)
        Notification.post(artifact=self, topic='metadata', text=description, subject=subject)

    def url(self):
        return self.app_config.url() + str(self.ticket_num) + '/'

    def shorthand_id(self):
        return '#' + str(self.ticket_num)

    def assigned_to_name(self):
        who = self.assigned_to
        if who in (None, User.anonymous()): return 'nobody'
        return who.display_name

    @property
    def attachments(self):
        return Attachment.by_metadata(ticket_id=self._id,type='attachment')

    def set_as_subticket_of(self, new_super_id):
        # For this to be generally useful we would have to check first that
        # new_super_id is not a sub_id (recursively) of self

        if self.super_id == new_super_id:
            return

        if self.super_id is not None:
            old_super = Ticket.query.get(_id=self.super_id, app_config_id=c.app.config._id)
            old_super.sub_ids = [id for id in old_super.sub_ids if id != self._id]
            old_super.dirty_sums(dirty_self=True)

        self.super_id = new_super_id

        if new_super_id is not None:
            new_super = Ticket.query.get(_id=new_super_id, app_config_id=c.app.config._id)
            if new_super.sub_ids is None:
                new_super.sub_ids = []
            if self._id not in new_super.sub_ids:
                new_super.sub_ids.append(self._id)
            new_super.dirty_sums(dirty_self=True)

    def recalculate_sums(self, super_sums=None):
        """Calculate custom fields of type 'sum' (if any) by recursing into subtickets (if any)."""
        if super_sums is None:
            super_sums = {}
            globals = Globals.query.get(app_config_id=c.app.config._id)
            for k in [cf.name for cf in globals.custom_fields or [] if cf.type=='sum']:
                super_sums[k] = float(0)

        # if there are no custom fields of type 'sum', we're done
        if not super_sums:
            return

        # if this ticket has no subtickets, use its field values directly
        if not self.sub_ids:
            for k in super_sums:
                try:
                    v = float(self.custom_fields.get(k, 0))
                except (TypeError, ValueError):
                    v = 0
                super_sums[k] += v

        # else recurse into subtickets
        else:
            sub_sums = {}
            for k in super_sums:
                sub_sums[k] = float(0)
            for id in self.sub_ids:
                subticket = Ticket.query.get(_id=id, app_config_id=c.app.config._id)
                subticket.recalculate_sums(sub_sums)
            for k, v in sub_sums.iteritems():
                self.custom_fields[k] = v
                super_sums[k] += v

    def dirty_sums(self, dirty_self=False):
        """From a changed ticket, climb the superticket chain to call recalculate_sums at the root."""
        root = self if dirty_self else None
        next_id = self.super_id
        while next_id is not None:
            root = Ticket.query.get(_id=next_id, app_config_id=c.app.config._id)
            next_id = root.super_id
        if root is not None:
            root.recalculate_sums()

    def update(self,ticket_form, tracker_globals):
        tags = (ticket_form.pop('tags', None) or '').split(',')
        if tags == ['']:
            tags = []
        labels = (ticket_form.pop('labels', None) or '').split(',')
        if labels == ['']:
            labels = []
        self.labels = labels
        h.tag_artifact(self, c.user, tags)
        custom_sums = set()
        other_custom_fields = set()
        for cf in tracker_globals.custom_fields or []:
            (custom_sums if cf.type=='sum' else other_custom_fields).add(cf.name)
            if cf.type == 'boolean' and 'custom_fields.'+cf.name not in ticket_form:
                self.custom_fields[cf.name] = 'False'
        for k, v in ticket_form.iteritems():
            if 'custom_fields.' in k:
                k = k.split('custom_fields.')[1]
            if k in custom_sums:
                # sums must be coerced to numeric type
                try:
                    self.custom_fields[k] = float(v)
                except (TypeError, ValueError):
                    self.custom_fields[k] = 0
            elif k in other_custom_fields:
                # strings are good enough for any other custom fields
                self.custom_fields[k] = v
            elif k == 'assigned_to':
                if v: self.assigned_to_id = bson.ObjectId(v)
            elif k != 'super_id':
                # if it's not a custom field, set it right on the ticket (but don't overwrite super_id)
                setattr(self, k, v)
        self.commit()
        # flush so we can participate in a subticket search (if any)
        session(self).flush()
        super_id = ticket_form.get('super_id')
        if super_id:
            self.set_as_subticket_of(bson.ObjectId(super_id))

    def __json__(self):
        return dict(
            _id=str(self._id),
            created_date=self.created_date,
            super_id=self.super_id,
            sub_ids=self.sub_ids,
            ticket_num=self.ticket_num,
            summary=self.summary,
            description=self.description,
            reported_by=self.reported_by_username,
            assigned_to=self.assigned_to_username,
            reported_by_id=self.reported_by_id and str(self.reported_by_id) or None,
            assigned_to_id=self.assigned_to_id and str(self.assigned_to_id) or None,
            milestone=self.milestone,
            status=self.status,
            custom_fields=self.custom_fields)

class Attachment(File):
    class __mongometa__:
        name = 'attachment.files'
        indexes = [
            'metadata.filename',
            'metadata.ticket_id' ]

    # Override the metadata schema here
    metadata=FieldProperty(dict(
            ticket_id=schema.ObjectId,
            app_config_id=schema.ObjectId,
            type=str,
            filename=str))

    @property
    def ticket(self):
        return Ticket.query.get(_id=self.metadata.ticket_id)

    def url(self):
        return self.ticket.url() + 'attachment/' + self.filename

MappedClass.compile_all()
<|endoftext|>


<commit_before>"""
    Unit tests for the ascii (n-column) reader
"""
import warnings
warnings.simplefilter("ignore")

import unittest
from DataLoader.loader import  Loader

import os.path

class abs_reader(unittest.TestCase):

    def setUp(self):
        self.loader = Loader()

    def test_checkdata(self):
        """
            Test .ABS file loaded as ascii
        """
        f = self.loader.load("ascii_test_1.txt")
        # The length of the data is 10
        self.assertEqual(len(f.x), 10)
        self.assertEqual(f.x[0],0.002618)
        self.assertEqual(f.x[9],0.0497)

    def test_truncated_1(self):
        """
            Test an ascii file with header and a
            comment line in the middle of the data section.
            The business rule says that we should stop
            reading at the first comment once the data
            section has started (and treat the comment
            as though it were the start of a footer).
        """
        # Test .ABS file loaded as ascii
        f = self.loader.load("ascii_test_2.txt")
        # The length of the data is 10
        self.assertEqual(len(f.x), 5)
        self.assertEqual(f.x[0],0.002618)
        self.assertEqual(f.x[4],0.02356)

    def test_truncated_2(self):
        """
            Test a 6-col ascii file with header and a
            line with only 2 columns in the middle of the data section.
            The business rule says that we should stop
            reading at the first inconsitent line.
        """
        # Test .ABS file loaded as ascii
        f = self.loader.load("ascii_test_3.txt")
        # The length of the data is 5
        self.assertEqual(len(f.x), 5)
        self.assertEqual(f.x[0],0.002618)
        self.assertEqual(f.x[4],0.02356)

    def test_truncated_3(self):
        """
            Test a 6-col ascii file with complex header and
            many lines with 2 or 2 columns in the middle of the data section.
            The business rule says that we should stop
            reading at the last line of header.
        """
        # Test .ABS file loaded as ascii
        f = self.loader.load("ascii_test_4.abs")
        # The length of the data is 5
        self.assertEqual(len(f.x), 5)
        self.assertEqual(f.x[0],0.02654)
        self.assertEqual(f.x[4],0.022254)

    def test_truncated_4(self):
        """
            Test mix of 6-col and 2-col.
            Only the last 5 2-col lines should be read.
        """
        # Test .ABS file loaded as ascii
        f = self.loader.load("ascii_test_5.txt")
        # The length of the data is 5
        self.assertEqual(len(f.x), 5)
        self.assertEqual(f.x[0],0.02879)
        self.assertEqual(f.x[4],0.0497)

if __name__ == '__main__':
    unittest.main()
   <commit_msg>changed test values<commit_after>"""
    Unit tests for the ascii (n-column) reader
"""
import warnings
warnings.simplefilter("ignore")

import unittest
from DataLoader.loader import  Loader

import os.path

class abs_reader(unittest.TestCase):

    def setUp(self):
        self.loader = Loader()

    def test_checkdata(self):
        """
            Test .ABS file loaded as ascii
        """
        f = self.loader.load("ascii_test_1.txt")
        # The length of the data is 10
        self.assertEqual(len(f.x), 10)
        self.assertEqual(f.x[0],0.002618)
        self.assertEqual(f.x[9],0.0497)

    def test_truncated_1(self):
        """
            Test an ascii file with header and a
            comment line in the middle of the data section.
            The business rule says that we should stop
            reading at the first comment once the data
            section has started (and treat the comment
            as though it were the start of a footer).
        """
        # Test .ABS file loaded as ascii
        f = self.loader.load("ascii_test_2.txt")
        # The length of the data is 10
        self.assertEqual(len(f.x), 5)
        self.assertEqual(f.x[0],0.002618)
        self.assertEqual(f.x[4],0.02356)

    def test_truncated_2(self):
        """
            Test a 6-col ascii file with header and a
            line with only 2 columns in the middle of the data section.
            The business rule says that we should stop
            reading at the first inconsitent line.
        """
        # Test .ABS file loaded as ascii
        f = self.loader.load("ascii_test_3.txt")
        # The length of the data is 5
        self.assertEqual(len(f.x), 5)
        self.assertEqual(f.x[0],0.002618)
        self.assertEqual(f.x[4],0.02356)

    def test_truncated_3(self):
        """
            Test a 6-col ascii file with complex header and
            many lines with 2 or 2 columns in the middle of the data section.
            The business rule says that we should stop
            reading at the last line of header.
        """
        # Test .ABS file loaded as ascii
        f = self.loader.load("ascii_test_4.abs")
        # The length of the data is 5
        self.assertEqual(len(f.x), 5)
        self.assertEqual(f.x[0],0.012654)
        self.assertEqual(f.x[4],0.02654)

    def test_truncated_4(self):
        """
            Test mix of 6-col and 2-col.
            Only the last 5 2-col lines should be read.
        """
        # Test .ABS file loaded as ascii
        f = self.loader.load("ascii_test_5.txt")
        # The length of the data is 5
        self.assertEqual(len(f.x), 5)
        self.assertEqual(f.x[0],0.02879)
        self.assertEqual(f.x[4],0.0497)

if __name__ == '__main__':
    unittest.main()
   <|endoftext|>


<commit_before>
'''
Pairwise comparisons between all sims at one timestep. Creates a distance
that may be compared to the other timesteps to look for clear trends in the
distance metric.
'''

from turbustat.statistics import stats_wrapper
from turbustat.data_reduction import Mask_and_Moments

from astropy.io.fits import getdata
from astropy.wcs import WCS
from spectral_cube import SpectralCube, LazyMask
import numpy as np
# from interruptible_pool import InterruptiblePool as Pool
from MPI import MPIPool
from itertools import izip, combinations, repeat
from pandas import DataFrame
from datetime import datetime

def pairwise(file_dict, pool, statistics=None, save=False,
             save_name='pairwise', save_path=None,
             add_noise=False, rms_noise=0.001):
    '''
    Create a distance matrix for a set of simulations.
    '''

    num = len(file_dict.keys())

    pos = file_dict.keys()

    output = pool.map(single_input,
                      zip(repeat(file_dict),
                          combinations(pos, 2),
                          repeat(statistics),
                          repeat(add_noise),
                          repeat(rms_noise)))

    dist_matrices = np.zeros((len(output[0][0]), num, num))

    for out in output:
        pos1 = pos.index(out[1])
        pos2 = pos.index(out[2])
        for i, stat in enumerate(out[0].keys()):
            dist_matrices[i, pos1, pos2] = out[0][stat]

    if save:
        stats = out[0].keys()

        if save_path is None:
            save_path = ""

        for i, stat in enumerate(stats):
            df = DataFrame(dist_matrices[i, :, :])
            df.to_csv(save_path+save_name+'_'+stat+'_distmat.csv')
    else:
        return dist_matrices

def timestep_wrapper(fid_files, des_files, pos, statistics, noise=False,
                     rms_noise=0.001):

    pos1, pos2 = pos

    print "On "+str(datetime.now())+" running %s %s" % (pos1, pos2)

    # Derive the property arrays assuming uniform noise (for sims)
    dataset1 = load_and_reduce(fid_files[pos1], add_noise=noise,
                               rms_noise=rms_noise)
    dataset2 = load_and_reduce(des_files[pos2], add_noise=noise,
                               rms_noise=rms_noise)

    distances = stats_wrapper(dataset1, dataset2,
                              statistics=statistics, multicore=True)
    return distances, pos1, pos2

def single_input(a):
    return timestep_wrapper(*a)

def load_and_reduce(filename, add_noise=False, rms_noise=0.001,
                    nsig=3):
    '''
    Load the cube in and derive the property arrays.
    '''

    if add_noise:
        if rms_noise is None:
            raise TypeError("Must specify value of rms noise.")

        cube, hdr = getdata(filename, header=True)

        from scipy.stats import norm
        cube += norm.rvs(0.0, rms_noise, cube.shape)

        sc = SpectralCube(data=cube, wcs=WCS(hdr))

        mask = LazyMask(np.isfinite, sc)
        sc = sc.with_mask(mask)

    else:
        sc = filename

    reduc = Mask_and_Moments(sc, scale=rms_noise)
    reduc.make_mask(mask=reduc.cube > nsig * reduc.scale)
    reduc.make_moments()
    reduc.make_moment_errors()

    return reduc.to_dict()

if __name__ == "__main__":

    import sys
    import glob

    folder = str(sys.argv[1])
    ncores = int(sys.argv[2])
    tstep = str(sys.argv[3])
    face = str(sys.argv[4])
    output_folder = str(sys.argv[5])

    # Grab all of the fits files, then sort them by fiducial, design,
    # then by number, then by the face

    fits_files = glob.glob(folder+"*.fits")

    # Search for all files that have the timestep in the name
    fits_tstep = []

    for fits in fits_files:
        if "00"+tstep in fits and "_0"+face+"_" in fits:
            fits_tstep.append(fits)

    # put into dictionary so we can accurately track which Fid/Des it's from

    des = ["D"+str(i) for i in range(32)]
    fid = ["F"+str(i) for i in range(5)]

    des_fid = des + fid

    tstep_dict = dict.fromkeys(des_fid)

    for key in tstep_dict.keys():
        sim_type = 'Design' if key[0] == 'D' else 'Fiducial'
        num = key[1:]

        for fits in fits_tstep:
            if sim_type+num+"_" in fits:
                tstep_dict[key] = fits
                break

    pool = MPIPool()
    if not pool.is_master():
        pool.wait()
        sys.exit(0)

    # pool = Pool(processes=ncores)

    pairwise(tstep_dict, pool, statistics=['Cramer', 'PCA'],
             save=True, save_name='SimSuite8_Design'+str(num)+"_"+str(face),
             save_path=output_folder)

    pool.close()
<commit_msg>Only compare fiducials, just to test<commit_after>
'''
Pairwise comparisons between all sims at one timestep. Creates a distance
that may be compared to the other timesteps to look for clear trends in the
distance metric.
'''

from turbustat.statistics import stats_wrapper
from turbustat.data_reduction import Mask_and_Moments

from astropy.io.fits import getdata
from astropy.wcs import WCS
from spectral_cube import SpectralCube, LazyMask
import numpy as np
# from interruptible_pool import InterruptiblePool as Pool
from MPI import MPIPool
from itertools import izip, combinations, repeat
from pandas import DataFrame
from datetime import datetime

def pairwise(file_dict, pool, statistics=None, save=False,
             save_name='pairwise', save_path=None,
             add_noise=False, rms_noise=0.001):
    '''
    Create a distance matrix for a set of simulations.
    '''

    num = len(file_dict.keys())

    pos = file_dict.keys()

    output = pool.map(single_input,
                      zip(repeat(file_dict),
                          combinations(pos, 2),
                          repeat(statistics),
                          repeat(add_noise),
                          repeat(rms_noise)))

    dist_matrices = np.zeros((len(output[0][0]), num, num))

    for out in output:
        pos1 = pos.index(out[1])
        pos2 = pos.index(out[2])
        for i, stat in enumerate(out[0].keys()):
            dist_matrices[i, pos1, pos2] = out[0][stat]

    if save:
        stats = out[0].keys()

        if save_path is None:
            save_path = ""

        for i, stat in enumerate(stats):
            df = DataFrame(dist_matrices[i, :, :])
            df.to_csv(save_path+save_name+'_'+stat+'_distmat.csv')
    else:
        return dist_matrices

def timestep_wrapper(fid_files, des_files, pos, statistics, noise=False,
                     rms_noise=0.001):

    pos1, pos2 = pos

    print "On "+str(datetime.now())+" running %s %s" % (pos1, pos2)

    # Derive the property arrays assuming uniform noise (for sims)
    dataset1 = load_and_reduce(fid_files[pos1], add_noise=noise,
                               rms_noise=rms_noise)
    dataset2 = load_and_reduce(des_files[pos2], add_noise=noise,
                               rms_noise=rms_noise)

    distances = stats_wrapper(dataset1, dataset2,
                              statistics=statistics, multicore=True)
    return distances, pos1, pos2

def single_input(a):
    return timestep_wrapper(*a)

def load_and_reduce(filename, add_noise=False, rms_noise=0.001,
                    nsig=3):
    '''
    Load the cube in and derive the property arrays.
    '''

    if add_noise:
        if rms_noise is None:
            raise TypeError("Must specify value of rms noise.")

        cube, hdr = getdata(filename, header=True)

        from scipy.stats import norm
        cube += norm.rvs(0.0, rms_noise, cube.shape)

        sc = SpectralCube(data=cube, wcs=WCS(hdr))

        mask = LazyMask(np.isfinite, sc)
        sc = sc.with_mask(mask)

    else:
        sc = filename

    reduc = Mask_and_Moments(sc, scale=rms_noise)
    reduc.make_mask(mask=reduc.cube > nsig * reduc.scale)
    reduc.make_moments()
    reduc.make_moment_errors()

    return reduc.to_dict()

if __name__ == "__main__":

    import sys
    import glob

    folder = str(sys.argv[1])
    ncores = int(sys.argv[2])
    tstep = str(sys.argv[3])
    face = str(sys.argv[4])
    output_folder = str(sys.argv[5])

    # Grab all of the fits files, then sort them by fiducial, design,
    # then by number, then by the face

    fits_files = glob.glob(folder+"*.fits")

    # Search for all files that have the timestep in the name
    fits_tstep = []

    for fits in fits_files:
        if "00"+tstep in fits and "_0"+face+"_" in fits:
            fits_tstep.append(fits)

    # put into dictionary so we can accurately track which Fid/Des it's from

    des = [] #["D"+str(i) for i in range(32)]
    fid = ["F"+str(i) for i in range(5)]

    des_fid = des + fid

    tstep_dict = dict.fromkeys(des_fid)

    for key in tstep_dict.keys():
        sim_type = 'Design' if key[0] == 'D' else 'Fiducial'
        num = key[1:]

        for fits in fits_tstep:
            if sim_type+num+"_" in fits:
                tstep_dict[key] = fits
                break

    pool = MPIPool(loadbalance=False)
    if not pool.is_master():
        pool.wait()
        sys.exit(0)

    # pool = Pool(processes=ncores)

    pairwise(tstep_dict, pool, statistics=['Cramer', 'PCA'],
             save=True, save_name='SimSuite8_Design'+str(num)+"_"+str(face),
             save_path=output_folder)

    pool.close()
<|endoftext|>