<commit_before>"""
sentry.plugins.bases.notify
~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import logging

from django import forms

from sentry.app import (
    digests,
    ratelimiter,
)
from sentry.digests import get_option_key as get_digest_option_key
from sentry.digests.notifications import (
    event_to_record,
    unsplit_key,
)
from sentry.plugins import Notification, Plugin
from sentry.models import (
    ProjectOption,
    UserOption,
)
from sentry.tasks.digests import deliver_digest

class NotificationConfigurationForm(forms.Form):
    pass

class BaseNotificationUserOptionsForm(forms.Form):
    def __init__(self, plugin, user, *args, **kwargs):
        self.plugin = plugin
        self.user = user
        super(BaseNotificationUserOptionsForm, self).__init__(*args, **kwargs)

    def get_title(self):
        return self.plugin.get_conf_title()

    def get_description(self):
        return ""

    def save(self):
        raise NotImplementedError

class NotificationPlugin(Plugin):
    description = ('Notify project members when a new event is seen for the first time, or when an '
                   'already resolved event has changed back to unresolved.')
    # site_conf_form = NotificationConfigurationForm
    project_conf_form = NotificationConfigurationForm

    def notify(self, notification):
        event = notification.event
        return self.notify_users(event.group, event)

    def rule_notify(self, event, futures):
        rules = []
        for future in futures:
            rules.append(future.rule)
            if not future.kwargs:
                continue
            raise NotImplementedError('The default behavior for notification de-duplication does not support args')

        project = event.group.project
        if hasattr(self, 'notify_digest') and digests.enabled(project):
            get_digest_option = lambda key: ProjectOption.objects.get_value(
                project,
                get_digest_option_key(self.get_conf_key(), key),
            )
            digest_key = unsplit_key(self, event.group.project)
            immediate_delivery = digests.add(
                digest_key,
                event_to_record(event, rules),
                increment_delay=get_digest_option('increment_delay'),
                maximum_delay=get_digest_option('maximum_delay'),
            )
            if immediate_delivery:
                deliver_digest.delay(digest_key)

        else:
            notification = Notification(event=event, rules=rules)
            self.notify(notification)

    def notify_users(self, group, event, fail_silently=False):
        raise NotImplementedError

    def notify_about_activity(self, activity):
        pass

    def get_sendable_users(self, project):
        conf_key = self.get_conf_key()

        alert_settings = dict(
            (o.user_id, int(o.value))
            for o in UserOption.objects.filter(
                project=project,
                key='%s:alert' % conf_key,
            )
        )

        disabled = set(u for u, v in alert_settings.iteritems() if v == 0)

        member_set = set(project.member_set.exclude(
            user__in=disabled,
        ).values_list('user', flat=True))

        # determine members default settings
        members_to_check = set(u for u in member_set if u not in alert_settings)
        if members_to_check:
            disabled = set(UserOption.objects.filter(
                key='subscribe_by_default',
                value='0',
                user__in=members_to_check,
            ).values_list('user', flat=True))
            member_set = filter(lambda x: x not in disabled, member_set)

        return member_set

    def __is_rate_limited(self, group, event):
        return ratelimiter.is_limited(
            project=group.project,
            key=self.get_conf_key(),
            limit=10,
        )

    def is_configured(self, project):
        raise NotImplementedError

    def should_notify(self, group, event):
        project = event.project
        if not self.is_configured(project=project):
            return False

        if group.is_muted():
            return False

        # If the plugin doesn't support digests or they are not enabled,
        # perform rate limit checks to support backwards compatibility with
        # older plugins.
        if not (hasattr(self, 'notify_digest') and digests.enabled(project)) and self.__is_rate_limited(group, event):
            logger = logging.getLogger('sentry.plugins.{0}'.format(self.get_conf_key()))
            logger.info('Notification for project %r dropped due to rate limiting', project)
            return False

        return True

    def test_configuration(self, project):
        from sentry.utils.samples import create_sample_event
        event = create_sample_event(project, platform='python')
        notification = Notification(event=event)
        return self.notify(notification)

    def get_notification_doc_html(self, **kwargs):
        return ""

# Backwards-compatibility
NotifyConfigurationForm = NotificationConfigurationForm
NotifyPlugin = NotificationPlugin
<commit_msg>Add logging for notifications (refs GH-2918)<commit_after>"""
sentry.plugins.bases.notify
~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, print_function

import logging

from django import forms

from sentry.app import (
    digests,
    ratelimiter,
)
from sentry.digests import get_option_key as get_digest_option_key
from sentry.digests.notifications import (
    event_to_record,
    unsplit_key,
)
from sentry.plugins import Notification, Plugin
from sentry.models import (
    ProjectOption,
    UserOption,
)
from sentry.tasks.digests import deliver_digest

class NotificationConfigurationForm(forms.Form):
    pass

class BaseNotificationUserOptionsForm(forms.Form):
    def __init__(self, plugin, user, *args, **kwargs):
        self.plugin = plugin
        self.user = user
        super(BaseNotificationUserOptionsForm, self).__init__(*args, **kwargs)

    def get_title(self):
        return self.plugin.get_conf_title()

    def get_description(self):
        return ""

    def save(self):
        raise NotImplementedError

class NotificationPlugin(Plugin):
    description = ('Notify project members when a new event is seen for the first time, or when an '
                   'already resolved event has changed back to unresolved.')
    # site_conf_form = NotificationConfigurationForm
    project_conf_form = NotificationConfigurationForm

    def notify(self, notification):
        self.logger.info('Notification dispatched [event=%s] [plugin=%s]',
                         notification.event.id, self.slug)
        event = notification.event
        return self.notify_users(event.group, event)

    def rule_notify(self, event, futures):
        rules = []
        for future in futures:
            rules.append(future.rule)
            if not future.kwargs:
                continue
            raise NotImplementedError('The default behavior for notification de-duplication does not support args')

        project = event.group.project
        if hasattr(self, 'notify_digest') and digests.enabled(project):
            get_digest_option = lambda key: ProjectOption.objects.get_value(
                project,
                get_digest_option_key(self.get_conf_key(), key),
            )
            digest_key = unsplit_key(self, event.group.project)
            immediate_delivery = digests.add(
                digest_key,
                event_to_record(event, rules),
                increment_delay=get_digest_option('increment_delay'),
                maximum_delay=get_digest_option('maximum_delay'),
            )
            if immediate_delivery:
                deliver_digest.delay(digest_key)

        else:
            notification = Notification(event=event, rules=rules)
            self.notify(notification)

    def notify_users(self, group, event, fail_silently=False):
        raise NotImplementedError

    def notify_about_activity(self, activity):
        pass

    def get_sendable_users(self, project):
        conf_key = self.get_conf_key()

        alert_settings = dict(
            (o.user_id, int(o.value))
            for o in UserOption.objects.filter(
                project=project,
                key='%s:alert' % conf_key,
            )
        )

        disabled = set(u for u, v in alert_settings.iteritems() if v == 0)

        member_set = set(project.member_set.exclude(
            user__in=disabled,
        ).values_list('user', flat=True))

        # determine members default settings
        members_to_check = set(u for u in member_set if u not in alert_settings)
        if members_to_check:
            disabled = set(UserOption.objects.filter(
                key='subscribe_by_default',
                value='0',
                user__in=members_to_check,
            ).values_list('user', flat=True))
            member_set = filter(lambda x: x not in disabled, member_set)

        return member_set

    def __is_rate_limited(self, group, event):
        return ratelimiter.is_limited(
            project=group.project,
            key=self.get_conf_key(),
            limit=10,
        )

    def is_configured(self, project):
        raise NotImplementedError

    def should_notify(self, group, event):
        project = event.project
        if not self.is_configured(project=project):
            return False

        if group.is_muted():
            return False

        # If the plugin doesn't support digests or they are not enabled,
        # perform rate limit checks to support backwards compatibility with
        # older plugins.
        if not (hasattr(self, 'notify_digest') and digests.enabled(project)) and self.__is_rate_limited(group, event):
            logger = logging.getLogger('sentry.plugins.{0}'.format(self.get_conf_key()))
            logger.info('Notification for project %r dropped due to rate limiting', project)
            return False

        return True

    def test_configuration(self, project):
        from sentry.utils.samples import create_sample_event
        event = create_sample_event(project, platform='python')
        notification = Notification(event=event)
        return self.notify(notification)

    def get_notification_doc_html(self, **kwargs):
        return ""

# Backwards-compatibility
NotifyConfigurationForm = NotificationConfigurationForm
NotifyPlugin = NotificationPlugin
<|endoftext|>


<commit_before><commit_msg>Enable search in news item view<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# This file is part of wger Workout Manager.
#
# wger Workout Manager is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# wger Workout Manager is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
import csv
import datetime
import logging

from django.db.models import Q
from django.contrib.auth.decorators import login_required
from django.contrib.auth.models import (
    Group,
    Permission,
    User
)
from django.core.urlresolvers import reverse, reverse_lazy
from django.http.response import (
    HttpResponseForbidden,
    HttpResponse,
    HttpResponseRedirect
)
from django.shortcuts import render, get_object_or_404
from django.utils.translation import ugettext as _
from django.utils.translation import ugettext_lazy
from django.views.generic import (
    ListView,
    DeleteView,
    CreateView,
    UpdateView
)

from wger.gym.forms import GymUserAddForm, GymUserPermisssionForm
from wger.gym.helpers import get_user_last_activity, is_any_gym_admin
from wger.gym.models import (
    Gym,
    GymAdminConfig,
    GymUserConfig
)
from wger.config.models import GymConfig as GlobalGymConfig
from wger.utils.generic_views import (
    WgerFormMixin,
    WgerDeleteMixin,
    WgerPermissionMixin
)
from wger.utils.helpers import password_generator

logger = logging.getLogger(__name__)

@login_required
def users(request, gym_pk):
    '''
    Exports all users in selected gym
    '''
    gym = get_object_or_404(Gym, pk=gym_pk)

    if not request.user.has_perm('gym.manage_gyms') \
            and not request.user.has_perm('gym.manage_gym'):
        return HttpResponseForbidden()

    if request.user.has_perm('gym.manage_gym') \
            and request.user.userprofile.gym != gym:
        return HttpResponseForbidden()

    # Crease CSV 'file'
    response = HttpResponse(content_type='text/csv')
    writer = csv.writer(response, delimiter='\t', quoting=csv.QUOTE_ALL)
    writer.writerow([_('Nr.'),
                     _('Username'),
                     _('First name'),
                     _('Last name'),
                     _('Gym')])
    for userprofile in gym.userprofile_set.all():
        user = userprofile.user
        writer.writerow([user.id,
                         user.username,
                         user.first_name,
                         user.last_name,
                         gym.name])

    # Send the data to the browser
    today = datetime.date.today()
    filename = 'User-data-gym-{gym}-{t.year}-{t.month:02d}-{t.day:02d}.csv'.format(t=today,
                                                                                gym=gym.id)
    response['Content-Disposition'] = 'attachment; filename={0}'.format(filename)
    response['Content-Length'] = len(response.content)
    return response
<commit_msg>Fix PEP8 error<commit_after># -*- coding: utf-8 -*-

# This file is part of wger Workout Manager.
#
# wger Workout Manager is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# wger Workout Manager is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
import csv
import datetime
import logging

from django.db.models import Q
from django.contrib.auth.decorators import login_required
from django.contrib.auth.models import (
    Group,
    Permission,
    User
)
from django.core.urlresolvers import reverse, reverse_lazy
from django.http.response import (
    HttpResponseForbidden,
    HttpResponse,
    HttpResponseRedirect
)
from django.shortcuts import render, get_object_or_404
from django.utils.translation import ugettext as _
from django.utils.translation import ugettext_lazy
from django.views.generic import (
    ListView,
    DeleteView,
    CreateView,
    UpdateView
)

from wger.gym.forms import GymUserAddForm, GymUserPermisssionForm
from wger.gym.helpers import get_user_last_activity, is_any_gym_admin
from wger.gym.models import (
    Gym,
    GymAdminConfig,
    GymUserConfig
)
from wger.config.models import GymConfig as GlobalGymConfig
from wger.utils.generic_views import (
    WgerFormMixin,
    WgerDeleteMixin,
    WgerPermissionMixin
)
from wger.utils.helpers import password_generator

logger = logging.getLogger(__name__)

@login_required
def users(request, gym_pk):
    '''
    Exports all users in selected gym
    '''
    gym = get_object_or_404(Gym, pk=gym_pk)

    if not request.user.has_perm('gym.manage_gyms') \
            and not request.user.has_perm('gym.manage_gym'):
        return HttpResponseForbidden()

    if request.user.has_perm('gym.manage_gym') \
            and request.user.userprofile.gym != gym:
        return HttpResponseForbidden()

    # Crease CSV 'file'
    response = HttpResponse(content_type='text/csv')
    writer = csv.writer(response, delimiter='\t', quoting=csv.QUOTE_ALL)
    writer.writerow([_('Nr.'),
                     _('Username'),
                     _('First name'),
                     _('Last name'),
                     _('Gym')])
    for userprofile in gym.userprofile_set.all():
        user = userprofile.user
        writer.writerow([user.id,
                         user.username,
                         user.first_name,
                         user.last_name,
                         gym.name])

    # Send the data to the browser
    today = datetime.date.today()
    filename = 'User-data-gym-{gym}-{t.year}-{t.month:02d}-{t.day:02d}.csv'.format(t=today,
                                                                                gym=gym.id)
    response['Content-Disposition'] = 'attachment; filename={0}'.format(filename)
    response['Content-Length'] = len(response.content)
    return response
<|endoftext|>


<commit_before>"""
sentry.web.frontend.generic
~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from django.views.generic import TemplateView as BaseTemplateView

from sentry.web.helpers import render_to_response

def static_media(request, **kwargs):
    """
    Serve static files below a given point in the directory structure.
    """
    from django.contrib.staticfiles.views import serve

    module = kwargs.get('module')
    path = kwargs.get('path', '')

    if module:
        path = '%s/%s' % (module, path)

    response = serve(request, path, insecure=True)

    # We need CORS for font files
    if path.endswith(('.eot', '.ttf', '.woff')):
        response['Access-Control-Allow-Origin'] = '*'
    return response

class TemplateView(BaseTemplateView):
    def render_to_response(self, context, **response_kwargs):
        return render_to_response(
            request=self.request,
            template=self.get_template_names(),
            context=context,
            **response_kwargs
        )
<commit_msg>Add CORS headers to js files for local dev<commit_after>"""
sentry.web.frontend.generic
~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import

from django.views.generic import TemplateView as BaseTemplateView

from sentry.web.helpers import render_to_response

def static_media(request, **kwargs):
    """
    Serve static files below a given point in the directory structure.
    """
    from django.contrib.staticfiles.views import serve

    module = kwargs.get('module')
    path = kwargs.get('path', '')

    if module:
        path = '%s/%s' % (module, path)

    response = serve(request, path, insecure=True)

    # We need CORS for font files
    if path.endswith(('.eot', '.ttf', '.woff', '.js')):
        response['Access-Control-Allow-Origin'] = '*'
    return response

class TemplateView(BaseTemplateView):
    def render_to_response(self, context, **response_kwargs):
        return render_to_response(
            request=self.request,
            template=self.get_template_names(),
            context=context,
            **response_kwargs
        )
<|endoftext|>


<commit_before>#!/usr/bin/env python

from viper.interactive.lexer import InteractiveLexer

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('-L', '--interactive-lexer', action='store_true', help='runs just the lexer in an interactive mode')
    args = parser.parse_args()

    if args.interactive_lexer:
        InteractiveLexer().cmdloop()
<commit_msg>Specify Python3 for interactive interpreter<commit_after>#!/usr/bin/env python3

from viper.interactive.lexer import InteractiveLexer

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('-L', '--interactive-lexer', action='store_true', help='runs just the lexer in an interactive mode')
    args = parser.parse_args()

    if args.interactive_lexer:
        InteractiveLexer().cmdloop()
<|endoftext|>


<commit_before><commit_msg>pull extra details from config vars<commit_after><|endoftext|>


<commit_before>import smtplib
import mimetypes
import itertools
import time
import sys
import os
import glob
import pandas as pd
import metatlas.metatlas_objects as metob
import multiprocessing as mp
import numpy as np

from datetime import datetime, time as dtime
from collections import defaultdict
from metatlas.mzml_loader import VERSION_TIMESTAMP
from email.mime.multipart import MIMEMultipart
from email.mime.base import MIMEBase
from email.mime.text import MIMEText
from email import encoders
from subprocess import check_output
from metatlas.helpers import metatlas_get_data_helper_fun as ma_data
from metatlas.metatlas_objects import find_invalid_runs
from metatlas.system_utils import send_mail

# TO-DO: have these vars be defined from external YML (probably nersc.yml?)

# the lower the tolerance, the more relevant data you might see
# at a cost of being less flexible with defined peaks
# default is 15
tolerance = 15
# the lower the threshold, the more "peaks" you'll see
threshold = 1e5
# change standard if needed. This is ABMBA
std = 229.981116596
std_neg = 227.966564596

run_times = {}

def check_metatlas():
    """
    Checks for invalid runs.
    """
    invalid_runs = find_invalid_runs(_override=True)

    if invalid_runs:
        grouped = defaultdict(list)
        for run in invalid_runs:
            grouped[run.username].append(run.mzml_file)
        for (username, filenames) in grouped.items():
            body = 'You have runs that are not longer accessible\n'
            body += 'To remove them from the database, run the following on ipython.nersc.gov:\n\n'
            body += 'from metatlas.metatlas_objects import find_invalid_runs, remove_objects\n'
            body += 'remove_objects(find_invalid_runs())\n\n'
            body += 'The invalid runs are:\n%s' % ('\n'.join(filenames))
            send_mail('Metatlas Runs are Invalid', username, body)

def yield_ppm(dataset, name, threshold, tolerance, mz_t, rt_max):
    """
    Helper for get_ppm. Given a name and a dataset, returns a filtered and bounded dataset for ppm under name,
    under specific parameters.
    """
    # bugs out if you divide by mz_t on the left...
    data_filter = (abs(dataset['mz'] - mz_t) * 1e6) <= tolerance * mz_t
    filtered = dataset[data_filter]
    bounded = filtered[filtered.i >= threshold]
    bounded = bounded.loc[:,list(bounded)]
    bounded[name] = (abs(bounded['mz'] - mz_t) * 1e6 / mz_t)
    if not bounded.empty and rt_max != -1:
        data_filter = abs(bounded.rt - rt_max) <= .5
        bounded = bounded[data_filter]
    return bounded

def get_ppms(dataset, tolerance=15, mz_std=229.981116596, threshold=1e4):
    """
    Given some dataset, gets applicable ppm and returns the resulting data.
    Can change various parameters if desired:
    - a different standard than ABMBA
    - tolerances for what is considered to be reasonable
    - a threshold for what is defined as a peak
    """
    # Switch off from pos data to neg data if needed
    ms1 = 'ms1_pos'
    ms2 = 'ms2_pos'
    data = dataset[ms1]
    if data.empty:
        mz_t = std_neg
        ms1 = 'ms1_neg'
        ms2 = 'ms2_neg'
    else:
        mz_t = mz_std

    # Looking through ms1_pos
    # MS 1
    data = dataset[ms1]

    ms1_max_i = -1
    rt_max = -1

    # get the max i and rt at max i
    filtered = data[(abs(data['mz'] - mz_t) * 1e6) <= tolerance * mz_t]
    tmp = filtered[filtered.i >= threshold]
    tmp = tmp.loc[:,list(tmp)]
    if not tmp.empty:
        max_i_row = tmp['i'].idxmax()
        ms1_max_i = tmp.loc[max_i_row].i
        rt_max = tmp.loc[max_i_row].rt

    d_1 = yield_ppm(data, 'ppm', threshold, tolerance, mz_t, rt_max)

    # Looking through ms2_pos
    #MS 2, pre
    data = dataset[ms2]
    data_filter = (abs(data['precursor_MZ'] - mz_t) * 1e6) <= tolerance * mz_t
    filtered = data[data_filter]
    thresh = filtered[filtered.i >= threshold]
    if rt_max != -1:
        thresh = thresh[abs(thresh.rt - rt_max) <= .5]

    uniques = thresh.drop_duplicates('precursor_MZ')
    d_2 = uniques.loc[:,list(uniques)]
    d_2['ppm'] = (abs(d_2['precursor_MZ'] - mz_t) * 1e6 / mz_t)

    # Double filter on mz and precursor_MZ
    # MS 2
    d_3 = yield_ppm(filtered, 'ppm', threshold, tolerance, mz_t, rt_max)

    return (d_1.reset_index(), d_2.reset_index(), d_3.reset_index(), ms1_max_i, rt_max)

# if neg file, then use std_neg
# ms1_pos should be empty

def data_verify(file_name):
    dataset = ma_data.df_container_from_metatlas_file(file_name)
    s = file_name.split('/')
    samples_dict = {}
    ppms = get_ppms(dataset, tolerance, std, threshold)
    samples_dict['file name'] = s[-1]
    samples_dict['ms1 ppm'] = ppms[0]['ppm'].tolist()
    samples_dict['ms1 intensity'] = ppms[0]['i'].tolist()
    samples_dict['ms2_precursor ppm'] = ppms[1]['ppm'].tolist()
    samples_dict['ms2 ppm'] = ppms[2]['ppm'].tolist()

    samples_dict['rt'] = ppms[0]['rt'].tolist()

    # extra stats and more data filtering
    samples_dict['ms1 ppm weighted mean'] = weighted_mean(ppms[0])
    samples_dict['ms2_precursor ppm weighted mean'] = weighted_mean(ppms[1]) #hacky
    samples_dict['ms2 ppm weighted mean'] = weighted_mean(ppms[2])

    samples_dict['ms1 max intensity'] = ppms[3]
    samples_dict['rt at max intensity'] = ppms[4]
    samples_dict['acquisition timestamp'] = run_times[u'/global%s' % file_name]
    samples_dict['acquisition time'] = datetime.fromtimestamp(samples_dict['acquisition timestamp']).strftime('%Y-%m-%d %H:%M:%S')

    for key in samples_dict.keys():
        if samples_dict[key] == [] or samples_dict[key] == -1:
            samples_dict[key] = 0

    return samples_dict

# Runs the files to create a list of dicts which contain info about
# the runs. Returns -1 if it failed
def process_files(files):
    samples = []
    if files:
        try:
            print 'found %d files' % len(files)
            p = mp.Pool(min(32, len(files)))
            samples = p.map(data_verify, files)
        finally:
            p.close()
            p.join()

            print 'done processing files'
        return samples
    return -1

# Clean up on a bunch of files to generate CSVs and a report
def clean_up(samples, files):
    directory = files[0].split('/')
    path = directory[len(directory) - 2]
    df = pd.DataFrame(samples)
    sorted_cols = ['file name', 'ms1 ppm', 'ms1 ppm weighted mean', 'ms1 intensity', 'ms1 max intensity']
    sorted_cols.extend(['rt', 'rt at max intensity', 'ms2_precursor ppm', 'ms2_precursor ppm weighted mean'])
    sorted_cols.extend(['ms2 ppm', 'ms2 ppm weighted mean', 'acquisition timestamp', 'acquisition time'])

    # get the full report
    df = df.loc[:,sorted_cols]
    df = df.sort_values('file name')
    # This directory should be modular in case we decide to move it
    csv_name = "/project/projectdirs/metatlas/projects/istd_logs/%s_run.csv" % path
    df.to_csv(csv_name, index = False)

    print 'begin clean up'
    # abridged report
    df = df.sort_values('file name')
    sorted_cols = ['file name', 'acquisition time', 'ms1 ppm weighted mean', 'ms1 max intensity', 'rt at max intensity']
    sorted_cols.extend(['ms2_precursor ppm weighted mean', 'ms2 ppm weighted mean'])
    df = df.loc[:,sorted_cols]
    for col in sorted_cols:
        if 'weighted mean' in col or 'rt at max' in col:
            for i in range(df.shape[0]):
                if df.loc[i, col] != 0:
                    df.loc[i, col] = '%.2f' % df[col][i]
        elif col == 'ms1 max intensity':
            for i in range(df.shape[0]):
                if df.loc[i, col] != 0:
                    df.loc[i, col] = '%.3e' % df[col][i]
    data_filter = df['file name'].str.contains('Blank')
    df_blanks = df[data_filter]
    df_non_blanks = df[~data_filter]
    print 'done clean up'
    return [csv_name, (df_non_blanks, df_blanks)]

# Calculates the weighted mean for some ppm and intensity data
def weighted_mean(df):
    i_sum = df['i'].sum()
    if i_sum:
        vm = df['i'].dot(df['ppm'])
        return vm / i_sum
    return -1

def convert_to_html(df):
    pd.options.display.max_colwidth = -1
    html_table = df.to_html(index = False)
    # hacky way to get css in the file
    html_table = html_table.replace('<table border="1"', '<table style="width: 100%;')
    html_table = html_table.replace('', '')
    html_table = html_table.replace('', '')
    html_table = html_table.replace('
', '
')
    return html_table

# make emails more modular
# sends an email of the run(s) done
def send_run_email(username, run_file, tables, standard='ABMBA'):
    sender = 'pasteur@nersc.gov'
    receivers = ['%s@nersc.gov' % username] #, '%s@nersc.gov' % 'bpb']
    # Build email
    msg = MIMEMultipart()
    msg['From'] = sender
    msg['To'] = ", ".join(receivers) # needs to be a string
    msg['Subject'] = "Run Information"

    template = "Hello %s,\n\n" % username
    template += "Here is a daily summary of your runs with %s from the last 3 days.\n" % standard
    template += "0's in the table indicate that the particular value was not detected."
    template += "Attached is a more verbose summary of the run in a .csv file.\n"
    # add more to the template if needed?

    text = MIMEText(template, 'plain')
    msg.attach(text)

    for title, table in itertools.izip(run_file, tables):
        s = title.replace('run_data/','').replace('_run.csv', '\n')
        num_files = len(table[0]) + len(table[1])
        txt = '
Run Report for %s:
' % s
        txt += '
Detected %d runs!
' % num_files
        text = MIMEText(txt, 'html')
        msg.attach(text)
        table_non_blank = convert_to_html(table[0])
        html = MIMEText(table_non_blank, 'html')
        msg.attach(html)
        text = MIMEText('
Blanks in %s:
' %s, 'html')
        msg.attach(text)
        table_blanks = convert_to_html(table[1])
        html = MIMEText(convert_to_html(table[1]), 'html')
        msg.attach(html)

    for f in run_file:
        ctype, encoding = mimetypes.guess_type(f)
        if ctype is None or encoding is not None:
            ctype = "application/octet-stream"

        maintype, subtype = ctype.split("/", 1)
        # Parser for generic files found on stackoverflow
        maintype, subtype = ctype.split("/", 1)
        fp = open(f, "rb")
        attachment = MIMEBase(maintype, subtype)
        attachment.set_payload(fp.read())
        fp.close()
        encoders.encode_base64(attachment)
        # another case where save path needs to be modular / in a .yml
        attachment.add_header("Content-Disposition", "attachment", filename=f.replace('/project/projectdirs/metatlas/projects/istd_logs/', ''))
        msg.attach(attachment)

    try:
        smtpObj = smtplib.SMTP('localhost')
        smtpObj.sendmail(sender, receivers, msg.as_string())
        sys.stdout.write("Successfully sent email to %s\n" % username)
        sys.stdout.flush()
    except smtplib.SMTPException:
        sys.stderr.write("Error: unable to send email to %s\n" % username)
        sys.stdout.flush()

# Run this to start the task
def run_checker():
    print 'starting task'
    start_time = time.time()
    origin_directory = '/project/projectdirs/metatlas/raw_data/'
    #pull users, email them their run info
    delta = 60*24*3 # 3 days to minutes
    check = 'find %s -name "*.h5" -mmin -%d -size +2k' % (origin_directory, delta)
    valid_files = check_output(check, shell=True).decode('utf-8').splitlines()
    valid_files = set(valid_files)

    files =  defaultdict(set) # dict of user name and the directory of their files
    for i in valid_files:
        s = i.split('/')
        usr = s[-3]
        directory = s[-2]
        files[usr].add(directory)

    print 'detected a total of %d files!\n' % len(valid_files)

    for username in files.keys():
        print 'observing files from', username
        csvs = []
        html_tables = []
        for directory in files[username]:
            print 'looking at', directory
            extension = '%s/%s/' % (username, directory)
            # list of all files in a given directory
            dir_files = glob.glob(os.path.join(origin_directory + extension,'*.h5'))
            # contains a list of dicts
            samples = process_files(dir_files)
            # a list containing a csv name and a tuple of the non-blank, blank dataframes
            info = clean_up(samples, dir_files)
            csvs.append(info[0])
            html_tables.append((info[1][0], info[1][1]))
        print 'attempting to send email'
        send_run_email('mtrinh', csvs, html_tables)
        print '\n' # for formatting purposes

    print 'finished task in %s seconds' % (time.time() - start_time)

# Integrity monitor does two major things:
#
if __name__ == '__main__':
    check_metatlas()

    runs = metob.retrieve('lcmsruns',username='*')

    for i in runs:
        run_times[i.hdf5_file] = i.acquisition_time

    run_checker()
<commit_msg>made it more PEP8<commit_after>import smtplib
import mimetypes
import itertools
import time
import sys
import os
import glob
import pandas as pd
import metatlas.metatlas_objects as metob
import multiprocessing as mp
import numpy as np

from datetime import datetime, time as dtime
from collections import defaultdict
from metatlas.mzml_loader import VERSION_TIMESTAMP
from email.mime.multipart import MIMEMultipart
from email.mime.base import MIMEBase
from email.mime.text import MIMEText
from email import encoders
from subprocess import check_output
from metatlas.helpers import metatlas_get_data_helper_fun as ma_data
from metatlas.metatlas_objects import find_invalid_runs
from metatlas.system_utils import send_mail

# TO-DO: have these vars be defined from external YML (probably nersc.yml?)

# the lower the tolerance, the more relevant data you might see
# at a cost of being less flexible with defined peaks
# default is 15
tolerance = 15
# the lower the threshold, the more "peaks" you'll see
threshold = 1e5
# change standard if needed. This is ABMBA
std = 229.981116596
std_neg = 227.966564596

run_times = {}
save_path = '/project/projectdirs/metatlas/projects/istd_logs/'

def check_metatlas():
    """
    Checks for invalid runs.
    """
    invalid_runs = find_invalid_runs(_override=True)

    if invalid_runs:
        grouped = defaultdict(list)
        for run in invalid_runs:
            grouped[run.username].append(run.mzml_file)
        for (username, filenames) in grouped.items():
            body = 'You have runs that are not longer accessible\n'
            body += 'To remove them from the database, run the following on ipython.nersc.gov:\n\n'
            body += 'from metatlas.metatlas_objects import find_invalid_runs, remove_objects\n'
            body += 'remove_objects(find_invalid_runs())\n\n'
            body += 'The invalid runs are:\n%s' % ('\n'.join(filenames))
            send_mail('Metatlas Runs are Invalid', username, body)

def yield_ppm(dataset, name, threshold, tolerance, mz_t, rt_max):
    """
    Helper for get_ppm. Given a name, a dataset, intensity threshold,
    ppm tolerance, time of highest mz, and retention time at that point,
    returns a filtered and bounded dataset for ppm under name,
    """
    # bugs out if you divide by mz_t on the left...
    data_filter = (abs(dataset['mz'] - mz_t) * 1e6) <= tolerance * mz_t
    filtered = dataset[data_filter]
    bounded = filtered[filtered.i >= threshold]
    bounded = bounded.loc[:, list(bounded)]
    bounded[name] = (abs(bounded['mz'] - mz_t) * 1e6 / mz_t)
    if not bounded.empty and rt_max != -1:
        data_filter = abs(bounded.rt - rt_max) <= .5
        bounded = bounded[data_filter]
    return bounded

def get_ppms(dataset, tolerance=15, mz_std=229.981116596, threshold=1e4):
    """
    Given some dataset, gets applicable ppm and returns the resulting data.
    Can change various parameters if desired:
    - a different standard than ABMBA
    - tolerances for what is considered to be reasonable
    - a threshold for what is defined as a peak
    """
    # # if neg file, then use std_neg: ms1_pos should be empty if so
    ms1 = 'ms1_pos'
    ms2 = 'ms2_pos'
    data = dataset[ms1]
    if data.empty:
        mz_t = std_neg
        ms1 = 'ms1_neg'
        ms2 = 'ms2_neg'
    else:
        mz_t = mz_std

    # Looking through ms1_pos
    # MS 1
    data = dataset[ms1]

    ms1_max_i = -1
    rt_max = -1

    # get the max i and rt at max i
    filtered = data[(abs(data['mz'] - mz_t) * 1e6) <= tolerance * mz_t]
    tmp = filtered[filtered.i >= threshold]
    tmp = tmp.loc[:, list(tmp)]
    if not tmp.empty:
        max_i_row = tmp['i'].idxmax()
        ms1_max_i = tmp.loc[max_i_row].i
        rt_max = tmp.loc[max_i_row].rt

    d_1 = yield_ppm(data, 'ppm', threshold, tolerance, mz_t, rt_max)

    # Looking through MS2, precursor
    data = dataset[ms2]
    data_filter = (abs(data['precursor_MZ'] - mz_t) * 1e6) <= tolerance * mz_t
    filtered = data[data_filter]
    thresh = filtered[filtered.i >= threshold]
    if rt_max != -1:
        thresh = thresh[abs(thresh.rt - rt_max) <= .5]

    uniques = thresh.drop_duplicates('precursor_MZ')
    d_2 = uniques.loc[:, list(uniques)]
    d_2['ppm'] = (abs(d_2['precursor_MZ'] - mz_t) * 1e6 / mz_t)

    # Double filter on mz and precursor_MZ
    # MS 2
    d_3 = yield_ppm(filtered, 'ppm', threshold, tolerance, mz_t, rt_max)

    return (d_1.reset_index(), d_2.reset_index(), d_3.reset_index(),
            ms1_max_i, rt_max)

def data_verify(file_name):
    dataset = ma_data.df_container_from_metatlas_file(file_name)
    s = file_name.split('/')
    samples_dict = {}
    ppms = get_ppms(dataset, tolerance, std, threshold)
    samples_dict['file name'] = s[-1]
    samples_dict['ms1 ppm'] = ppms[0]['ppm'].tolist()
    samples_dict['ms1 intensity'] = ppms[0]['i'].tolist()
    samples_dict['ms2_precursor ppm'] = ppms[1]['ppm'].tolist()
    samples_dict['ms2 ppm'] = ppms[2]['ppm'].tolist()

    samples_dict['rt'] = ppms[0]['rt'].tolist()

    # extra stats and more data filtering
    samples_dict['ms1 ppm weighted mean'] = weighted_mean(ppms[0])
    # hacky
    samples_dict['ms2_precursor ppm weighted mean'] = weighted_mean(ppms[1])
    samples_dict['ms2 ppm weighted mean'] = weighted_mean(ppms[2])

    samples_dict['ms1 max intensity'] = ppms[3]
    samples_dict['rt at max intensity'] = ppms[4]
    samples_dict['acquisition timestamp'] = run_times[u'/global%s' % file_name]
    samples_dict['acquisition time'] = datetime.fromtimestamp(
        samples_dict['acquisition timestamp']).strftime('%Y-%m-%d %H:%M:%S')

    for key in samples_dict.keys():
        if samples_dict[key] == [] or samples_dict[key] == -1:
            samples_dict[key] = 0

    return samples_dict

def process_files(files):
    """
    Goes through the files and returns a list of dicts which contain info about
    the runs. Returns -1 if it failed.
    """
    samples = []
    if files:
        try:
            print 'found %d files' % len(files)
            p = mp.Pool(min(32, len(files)))
            samples = p.map(data_verify, files)
        finally:
            p.close()
            p.join()

            print 'done processing files'
        return samples
    return -1

def clean_up(samples, files):
    """
    Clean up on a bunch of files to generate CSVs and a report.
    """
    directory = files[0].split('/')
    path = directory[len(directory) - 2]
    df = pd.DataFrame(samples)
    sorted_cols = ['file name', 'ms1 ppm', 'ms1 ppm weighted mean',
                   'ms1 intensity', 'ms1 max intensity']
    sorted_cols.extend(['rt', 'rt at max intensity', 'ms2_precursor ppm',
                        'ms2_precursor ppm weighted mean'])
    sorted_cols.extend(['ms2 ppm', 'ms2 ppm weighted mean',
                        'acquisition timestamp', 'acquisition time'])

    # get the full report
    df = df.loc[:, sorted_cols]
    df = df.sort_values('file name')
    csv_name = save_path + '%s_run.csv' % path
    df.to_csv(csv_name, index=False)

    print 'begin clean up'
    # abridged report
    df = df.sort_values('file name')
    sorted_cols = ['file name', 'acquisition time', 'ms1 ppm weighted mean',
                   'ms1 max intensity', 'rt at max intensity']
    sorted_cols.extend(['ms2_precursor ppm weighted mean',
                        'ms2 ppm weighted mean'])
    df = df.loc[:, sorted_cols]
    for col in sorted_cols:
        if 'weighted mean' in col or 'rt at max' in col:
            for i in range(df.shape[0]):
                if df.loc[i, col] != 0:
                    df.loc[i, col] = '%.2f' % df[col][i]
        elif col == 'ms1 max intensity':
            for i in range(df.shape[0]):
                if df.loc[i, col] != 0:
                    df.loc[i, col] = '%.3e' % df[col][i]
    data_filter = df['file name'].str.contains('Blank')
    df_blanks = df[data_filter]
    df_non_blanks = df[~data_filter]
    print 'done clean up'
    return [csv_name, (df_non_blanks, df_blanks)]

def weighted_mean(df):
    """
    Calculates the weighted mean for some ppm and intensity data.
    """
    i_sum = df['i'].sum()
    if i_sum:
        vm = df['i'].dot(df['ppm'])
        return vm / i_sum
    return -1

def convert_to_html(df):
    """
    Converts a dataframe into a styled html table for emails.
    """
    pd.options.display.max_colwidth = -1
    html = df.to_html(index=False)
    # hacky way to get css in the file
    html = html.replace('<table border="1"',
                        '<table style="width: 100%;')
    html = html.replace('',
                        '')
    html = html.replace('',
                        '')
    html = html.replace('
',
                        '
')
    return html

def send_run_email(username, run_file, tables, standard='ABMBA'):
    """
    Sends an email of the run(s) done and packages the generated CSV(s).
    """
    sender = 'pasteur@nersc.gov'
    receivers = ['%s@nersc.gov' % username, '%s@nersc.gov' % 'bpb']
    # Build email
    msg = MIMEMultipart()
    msg['From'] = sender
    msg['To'] = ", ".join(receivers)  # needs to be a string
    msg['Subject'] = "Run Information"

    template = "Hello %s,\n\n" % username
    template += "Here is a daily summary of your runs with %s from the last 3 days.\n" % standard
    template += "0's in the table indicate that the particular value was not detected."
    template += "Attached is a more verbose summary of the run in a .csv file.\n"
    # add more to the template if needed?

    text = MIMEText(template, 'plain')
    msg.attach(text)

    for title, table in itertools.izip(run_file, tables):
        s = title.replace('run_data/', '').replace('_run.csv', '\n')
        num_files = len(table[0]) + len(table[1])
        txt = '
Run Report for %s:
' % s
        txt += '
Detected %d runs!
' % num_files
        text = MIMEText(txt, 'html')
        msg.attach(text)
        table_non_blank = convert_to_html(table[0])
        html = MIMEText(table_non_blank, 'html')
        msg.attach(html)
        text = MIMEText('
Blanks in %s:
' % s, 'html')
        msg.attach(text)
        table_blanks = convert_to_html(table[1])
        html = MIMEText(convert_to_html(table[1]), 'html')
        msg.attach(html)

    for f in run_file:
        ctype, encoding = mimetypes.guess_type(f)
        if ctype is None or encoding is not None:
            ctype = "application/octet-stream"

        maintype, subtype = ctype.split("/", 1)
        # Parser for generic files found on stackoverflow
        maintype, subtype = ctype.split("/", 1)
        fp = open(f, "rb")
        attachment = MIMEBase(maintype, subtype)
        attachment.set_payload(fp.read())
        fp.close()
        encoders.encode_base64(attachment)
        # another case where save path needs to be modular / in a .yml
        attachment.add_header("Content-Disposition", "attachment",
                              filename=f.replace(save_path, ''))
        msg.attach(attachment)

    try:
        smtpObj = smtplib.SMTP('localhost')
        smtpObj.sendmail(sender, receivers, msg.as_string())
        sys.stdout.write("Successfully sent email to %s\n" % username)
        sys.stdout.flush()
    except smtplib.SMTPException:
        sys.stderr.write("Error: unable to send email to %s\n" % username)
        sys.stdout.flush()

# Run this to start the task
def run_checker():
    print 'starting task'
    start_time = time.time()
    origin_directory = '/project/projectdirs/metatlas/raw_data/'
    # pull users, email them their run info
    delta = 60*24*3  # 3 days to minutes
    check = 'find %s -name "*.h5" -mmin -%d -size +2k' % (
            origin_directory, delta)
    valid_files = check_output(check, shell=True).decode('utf-8').splitlines()
    valid_files = set(valid_files)

    # dict of user name and the directory of their files
    files = defaultdict(set)
    for i in valid_files:
        s = i.split('/')
        usr = s[-3]
        directory = s[-2]
        files[usr].add(directory)

    print 'detected a total of %d files!\n' % len(valid_files)

    for username in files.keys():
        print 'observing files from', username
        csvs = []
        html_tables = []
        for directory in files[username]:
            print 'looking at', directory
            extension = '%s/%s/' % (username, directory)
            # list of all files in a given directory
            dir_files = glob.glob(os.path.join(origin_directory +
                                  extension, '*.h5'))
            # contains a list of dicts
            samples = process_files(dir_files)
            # a list containing a csv name
            # and a tuple of the non-blank, blank dataframes
            info = clean_up(samples, dir_files)
            csvs.append(info[0])
            html_tables.append((info[1][0], info[1][1]))
        print 'attempting to send email'
        send_run_email('mtrinh', csvs, html_tables)
        print '\n'  # for formatting purposes

    print 'finished task in %s seconds' % (time.time() - start_time)

# Integrity monitor does two major things:
# - Checks the database for invalid files and warns the user.
# - Runs through all the raw data collected in the past 3 days and gives the
#   user a general report of the data collected.
if __name__ == '__main__':
    check_metatlas()

    runs = metob.retrieve('lcmsruns', username='*')

    for i in runs:
        run_times[i.hdf5_file] = i.acquisition_time

    run_checker()
<|endoftext|>


<commit_before>from .registry import Registry

def action(f):
    """
        mark a method as being an action.
    """
    f.action = True
    return f

class ActionRegistry(dict):
    def register(self, handler, action, path=None, spoke=None):
        if action not in self:
            self[action] = []
        self[action].append((handler, path, spoke))

    def get(self, action, path=None, spoke=None):
        """
            Action resolution is as follows:
            - A handler is registered on an action and optionally a spoke and
              path
            - spoke and path have to match if specified
            if there are no entries at all, find a handler on the spoke
            itself.

            To consider: add priority when registering action

            Een action / handler registreer je in een bepaalde context:
            - globaal (geld in iedere context)
            - voor bepaalde spoke
            - voor bepaald path
            - spoke en path

            Vervolgens zoek je een handler in die context. Als je een nauwkeurige
            context specificeert, dan verwacht

            Een action die op path P en spoke S geregistreerd is matcht dus
            niet op path P' en spoke S
        """
        entries = super(ActionRegistry, self).get(action)
        if entries:
            for (h, epath, espoke) in entries:
                if epath and path != epath:
                    continue
                if espoke and spoke != espoke:
                    continue
                return h

        ## give up if there's no explicit spoke context passed
        if not spoke:
            return None

        classhandler = getattr(spoke, action, None)
        if classhandler and getattr(classhandler, 'action', False):
            return classhandler
        return None

action_registry = Registry(ActionRegistry())
<commit_msg>Allow action to be bound to class in stead of instance<commit_after>from .registry import Registry

def action(f):
    """
        mark a method as being an action.
    """
    f.action = True
    return f

class ActionRegistry(dict):
    def register(self, handler, action, path=None, spoke=None):
        if action not in self:
            self[action] = []
        self[action].append((handler, path, spoke))

    def get(self, action, path=None, spoke=None):
        """
            Action resolution is as follows:
            - A handler is registered on an action and optionally a spoke and
              path
            - spoke and path have to match if specified
            if there are no entries at all, find a handler on the spoke
            itself.

            To consider: add priority when registering action

            Een action / handler registreer je in een bepaalde context:
            - globaal (geld in iedere context)
            - voor bepaalde spoke
            - voor bepaald path
            - spoke en path

            Vervolgens zoek je een handler in die context. Als je een nauwkeurige
            context specificeert, dan verwacht

            Een action die op path P en spoke S geregistreerd is matcht dus
            niet op path P' en spoke S
        """
        entries = super(ActionRegistry, self).get(action)

        if entries:
            ## Match spoke against an actual instance first
            for (h, epath, espoke) in entries:
                if epath and path != epath:
                    continue
                if espoke and spoke != espoke:
                    continue
                return h
            ## and then against a spoke type class
            for (h, epath, espoke) in entries:
                if epath and path != epath:
                    continue
                if espoke and espoke != spoke.__class__:
                    continue
                return h

        ## give up if there's no explicit spoke context passed
        if not spoke:
            return None

        classhandler = getattr(spoke, action, None)
        if classhandler and getattr(classhandler, 'action', False):
            return classhandler
        return None

action_registry = Registry(ActionRegistry())
<|endoftext|>


<commit_before>#
# __     ___       ____
# \ \   / (_)_ __ / ___| ___
#  \ \ / /| | '__| |  _ / _ \
#   \ V / | | |  | |_| | (_) |
#    \_/  |_|_|   \____|\___/
#
#   Tiny Python ORM for MySQL
#
#   E-mail : nz2324@126.com
#
#   URL : https://virgo.readthedocs.org/
#
#   Licence : BSD
#
#
# Permission to use, copy, modify,
# and distribute this software for any purpose with
# or without fee is hereby granted,
# provided that the above copyright notice
# and this permission notice appear in all copies.
#

import re
import MySQLdb
import MySQLdb.cursors

# regular expression to get matched rows number from cursor's info
VG_RowsMatchedRe = re.compile(r'Rows matched: (\d+)')

# marks for operators
OP_LT = 1
OP_LE = 2
OP_GT = 3
OP_GE = 4
OP_EQ = 5
OP_NE = 6
OP_ADD = 7
OP_AND = 8
OP_OR = 9
OP_LIKE = 10

class Database(object):
    """Manage Database connection"""

    # configs for connection with default value
    configs = {
        "host": "localhost",
        "port": 3306,
        "db": "",
        "user": "",
        "passwd": "",
        "charset": "utf8"
    }

    # debuge mode?
    debug = True

    # It is strongly recommended that you set this True
    autocommit = True

    # MySQL connection object, you should use method get_conn to get it
    conn = None

    # record query times
    query_times = 0

    # record SQL last time executed
    SQL = None

    @classmethod
    def config(cls, debug=True, autocommit=True, **configs):
        """
        Configure the database connection.

        The connection will be auto established with these configs.

        Keyword parameters for this method:

          host
            string, host to connect

          user
            string, user to connect as

          passwd
            string, password to use

          db
            string, database to use

          port
            integer, TCP/IP port to connect to

          charset
            string, charset of connection

        See the MySQLdb documentation for more infomation,
        the parameters of MySQLdb.connect are all supported.
        """
        cls.configs.update(configs)
        cls.debug = debug
        cls.autocommit = autocommit

    @classmethod
    def connect(cls):
        """
        Connect to database, singleton pattern.
        This will new one conn object.
        """
        cls.conn = MySQLdb.connect(
            cursorclass=MySQLdb.cursors.DictCursor,
            **cls.configs
        )
        cls.conn.autocommit(cls.autocommit)

    @classmethod
    def get_conn(cls):
        """
        Get MySQL connection object.
        if the conn is open and working, return it,
        else, new one and return it.
        """
        # singleton
        if not cls.conn or not cls.conn.open:
            cls.connect()

        try:
            cls.conn.ping()  # ping to test if the connection is working
        except MySQLdb.OperationalError:
            cls.connect()

        return cls.conn

    @classmethod
    def execute(cls, SQL):
        """
        Execute one SQL command.

        Parameter:
          SQL
            string, SQL command to run.
        """
        cursor = cls.get_conn().cursor()

        try:
            cursor.execute(SQL)
        except Exception, e:
            if cls.debug:  # if debug, report the SQL
                print "SQL:", SQL
            raise e

        cls.query_times += 1
        cls.SQL = SQL

        # add attribute 'matchedRows' to cursor.store query matched rows number
        cursor.matchedRows = int(
            VG_RowsMatchedRe.search(cursor._info).group(1)
        ) if cursor._info else int(cursor.rowcount)

        return cursor

class Leaf(object):

    def _e(op):
        def e(self, right):
            return Expr(self, right, op)
        return e

    __lt__ = _e(OP_LT)

    __le__ = _e(OP_LE)

    __gt__ = _e(OP_GT)

    __ge__ = _e(OP_GE)

    __eq__ = _e(OP_EQ)

    __ne__ = _e(OP_NE)

    __add__ = _e(OP_ADD)

    __and__ = _e(OP_AND)

    __or__ = _e(OP_OR)

class Expr(Leaf):
    """
    Expression object.

    You need't new an expression in this way: myexpr = Expr(left, right, op)

    Use expression like this:
      User.id == 4
      User.name == "Amy"
    """

    def __init__(self, left, right, op):
        self.left = left
        self.right = right
        self.op = op

class EqExpr(Expr):
    """
    Equal exception.

    eg.
      User.id == 1
    """
    def __init__(self, left, right):
        super(EqExpr, self).__init__(left, right, OP_EQ)

# descriptor for Field objects
class FieldDescriptor(object):

    def __init__(self, field):
        self.name = field.name
        self.field = field

    def __get__(self, instance, type=None):
        if instance:
            return instance.data[self.name]
        return self.field

    def __set__(self, instance, value):
        instance.data[self.name] = value

class Field(Leaf):
    """
    Field object.

    Field examples: User.name, User.age ..
    """

    def __init__(self, is_primarykey=False, is_foreignkey=False):
        self.is_primarykey = is_primarykey
        self.is_foreignkey = is_foreignkey

    # describe model's attr
    def describe(self, name, model):
        self.name = name
        self.model = model
        # fullname eg. : User.id 's fullname is "user.id"
        self.fullname = self.model.table_name + "." + self.name
        # describe the attribute, reload its access control of writing, reading
        setattr(model, name, FieldDescriptor(self))

    def __eq__(self, right):  # produce equal expressions
        return EqExpr(self, right)

    def like(self, right):
        return Expr(self, right, OP_LIKE)

class PrimaryKey(Field):
    """
    PrimaryKey object.

    PrimaryKey example: User.id
    """

    def __init__(self):
        super(PrimaryKey, self).__init__(is_primarykey=True)

class ForeignKey(Field):
    """
    ForeignKey object.
    ForeignKey example: Post.user_id = ForeignKey(point_to=User.id)

    Parameter:
      point_to
        PrimaryKey object, the primary key this foreignkey referenced to.
    """

    def __init__(self, point_to):
        super(ForeignKey, self).__init__(is_foreignkey=True)
        self.point_to = point_to

class Compiler(object):
    """
    Compile expressions and sequence of methods to SQL string(s).
    """

    # operator mapping
    OP_MAPPING = {
        OP_LT: " < ",
        OP_LE: " <= ",
        OP_GT: " > ",
        OP_GE: " >= ",
        OP_EQ: " = ",
        OP_NE: " <> ",
        OP_ADD: " + ",
        OP_AND: " and ",
        OP_OR: " or ",
        OP_LIKE: "like"
    }

    expr_cache = {}  # dict to store parsed expressions {expr: string}

    # parse expressions to string
    @staticmethod
    def parse_expr(expr):

        # first check cache
        cache = Compiler.expr_cache
        if expr in cache:
            return cache[expr]

        l, op, r = expr.left, expr.op, expr.right

        dct = {l: None, r: None}

        for side in dct.keys():

            if isinstance(side, Field):
                dct[side] = side.fullname
            elif isinstance(side, Expr):
                dct[side] = Compiler.parse_expr(side)
            else:  # string or numbers
                # cast to string and escape this string
                dct[side] = "'" + MySQLdb.escape_string(str(side)) + "'"

        # dont forget to set cache
        cache[expr] = string = dct[l] + Compiler.OP_MAPPING[op] + dct[r]
        return string

class MetaModel(type):  # metaclass for 'single Model'

    def __init__(cls, name, bases, attrs):

        # use lowercase of clsname as table name
        cls.table_name = cls.__name__.lower()
        # {field name: filed}
        fields = {}
        # PrimaryKey object
        primarykey = None

        # foreach filed, describe it and find the primarykey
        for name, attr in cls.__dict__.iteritems():
            if isinstance(attr, Field):
                attr.describe(name, cls)
                fields[name] = attr
                if attr.is_primarykey:
                    primarykey = attr

        if primarykey is None:  # if primarykey not found
            primarykey = PrimaryKey()  # then we new one primarykey: 'id'
            primarykey.describe("id", cls)
            fields['id'] = primarykey

        cls.fields = fields
        cls.primarykey = primarykey

class Model(object):
    """
    Model object.

    Tables are mapped to models.

    """

    __metaclass__ = MetaModel
<commit_msg>add in ,rename between to Between<commit_after>#
# __     ___       ____
# \ \   / (_)_ __ / ___| ___
#  \ \ / /| | '__| |  _ / _ \
#   \ V / | | |  | |_| | (_) |
#    \_/  |_|_|   \____|\___/
#
#   Tiny Python ORM for MySQL
#
#   E-mail : nz2324@126.com
#
#   URL : https://virgo.readthedocs.org/
#
#   Licence : BSD
#
#
# Permission to use, copy, modify,
# and distribute this software for any purpose with
# or without fee is hereby granted,
# provided that the above copyright notice
# and this permission notice appear in all copies.
#

import re
import MySQLdb
import MySQLdb.cursors

# regular expression to get matched rows number from cursor's info
VG_RowsMatchedRe = re.compile(r'Rows matched: (\d+)')

# marks for operators
OP_LT = 1
OP_LE = 2
OP_GT = 3
OP_GE = 4
OP_EQ = 5
OP_NE = 6
OP_ADD = 7
OP_AND = 8
OP_OR = 9
OP_LIKE = 10
OP_BETWEEN = 11
OP_IN = 12

class Database(object):
    """Manage Database connection"""

    # configs for connection with default value
    configs = {
        "host": "localhost",
        "port": 3306,
        "db": "",
        "user": "",
        "passwd": "",
        "charset": "utf8"
    }

    # debuge mode?
    debug = True

    # It is strongly recommended that you set this True
    autocommit = True

    # MySQL connection object, you should use method get_conn to get it
    conn = None

    # record query times
    query_times = 0

    # record SQL last time executed
    SQL = None

    @classmethod
    def config(cls, debug=True, autocommit=True, **configs):
        """
        Configure the database connection.

        The connection will be auto established with these configs.

        Keyword parameters for this method:

          host
            string, host to connect

          user
            string, user to connect as

          passwd
            string, password to use

          db
            string, database to use

          port
            integer, TCP/IP port to connect to

          charset
            string, charset of connection

        See the MySQLdb documentation for more infomation,
        the parameters of MySQLdb.connect are all supported.
        """
        cls.configs.update(configs)
        cls.debug = debug
        cls.autocommit = autocommit

    @classmethod
    def connect(cls):
        """
        Connect to database, singleton pattern.
        This will new one conn object.
        """
        cls.conn = MySQLdb.connect(
            cursorclass=MySQLdb.cursors.DictCursor,
            **cls.configs
        )
        cls.conn.autocommit(cls.autocommit)

    @classmethod
    def get_conn(cls):
        """
        Get MySQL connection object.
        if the conn is open and working, return it,
        else, new one and return it.
        """
        # singleton
        if not cls.conn or not cls.conn.open:
            cls.connect()

        try:
            cls.conn.ping()  # ping to test if the connection is working
        except MySQLdb.OperationalError:
            cls.connect()

        return cls.conn

    @classmethod
    def execute(cls, SQL):
        """
        Execute one SQL command.

        Parameter:
          SQL
            string, SQL command to run.
        """
        cursor = cls.get_conn().cursor()

        try:
            cursor.execute(SQL)
        except Exception, e:
            if cls.debug:  # if debug, report the SQL
                print "SQL:", SQL
            raise e

        cls.query_times += 1
        cls.SQL = SQL

        # add attribute 'matchedRows' to cursor.store query matched rows number
        cursor.matchedRows = int(
            VG_RowsMatchedRe.search(cursor._info).group(1)
        ) if cursor._info else int(cursor.rowcount)

        return cursor

class Leaf(object):

    def _e(op):
        def e(self, right):
            return Expr(self, right, op)
        return e

    __lt__ = _e(OP_LT)

    __le__ = _e(OP_LE)

    __gt__ = _e(OP_GT)

    __ge__ = _e(OP_GE)

    __eq__ = _e(OP_EQ)

    __ne__ = _e(OP_NE)

    __add__ = _e(OP_ADD)

    __and__ = _e(OP_AND)

    __or__ = _e(OP_OR)

class Expr(Leaf):
    """
    Expression object.

    You need't new an expression in this way: myexpr = Expr(left, right, op)

    Use expression like this:
      User.id == 4
      User.name == "Amy"
    """

    def __init__(self, left, right, op):
        self.left = left
        self.right = right
        self.op = op

class EqExpr(Expr):
    """
    Equal exception.

    eg.
      User.id == 1
    """
    def __init__(self, left, right):
        super(EqExpr, self).__init__(left, right, OP_EQ)

# descriptor for Field objects
class FieldDescriptor(object):

    def __init__(self, field):
        self.name = field.name
        self.field = field

    def __get__(self, instance, type=None):
        if instance:
            return instance.data[self.name]
        return self.field

    def __set__(self, instance, value):
        instance.data[self.name] = value

class Field(Leaf):
    """
    Field object.

    Field examples: User.name, User.age ..
    """

    def __init__(self, is_primarykey=False, is_foreignkey=False):
        self.is_primarykey = is_primarykey
        self.is_foreignkey = is_foreignkey

    # describe model's attr
    def describe(self, name, model):
        self.name = name
        self.model = model
        # fullname eg. : User.id 's fullname is "user.id"
        self.fullname = self.model.table_name + "." + self.name
        # describe the attribute, reload its access control of writing, reading
        setattr(model, name, FieldDescriptor(self))

    def __eq__(self, right):  # produce equal expressions
        return EqExpr(self, right)

    def like(self, pattern):
        """
        eg. User.name.like("Amy%")
        """
        return Expr(self, pattern, OP_LIKE)

    def Between(self, value1, value2):
        """
        eg. User.id.Bettwen(3, 7)
        """
        return Expr(self, (value1, value2), OP_BETWEEN)

    def In(self, *values):
        """
        eg. User.id.In(1, 2, 3, 4, 5)
        """
        return Expr(self, values, OP_IN)

class PrimaryKey(Field):
    """
    PrimaryKey object.

    PrimaryKey example: User.id
    """

    def __init__(self):
        super(PrimaryKey, self).__init__(is_primarykey=True)

class ForeignKey(Field):
    """
    ForeignKey object.
    ForeignKey example: Post.user_id = ForeignKey(point_to=User.id)

    Parameter:
      point_to
        PrimaryKey object, the primary key this foreignkey referenced to.
    """

    def __init__(self, point_to):
        super(ForeignKey, self).__init__(is_foreignkey=True)
        self.point_to = point_to

class Compiler(object):
    """
    Compile expressions and sequence of methods to SQL string(s).
    """

    # operator mapping
    OP_MAPPING = {
        OP_LT: " < ",
        OP_LE: " <= ",
        OP_GT: " > ",
        OP_GE: " >= ",
        OP_EQ: " = ",
        OP_NE: " <> ",
        OP_ADD: " + ",
        OP_AND: " and ",
        OP_OR: " or ",
        OP_LIKE: "like"
    }

    expr_cache = {}  # dict to store parsed expressions {expr: string}

    # parse one side of expression to string
    @staticmethod
    def __parse_expr_one_side(side):

        if isinstance(side, Field):
            return side.fullname
        elif isinstance(side, Expr):
            return Compiler.parse_expr(side)
        else:  # string or numbers
            escapestr = MySQLdb.escape_string(str(side))
            return (
                "'" + escapestr + "'" if isinstance(side, str) else escapestr
            )

    # parse expressions to string
    @staticmethod
    def parse_expr(expr):

        # first check cache
        cache = Compiler.expr_cache
        if expr in cache:
            return cache[expr]

        l, op, r = expr.left, expr.op, expr.right

        OP_MAPPING = Compiler.OP_MAPPING

        string = None

        tostr = Compiler.__parse_expr_one_side

        if op in OP_MAPPING:
            string = tostr(l) + OP_MAPPING[op] + tostr(r)
        elif op is OP_BETWEEN:
            string = (
                tostr(l) + " between " + tostr(r[0]) + " and " + tostr(r[1])
            )
        elif op is OP_IN:
            valuestr = ", ".join(tostr(value) for value in r)
            string = (
                tostr(l) + " in " + "(" + valuestr +")"
            )

        # dont forget to set cache
        cache[expr] = string
        return string

class MetaModel(type):  # metaclass for 'single Model'

    def __init__(cls, name, bases, attrs):

        # use lowercase of clsname as table name
        cls.table_name = cls.__name__.lower()
        # {field name: filed}
        fields = {}
        # PrimaryKey object
        primarykey = None

        # foreach filed, describe it and find the primarykey
        for name, attr in cls.__dict__.iteritems():
            if isinstance(attr, Field):
                attr.describe(name, cls)
                fields[name] = attr
                if attr.is_primarykey:
                    primarykey = attr

        if primarykey is None:  # if primarykey not found
            primarykey = PrimaryKey()  # then we new one primarykey: 'id'
            primarykey.describe("id", cls)
            fields['id'] = primarykey

        cls.fields = fields
        cls.primarykey = primarykey

class Model(object):
    """
    Model object.

    Tables are mapped to models.

    """

    __metaclass__ = MetaModel
<|endoftext|>


<commit_before># Copyright (c) 2008-2015 MetPy Developers.
# Distributed under the terms of the BSD 3-Clause License.
# SPDX-License-Identifier: BSD-3-Clause

import glob
import logging
import os.path

import numpy as np

from metpy.io.nexrad import Level2File, Level3File, is_precip_mode
from metpy.cbook import get_test_data

# Turn off the warnings for tests
logging.getLogger("metpy.io.nexrad").setLevel(logging.CRITICAL)

def test_level3_generator():
    datadir = get_test_data('nids', as_file_obj=False)
    for fname in glob.glob(os.path.join(datadir, 'K???_*')):
        yield read_level3_file, fname

def read_level3_file(fname):
    Level3File(fname)

# 1999 file tests old message 1
# KFTG tests bzip compression and newer format for a part of message 31
# KTLX 2015 has missing segments for message 18, which was causing exception
level2_files = ['KTLX20130520_201643_V06.gz', 'KTLX19990503_235621.gz',
                'Level2_KFTG_20150430_1419.ar2v', 'KTLX20150530_000802_V06.bz2']

def test_level2_generator():
    for fname in level2_files:
        yield read_level2_file, get_test_data(fname, as_file_obj=False)

def read_level2_file(fname):
    Level2File(fname)

class TestLevel2(object):
    def test_fobj(self):
        Level2File(get_test_data('Level2_KFTG_20150430_1419.ar2v'))

class TestLevel3(object):
    def test_basic(self):
        Level3File(get_test_data('nids/Level3_FFC_N0Q_20140407_1805.nids', as_file_obj=False))

    def test_nwstg(self):
        Level3File(get_test_data('nids/sn.last', as_file_obj=False))

    def test_fobj(self):
        Level3File(get_test_data('nids/Level3_FFC_N0Q_20140407_1805.nids'))

class TestPrecipMode(object):
    def test21(self):
        assert is_precip_mode(21), 'VCP 21 is precip'

    def test11(self):
        assert is_precip_mode(11), 'VCP 11 is precip'

    def test31(self):
        assert not is_precip_mode(31), 'VCP 31 is not precip'

class TestZipCalls(object):
    def test_msg15(self):
        f = Level2File(get_test_data('KTLX20130520_201643_V06.gz', as_file_obj=False))
        data = f.clutter_filter_map['data']
        assert isinstance(data[0][0], list)

    def test_tracks(self):
        f = Level3File(get_test_data('nids/KOUN_SDUS34_NSTTLX_201305202016'))
        for data in f.sym_block[0]:
            if 'track' in data:
                x, y = np.array(data['track']).T
                assert len(x)
                assert len(y)

    def test_vector_packet(self):
        f = Level3File(get_test_data('nids/KOUN_SDUS64_NHITLX_201305202016'))
        for page in f.graph_pages:
            for item in page:
                if 'vectors' in item:
                    x1, x2, y1, y2 = np.array(item['vectors']).T
                    assert len(x1)
                    assert len(x2)
                    assert len(y1)
                    assert len(y2)
<commit_msg>MNT: Refactor NEXRAD tests.<commit_after># Copyright (c) 2008-2015 MetPy Developers.
# Distributed under the terms of the BSD 3-Clause License.
# SPDX-License-Identifier: BSD-3-Clause

import glob
import logging
import os.path

import pytest

import numpy as np

from metpy.io.nexrad import Level2File, Level3File, is_precip_mode
from metpy.cbook import get_test_data

# Turn off the warnings for tests
logging.getLogger("metpy.io.nexrad").setLevel(logging.CRITICAL)

#
# NEXRAD Level 2 Tests
#

# 1999 file tests old message 1
# KFTG tests bzip compression and newer format for a part of message 31
# KTLX 2015 has missing segments for message 18, which was causing exception
level2_files = ['KTLX20130520_201643_V06.gz', 'KTLX19990503_235621.gz',
                'Level2_KFTG_20150430_1419.ar2v', 'KTLX20150530_000802_V06.bz2']

@pytest.mark.parametrize('fname', level2_files)
def test_level2(fname):
    'Test reading NEXRAD level 2 files from the filename'
    Level2File(get_test_data(fname, as_file_obj=False))

def test_level2_fobj():
    'Test reading NEXRAD level2 data from a file object'
    Level2File(get_test_data('Level2_KFTG_20150430_1419.ar2v'))

#
# NIDS/Level 3 Tests
#

nexrad_nids_files = glob.glob(os.path.join(get_test_data('nids', as_file_obj=False), 'K???_*'))

@pytest.mark.parametrize('fname', nexrad_nids_files)
def test_level3_files(fname):
    Level3File(fname)

class TestLevel3(object):
    def test_basic(self):
        Level3File(get_test_data('nids/Level3_FFC_N0Q_20140407_1805.nids', as_file_obj=False))

    def test_nwstg(self):
        Level3File(get_test_data('nids/sn.last', as_file_obj=False))

    def test_fobj(self):
        Level3File(get_test_data('nids/Level3_FFC_N0Q_20140407_1805.nids'))

class TestPrecipMode(object):
    def test21(self):
        assert is_precip_mode(21), 'VCP 21 is precip'

    def test11(self):
        assert is_precip_mode(11), 'VCP 11 is precip'

    def test31(self):
        assert not is_precip_mode(31), 'VCP 31 is not precip'

class TestZipCalls(object):
    def test_msg15(self):
        f = Level2File(get_test_data('KTLX20130520_201643_V06.gz', as_file_obj=False))
        data = f.clutter_filter_map['data']
        assert isinstance(data[0][0], list)

    def test_tracks(self):
        f = Level3File(get_test_data('nids/KOUN_SDUS34_NSTTLX_201305202016'))
        for data in f.sym_block[0]:
            if 'track' in data:
                x, y = np.array(data['track']).T
                assert len(x)
                assert len(y)

    def test_vector_packet(self):
        f = Level3File(get_test_data('nids/KOUN_SDUS64_NHITLX_201305202016'))
        for page in f.graph_pages:
            for item in page:
                if 'vectors' in item:
                    x1, x2, y1, y2 = np.array(item['vectors']).T
                    assert len(x1)
                    assert len(x2)
                    assert len(y1)
                    assert len(y2)
<|endoftext|>


<commit_before>from lxml import etree
import sys

infile = sys.argv[1]
outfile = sys.argv[2]
f = open(outfile, 'w')

tree = etree.parse(infile)
items = tree.findall('channel/item')

month_dict = {}
month_dict['Jan'] = '01'
month_dict['Feb'] = '02'
month_dict['Mar'] = '03'
month_dict['Apr'] = '04'
month_dict['May'] = '05'
month_dict['Jun'] = '06'
month_dict['Jul'] = '07'
month_dict['Aug'] = '08'
month_dict['Sep'] = '09'
month_dict['Oct'] = '10'
month_dict['Nov'] = '11'
month_dict['Dec'] = '12'

for item in items:

    pub_date = ""
    title = ""
    content = ""

    for element in item.iter():

        text = element.text

        if element.tag == "pubDate":
            pub_date = element.text

            # convert wordpress date format to jrnl date format
            split_date = pub_date.split()
            pub_date = "%s-%s-%s %s" % (month_dict[split_date[2]], split_date[1], split_date[3], split_date[4][:5])

        elif element.tag == "title":
            title = element.text
        elif element.tag == "{http://purl.org/rss/1.0/modules/content/}encoded":
            content = element.text

    f.write("%s %s\n%s\n\n" % (pub_date, title, content))

f.close()
<commit_msg>Moved year to the front in date.<commit_after>from lxml import etree
import sys

infile = sys.argv[1]
outfile = sys.argv[2]
f = open(outfile, 'w')

tree = etree.parse(infile)
items = tree.findall('channel/item')

month_dict = {}
month_dict['Jan'] = '01'
month_dict['Feb'] = '02'
month_dict['Mar'] = '03'
month_dict['Apr'] = '04'
month_dict['May'] = '05'
month_dict['Jun'] = '06'
month_dict['Jul'] = '07'
month_dict['Aug'] = '08'
month_dict['Sep'] = '09'
month_dict['Oct'] = '10'
month_dict['Nov'] = '11'
month_dict['Dec'] = '12'

for item in items:

    pub_date = ""
    title = ""
    content = ""

    for element in item.iter():

        text = element.text

        if element.tag == "pubDate":
            pub_date = element.text

            # convert wordpress date format to jrnl date format
            split_date = pub_date.split()
            pub_date = "%s-%s-%s %s" % (split_date[3], month_dict[split_date[2]], split_date[1], split_date[4][:5])

        elif element.tag == "title":
            title = element.text
        elif element.tag == "{http://purl.org/rss/1.0/modules/content/}encoded":
            content = element.text

    f.write("%s %s\n%s\n\n" % (pub_date, title, content))

f.close()
<|endoftext|>


<commit_before>
from cStringIO import StringIO
import os
from string import punctuation
from urllib import unquote
from zipfile import ZipFile

from django.core.files.base import ContentFile
from django.core.files.storage import default_storage
from django.db import models
from django.utils.translation import ugettext_lazy as _

from mezzanine.conf import settings
from mezzanine.core.fields import FileField
from mezzanine.core.models import Orderable, RichText
from mezzanine.pages.models import Page
from mezzanine.utils.importing import import_dotted_path

# Set the directory where gallery images are uploaded to,
# either MEDIA_ROOT + 'galleries', or filebrowser's upload
# directory if being used.
GALLERIES_UPLOAD_DIR = "galleries"
if settings.PACKAGE_NAME_FILEBROWSER in settings.INSTALLED_APPS:
    fb_settings = "%s.settings" % settings.PACKAGE_NAME_FILEBROWSER
    try:
        GALLERIES_UPLOAD_DIR = import_dotted_path(fb_settings).DIRECTORY
    except ImportError:
        pass

class Gallery(Page, RichText):
    """
    Page bucket for gallery photos.
    """

    zip_import = models.FileField(upload_to="galleries", blank=True,
                    verbose_name=_("Zip import"),
                    help_text=_("Upload a zip file containing images, and "
                                "they'll be imported into this gallery."))

    class Meta:
        verbose_name = _("Gallery")
        verbose_name_plural = _("Galleries")

    def save(self, delete_zip_import=True, *args, **kwargs):
        """
        If a zip file is uploaded, extract any images from it and add
        them to the gallery, before removing the zip file.
        """
        super(Gallery, self).save(*args, **kwargs)
        if self.zip_import:
            zip_file = ZipFile(self.zip_import)
            # import PIL in either of the two ways it can end up installed.
            try:
                from PIL import Image
            except ImportError:
                import Image
            for name in zip_file.namelist():
                data = zip_file.read(name)
                try:
                    image = Image.open(StringIO(data))
                    image.load()
                    image = Image.open(StringIO(data))
                    image.verify()
                except:
                    continue
                path = os.path.join(GALLERIES_UPLOAD_DIR, self.slug,
                                    name.decode("utf-8"))
                try:
                    saved_path = default_storage.save(path, ContentFile(data))
                except UnicodeEncodeError:
                    from warnings import warn
                    warn("A file was saved that contains unicode "
                         "characters in its path, but somehow the current "
                         "locale does not support utf-8. You may need to set "
                         "'LC_ALL' to a correct value, eg: 'en_US.UTF-8'.")
                    path = os.path.join(GALLERIES_UPLOAD_DIR, self.slug,
                                        unicode(name, errors="ignore"))
                    saved_path = default_storage.save(path, ContentFile(data))
                self.images.add(GalleryImage(file=saved_path))
            if delete_zip_import:
                zip_file.close()
                self.zip_import.delete(save=True)

class GalleryImage(Orderable):

    gallery = models.ForeignKey("Gallery", related_name="images")
    file = FileField(_("File"), max_length=200, upload_to="galleries",
                                                           format="Image")
    description = models.CharField(_("Description"), max_length=1000,
                                                           blank=True)

    class Meta:
        verbose_name = _("Image")
        verbose_name_plural = _("Images")

    def __unicode__(self):
        return self.description

    def save(self, *args, **kwargs):
        """
        If no description is given when created, create one from the
        file name.
        """
        if not self.id and not self.description:
            name = unquote(self.file.url).split("/")[-1].rsplit(".", 1)[0]
            name = name.replace("'", "")
            name = "".join([c if c not in punctuation else " " for c in name])
            # str.title() doesn't deal with unicode very well.
            # http://bugs.python.org/issue6412
            name = "".join([s.upper() if i == 0 or name[i - 1] == " " else s
                            for i, s in enumerate(name)])
            self.description = name
        super(GalleryImage, self).save(*args, **kwargs)
<commit_msg>gboshtools#10 Changing file names within the Media Manager breaks references to gallery images<commit_after>from filebrowser_safe.views import filebrowser_post_rename
from filebrowser_safe import settings as filebrowser_settings

from cStringIO import StringIO
import os
from string import punctuation
from urllib import unquote
from zipfile import ZipFile

from django.core.files.base import ContentFile
from django.core.files.storage import default_storage
from django.db import models
from django.dispatch import receiver
from django.utils.translation import ugettext_lazy as _

from mezzanine.conf import settings
from mezzanine.core.fields import FileField
from mezzanine.core.models import Orderable, RichText
from mezzanine.pages.models import Page
from mezzanine.utils.importing import import_dotted_path

# Set the directory where gallery images are uploaded to,
# either MEDIA_ROOT + 'galleries', or filebrowser's upload
# directory if being used.
GALLERIES_UPLOAD_DIR = "galleries"
if settings.PACKAGE_NAME_FILEBROWSER in settings.INSTALLED_APPS:
    fb_settings = "%s.settings" % settings.PACKAGE_NAME_FILEBROWSER
    try:
        GALLERIES_UPLOAD_DIR = import_dotted_path(fb_settings).DIRECTORY
    except ImportError:
        pass

class Gallery(Page, RichText):
    """
    Page bucket for gallery photos.
    """

    zip_import = models.FileField(upload_to="galleries", blank=True,
                    verbose_name=_("Zip import"),
                    help_text=_("Upload a zip file containing images, and "
                                "they'll be imported into this gallery."))

    class Meta:
        verbose_name = _("Gallery")
        verbose_name_plural = _("Galleries")

    def save(self, delete_zip_import=True, *args, **kwargs):
        """
        If a zip file is uploaded, extract any images from it and add
        them to the gallery, before removing the zip file.
        """
        super(Gallery, self).save(*args, **kwargs)
        if self.zip_import:
            zip_file = ZipFile(self.zip_import)
            # import PIL in either of the two ways it can end up installed.
            try:
                from PIL import Image
            except ImportError:
                import Image
            for name in zip_file.namelist():
                data = zip_file.read(name)
                try:
                    image = Image.open(StringIO(data))
                    image.load()
                    image = Image.open(StringIO(data))
                    image.verify()
                except:
                    continue
                path = os.path.join(GALLERIES_UPLOAD_DIR, self.slug,
                                    name.decode("utf-8"))
                try:
                    saved_path = default_storage.save(path, ContentFile(data))
                except UnicodeEncodeError:
                    from warnings import warn
                    warn("A file was saved that contains unicode "
                         "characters in its path, but somehow the current "
                         "locale does not support utf-8. You may need to set "
                         "'LC_ALL' to a correct value, eg: 'en_US.UTF-8'.")
                    path = os.path.join(GALLERIES_UPLOAD_DIR, self.slug,
                                        unicode(name, errors="ignore"))
                    saved_path = default_storage.save(path, ContentFile(data))
                self.images.add(GalleryImage(file=saved_path))
            if delete_zip_import:
                zip_file.close()
                self.zip_import.delete(save=True)

class GalleryImage(Orderable):

    gallery = models.ForeignKey("Gallery", related_name="images")
    file = FileField(_("File"), max_length=200, upload_to="galleries",
                                                           format="Image")
    description = models.CharField(_("Description"), max_length=1000,
                                                           blank=True)

    class Meta:
        verbose_name = _("Image")
        verbose_name_plural = _("Images")

    def __unicode__(self):
        return self.description

    def save(self, *args, **kwargs):
        """
        If no description is given when created, create one from the
        file name.
        """
        if not self.id and not self.description:
            name = unquote(self.file.url).split("/")[-1].rsplit(".", 1)[0]
            name = name.replace("'", "")
            name = "".join([c if c not in punctuation else " " for c in name])
            # str.title() doesn't deal with unicode very well.
            # http://bugs.python.org/issue6412
            name = "".join([s.upper() if i == 0 or name[i - 1] == " " else s
                            for i, s in enumerate(name)])
            self.description = name
        super(GalleryImage, self).save(*args, **kwargs)

@receiver(filebrowser_post_rename)
def on_filebrowser_image_renamed(sender, **kwargs):
    old_file = "%s%s" % (GALLERIES_UPLOAD_DIR, kwargs['filename'])
    new_file = "%s%s" % (GALLERIES_UPLOAD_DIR, kwargs['new_filename'])
    for gallery_image in GalleryImage.objects.filter(file=old_file):
        gallery_image.file = new_file
        gallery_image.save()
<|endoftext|>