<commit_before>import json

class filter_history(object):

    def process(self,input,output,filter_spec):

        out = None

        for line in file(input):
            data = json.loads(line)
            model = data.get('model')
            field = data.get('field')
            created = data.get('created',False)

            model_spec = filter_spec.get(model)
            if model_spec is not None:
                ok = False
                if created and model_spec.get('created',False)
                    ok = True
                elif not created:
                    allowed_fields = model_spec.get('fields',[])
                    ok = field in allowed_fields

                if ok:
                    if out is None:
                        out = file(output,'w')
                    out.write(line.strip() + '\n')
<commit_msg>syntax error<commit_after>import json

class filter_history(object):

    def process(self,input,output,filter_spec):

        out = None

        for line in file(input):
            data = json.loads(line)
            model = data.get('model')
            field = data.get('field')
            created = data.get('created',False)

            model_spec = filter_spec.get(model)
            if model_spec is not None:
                ok = False
                if created and model_spec.get('created',False):
                    ok = True
                elif not created:
                    allowed_fields = model_spec.get('fields',[])
                    ok = field in allowed_fields

                if ok:
                    if out is None:
                        out = file(output,'w')
                    out.write(line.strip() + '\n')
<|endoftext|>


<commit_before>from devilry_settings import *

########################################################################
#
# Defaults for django settings
# - See: https://docs.djangoproject.com/en/dev/ref/settings/
#
########################################################################

DEBUG = False
TEMPLATE_DEBUG = DEBUG

TIME_ZONE = 'Europe/Oslo'
LANGUAGE_CODE = 'en-us'
SITE_ID = 1
USE_I18N = True
USE_L10N = True
DATETIME_FORMAT = "N j, Y, H:i"
LOGIN_URL = '/authenticate/login'
STATIC_URL = '/static/'
STATIC_ROOT = 'static'
DATABASES = {}
EMAIL_SUBJECT_PREFIX = '[Devilry] '
ROOT_URLCONF = 'devilry_settings.default_root_urlconf'
AUTH_PROFILE_MODULE = 'core.DevilryUserProfile'

INSTALLED_APPS = ['django.contrib.markup',
                  'django.contrib.sessions',
                  'django.contrib.auth',
                  'django.contrib.contenttypes',
                  'django.contrib.staticfiles',
                  'django.contrib.sessions',
                  'django.contrib.messages',
                  'django.contrib.admin',
                  'errortemplates',
                  'djangorestframework',
                  'gunicorn',

                  'devilry.apps.core',
                  'devilry.apps.theme',
                  'devilry.apps.extjshelpers',
                  'devilry.apps.extjsux',
                  'devilry.apps.developertools',
                  'devilry.apps.jsfiledownload',

                  'devilry.apps.approved_gradeeditor',
                  'devilry.apps.manual_gradeeditor',
                  'devilry.apps.autograde_gradeeditor',
                  'devilry.apps.basicform_gradeeditor',
                  'devilry.apps.commentform_gradeeditor',

                  'devilry.apps.statistics',
                  'devilry.apps.markup',
                  'devilry.apps.student',
                  'devilry.apps.examiner',
                  'devilry.apps.administrator',
                  'devilry.apps.superadmin',
                  'devilry.apps.authenticate',
                  'devilry.apps.gradeeditors',
                  'devilry.apps.send_email_to_students',

                  'devilry_usersearch',
                  'devilry_useradmin',
                  'devilry_settings'
                 ]

TEMPLATE_CONTEXT_PROCESSORS = ("django.contrib.auth.context_processors.auth",
                               "django.core.context_processors.debug",
                               "django.core.context_processors.i18n",
                               "django.core.context_processors.media",
                               "django.core.context_processors.debug",
                               'django.contrib.messages.context_processors.messages',
                               'devilry.apps.theme.templatecontext.template_variables')

MIDDLEWARE_CLASSES = ['django.middleware.common.CommonMiddleware',
                      'django.contrib.sessions.middleware.SessionMiddleware',
                      'django.contrib.auth.middleware.AuthenticationMiddleware',
                      'django.middleware.transaction.TransactionMiddleware',
                      'django.contrib.messages.middleware.MessageMiddleware',
                      'devilry.utils.logexceptionsmiddleware.TracebackLoggingMiddleware']

########################################################################
#
# Defaults for settings defined by Devilry.
#
########################################################################

# Make sure this does not end with / (i.e. '' means / is the main page).
# DEVILRY_URLPATH_PREFIX = '/django/devilry'
DEVILRY_URLPATH_PREFIX = ''

# The default grade-plugin:
DEVILRY_DEFAULT_GRADEEDITOR='approved'

DEVILRY_STATIC_URL = '/static' # Must not end in / (this means that '' is the server root)
DEVILRY_THEME_URL = DEVILRY_STATIC_URL + '/theme/themes/devilry'
DEVILRY_EXTJS_URL = DEVILRY_STATIC_URL + '/extjs4'
DEVILRY_MATHJAX_URL = 'https://d3eoax9i5htok0.cloudfront.net/mathjax/latest/MathJax.js'
DEVILRY_LOGOUT_URL = '/authenticate/logout'
DEVILRY_HELP_URL = 'https://github.com/devilry/devilry-django/wiki'

#Set max file size to 5MB. Files greater than this size are split into chunks of this size.
DEVILRY_MAX_ARCHIVE_CHUNK_SIZE = 5000000

DEVILRY_SEND_EMAIL_TO_USERS = True
DEVILRY_EMAIL_SUBJECT_PREFIX_ADMIN = '[devilry-admin] '
DEVILRY_EMAIL_SIGNATURE = "This is a message from the Devilry assignment delivery system. " \
                  "Please do not respond to this email."

DEVILRY_DELIVERY_STORE_BACKEND = 'devilry.apps.core.deliverystore.FsHierDeliveryStore'
DEVILRY_FSHIERDELIVERYSTORE_INTERVAL = 1000
DEVILRY_SYNCSYSTEM = 'YOUR MASTER SYSTEM HERE'

## Email pattern. Set this, and add 'devilry.apps.autoset_empty_email_by_username' to INSTALLED_APPS
## to automatically set email to "<username>@DEVILRY_DEFAULT_EMAIL_SUFFIX" if it is not set when a user is saved.
#DEVILRY_DEFAULT_EMAIL_SUFFIX = 'example.com'

## When sorting by fullname, would you like to sort by last name? Currently
## only affects the overview over an entire period.
DEVILRY_SORT_FULL_NAME_BY_LASTNAME = True

## Messages that are displayed in the 3 dashboards for users with no permission to the dashboard
## The body of each message can contain html. For example, you can add an email link using: admin@example.com
DEVILRY_STUDENT_NO_PERMISSION_MSG = {'title': 'No published assignments',
                                     'body': 'You are not registered as a student on any assignments in Devilry. This is usually because you subject/course administrator has not published any assignments yet. Contact your subject/course administrator if this is wrong.'}
DEVILRY_EXAMINER_NO_PERMISSION_MSG = {'title': 'You are not an examiner',
                                      'body': 'You are not registered as an examiner on any publshed assignments in Devilry. If this is wrong, please contact the subject/course administrator.'}
DEVILRY_ADMINISTRATOR_NO_PERMISSION_MSG = {'title': 'You are not an administrator',
                                           'body': 'You are not registered as an administrator on any Node, Subject/Course, Period/Semester or Assignment in Devilry. If this is wrong, please contact the system administrator.'}

## Deadline handling method:
## 0: Soft deadlines
## 1: Hard deadlines
DEFAULT_DEADLINE_HANDLING_METHOD = 0

#################################################
# Settings without a default value
#################################################

## If we use django to serve static files, we need this setting to define
## where they are located. Static files are located in the devilry/static/
## directory in the devilry source repository.
#DEVILRY_STATIC_ROOT = '/path/to/static'
#DEVILRY_SYSTEM_ADMIN_EMAIL = 'devilry-support@example.com'

<commit_msg>devilry_settings: Cleanup.<commit_after>

########################################################################
#
# Defaults for django settings
# - See: https://docs.djangoproject.com/en/dev/ref/settings/
#
########################################################################

DEBUG = False
TEMPLATE_DEBUG = DEBUG

TIME_ZONE = 'Europe/Oslo'
LANGUAGE_CODE = 'en-us'
SITE_ID = 1
USE_I18N = True
USE_L10N = True
DATETIME_FORMAT = "N j, Y, H:i"
LOGIN_URL = '/authenticate/login'
STATIC_URL = '/static/'
STATIC_ROOT = 'static'
DATABASES = {}
EMAIL_SUBJECT_PREFIX = '[Devilry] '
ROOT_URLCONF = 'devilry_settings.default_root_urlconf'
AUTH_PROFILE_MODULE = 'core.DevilryUserProfile'

INSTALLED_APPS = ['django.contrib.markup',
                  'django.contrib.sessions',
                  'django.contrib.auth',
                  'django.contrib.contenttypes',
                  'django.contrib.staticfiles',
                  'django.contrib.sessions',
                  'django.contrib.messages',
                  'django.contrib.admin',
                  'errortemplates',
                  'djangorestframework',
                  'gunicorn',

                  'devilry.apps.core',
                  'devilry.apps.theme',
                  'devilry.apps.extjshelpers',
                  'devilry.apps.extjsux',
                  'devilry.apps.developertools',
                  'devilry.apps.jsfiledownload',

                  'devilry.apps.approved_gradeeditor',
                  'devilry.apps.manual_gradeeditor',
                  'devilry.apps.autograde_gradeeditor',
                  'devilry.apps.basicform_gradeeditor',
                  'devilry.apps.commentform_gradeeditor',

                  'devilry.apps.statistics',
                  'devilry.apps.markup',
                  'devilry.apps.student',
                  'devilry.apps.examiner',
                  'devilry.apps.administrator',
                  'devilry.apps.superadmin',
                  'devilry.apps.authenticate',
                  'devilry.apps.gradeeditors',
                  'devilry.apps.send_email_to_students',

                  'devilry_usersearch',
                  'devilry_useradmin',
                  'devilry_settings'
                 ]

TEMPLATE_CONTEXT_PROCESSORS = ("django.contrib.auth.context_processors.auth",
                               "django.core.context_processors.debug",
                               "django.core.context_processors.i18n",
                               "django.core.context_processors.media",
                               "django.core.context_processors.debug",
                               'django.contrib.messages.context_processors.messages',
                               'devilry.apps.theme.templatecontext.template_variables')

MIDDLEWARE_CLASSES = ['django.middleware.common.CommonMiddleware',
                      'django.contrib.sessions.middleware.SessionMiddleware',
                      'django.contrib.auth.middleware.AuthenticationMiddleware',
                      'django.middleware.transaction.TransactionMiddleware',
                      'django.contrib.messages.middleware.MessageMiddleware',
                      'devilry.utils.logexceptionsmiddleware.TracebackLoggingMiddleware']

########################################################################
#
# Defaults for settings defined by Devilry.
#
########################################################################

# Make sure this does not end with / (i.e. '' means / is the main page).
# DEVILRY_URLPATH_PREFIX = '/django/devilry'
DEVILRY_URLPATH_PREFIX = ''

# The default grade-plugin:
DEVILRY_DEFAULT_GRADEEDITOR='approved'

DEVILRY_STATIC_URL = '/static' # Must not end in / (this means that '' is the server root)
DEVILRY_THEME_URL = DEVILRY_STATIC_URL + '/theme/themes/devilry'
DEVILRY_EXTJS_URL = DEVILRY_STATIC_URL + '/extjs4'
DEVILRY_MATHJAX_URL = 'https://d3eoax9i5htok0.cloudfront.net/mathjax/latest/MathJax.js'
DEVILRY_LOGOUT_URL = '/authenticate/logout'
DEVILRY_HELP_URL = 'https://github.com/devilry/devilry-django/wiki'

#Set max file size to 5MB. Files greater than this size are split into chunks of this size.
DEVILRY_MAX_ARCHIVE_CHUNK_SIZE = 5000000

DEVILRY_SEND_EMAIL_TO_USERS = True
DEVILRY_EMAIL_SUBJECT_PREFIX_ADMIN = '[devilry-admin] '
DEVILRY_EMAIL_SIGNATURE = "This is a message from the Devilry assignment delivery system. " \
                  "Please do not respond to this email."

DEVILRY_DELIVERY_STORE_BACKEND = 'devilry.apps.core.deliverystore.FsHierDeliveryStore'
DEVILRY_FSHIERDELIVERYSTORE_INTERVAL = 1000
DEVILRY_SYNCSYSTEM = 'YOUR MASTER SYSTEM HERE'

## Email pattern. Set this, and add 'devilry.apps.autoset_empty_email_by_username' to INSTALLED_APPS
## to automatically set email to "<username>@DEVILRY_DEFAULT_EMAIL_SUFFIX" if it is not set when a user is saved.
#DEVILRY_DEFAULT_EMAIL_SUFFIX = 'example.com'

## When sorting by fullname, would you like to sort by last name? Currently
## only affects the overview over an entire period.
DEVILRY_SORT_FULL_NAME_BY_LASTNAME = True

## Messages that are displayed in the 3 dashboards for users with no permission to the dashboard
## The body of each message can contain html. For example, you can add an email link using: admin@example.com
DEVILRY_STUDENT_NO_PERMISSION_MSG = {'title': 'No published assignments',
                                     'body': 'You are not registered as a student on any assignments in Devilry. This is usually because you subject/course administrator has not published any assignments yet. Contact your subject/course administrator if this is wrong.'}
DEVILRY_EXAMINER_NO_PERMISSION_MSG = {'title': 'You are not an examiner',
                                      'body': 'You are not registered as an examiner on any publshed assignments in Devilry. If this is wrong, please contact the subject/course administrator.'}
DEVILRY_ADMINISTRATOR_NO_PERMISSION_MSG = {'title': 'You are not an administrator',
                                           'body': 'You are not registered as an administrator on any Node, Subject/Course, Period/Semester or Assignment in Devilry. If this is wrong, please contact the system administrator.'}

## Deadline handling method:
## 0: Soft deadlines
## 1: Hard deadlines
DEFAULT_DEADLINE_HANDLING_METHOD = 0

#################################################
# Settings without a default value
#################################################

## If we use django to serve static files, we need this setting to define
## where they are located. Static files are located in the devilry/static/
## directory in the devilry source repository.
#DEVILRY_STATIC_ROOT = '/path/to/static'
#DEVILRY_SYSTEM_ADMIN_EMAIL = 'devilry-support@example.com'

<|endoftext|>


<commit_before>#!/usr/bin/evn python
# encoding: utf-8

from pprof.project import ProjectFactory, log
from pprof.settings import config
from group import PprofGroup

from os import path
from plumbum import FG, local
from plumbum.cmd import chmod

class Ruby(PprofGroup):

    class Factory:

        def create(self, exp):
            return Ruby(exp, "ruby", "compilation")
    ProjectFactory.addFactory("Ruby", Factory())

    src_dir = "ruby-2.2.2"
    src_file = src_dir + ".tar.gz"
    src_uri = "http://cache.ruby-lang.org/pub/ruby/2.2/" + src_file

    def download(self):
        from pprof.utils.downloader import Wget
        from plumbum.cmd import tar

        with local.cwd(self.builddir):
            Wget(self.src_uri, self.src_file)
            tar("xfz", self.src_file)

    def configure(self):
        from pprof.utils.compiler import clang, clang_cxx
        ruby_dir = path.join(self.builddir, self.src_dir)
        with local.cwd(ruby_dir):
            with local.env(CC=str(clang()), CXX=str(clang_cxx),
                           CFLAGS=" ".join(self.cflags),
                           LIBS=" ".join(self.ldflags)):
                configure = local["./configure"]
                configure["--with-static-linked-ext", "--disable-shared"] & FG

    def build(self):
        from plumbum.cmd import make, ln

        ruby_dir = path.join(self.builddir, self.src_dir)
        with local.cwd(ruby_dir):
            make & FG

    def run_tests(self, experiment):
        from plumbum.cmd import ruby, echo, chmod
        from pprof.project import wrap

        ruby_dir = path.join(self.builddir, self.src_dir)
        exp = wrap(path.join(ruby_dir, "ruby"), experiment)

        with local.env(RUBYOPT=""):
            ruby[path.join(self.testdir, "benchmark", "run.rb"),
                 "--ruby=\"" + str(exp) + "\"",
                 "--opts=\"-I" + path.join(self.testdir, "lib") +
                 " -I" + path.join(self.testdir, ".") +
                 " -I" + path.join(self.testdir, ".ext", "common") +
                 "\"", "-r"] & FG
<commit_msg>ruby: enable parallel build<commit_after>#!/usr/bin/evn python
# encoding: utf-8

from pprof.project import ProjectFactory, log
from pprof.settings import config
from group import PprofGroup

from os import path
from plumbum import FG, local
from plumbum.cmd import chmod

class Ruby(PprofGroup):

    class Factory:

        def create(self, exp):
            return Ruby(exp, "ruby", "compilation")
    ProjectFactory.addFactory("Ruby", Factory())

    src_dir = "ruby-2.2.2"
    src_file = src_dir + ".tar.gz"
    src_uri = "http://cache.ruby-lang.org/pub/ruby/2.2/" + src_file

    def download(self):
        from pprof.utils.downloader import Wget
        from plumbum.cmd import tar

        with local.cwd(self.builddir):
            Wget(self.src_uri, self.src_file)
            tar("xfz", self.src_file)

    def configure(self):
        from pprof.utils.compiler import clang, clang_cxx
        ruby_dir = path.join(self.builddir, self.src_dir)
        with local.cwd(ruby_dir):
            with local.env(CC=str(clang()), CXX=str(clang_cxx),
                           CFLAGS=" ".join(self.cflags),
                           LIBS=" ".join(self.ldflags)):
                configure = local["./configure"]
                configure["--with-static-linked-ext", "--disable-shared"] & FG

    def build(self):
        from plumbum.cmd import make, ln

        ruby_dir = path.join(self.builddir, self.src_dir)
        with local.cwd(ruby_dir):
            make("-j", config["jobs"])

    def run_tests(self, experiment):
        from plumbum.cmd import ruby, echo, chmod
        from pprof.project import wrap

        ruby_dir = path.join(self.builddir, self.src_dir)
        exp = wrap(path.join(ruby_dir, "ruby"), experiment)

        with local.env(RUBYOPT=""):
            ruby[path.join(self.testdir, "benchmark", "run.rb"),
                 "--ruby=\"" + str(exp) + "\"",
                 "--opts=\"-I" + path.join(self.testdir, "lib") +
                 " -I" + path.join(self.testdir, ".") +
                 " -I" + path.join(self.testdir, ".ext", "common") +
                 "\"", "-r"] & FG
<|endoftext|>


<commit_before>#!/usr/bin/env python

from .os import detect_os

def legitimize(text, os=detect_os()):
    """Converts a string to a valid filename.
    """

    # POSIX systems
    text = text.translate({
        0: None,
        ord('/'): '-',
        ord('|'): '-',
    })

    if os == 'windows' or os == 'cygwin' or os == 'wsl':
        # Windows (non-POSIX namespace)
        text = text.translate({
            # Reserved in Windows VFAT and NTFS
            ord(':'): '-',
            ord('*'): '-',
            ord('?'): '-',
            ord('\\'): '-',
            ord('\"'): '\'',
            # Reserved in Windows VFAT
            ord('+'): '-',
            ord('<'): '-',
            ord('>'): '-',
            ord('['): '(',
            ord(']'): ')',
        })
    else:
        # *nix
        if os == 'mac':
            # Mac OS HFS+
            text = text.translate({
                ord(':'): '-',
            })

        # Remove leading .
        if text.startswith("."):
            text = text[1:]

    text = text[:80] # Trim to 82 Unicode characters long
    return text
<commit_msg>util.fs: \t not allowed in FAT (close #2646)<commit_after>#!/usr/bin/env python

from .os import detect_os

def legitimize(text, os=detect_os()):
    """Converts a string to a valid filename.
    """

    # POSIX systems
    text = text.translate({
        0: None,
        ord('/'): '-',
        ord('|'): '-',
    })

    # FIXME: do some filesystem detection
    if os == 'windows' or os == 'cygwin' or os == 'wsl':
        # Windows (non-POSIX namespace)
        text = text.translate({
            # Reserved in Windows VFAT and NTFS
            ord(':'): '-',
            ord('*'): '-',
            ord('?'): '-',
            ord('\\'): '-',
            ord('\"'): '\'',
            # Reserved in Windows VFAT
            ord('+'): '-',
            ord('<'): '-',
            ord('>'): '-',
            ord('['): '(',
            ord(']'): ')',
            ord('\t'): ' ',
        })
    else:
        # *nix
        if os == 'mac':
            # Mac OS HFS+
            text = text.translate({
                ord(':'): '-',
            })

        # Remove leading .
        if text.startswith("."):
            text = text[1:]

    text = text[:80] # Trim to 82 Unicode characters long
    return text
<|endoftext|>


<commit_before>#!/usr/bin/env python

script_name = 'you-get'
__version__ = '0.4.1302'
<commit_msg>version 0.4.1314<commit_after>#!/usr/bin/env python

script_name = 'you-get'
__version__ = '0.4.1314'
<|endoftext|>


<commit_before># Standard Libary
import logging
import uuid

# Third-Party
from django_fsm import FSMIntegerField
from django_fsm import transition
from django_fsm_log.decorators import fsm_log_by
from django_fsm_log.decorators import fsm_log_description
from dry_rest_permissions.generics import allow_staff_or_superuser
from dry_rest_permissions.generics import authenticated_users
from model_utils import Choices
from model_utils.models import TimeStampedModel
from django_fsm_log.models import StateLog
from django.contrib.contenttypes.fields import GenericRelation

# Django
from django.db import models
from django.utils.functional import cached_property

# First-Party
from api.managers import MemberManager

log = logging.getLogger(__name__)

class Member(TimeStampedModel):
    id = models.UUIDField(
        primary_key=True,
        default=uuid.uuid4,
        editable=False,
    )
    STATUS = Choices(
        (-10, 'inactive', 'Inactive',),
        (0, 'new', 'New',),
        (10, 'active', 'Active',),
    )

    status = FSMIntegerField(
        help_text="""DO NOT CHANGE MANUALLY unless correcting a mistake.  Use the buttons to change state.""",
        choices=STATUS,
        default=STATUS.new,
    )

    MEM_STATUS = Choices(
        (10, 'active', 'Active',),
        (20, 'active_internal', 'Active Internal',),
        (30, 'active_licensed', 'Active Licensed',),
        (40, 'cancelled', 'Cancelled',),
        (50, 'closed', 'Closed',),
        (60, 'closed_merged', 'Closed Merged',),
        (70, 'closed_revoked', 'Closed Revoked',),
        (80, 'closed_voluntary', 'Closed Voluntary',),
        (90, 'expelled', 'Expelled',),
        (100, 'expired', 'Expired',),
        (105, 'expired_licensed', 'Expired Licensed',),
        (110, 'lapsed', 'Lapsed',),
        (120, 'not_approved', 'Not Approved',),
        (130, 'pending', 'Pending',),
        (140, 'pending_voluntary', 'Pending Voluntary',),
        (150, 'suspended', 'Suspended',),
        (160, 'suspended_membership', 'Suspended Membership',),
        (170, 'awaiting_payment', 'Awaiting Payment',),
    )

    mem_status = models.IntegerField(
        choices=MEM_STATUS,
        null=True,
        blank=True,
    )

    SUB_STATUS = Choices(
        (10, 'active', 'Active',),
        (20, 'expired', 'Expired',),
        (30, 'pending', 'Pending',),
        (40, 'lapsedRenew', 'Lapsed',),
        (50, 'cancelled', 'Cancelled',),
        (60, 'swapped', 'Swapped',),
    )

    sub_status = models.IntegerField(
        choices=SUB_STATUS,
        null=True,
        blank=True,
    )

    MEM_CODE = Choices(
        (10, 'RG', 'RG Regular'),
        (20, 'R5', 'R5 Regular 50 Year'),
        (30, 'SN', 'SN Senior'),
        (40, 'S5', 'S5 Senior 50 Year'),
        (50, 'SL', 'SL Senior Legacy'),
        (60, 'Y1', 'Y1 Youth Initial'),
        (70, 'Y2', 'Y2 Youth Subsequent'),
        (80, 'LF', 'LF Lifetime Regular'),
        (90, 'L5', 'L5 Lifetime 50 Year'),
        (100, 'LY', 'LY Lifetime Youth'),
        (110, 'LS', 'LS Lifetime Senior'),
        (120, 'AS', 'AS Associate'),
    )

    mem_code = models.IntegerField(
        choices=MEM_CODE,
        null=True,
        blank=True,
    )

    inactive_date = models.DateField(
        null=True,
        blank=True,
    )

    INACTIVE_REASON = Choices(
        (1, 'Non_renewal', 'Non-Renewal'),
        (2, 'Renewed', 'Renewed'),
        (3, 'NotCancelled', 'Not Cancelled'),
        (4, 'Non_Payment', 'Non-Payment'),
        (5, 'Expired', 'Expired'),
        (6, 'Deceased', 'Deceased'),
        (7, 'changedOption', 'changedOption'),
        (8, 'Other', 'Other'),
        (9, 'cancelled', 'cancelled'),
        (10, 'Transferred', 'Transferred'),
        (11, 'swappedChapter', 'swappedChapter'),
        (12, 'swapped', 'swapped'),
    )

    inactive_reason = models.IntegerField(
        choices=INACTIVE_REASON,
        null=True,
        blank=True,
    )

    PART = Choices(
        # (-1, 'director', 'Director'),
        (1, 'tenor', 'Tenor'),
        (2, 'lead', 'Lead'),
        (3, 'baritone', 'Baritone'),
        (4, 'bass', 'Bass'),
    )

    part = models.IntegerField(
        choices=PART,
        null=True,
        blank=True,
    )

    established_date = models.DateField(
        null=True,
        blank=True,
    )

    current_through = models.DateField(
        null=True,
        blank=True,
    )

    mc_pk = models.UUIDField(
        null=True,
        blank=True,
        unique=True,
        db_index=True,
    )

    # Properties
    @cached_property
    def is_mc(self):
        return bool(self.mc_pk)

    # FKs
    group = models.ForeignKey(
        'Group',
        related_name='members',
        on_delete=models.CASCADE,
    )

    person = models.ForeignKey(
        'Person',
        related_name='members',
        on_delete=models.CASCADE,
    )

    subscription = models.ForeignKey(
        'Subscription',
        related_name='members',
        on_delete=models.CASCADE,
        null=True,
        blank=True,
    )

    # Relations
    statelogs = GenericRelation(
        StateLog,
        related_query_name='members',
    )

    # Internals
    objects = MemberManager()

    class Meta:
        unique_together = (
            ('group', 'person',),
        )

    class JSONAPIMeta:
        resource_name = "member"

    def __str__(self):
        return str(self.id)

    def clean(self):
        pass
        # if self.is_admin:
        #     try:
        #         is_active = self.person.user.is_active
        #     except ObjectDoesNotExist:
        #         raise ValidationError(
        #             {'is_admin': 'Admin must User account.'}
        #         )
        #     if not is_active:
        #         raise ValidationError(
        #             {'is_admin': 'Admin User account must be active.'}
        #         )

    # Permissions
    @staticmethod
    @allow_staff_or_superuser
    @authenticated_users
    def has_read_permission(request):
        return True

    @allow_staff_or_superuser
    @authenticated_users
    def has_object_read_permission(self, request):
        return True

    @staticmethod
    @allow_staff_or_superuser
    @authenticated_users
    def has_write_permission(request):
        return any([
            request.user.is_group_manager,
        ])

    @allow_staff_or_superuser
    @authenticated_users
    def has_object_write_permission(self, request):
        return any([
            all([
                self.group.officers.filter(
                    person__user=request.user,
                    status__gt=0,
                ),
                self.mc_pk == None,
            ]),
        ])

    # Transitions
    @fsm_log_by
    @fsm_log_description
    @transition(field=status, source='*', target=STATUS.active)
    def activate(self, description=None, *args, **kwargs):
        """Activate the Member."""
        return

    @fsm_log_by
    @fsm_log_description
    @transition(field=status, source='*', target=STATUS.inactive)
    def deactivate(self, description=None, *args, **kwargs):
        """Deactivate the Member."""
        return
<commit_msg>Add direct MC Check<commit_after># Standard Libary
import logging
import uuid

# Third-Party
from django_fsm import FSMIntegerField
from django_fsm import transition
from django_fsm_log.decorators import fsm_log_by
from django_fsm_log.decorators import fsm_log_description
from dry_rest_permissions.generics import allow_staff_or_superuser
from dry_rest_permissions.generics import authenticated_users
from model_utils import Choices
from model_utils.models import TimeStampedModel
from django_fsm_log.models import StateLog
from django.contrib.contenttypes.fields import GenericRelation
from django.utils.timezone import localdate

# Django
from django.db import models
from django.db.models import Q
from django.utils.functional import cached_property
from django.apps import apps

# First-Party
from api.managers import MemberManager

log = logging.getLogger(__name__)

class Member(TimeStampedModel):
    id = models.UUIDField(
        primary_key=True,
        default=uuid.uuid4,
        editable=False,
    )
    STATUS = Choices(
        (-10, 'inactive', 'Inactive',),
        (0, 'new', 'New',),
        (10, 'active', 'Active',),
    )

    status = FSMIntegerField(
        help_text="""DO NOT CHANGE MANUALLY unless correcting a mistake.  Use the buttons to change state.""",
        choices=STATUS,
        default=STATUS.new,
    )

    MEM_STATUS = Choices(
        (10, 'active', 'Active',),
        (20, 'active_internal', 'Active Internal',),
        (30, 'active_licensed', 'Active Licensed',),
        (40, 'cancelled', 'Cancelled',),
        (50, 'closed', 'Closed',),
        (60, 'closed_merged', 'Closed Merged',),
        (70, 'closed_revoked', 'Closed Revoked',),
        (80, 'closed_voluntary', 'Closed Voluntary',),
        (90, 'expelled', 'Expelled',),
        (100, 'expired', 'Expired',),
        (105, 'expired_licensed', 'Expired Licensed',),
        (110, 'lapsed', 'Lapsed',),
        (120, 'not_approved', 'Not Approved',),
        (130, 'pending', 'Pending',),
        (140, 'pending_voluntary', 'Pending Voluntary',),
        (150, 'suspended', 'Suspended',),
        (160, 'suspended_membership', 'Suspended Membership',),
        (170, 'awaiting_payment', 'Awaiting Payment',),
    )

    mem_status = models.IntegerField(
        choices=MEM_STATUS,
        null=True,
        blank=True,
    )

    SUB_STATUS = Choices(
        (10, 'active', 'Active',),
        (20, 'expired', 'Expired',),
        (30, 'pending', 'Pending',),
        (40, 'lapsedRenew', 'Lapsed',),
        (50, 'cancelled', 'Cancelled',),
        (60, 'swapped', 'Swapped',),
    )

    sub_status = models.IntegerField(
        choices=SUB_STATUS,
        null=True,
        blank=True,
    )

    MEM_CODE = Choices(
        (10, 'RG', 'RG Regular'),
        (20, 'R5', 'R5 Regular 50 Year'),
        (30, 'SN', 'SN Senior'),
        (40, 'S5', 'S5 Senior 50 Year'),
        (50, 'SL', 'SL Senior Legacy'),
        (60, 'Y1', 'Y1 Youth Initial'),
        (70, 'Y2', 'Y2 Youth Subsequent'),
        (80, 'LF', 'LF Lifetime Regular'),
        (90, 'L5', 'L5 Lifetime 50 Year'),
        (100, 'LY', 'LY Lifetime Youth'),
        (110, 'LS', 'LS Lifetime Senior'),
        (120, 'AS', 'AS Associate'),
    )

    mem_code = models.IntegerField(
        choices=MEM_CODE,
        null=True,
        blank=True,
    )

    inactive_date = models.DateField(
        null=True,
        blank=True,
    )

    INACTIVE_REASON = Choices(
        (1, 'Non_renewal', 'Non-Renewal'),
        (2, 'Renewed', 'Renewed'),
        (3, 'NotCancelled', 'Not Cancelled'),
        (4, 'Non_Payment', 'Non-Payment'),
        (5, 'Expired', 'Expired'),
        (6, 'Deceased', 'Deceased'),
        (7, 'changedOption', 'changedOption'),
        (8, 'Other', 'Other'),
        (9, 'cancelled', 'cancelled'),
        (10, 'Transferred', 'Transferred'),
        (11, 'swappedChapter', 'swappedChapter'),
        (12, 'swapped', 'swapped'),
    )

    inactive_reason = models.IntegerField(
        choices=INACTIVE_REASON,
        null=True,
        blank=True,
    )

    PART = Choices(
        # (-1, 'director', 'Director'),
        (1, 'tenor', 'Tenor'),
        (2, 'lead', 'Lead'),
        (3, 'baritone', 'Baritone'),
        (4, 'bass', 'Bass'),
    )

    part = models.IntegerField(
        choices=PART,
        null=True,
        blank=True,
    )

    established_date = models.DateField(
        null=True,
        blank=True,
    )

    current_through = models.DateField(
        null=True,
        blank=True,
    )

    mc_pk = models.UUIDField(
        null=True,
        blank=True,
        unique=True,
        db_index=True,
    )

    # Properties
    @cached_property
    def is_mc(self):
        return bool(self.mc_pk)

    # FKs
    group = models.ForeignKey(
        'Group',
        related_name='members',
        on_delete=models.CASCADE,
    )

    person = models.ForeignKey(
        'Person',
        related_name='members',
        on_delete=models.CASCADE,
    )

    subscription = models.ForeignKey(
        'Subscription',
        related_name='members',
        on_delete=models.CASCADE,
        null=True,
        blank=True,
    )

    # Relations
    statelogs = GenericRelation(
        StateLog,
        related_query_name='members',
    )

    # Internals
    objects = MemberManager()

    class Meta:
        unique_together = (
            ('group', 'person',),
        )

    class JSONAPIMeta:
        resource_name = "member"

    def __str__(self):
        return str(self.id)

    def clean(self):
        pass
        # if self.is_admin:
        #     try:
        #         is_active = self.person.user.is_active
        #     except ObjectDoesNotExist:
        #         raise ValidationError(
        #             {'is_admin': 'Admin must User account.'}
        #         )
        #     if not is_active:
        #         raise ValidationError(
        #             {'is_admin': 'Admin User account must be active.'}
        #         )

    # Methods
    def mc_check(self):
        Join = apps.get_model('bhs.join')
        if not self.mc_pk:
            return ValueError("Not a MC Record")
        status = bool(Join.objects.filter(
            Q(inactive_date=None) |
            Q(
                inactive_date__gt=localdate(),
                subscription__status='active',
            ),
            structure__id=self.group.mc_pk,
            subscription__human__id=self.person.mc_pk,
        ))
        if status and self.status != self.STATUS.active:
            self.activate()
            self.save()
            return "Activated"
        if not status and self.status != self.STATUS.inactive:
            self.deactivate()
            self.save()
            return "Deactivated"
        return "No change"

    # Permissions
    @staticmethod
    @allow_staff_or_superuser
    @authenticated_users
    def has_read_permission(request):
        return True

    @allow_staff_or_superuser
    @authenticated_users
    def has_object_read_permission(self, request):
        return True

    @staticmethod
    @allow_staff_or_superuser
    @authenticated_users
    def has_write_permission(request):
        return any([
            request.user.is_group_manager,
        ])

    @allow_staff_or_superuser
    @authenticated_users
    def has_object_write_permission(self, request):
        return any([
            all([
                self.group.officers.filter(
                    person__user=request.user,
                    status__gt=0,
                ),
                self.mc_pk == None,
            ]),
        ])

    # Transitions
    @fsm_log_by
    @fsm_log_description
    @transition(field=status, source='*', target=STATUS.active)
    def activate(self, description=None, *args, **kwargs):
        """Activate the Member."""
        return

    @fsm_log_by
    @fsm_log_description
    @transition(field=status, source='*', target=STATUS.inactive)
    def deactivate(self, description=None, *args, **kwargs):
        """Deactivate the Member."""
        return
<|endoftext|>


<commit_before>from flask import request, redirect
import requests
from ..utils.status import get_status
from ..utils.reminders import create_reminder
from ..utils.twilioutil import send_text
import twilio.twiml
from twilio.rest import TwilioRestClient
import json
import dateutil.parser
from flask import request
from project.models import Phone

ACCOUNT_SID = "ayylmao"
AUTH_TOKEN = "ayylmao"

def call():
    resp = twilio.twiml.Response()
    phone = request.form.get("From", "")
    phon = Phone.query.filter(Phone.phone_number == phone).first()
    if phon is None:
        send_text(phone, "Register an account at http://queri.me/login/")
        with resp.gather(numDigits=1, action="/internal/handle-key-2", method="POST") as g:
            g.say("Please register your phone number at our website. Check your text messages for a link.")
        return str(resp)

    resp.say("Hello!")
    with resp.gather(numDigits=1, action="/internal/handle-key", method="POST") as g:
        g.say("To get the status of a machine or set a reminder, press 1.")
    return str(resp)

def handle_key_2():
    digit_pressed = request.values.get('Digits', None)
    resp = twilio.twiml.Response()
    if digit_pressed == "0":
        resp.play("http://a.tumblr.com/tumblr_mascpn4kyJ1qejfr7o1.mp3")
    elif digit_pressed == "4":
        resp.play("http://queri.me/static/MLG.mp3")
    elif digit_pressed == "2":
        # 22884646#*
        resp.gather(numDigits=9, action="/internal/konami", method="POST", finishOnKey='')
    else:
        return redirect('/internal/call')
    return str(resp)

def konami():
    digits_pressed = request.values.get('Digits', None)
    resp = twilio.twiml.Response()
    if digits_pressed == "2884646#*":
        resp.play("http://queri.me/static/contra.mp3")
        return str(resp)
    return ''

def handle_key():
    digit_pressed = request.values.get('Digits', None)
    resp = twilio.twiml.Response()
    if digit_pressed == "1":
        resp.say("Make your request, then press any key.")
        resp.record(timeout=10, action='http://queri.me/internal/rec')
    elif digit_pressed == "0":
        resp.play("http://a.tumblr.com/tumblr_mascpn4kyJ1qejfr7o1.mp3")
    elif digit_pressed == "4":
        resp.play("http://queri.me/static/MLG.mp3")
    else:
        return redirect('/internal/call')
    return str(resp)

def do_wit(body, phone, recording=False):
    if not recording:
        wit = requests.get('https://api.wit.ai/message?v=20140905&q=%s' % body, headers={'Authorization':'Bearer L3VB34V6YTDFO4BRXNDQNAYMVOOF4BHB'}).text
    if recording:
        wit = requests.post('https://api.wit.ai/speech', headers={'Authorization':'Bearer L3VB34V6YTDFO4BRXNDQNAYMVOOF4BHB', 'Content-Type': 'audio/wav'}, data=body).text
    print wit
    jso = json.loads(wit)
    print str(jso)
    intent = jso['outcomes'][0]['intent']
    if intent == 'get_status':
        m = get_status(wit, phone)
    elif intent == 'remind':
        entities = jso['outcomes'][0]['entities']
        date = dateutil.parser.parse(entities['time'][0]['value']['from'])
        text = entities['message'][0]['value']
        m = create_reminder(date, text, phone)
    else:
        m = "Hmm? Try again please :("
    return m

def text():
    b = request.form.get('Body','')
    phone = request.form.get('From','')
    m = do_wit(b, phone)

    # Send to wit.ai for processing
    resp = twilio.twiml.Response()
    resp.message(m)
    return str(resp)

def rec():
    resp = twilio.twiml.Response()
    print repr(request.form)
    recording = requests.get(request.form.get('RecordingUrl',''), stream=True).content
    m = do_wit(recording,request.form.get('From',''),recording=True)
    resp.say(m)
    return str(resp)
<commit_msg>shrek<commit_after>from flask import request, redirect
import requests
from ..utils.status import get_status
from ..utils.reminders import create_reminder
from ..utils.twilioutil import send_text
import twilio.twiml
from twilio.rest import TwilioRestClient
import json
import dateutil.parser
from flask import request
from project.models import Phone

ACCOUNT_SID = "ayylmao"
AUTH_TOKEN = "ayylmao"

def call():
    resp = twilio.twiml.Response()
    phone = request.form.get("From", "")
    phon = Phone.query.filter(Phone.phone_number == phone).first()
    if phon is None or not phon.is_confirmed:
        send_text(phone, "Register an account at http://queri.me/login/")
        with resp.gather(numDigits=1, action="/internal/handle-key-2", method="POST") as g:
            g.say("Please register your phone number at our website. Check your text messages for a link.")
        return str(resp)

    resp.say("Hello!")
    with resp.gather(numDigits=1, action="/internal/handle-key", method="POST") as g:
        g.say("To get the status of a machine or set a reminder, press 1.")
    return str(resp)

def handle_key_2():
    digit_pressed = request.values.get('Digits', None)
    resp = twilio.twiml.Response()
    if digit_pressed == "0":
        resp.play("http://a.tumblr.com/tumblr_mascpn4kyJ1qejfr7o1.mp3")
    elif digit_pressed == "4":
        resp.play("http://queri.me/static/MLG.mp3")
    elif digit_pressed == "2":
        # 22884646#*
        resp.gather(numDigits=9, action="/internal/konami", method="POST", finishOnKey='')
    else:
        return redirect('/internal/call')
    return str(resp)

def konami():
    digits_pressed = request.values.get('Digits', None)
    resp = twilio.twiml.Response()
    if digits_pressed == "2884646#*":
        resp.play("http://queri.me/static/contra.mp3")
        return str(resp)
    return ''

def handle_key():
    digit_pressed = request.values.get('Digits', None)
    resp = twilio.twiml.Response()
    if digit_pressed == "1":
        resp.say("Make your request, then press any key.")
        resp.record(timeout=10, action='http://queri.me/internal/rec')
    elif digit_pressed == "0":
        resp.play("http://a.tumblr.com/tumblr_mascpn4kyJ1qejfr7o1.mp3")
    elif digit_pressed == "4":
        resp.play("http://queri.me/static/MLG.mp3")
    else:
        return redirect('/internal/call')
    return str(resp)

def do_wit(body, phone, recording=False):
    if not recording:
        wit = requests.get('https://api.wit.ai/message?v=20140905&q=%s' % body, headers={'Authorization':'Bearer L3VB34V6YTDFO4BRXNDQNAYMVOOF4BHB'}).text
    if recording:
        wit = requests.post('https://api.wit.ai/speech', headers={'Authorization':'Bearer L3VB34V6YTDFO4BRXNDQNAYMVOOF4BHB', 'Content-Type': 'audio/wav'}, data=body).text
    print wit
    jso = json.loads(wit)
    print str(jso)
    intent = jso['outcomes'][0]['intent']
    if intent == 'get_status':
        m = get_status(wit, phone)
    elif intent == 'remind':
        entities = jso['outcomes'][0]['entities']
        date = dateutil.parser.parse(entities['time'][0]['value']['from'])
        text = entities['message'][0]['value']
        m = create_reminder(date, text, phone)
    else:
        m = "Hmm? Try again please :("
    return m

def text():
    b = request.form.get('Body','')
    phone = request.form.get('From','')
    m = do_wit(b, phone)

    # Send to wit.ai for processing
    resp = twilio.twiml.Response()
    resp.message(m)
    return str(resp)

def rec():
    resp = twilio.twiml.Response()
    print repr(request.form)
    recording = requests.get(request.form.get('RecordingUrl',''), stream=True).content
    m = do_wit(recording,request.form.get('From',''),recording=True)
    resp.say(m)
    return str(resp)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

""" License

    Copyright (C) 2013 YunoHost

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published
    by the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program; if not, see http://www.gnu.org/licenses

"""

""" yunohost_domain.py

    Manage domains
"""
import os
import sys
import datetime
import re
import shutil
import json
import yaml
import errno
from urllib import urlopen

from moulinette.core import MoulinetteError

def domain_list(auth, raw=False, filter=None, limit=None, offset=None):
    """
    List domains

    Keyword argument:
        raw -- Return domains as a bash-usable list instead of JSON
        filter -- LDAP filter used to search
        offset -- Starting number for domain fetching
        limit -- Maximum number of domain fetched

    """
    result_list = []

    # Set default arguments values
    if offset is None:
        offset = 0
    if limit is None:
        limit = 1000
    if filter is None:
        filter = 'virtualdomain=*'

    result = auth.search('ou=domains,dc=yunohost,dc=org', filter, ['virtualdomain'])

    if len(result) > offset and limit > 0:
        for domain in result[offset:offset+limit]:
            result_list.append(domain['virtualdomain'][0])

    if raw:
        for domain in result_list:
            print domain
    else:
        return { 'domains': result_list }

def domain_add(auth, domain, dyndns=False):
    """
    Create a custom domain

    Keyword argument:
        domain -- Domain name to add
        dyndns -- Subscribe to DynDNS

    """
    from yunohost.service import service_regenconf
    from yunohost.hook import hook_callback

    attr_dict = { 'objectClass' : ['mailDomain', 'top'] }
    try:
        ip = str(urlopen('http://ip.yunohost.org').read())
    except IOError:
        ip = "127.0.0.1"
    now = datetime.datetime.now()
    timestamp = str(now.year) + str(now.month) + str(now.day)

    if domain in domain_list(auth)['domains']:
        raise MoulinetteError(errno.EEXIST, m18n.n('domain_exists'))

    # DynDNS domain
    if dyndns:
        if len(domain.split('.')) < 3:
            raise MoulinetteError(errno.EINVAL, m18n.n('domain_dyndns_invalid'))
        import requests
        from yunohost.dyndns import dyndns_subscribe

        try:
            r = requests.get('https://dyndns.yunohost.org/domains')
        except ConnectionError:
            pass
        else:
            dyndomains = json.loads(r.text)
            dyndomain  = '.'.join(domain.split('.')[1:])
            if dyndomain in dyndomains:
                if os.path.exists('/etc/cron.d/yunohost-dyndns'):
                    raise MoulinetteError(errno.EPERM,
                                          m18n.n('domain_dyndns_already_subscribed'))
                dyndns_subscribe(domain=domain)
            else:
                raise MoulinetteError(errno.EINVAL,
                                      m18n.n('domain_dyndns_root_unknown'))

    try:
        # Commands
        ssl_dir = '/usr/share/yunohost/yunohost-config/ssl/yunoCA'
        ssl_domain_path  = '/etc/yunohost/certs/%s' % domain
        with open('%s/serial' % ssl_dir, 'r') as f:
            serial = f.readline().rstrip()
        try: os.listdir(ssl_domain_path)
        except OSError: os.makedirs(ssl_domain_path)

        command_list = [
            'cp %s/openssl.cnf %s'                               % (ssl_dir, ssl_domain_path),
            'sed -i "s/yunohost.org/%s/g" %s/openssl.cnf'        % (domain, ssl_domain_path),
            'openssl req -new -config %s/openssl.cnf -days 3650 -out %s/certs/yunohost_csr.pem -keyout %s/certs/yunohost_key.pem -nodes -batch'
            % (ssl_domain_path, ssl_dir, ssl_dir),
            'openssl ca -config %s/openssl.cnf -days 3650 -in %s/certs/yunohost_csr.pem -out %s/certs/yunohost_crt.pem -batch'
            % (ssl_domain_path, ssl_dir, ssl_dir),
            'ln -s /etc/ssl/certs/ca-yunohost_crt.pem %s/ca.pem' % ssl_domain_path,
            'cp %s/certs/yunohost_key.pem    %s/key.pem'         % (ssl_dir, ssl_domain_path),
            'cp %s/newcerts/%s.pem %s/crt.pem'                   % (ssl_dir, serial, ssl_domain_path),
            'chmod 755 %s'                                       % ssl_domain_path,
            'chmod 640 %s/key.pem'                               % ssl_domain_path,
            'chmod 640 %s/crt.pem'                               % ssl_domain_path,
            'chmod 600 %s/openssl.cnf'                           % ssl_domain_path,
            'chown root:metronome %s/key.pem'                    % ssl_domain_path,
            'chown root:metronome %s/crt.pem'                    % ssl_domain_path,
            'cat %s/ca.pem >> %s/crt.pem'                        % (ssl_domain_path, ssl_domain_path)
        ]

        for command in command_list:
            if os.system(command) != 0:
                raise MoulinetteError(errno.EIO,
                                      m18n.n('domain_cert_gen_failed'))

        try:
            auth.validate_uniqueness({ 'virtualdomain': domain })
        except MoulinetteError:
            raise MoulinetteError(errno.EEXIST, m18n.n('domain_exists'))

        attr_dict['virtualdomain'] = domain

        if not auth.add('virtualdomain=%s,ou=domains' % domain, attr_dict):
            raise MoulinetteError(errno.EIO, m18n.n('domain_creation_failed'))

        try:
            with open('/etc/yunohost/installed', 'r') as f:
                service_regenconf(service='nginx')
                service_regenconf(service='metronome')
                service_regenconf(service='dnsmasq')
                os.system('yunohost app ssowatconf > /dev/null 2>&1')
        except IOError: pass
    except:
        # Force domain removal silently
        try: domain_remove(auth, domain, True)
        except: pass
        raise

    hook_callback('post_domain_add', args=[domain])

    msignals.display(m18n.n('domain_created'), 'success')

def domain_remove(auth, domain, force=False):
    """
    Delete domains

    Keyword argument:
        domain -- Domain to delete
        force -- Force the domain removal

    """
    from yunohost.service import service_regenconf
    from yunohost.hook import hook_callback

    if not force and domain not in domain_list(auth)['domains']:
        raise MoulinetteError(errno.EINVAL, m18n.n('domain_unknown'))

    # Check if apps are installed on the domain
    for app in os.listdir('/etc/yunohost/apps/'):
        with open('/etc/yunohost/apps/' + app +'/settings.yml') as f:
            try:
                app_domain = yaml.load(f)['domain']
            except:
                continue
            else:
                if app_domain == domain:
                    raise MoulinetteError(errno.EPERM,
                                          m18n.n('domain_uninstall_app_first'))

    if auth.remove('virtualdomain=' + domain + ',ou=domains') or force:
        os.system('rm -rf /etc/yunohost/certs/%s' % domain)
    else:
        raise MoulinetteError(errno.EIO, m18n.n('domain_deletion_failed'))

    service_regenconf(service='nginx')
    service_regenconf(service='metronome')
    service_regenconf(service='dnsmasq')
    os.system('yunohost app ssowatconf > /dev/null 2>&1')

    hook_callback('post_domain_remove', args=[domain])

    msignals.display(m18n.n('domain_deleted'), 'success')

def domain_dns_conf(domain, ttl=None):
    """
    Generate DNS configuration for a domain

    Keyword argument:
        domain -- Domain name
        ttl -- Time to live
    """

    ttl = 3600 if ttl is None else ttl

    ip4 = urlopen("http://ip.yunohost.org").read().strip()

    result = "@ {ttl} IN A {ip4}\n* {ttl} IN A {ip4}\n".format(ttl=ttl, ip4=ip4)

    ip6 = None

    try:
        ip6 = urlopen("http://ip6.yunohost.org").read().strip()
    except Exception:
        pass
    else:
        result += "@ {ttl} IN AAAA {ip6}\n* {ttl} IN AAAA {ip6}\n".format(ttl=ttl, ip6=ip6)

    result += "\n_xmpp-client._tcp {ttl} IN SRV 0 5 5222 {domain}.\n_xmpp-server._tcp {ttl} IN SRV 0 5 5269 {domain}.\n".format(ttl=ttl, domain=domain)

    result += "muc {ttl} IN CNAME @\npubsub {ttl} IN CNAME @\nvjud {ttl} IN CNAME @\n\n".format(ttl=ttl)

    result += "@ {ttl} IN MX 10 {domain}.\n".format(ttl=ttl, domain=domain)

    if ip6 is None:
        result += '@ {ttl} IN TXT "v=spf1 a mx ip4:{ip4} -all"\n'.format(ttl=ttl, ip4=ip4)
    else:
        result += '@ {ttl} IN TXT "v=spf1 a mx ip4:{ip4} ip6:{ip6} -all"\n'.format(ttl=ttl, ip4=ip4, ip6=ip6)

    return result
<commit_msg>[enh] Add DKIM DNS record in domain_dns_conf (close #198)<commit_after># -*- coding: utf-8 -*-

""" License

    Copyright (C) 2013 YunoHost

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published
    by the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program; if not, see http://www.gnu.org/licenses

"""

""" yunohost_domain.py

    Manage domains
"""
import os
import sys
import datetime
import re
import shutil
import json
import yaml
import errno
from urllib import urlopen

from moulinette.core import MoulinetteError

def domain_list(auth, raw=False, filter=None, limit=None, offset=None):
    """
    List domains

    Keyword argument:
        raw -- Return domains as a bash-usable list instead of JSON
        filter -- LDAP filter used to search
        offset -- Starting number for domain fetching
        limit -- Maximum number of domain fetched

    """
    result_list = []

    # Set default arguments values
    if offset is None:
        offset = 0
    if limit is None:
        limit = 1000
    if filter is None:
        filter = 'virtualdomain=*'

    result = auth.search('ou=domains,dc=yunohost,dc=org', filter, ['virtualdomain'])

    if len(result) > offset and limit > 0:
        for domain in result[offset:offset+limit]:
            result_list.append(domain['virtualdomain'][0])

    if raw:
        for domain in result_list:
            print domain
    else:
        return { 'domains': result_list }

def domain_add(auth, domain, dyndns=False):
    """
    Create a custom domain

    Keyword argument:
        domain -- Domain name to add
        dyndns -- Subscribe to DynDNS

    """
    from yunohost.service import service_regenconf
    from yunohost.hook import hook_callback

    attr_dict = { 'objectClass' : ['mailDomain', 'top'] }
    try:
        ip = str(urlopen('http://ip.yunohost.org').read())
    except IOError:
        ip = "127.0.0.1"
    now = datetime.datetime.now()
    timestamp = str(now.year) + str(now.month) + str(now.day)

    if domain in domain_list(auth)['domains']:
        raise MoulinetteError(errno.EEXIST, m18n.n('domain_exists'))

    # DynDNS domain
    if dyndns:
        if len(domain.split('.')) < 3:
            raise MoulinetteError(errno.EINVAL, m18n.n('domain_dyndns_invalid'))
        import requests
        from yunohost.dyndns import dyndns_subscribe

        try:
            r = requests.get('https://dyndns.yunohost.org/domains')
        except ConnectionError:
            pass
        else:
            dyndomains = json.loads(r.text)
            dyndomain  = '.'.join(domain.split('.')[1:])
            if dyndomain in dyndomains:
                if os.path.exists('/etc/cron.d/yunohost-dyndns'):
                    raise MoulinetteError(errno.EPERM,
                                          m18n.n('domain_dyndns_already_subscribed'))
                dyndns_subscribe(domain=domain)
            else:
                raise MoulinetteError(errno.EINVAL,
                                      m18n.n('domain_dyndns_root_unknown'))

    try:
        # Commands
        ssl_dir = '/usr/share/yunohost/yunohost-config/ssl/yunoCA'
        ssl_domain_path  = '/etc/yunohost/certs/%s' % domain
        with open('%s/serial' % ssl_dir, 'r') as f:
            serial = f.readline().rstrip()
        try: os.listdir(ssl_domain_path)
        except OSError: os.makedirs(ssl_domain_path)

        command_list = [
            'cp %s/openssl.cnf %s'                               % (ssl_dir, ssl_domain_path),
            'sed -i "s/yunohost.org/%s/g" %s/openssl.cnf'        % (domain, ssl_domain_path),
            'openssl req -new -config %s/openssl.cnf -days 3650 -out %s/certs/yunohost_csr.pem -keyout %s/certs/yunohost_key.pem -nodes -batch'
            % (ssl_domain_path, ssl_dir, ssl_dir),
            'openssl ca -config %s/openssl.cnf -days 3650 -in %s/certs/yunohost_csr.pem -out %s/certs/yunohost_crt.pem -batch'
            % (ssl_domain_path, ssl_dir, ssl_dir),
            'ln -s /etc/ssl/certs/ca-yunohost_crt.pem %s/ca.pem' % ssl_domain_path,
            'cp %s/certs/yunohost_key.pem    %s/key.pem'         % (ssl_dir, ssl_domain_path),
            'cp %s/newcerts/%s.pem %s/crt.pem'                   % (ssl_dir, serial, ssl_domain_path),
            'chmod 755 %s'                                       % ssl_domain_path,
            'chmod 640 %s/key.pem'                               % ssl_domain_path,
            'chmod 640 %s/crt.pem'                               % ssl_domain_path,
            'chmod 600 %s/openssl.cnf'                           % ssl_domain_path,
            'chown root:metronome %s/key.pem'                    % ssl_domain_path,
            'chown root:metronome %s/crt.pem'                    % ssl_domain_path,
            'cat %s/ca.pem >> %s/crt.pem'                        % (ssl_domain_path, ssl_domain_path)
        ]

        for command in command_list:
            if os.system(command) != 0:
                raise MoulinetteError(errno.EIO,
                                      m18n.n('domain_cert_gen_failed'))

        try:
            auth.validate_uniqueness({ 'virtualdomain': domain })
        except MoulinetteError:
            raise MoulinetteError(errno.EEXIST, m18n.n('domain_exists'))

        attr_dict['virtualdomain'] = domain

        if not auth.add('virtualdomain=%s,ou=domains' % domain, attr_dict):
            raise MoulinetteError(errno.EIO, m18n.n('domain_creation_failed'))

        try:
            with open('/etc/yunohost/installed', 'r') as f:
                service_regenconf(service='nginx')
                service_regenconf(service='metronome')
                service_regenconf(service='dnsmasq')
                os.system('yunohost app ssowatconf > /dev/null 2>&1')
        except IOError: pass
    except:
        # Force domain removal silently
        try: domain_remove(auth, domain, True)
        except: pass
        raise

    hook_callback('post_domain_add', args=[domain])

    msignals.display(m18n.n('domain_created'), 'success')

def domain_remove(auth, domain, force=False):
    """
    Delete domains

    Keyword argument:
        domain -- Domain to delete
        force -- Force the domain removal

    """
    from yunohost.service import service_regenconf
    from yunohost.hook import hook_callback

    if not force and domain not in domain_list(auth)['domains']:
        raise MoulinetteError(errno.EINVAL, m18n.n('domain_unknown'))

    # Check if apps are installed on the domain
    for app in os.listdir('/etc/yunohost/apps/'):
        with open('/etc/yunohost/apps/' + app +'/settings.yml') as f:
            try:
                app_domain = yaml.load(f)['domain']
            except:
                continue
            else:
                if app_domain == domain:
                    raise MoulinetteError(errno.EPERM,
                                          m18n.n('domain_uninstall_app_first'))

    if auth.remove('virtualdomain=' + domain + ',ou=domains') or force:
        os.system('rm -rf /etc/yunohost/certs/%s' % domain)
    else:
        raise MoulinetteError(errno.EIO, m18n.n('domain_deletion_failed'))

    service_regenconf(service='nginx')
    service_regenconf(service='metronome')
    service_regenconf(service='dnsmasq')
    os.system('yunohost app ssowatconf > /dev/null 2>&1')

    hook_callback('post_domain_remove', args=[domain])

    msignals.display(m18n.n('domain_deleted'), 'success')

def domain_dns_conf(domain, ttl=None):
    """
    Generate DNS configuration for a domain

    Keyword argument:
        domain -- Domain name
        ttl -- Time to live
    """

    ttl = 3600 if ttl is None else ttl

    ip4 = urlopen("http://ip.yunohost.org").read().strip()

    result = "@ {ttl} IN A {ip4}\n* {ttl} IN A {ip4}\n".format(ttl=ttl, ip4=ip4)

    ip6 = None

    try:
        ip6 = urlopen("http://ip6.yunohost.org").read().strip()
    except Exception:
        pass
    else:
        result += "@ {ttl} IN AAAA {ip6}\n* {ttl} IN AAAA {ip6}\n".format(ttl=ttl, ip6=ip6)

    result += "\n_xmpp-client._tcp {ttl} IN SRV 0 5 5222 {domain}.\n_xmpp-server._tcp {ttl} IN SRV 0 5 5269 {domain}.\n".format(ttl=ttl, domain=domain)

    result += "muc {ttl} IN CNAME @\npubsub {ttl} IN CNAME @\nvjud {ttl} IN CNAME @\n\n".format(ttl=ttl)

    result += "@ {ttl} IN MX 10 {domain}.\n".format(ttl=ttl, domain=domain)

    if ip6 is None:
        result += '@ {ttl} IN TXT "v=spf1 a mx ip4:{ip4} -all"\n'.format(ttl=ttl, ip4=ip4)
    else:
        result += '@ {ttl} IN TXT "v=spf1 a mx ip4:{ip4} ip6:{ip6} -all"\n'.format(ttl=ttl, ip4=ip4, ip6=ip6)

    try:
        with open('/etc/dkim/{domain}.mail.txt'.format(domain=domain)) as f:
            dkim_content = f.read()
    except IOError:
        pass
    else:
        dkim = re.match((
            r'^(?P<domain>[a-z_\-\.]+)[\s]+([0-9]+[\s]+)?IN[\s]+TXT[\s]+[^"]*'
            '(?=.*(;[\s]*|")v=(?P<v>[^";]+))'
            '(?=.*(;[\s]*|")k=(?P<k>[^";]+))'
            '(?=.*(;[\s]*|")p=(?P<p>[^";]+))'), dkim_content, re.M|re.S
        )
        if dkim:
            result += '{domain} {ttl} IN TXT "v={v}; k={k}; p={p}"'.format(
                domain=dkim.group('domain'), ttl=ttl,
                v=dkim.group('v'), k=dkim.group('k'), p=dkim.group('p')
            )

    return result
<|endoftext|>


<commit_before>import sys
import getopt
import os
import csv
import cPickle
import gzip
import theano
import time, PIL.Image
import numpy as np

def get_git_home():
    testpath = '.'
    while not '.git' in os.listdir(testpath) and not os.path.abspath(testpath) == '/':
        testpath = os.path.sep.join(('..', testpath))
    if not os.path.abspath(testpath) == '/':
        return os.path.abspath(testpath)
    else:
        raise ValueError, "Not in git repository"

HOME = get_git_home()
sys.path.append(HOME)

from hashmapd.load_config import LoadConfig, DefaultConfig, dict_to_cfg

TRAINING_FILE = "training_data"
VALIDATION_FILE = "validation_data"
TESTING_FILE = "testing_data"
PICKLED_TYPE = ".pkl.gz"
#
#def read_user_word_pixels(cfg):
#    """
#    Reads in data from user_word_vectors.csv
#    """
#    print "attempting to read " + cfg.input.csv_data
#
#    raw_pixels = np.zeros((cfg.input.number_of_examples, cfg.shape.input_vector_length), dtype=theano.config.floatX) #store as float so that normalized_counts uses float math
#
#    vectorReader = csv.DictReader(open(cfg.input.csv_data, 'rb'), delimiter=',')
#    iter=0
#    for row in vectorReader:
#        if iter % 100==0:
#            print 'reading row '+ str(iter) + '..' #MKT: presumably a nicer way to do this
#        iter += 1
#        user_id = int(row['user_id'])
#        word_id = int(row['word_id'])
#        pixel = float(row['pixel'])
#        raw_pixels[user_id,word_id] = pixel
#
#    #total_user_pixels = raw_pixels.sum(axis=1)
#    #normalized_pixels = (raw_pixels.transpose()/total_user_pixels).transpose()
#
#    print 'done reading input'
#    print raw_pixels
#    return raw_pixels
#
def read_user_word_counts(cfg):
    """
    Reads in data from user_word_vectors.csv
    """
    print "attempting to read " + cfg.raw.csv_data

    raw_counts = np.zeros((cfg.input.number_of_examples, cfg.shape.input_vector_length), dtype=theano.config.floatX) #store as float so that normalized_counts uses float math
    vectorReader = csv.DictReader(open(cfg.raw.csv_data, 'rb'), delimiter=',')
    iter=0
    for row in vectorReader:
        if iter % 10000==0:  #MKT: presumably a nicer way to do this ?
            print 'reading row '+ str(iter) + '..'
        iter += 1
        user_id = int(row['user_id'])-1
        word_id = int(row['word_id'])-1
        count = int(row['count'])
        raw_counts[user_id,word_id] = count

    print 'done reading input'
    return raw_counts

def read_user_labels(cfg):
    """
    Reads in data from fake_txt_users.csv
    """
    print "attempting to read " + cfg.raw.users_file

    users = []
    vectorReader = csv.DictReader(open(cfg.raw.users_file, 'rb'), delimiter=',')
    iter=0
    for row in vectorReader:
        if iter % 10000==0:  #MKT: presumably a nicer way to do this ?
            print 'reading row '+ str(iter) + '..'
        iter += 1
        user_id = int(row['user_id'])
        user_name = row['screen']
        users.append((user_id, user_name))

    print 'done reading input'
    return dict(users)

def validate_config(cfg):

    num_examples = cfg.input.number_of_examples
    train_cutoff = cfg.input.number_for_training
    validate_cutoff = train_cutoff+cfg.input.number_for_validation
    test_cutoff = validate_cutoff+cfg.input.number_for_testing

    assert test_cutoff<=num_examples, \
        "total of number of examples should be more than cases for train, validate and test"
    assert 0<train_cutoff, \
        "config fail, number_for_training needs to be > 0"
    assert train_cutoff<=validate_cutoff, \
        "config fail, number_for_validation should be > 0"
    assert validate_cutoff<=test_cutoff, \
        "config fail, number_for_testing should be >= 0"

def get_filename(name, number, filetype=PICKLED_TYPE, extsep='.'):
    return os.path.join("data", "%s_%s%s"%(name, number, filetype))

def normalize_and_output_pickled_data(cfg, raw_counts, user_labels):

    print "outputting full data set"

    num_examples = cfg.input.number_of_examples
    train_cutoff = cfg.input.number_for_training
    validate_cutoff = train_cutoff+cfg.input.number_for_validation
    test_cutoff = validate_cutoff+cfg.input.number_for_testing
    batch_size = cfg.train.train_batch_size

    print raw_counts
    train_set_x = raw_counts[0:train_cutoff]
    train_sums = train_set_x.sum(axis=1)
    mean_doc_size = train_sums.mean()
    print train_set_x
    print train_sums

    if (cfg.input.number_for_validation ==0):
        print 'WARNING: no examples set aside for validation, copying train set data for validation (as a quick hack only)'
        valid_set_x = train_set_x
    else:
        valid_set_x = raw_counts[train_cutoff:validate_cutoff]
    valid_sums = valid_set_x.sum(axis=1)

    if (cfg.input.number_for_testing ==0):
        print 'WARNING: no examples set aside for testing, copying validation set data for training (as a quick hack only)'
        test_set_x = valid_set_x
    else:
        test_set_x = raw_counts[validate_cutoff:test_cutoff]
    test_sums = test_set_x.sum(axis=1)

    print '...  pickling and zipping train/validate/test data to data directory'

    train_file = gzip.open(get_filename(TRAINING_FILE, 0),'wb')
    valid_file = gzip.open(get_filename(VALIDATION_FILE, 0),'wb')
    test_file = gzip.open(get_filename(TESTING_FILE, 0), 'wb')

    if (cfg.train.first_layer_type=='poisson'):
        cPickle.dump((train_set_x,train_sums,np.zeros(train_sums.shape,dtype=theano.config.floatX)), train_file, cPickle.HIGHEST_PROTOCOL)
        cPickle.dump((valid_set_x,valid_sums,np.zeros(valid_sums.shape,dtype=theano.config.floatX)), valid_file, cPickle.HIGHEST_PROTOCOL)
        cPickle.dump((test_set_x,test_sums,np.zeros(test_sums.shape,dtype=theano.config.floatX)), test_file, cPickle.HIGHEST_PROTOCOL)
    else:
        cPickle.dump((normalize_data_x(train_set_x,train_sums,'training'),train_sums,np.zeros(train_sums.shape,dtype=theano.config.floatX)), train_file, cPickle.HIGHEST_PROTOCOL)
        cPickle.dump((normalize_data_x(valid_set_x,valid_sums,'validation'),valid_sums,np.zeros(valid_sums.shape,dtype=theano.config.floatX)), valid_file, cPickle.HIGHEST_PROTOCOL)
        cPickle.dump((normalize_data_x(test_set_x,test_sums,'testing'),test_sums,np.zeros(test_sums.shape,dtype=theano.config.floatX)), test_file, cPickle.HIGHEST_PROTOCOL)

    train_file.close()
    valid_file.close()
    test_file.close()

    data_info = {'training_prefix': os.path.join('data', TRAINING_FILE),
        'n_training_files': 1,
        'n_training_batches':train_cutoff/batch_size,
        'validation_prefix':  os.path.join('data', VALIDATION_FILE),
        'n_validation_files': 1,
        'n_validation_batches': validate_cutoff/batch_size,
        'testing_prefix':  os.path.join('data', TESTING_FILE),
        'n_testing_files': 1,
        'n_testing_batches': test_cutoff/batch_size,
        'batches_per_file': (train_cutoff+validate_cutoff+test_cutoff)/batch_size,
        'mean_doc_size': mean_doc_size,
    }

    dict_to_cfg(data_info, 'info', 'data.cfg')

    print '...  pickling and zipping render_data to '+ cfg.input.render_data
    render_data = normalize_data_x(train_set_x,train_sums,'training')[0:num_examples]
    f = gzip.open(cfg.input.render_data,'wb')

    render_file_has_labels = cfg.input.render_data_has_labels
    data = render_data
    if (render_file_has_labels):
        labels = user_labels.items()
        labels.sort()
        labels = labels[0:num_examples] #FIXME makes dangerous assumption that user_labels has same ordering as render_data
        data = (render_data, labels)

    cPickle.dump(data,f, cPickle.HIGHEST_PROTOCOL)

    print data
    f.close()

def normalize_data_x(data_x,sums_x,name):
    for idx in xrange(len(sums_x)):
        if sums_x[idx] == 0.:
            print 'input for '+name+' user_id %i has all elements zero will not attempt to normalize '%idx
            sums_x[idx] = 1

    return (data_x.transpose()/sums_x).transpose()

if __name__ == '__main__':
    from optparse import OptionParser

    parser = OptionParser()
    parser.add_option("-f", "--file", dest="config", default="config",
        help="Path of the config file to use")
    (options, args) = parser.parse_args()

    cfg = LoadConfig(options.config)
    validate_config(cfg)

    if (cfg.input.csv_contains_counts):
        #read in csv and normalize
        raw_counts = read_user_word_counts(cfg)

    if (cfg.input.csv_contains_pixels):
        #read in pixel values between 0 and 1
        raw_counts = read_user_word_pixels(cfg)

    if (raw_counts == None):
        print "You need to set either cfg.input.csv_contains_pixels or cfg.input.csv_contains_counts to True otherwise not sure how to process data"

    render_file_has_labels = cfg.input.render_data_has_labels

    user_labels = None
    if (render_file_has_labels):
        user_labels = read_user_labels(cfg)

    #output into pickled data files
    normalize_and_output_pickled_data(cfg,raw_counts, user_labels)

<commit_msg>Fixed off-by-one user/label rotation in fake_txt<commit_after>import sys
import getopt
import os
import csv
import cPickle
import gzip
import theano
import time, PIL.Image
import numpy as np

def get_git_home():
    testpath = '.'
    while not '.git' in os.listdir(testpath) and not os.path.abspath(testpath) == '/':
        testpath = os.path.sep.join(('..', testpath))
    if not os.path.abspath(testpath) == '/':
        return os.path.abspath(testpath)
    else:
        raise ValueError, "Not in git repository"

HOME = get_git_home()
sys.path.append(HOME)

from hashmapd.load_config import LoadConfig, DefaultConfig, dict_to_cfg

TRAINING_FILE = "training_data"
VALIDATION_FILE = "validation_data"
TESTING_FILE = "testing_data"
PICKLED_TYPE = ".pkl.gz"
#
#def read_user_word_pixels(cfg):
#    """
#    Reads in data from user_word_vectors.csv
#    """
#    print "attempting to read " + cfg.input.csv_data
#
#    raw_pixels = np.zeros((cfg.input.number_of_examples, cfg.shape.input_vector_length), dtype=theano.config.floatX) #store as float so that normalized_counts uses float math
#
#    vectorReader = csv.DictReader(open(cfg.input.csv_data, 'rb'), delimiter=',')
#    iter=0
#    for row in vectorReader:
#        if iter % 100==0:
#            print 'reading row '+ str(iter) + '..' #MKT: presumably a nicer way to do this
#        iter += 1
#        user_id = int(row['user_id'])
#        word_id = int(row['word_id'])
#        pixel = float(row['pixel'])
#        raw_pixels[user_id,word_id] = pixel
#
#    #total_user_pixels = raw_pixels.sum(axis=1)
#    #normalized_pixels = (raw_pixels.transpose()/total_user_pixels).transpose()
#
#    print 'done reading input'
#    print raw_pixels
#    return raw_pixels
#
def read_user_word_counts(cfg):
    """
    Reads in data from user_word_vectors.csv
    """
    print "attempting to read " + cfg.raw.csv_data

    raw_counts = np.zeros((cfg.input.number_of_examples, cfg.shape.input_vector_length), dtype=theano.config.floatX) #store as float so that normalized_counts uses float math
    vectorReader = csv.DictReader(open(cfg.raw.csv_data, 'rb'), delimiter=',')
    iter=0
    for row in vectorReader:
        if iter % 10000==0:  #MKT: presumably a nicer way to do this ?
            print 'reading row '+ str(iter) + '..'
        iter += 1
        user_id = int(row['user_id'])
        word_id = int(row['word_id'])
        count = int(row['count'])
        assert user_id >= 0 and word_id >= 0 and count >= 0
        raw_counts[user_id,word_id] = count

    print 'done reading input'
    return raw_counts

def read_user_labels(cfg):
    """
    Reads in data from fake_txt_users.csv
    """
    print "attempting to read " + cfg.raw.users_file

    users = []
    vectorReader = csv.DictReader(open(cfg.raw.users_file, 'rb'), delimiter=',')
    iter=0
    for row in vectorReader:
        if iter % 10000==0:  #MKT: presumably a nicer way to do this ?
            print 'reading row '+ str(iter) + '..'
        iter += 1
        user_id = int(row['user_id'])
        user_name = row['screen']
        users.append((user_id, user_name))

    print 'done reading input'
    return dict(users)

def validate_config(cfg):

    num_examples = cfg.input.number_of_examples
    train_cutoff = cfg.input.number_for_training
    validate_cutoff = train_cutoff+cfg.input.number_for_validation
    test_cutoff = validate_cutoff+cfg.input.number_for_testing

    assert test_cutoff<=num_examples, \
        "total of number of examples should be more than cases for train, validate and test"
    assert 0<train_cutoff, \
        "config fail, number_for_training needs to be > 0"
    assert train_cutoff<=validate_cutoff, \
        "config fail, number_for_validation should be > 0"
    assert validate_cutoff<=test_cutoff, \
        "config fail, number_for_testing should be >= 0"

def get_filename(name, number, filetype=PICKLED_TYPE, extsep='.'):
    return os.path.join("data", "%s_%s%s"%(name, number, filetype))

def normalize_and_output_pickled_data(cfg, raw_counts, user_labels):

    print "outputting full data set"

    num_examples = cfg.input.number_of_examples
    train_cutoff = cfg.input.number_for_training
    validate_cutoff = train_cutoff+cfg.input.number_for_validation
    test_cutoff = validate_cutoff+cfg.input.number_for_testing
    batch_size = cfg.train.train_batch_size

    print raw_counts
    train_set_x = raw_counts[0:train_cutoff]
    train_sums = train_set_x.sum(axis=1)
    mean_doc_size = train_sums.mean()
    print train_set_x
    print train_sums

    if (cfg.input.number_for_validation ==0):
        print 'WARNING: no examples set aside for validation, copying train set data for validation (as a quick hack only)'
        valid_set_x = train_set_x
    else:
        valid_set_x = raw_counts[train_cutoff:validate_cutoff]
    valid_sums = valid_set_x.sum(axis=1)

    if (cfg.input.number_for_testing ==0):
        print 'WARNING: no examples set aside for testing, copying validation set data for training (as a quick hack only)'
        test_set_x = valid_set_x
    else:
        test_set_x = raw_counts[validate_cutoff:test_cutoff]
    test_sums = test_set_x.sum(axis=1)

    print '...  pickling and zipping train/validate/test data to data directory'

    train_file = gzip.open(get_filename(TRAINING_FILE, 0),'wb')
    valid_file = gzip.open(get_filename(VALIDATION_FILE, 0),'wb')
    test_file = gzip.open(get_filename(TESTING_FILE, 0), 'wb')

    if (cfg.train.first_layer_type=='poisson'):
        cPickle.dump((train_set_x,train_sums,np.zeros(train_sums.shape,dtype=theano.config.floatX)), train_file, cPickle.HIGHEST_PROTOCOL)
        cPickle.dump((valid_set_x,valid_sums,np.zeros(valid_sums.shape,dtype=theano.config.floatX)), valid_file, cPickle.HIGHEST_PROTOCOL)
        cPickle.dump((test_set_x,test_sums,np.zeros(test_sums.shape,dtype=theano.config.floatX)), test_file, cPickle.HIGHEST_PROTOCOL)
    else:
        cPickle.dump((normalize_data_x(train_set_x,train_sums,'training'),train_sums,np.zeros(train_sums.shape,dtype=theano.config.floatX)), train_file, cPickle.HIGHEST_PROTOCOL)
        cPickle.dump((normalize_data_x(valid_set_x,valid_sums,'validation'),valid_sums,np.zeros(valid_sums.shape,dtype=theano.config.floatX)), valid_file, cPickle.HIGHEST_PROTOCOL)
        cPickle.dump((normalize_data_x(test_set_x,test_sums,'testing'),test_sums,np.zeros(test_sums.shape,dtype=theano.config.floatX)), test_file, cPickle.HIGHEST_PROTOCOL)

    train_file.close()
    valid_file.close()
    test_file.close()

    data_info = {'training_prefix': os.path.join('data', TRAINING_FILE),
        'n_training_files': 1,
        'n_training_batches':train_cutoff/batch_size,
        'validation_prefix':  os.path.join('data', VALIDATION_FILE),
        'n_validation_files': 1,
        'n_validation_batches': validate_cutoff/batch_size,
        'testing_prefix':  os.path.join('data', TESTING_FILE),
        'n_testing_files': 1,
        'n_testing_batches': test_cutoff/batch_size,
        'batches_per_file': (train_cutoff+validate_cutoff+test_cutoff)/batch_size,
        'mean_doc_size': mean_doc_size,
    }

    dict_to_cfg(data_info, 'info', 'data.cfg')

    print '...  pickling and zipping render_data to '+ cfg.input.render_data
    render_data = normalize_data_x(train_set_x,train_sums,'training')[0:num_examples]
    f = gzip.open(cfg.input.render_data,'wb')

    render_file_has_labels = cfg.input.render_data_has_labels
    data = render_data
    if (render_file_has_labels):
        labels = user_labels.items()
        labels.sort()
        labels = labels[0:num_examples] #FIXME makes dangerous assumption that user_labels has same ordering as render_data
        data = (render_data, labels)

    cPickle.dump(data,f, cPickle.HIGHEST_PROTOCOL)

    print data
    f.close()

def normalize_data_x(data_x,sums_x,name):
    for idx in xrange(len(sums_x)):
        if sums_x[idx] == 0.:
            print 'input for '+name+' user_id %i has all elements zero will not attempt to normalize '%idx
            sums_x[idx] = 1

    return (data_x.transpose()/sums_x).transpose()

if __name__ == '__main__':
    from optparse import OptionParser

    parser = OptionParser()
    parser.add_option("-f", "--file", dest="config", default="config",
        help="Path of the config file to use")
    (options, args) = parser.parse_args()

    cfg = LoadConfig(options.config)
    validate_config(cfg)

    if (cfg.input.csv_contains_counts):
        #read in csv and normalize
        raw_counts = read_user_word_counts(cfg)

    if (cfg.input.csv_contains_pixels):
        #read in pixel values between 0 and 1
        raw_counts = read_user_word_pixels(cfg)

    if (raw_counts == None):
        print "You need to set either cfg.input.csv_contains_pixels or cfg.input.csv_contains_counts to True otherwise not sure how to process data"

    render_file_has_labels = cfg.input.render_data_has_labels

    user_labels = None
    if (render_file_has_labels):
        user_labels = read_user_labels(cfg)

    #output into pickled data files
    normalize_and_output_pickled_data(cfg,raw_counts, user_labels)

<|endoftext|>


<commit_before>__version__ = "4.0m2"
<commit_msg>update version to 4.0m2a<commit_after>__version__ = "4.0m2a"
<|endoftext|>


<commit_before><commit_msg>Corresponding variable control tests.<commit_after><|endoftext|>


<commit_before>from unittest import mock
from django.test import TestCase, override_settings

from promgen import models
from promgen.sender.ikasan import SenderIkasan
from promgen.tests import TEST_ALERT, TEST_SETTINGS

_MESSAGE = '''node_down foo-BETA testhost.localhost:9100 node resolved

description: testhost.localhost:9100 of job node has been down for more than 5 minutes.
summary: Instance testhost.localhost:9100 down

Prometheus: https://monitoring.promehteus.localhost/graph#%5B%7B%22expr%22%3A%22up%20%3D%3D%200%22%2C%22tab%22%3A0%7D%5D
Alert Manager: https://am.promehteus.localhost'''

class IkasanTest(TestCase):
    @mock.patch('django.db.models.signals.post_save', mock.Mock())
    def setUp(self):
        self.service = models.Service.objects.create(name='Service 1')
        self.project = models.Project.objects.create(name='Project 1', service=self.service)
        self.sender = models.Sender.objects.create(
            project=self.project,
            sender='promgen.sender.ikasan',
            value='#',
        )

    @override_settings(PROMGEN=TEST_SETTINGS)
    @mock.patch('requests.post')
    def test_project(self, mock_post):
        self.assertEqual(SenderIkasan().send(TEST_ALERT), 1)
        mock_post.assert_called_once_with('http://ikasan.example', {
            'color': 'green',
            'channel': '#',
            'message': _MESSAGE}
        )
<commit_msg>Convert promgen.sender.ikasan to class<commit_after>from unittest import mock

from django.contrib.contenttypes.models import ContentType
from django.test import TestCase, override_settings

from promgen import models
from promgen.sender.ikasan import SenderIkasan
from promgen.tests import TEST_ALERT, TEST_SETTINGS

_MESSAGE = '''node_down foo-BETA testhost.localhost:9100 node resolved

description: testhost.localhost:9100 of job node has been down for more than 5 minutes.
summary: Instance testhost.localhost:9100 down

Prometheus: https://monitoring.promehteus.localhost/graph#%5B%7B%22expr%22%3A%22up%20%3D%3D%200%22%2C%22tab%22%3A0%7D%5D
Alert Manager: https://am.promehteus.localhost'''

class IkasanTest(TestCase):
    @mock.patch('django.db.models.signals.post_save', mock.Mock())
    def setUp(self):
        project_type = ContentType.objects.get_by_natural_key('promgen', 'Project')
        self.service = models.Service.objects.create(name='Service 1')
        self.project = models.Project.objects.create(name='Project 1', service=self.service)
        self.sender = models.Sender.objects.create(
            object_id=self.project.id,
            content_type_id=project_type.id,
            sender='promgen.sender.ikasan',
            value='#',
        )

    @override_settings(PROMGEN=TEST_SETTINGS)
    @mock.patch('requests.post')
    def test_project(self, mock_post):
        self.assertEquals(SenderIkasan().send(TEST_ALERT), 1)
        mock_post.assert_called_once_with('http://ikasan.example', {
            'color': 'green',
            'channel': '#',
            'message': _MESSAGE}
        )
<|endoftext|>