<commit_before>from django.conf import settings

from subscription.models import UserSubscription

SUBSCRIPTION_NAMES = (
    'AstroBin Lite',
    'AstroBin Premium',
    'AstroBin Lite (autorenew)',
    'AstroBin Premium (autorenew)'
)

def premium_get_usersubscription(user):
    us = UserSubscription.objects.filter(
        user = user,
        subscription__name__in = SUBSCRIPTION_NAMES,
    )

    if us.count() > 0:
        us = us[0]
    else:
        return None

    return us

def premium_get_valid_usersubscription(user):
    us = UserSubscription.objects.filter(
        user = user,
        active = True,
        subscription__name__in = SUBSCRIPTION_NAMES,)

    if us.count() > 0:
        us = us[0]
    else:
        return None

    if not us.valid():
        return None

    return us

def premium_get_invalid_usersubscription(user):
    us = premium_get_usersubscription(user)
    if not us.valid():
        return us
    return None

def premium_used_percent(user):
    s = premium_get_valid_usersubscription(user)
    counter = user.userprofile.premium_counter
    percent = 100

    if s is None:
        # User is on Free, or their subscription is inactive, cancelled or
        # expired.
        percent = counter / float(settings.PREMIUM_MAX_IMAGES_FREE) * 100

    elif s.subscription.name == "AstroBin Lite":
        percent = counter / float(settings.PREMIUM_MAX_IMAGES_LITE) * 100

    elif s.subscription.name == "AstroBin Premium":
        percent = -1

    if percent > 100:
        percent = 100

    return percent

def premium_progress_class(p):
    if p < 90: return 'progress-success'
    if p > 97: return 'progress-danger'
    return 'progress-warning'

def premium_user_has_subscription(user):
    us = premium_get_usersubscription(user)
    return us is not None

def premium_user_has_valid_subscription(user):
    us = premium_get_valid_usersubscription(user)
    return us is not None

def premium_user_has_invalid_subscription(user):
    us = premium_get_invalid_usersubscription(user)
    return us is not None
<commit_msg>Fix premium_used_percent to use sub group name instead of name<commit_after>from django.conf import settings

from subscription.models import UserSubscription

SUBSCRIPTION_NAMES = (
    'AstroBin Lite',
    'AstroBin Premium',
    'AstroBin Lite (autorenew)',
    'AstroBin Premium (autorenew)'
)

def premium_get_usersubscription(user):
    us = UserSubscription.objects.filter(
        user = user,
        subscription__name__in = SUBSCRIPTION_NAMES,
    )

    if us.count() > 0:
        us = us[0]
    else:
        return None

    return us

def premium_get_valid_usersubscription(user):
    us = UserSubscription.objects.filter(
        user = user,
        active = True,
        subscription__name__in = SUBSCRIPTION_NAMES,)

    if us.count() > 0:
        us = us[0]
    else:
        return None

    if not us.valid():
        return None

    return us

def premium_get_invalid_usersubscription(user):
    us = premium_get_usersubscription(user)
    if not us.valid():
        return us
    return None

def premium_used_percent(user):
    s = premium_get_valid_usersubscription(user)
    counter = user.userprofile.premium_counter
    percent = 100

    if s is None:
        # User is on Free, or their subscription is inactive, cancelled or
        # expired.
        percent = counter / float(settings.PREMIUM_MAX_IMAGES_FREE) * 100

    elif s.subscription.group.name == "astrobin_lite":
        percent = counter / float(settings.PREMIUM_MAX_IMAGES_LITE) * 100

    elif s.subscription.group.name == "astrobin_premium":
        percent = -1

    if percent > 100:
        percent = 100

    return percent

def premium_progress_class(p):
    if p < 90: return 'progress-success'
    if p > 97: return 'progress-danger'
    return 'progress-warning'

def premium_user_has_subscription(user):
    us = premium_get_usersubscription(user)
    return us is not None

def premium_user_has_valid_subscription(user):
    us = premium_get_valid_usersubscription(user)
    return us is not None

def premium_user_has_invalid_subscription(user):
    us = premium_get_invalid_usersubscription(user)
    return us is not None
<|endoftext|>


<commit_before>"""
OAuth related API views and routes
"""

import json
import re

from functools import wraps
from urllib.parse import urlencode

from flask import abort, flash, redirect, request, Response, url_for
from flask_login import login_user
from flask_security import current_user, login_required
from flask_security.utils import url_for_security
from sqlalchemy.orm.exc import MultipleResultsFound, NoResultFound

from dockci.models.auth import OAuthToken, User
from dockci.server import (APP,
                           CONFIG,
                           DB,
                           OAUTH_APPS,
                           OAUTH_APPS_SCOPE_SERIALIZERS,
                           )
from dockci.util import ext_url_for, get_token_for

RE_VALID_OAUTH = re.compile(r'^[a-z]+$')

class OAuthRegError(Exception):
    """ Exception for when OAuth registration fails for some reason """
    def __init__(self, reason):
        super(OAuthRegError, self).__init__()
        self.reason = reason

def check_oauth_enabled(name):
    """
    Check config to see if a given OAuth service is configured, and enabled for
    register or login actions
    """
    if not RE_VALID_OAUTH.match(name):
        return (False,) * 3

    return (
        getattr(CONFIG, '%s_enabled' % name),
        getattr(CONFIG, 'security_registerable_%s' % name),
        getattr(CONFIG, 'security_login_%s' % name),
    )

def oauth_redir(return_to=None):
    """ Get the OAuth redirection URL """
    try:
        if return_to is None:
            return_to = request.args['return_to']

    except KeyError:
        return_to = url_for('index_view')

    return redirect(return_to)

@APP.route('/login/<name>')
def oauth_login(name):
    """ Entry point for OAuth logins """
    try:
        if name not in OAUTH_APPS:
            raise OAuthRegError("%s auth not available" % name.title())

        else:
            configured, _, login = check_oauth_enabled(name)

            if not (configured and login):
                raise OAuthRegError("%s login not enabled" % name.title())

            return oauth_response(OAUTH_APPS[name])

    except OAuthRegError as ex:
        flash(ex.reason, 'danger')

    return oauth_redir(url_for_security('login'))

@APP.route('/oauth-authorized/<name>')
def oauth_authorized(name):
    """
    Callback for oauth authorizations. Handles adding the token to the logged
    in user, errors, as well as redirecting back to the original page
    """
    try:
        try:
            oauth_app = OAUTH_APPS[name]

        except KeyError:
            raise OAuthRegError("%s auth not available" % name.title())

        else:
            configured, register, login = check_oauth_enabled(name)

            if not configured:
                raise OAuthRegError("%s auth not enabled" % name.title())

            resp = oauth_app.authorized_response()

        if resp is None or 'error' in resp:
            flash("{name}: {message}".format(
                name=name.title(),
                message=request.args['error_description'],
            ), 'danger')

        else:
            if current_user.is_authenticated():
                user = current_user
                oauth_token = get_oauth_token(name, resp)

            elif not (register or login):
                raise OAuthRegError(
                    "Registration and login disabled for %s" % name.title())

            else:
                user, oauth_token = user_from_oauth(name, resp)

            if user.id is None and not register:
                raise OAuthRegError(
                    "Registration disabled for %s" % name.title())
            elif user.id is not None and not login:
                raise OAuthRegError(
                    "Login disabled for %s" % name.title())

            associate_user(name, user, oauth_token)

    except OAuthRegError as ex:
        flash(ex.reason, 'danger')

    return oauth_redir()

def associate_user(name, user, oauth_token):
    """
    Given a user, and an OAuth Token, associate the 2. This will ensure that
    the token isn't in use by a different user, erase old tokens of the same
    service, associate the token with the user, then log the user in.

    The DB session will be committed, and a flash message displayed
    """
    if user is None:
        raise OAuthRegError("Couldn't retrieve user "
                            "details from %s" % name.title())

    # Delete other tokens if the user exists, and token is new
    if user.id is not None and oauth_token.id is None:
        user.oauth_tokens.filter_by(service=name).delete(
            synchronize_session=False,
        )

    if oauth_token.user is None:
        user.oauth_tokens.append(oauth_token)

    elif oauth_token.user != user:
        raise OAuthRegError("An existing user is already associated "
                            "with that %s account" % name.title())

    DB.session.add(oauth_token)
    DB.session.add(user)
    DB.session.commit()

    flash(u"Connected to %s" % name.title(), 'success')
    login_user(user)

def get_oauth_token(name, response):
    """
    Retrieve an ``OAuthToken`` for the response. If a token exists with the
    same service name, and key then we update it with the new details
    """
    try:
        oauth_token = OAuthToken.query.filter_by(
            service=name,
            key=response['access_token'],
        ).one()

    except NoResultFound:
        return create_oauth_token(name, response)

    except MultipleResultsFound:
        raise OAuthRegError(
            "Multiple accounts associated with this "
            "%s account. This shouldn't happen" % name.title()
        )

    else:
        oauth_token.update_details_from(
            create_oauth_token(name, response)
        )
        return oauth_token

def create_oauth_token(name, response):
    """ Create a new ``OAuthToken`` from an OAuth response """
    return OAuthToken(
        service=name,
        key=response['access_token'],
        secret='',
        scope=OAUTH_APPS_SCOPE_SERIALIZERS[name](response['scope'])
    )

def user_from_oauth(name, response):
    """
    Given an OAuth response, extrapolate a ``User`` and an ``OAuthToken``.

    First, if a token exists, we get it's user. Otherwise, we look up the email
    address from the service and look for a matching user. If there is no user
    registered ith the email, we create a new user and return it (uncommitted)

    If the email can't be retrieved from the service, ``OAuthRegError`` raises

    If a user exists with the email from the service, ``OAuthRegError`` raises
    """
    oauth_app = OAUTH_APPS[name]
    oauth_token = get_oauth_token(name, response)
    oauth_token_tuple = (oauth_token.key, oauth_token.secret)

    if oauth_token.id is not None:
        return oauth_token.user, oauth_token

    user_data = oauth_app.get('/user', token=oauth_token_tuple).data
    user_email = user_data['email']

    if user_email is None:
        raise OAuthRegError(
            "Couldn't get email address from %s" % name.title())

    try:
        user_by_email = User.query.filter_by(email=user_email).one()

    except NoResultFound:
        pass

    else:
        if user_by_email.oauth_tokens.filter_by(service=name).count() > 0:
            return user_by_email, oauth_token

        else:
            raise OAuthRegError("A user is already registered "
                                "with the email '%s'" % user_email)

    return User(
        email=user_email,
        active=True,
    ), oauth_token

def oauth_response(oauth_app):
    """
    Build an authorization for the given OAuth service, and return the response
    """
    return_to = request.args.get('return_to', None)
    base_url = ext_url_for(
        'oauth_authorized', name=oauth_app.name,
    )
    if return_to is not None:
        callback_uri = '{base_url}?{query}'.format(
            base_url=base_url,
            query=urlencode({'return_to': return_to})
        )

    else:
        callback_uri = base_url

    return oauth_app.authorize(callback=callback_uri)

def oauth_required(acceptable=None, force_name=None):
    """
    Wrap the view in oauth functionality to make sure there's an acceptable
    token, before moving on to the view
    """
    if force_name is not None and acceptable is not None:
        if force_name not in acceptable:
            raise ValueError(
                "Name has been forced to '%s', but this is never acceptable "
                "for set %s",
                force_name,
                acceptable,
            )

    def outer(func):
        """
        Wrap func in the inner check, and possibly force_name helper
        """
        @login_required
        @wraps(func)
        def inner_with_name(name, *args, **kwargs):
            """
            Wrap a view function to provide the redirect/return_to
            functionality on current user having no oauth
            """
            if name not in acceptable:
                return abort(404)

            oauth_app = OAUTH_APPS[name]
            if not get_token_for(oauth_app):
                resp = oauth_response(oauth_app)
                return Response(json.dumps({'redirect': resp.location}),
                                mimetype='application/json')

            else:
                if force_name is not None:
                    return func(*args, **kwargs)

                else:
                    return func(name, *args, **kwargs)

        @wraps(inner_with_name)
        def inner_forced_name(*args, **kwargs):
            """
            Wrap inner_with_name to add the name as the value given in the
            force_name param earlier
            """
            return inner_with_name(force_name, *args, **kwargs)

        if force_name is None:
            return inner_with_name

        else:
            return inner_forced_name

    return outer
<commit_msg>OAuth redir param is next, add better error handling<commit_after>"""
OAuth related API views and routes
"""

import json
import logging
import re

from functools import wraps
from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse

from flask import abort, flash, redirect, request, Response, url_for
from flask_login import login_user
from flask_security import current_user, login_required
from flask_security.utils import url_for_security
from sqlalchemy.orm.exc import MultipleResultsFound, NoResultFound

from dockci.models.auth import OAuthToken, User
from dockci.server import (APP,
                           CONFIG,
                           DB,
                           OAUTH_APPS,
                           OAUTH_APPS_SCOPE_SERIALIZERS,
                           )
from dockci.util import ext_url_for, get_token_for

RE_VALID_OAUTH = re.compile(r'^[a-z]+$')

USER_API_PATHS = {
    'github': '/user',
    'gitlab': '/api/v3/user',
}

class OAuthRegError(Exception):
    """ Exception for when OAuth registration fails for some reason """
    def __init__(self, reason):
        super(OAuthRegError, self).__init__()
        self.reason = reason

def check_oauth_enabled(name):
    """
    Check config to see if a given OAuth service is configured, and enabled for
    register or login actions
    """
    if not RE_VALID_OAUTH.match(name):
        return (False,) * 3

    return (
        getattr(CONFIG, '%s_enabled' % name),
        getattr(CONFIG, 'security_registerable_%s' % name),
        getattr(CONFIG, 'security_login_%s' % name),
    )

def oauth_redir(next_url=None):
    """ Get the OAuth redirection URL """
    if next_url is None:
        next_url = request.args.get('next') or url_for('index_view')

    return redirect(next_url)

@APP.route('/login/<name>')
def oauth_login(name):
    """ Entry point for OAuth logins """
    try:
        if name not in OAUTH_APPS:
            raise OAuthRegError("%s auth not available" % name.title())

        else:
            configured, _, login = check_oauth_enabled(name)

            if not (configured and login):
                raise OAuthRegError("%s login not enabled" % name.title())

            return oauth_response(OAUTH_APPS[name])

    except OAuthRegError as ex:
        flash(ex.reason, 'danger')

    return oauth_redir(url_for_security('login'))

@APP.route('/oauth-authorized/<name>')
def oauth_authorized(name):
    """
    Callback for oauth authorizations. Handles adding the token to the logged
    in user, errors, as well as redirecting back to the original page
    """
    try:
        try:
            oauth_app = OAUTH_APPS[name]

        except KeyError:
            raise OAuthRegError("%s auth not available" % name.title())

        else:
            configured, register, login = check_oauth_enabled(name)

            if not configured:
                raise OAuthRegError("%s auth not enabled" % name.title())

            resp = oauth_app.authorized_response()

        if isinstance(resp, Exception):
            try:
                logging.error('OAuth exception data: %s', resp.data)
            except AttributeError:
                pass

            raise resp

        if resp is None or 'error' in resp:
            flash("{name}: {message}".format(
                name=name.title(),
                message=request.args['error_description'],
            ), 'danger')

        else:
            if current_user.is_authenticated():
                user = current_user
                oauth_token = get_oauth_token(name, resp)

            elif not (register or login):
                raise OAuthRegError(
                    "Registration and login disabled for %s" % name.title())

            else:
                user, oauth_token = user_from_oauth(name, resp)

            if user.id is None and not register:
                raise OAuthRegError(
                    "Registration disabled for %s" % name.title())
            elif user.id is not None and not login:
                raise OAuthRegError(
                    "Login disabled for %s" % name.title())

            associate_user(name, user, oauth_token)

    except OAuthRegError as ex:
        flash(ex.reason, 'danger')

    return oauth_redir()

def associate_user(name, user, oauth_token):
    """
    Given a user, and an OAuth Token, associate the 2. This will ensure that
    the token isn't in use by a different user, erase old tokens of the same
    service, associate the token with the user, then log the user in.

    The DB session will be committed, and a flash message displayed
    """
    if user is None:
        raise OAuthRegError("Couldn't retrieve user "
                            "details from %s" % name.title())

    # Delete other tokens if the user exists, and token is new
    if user.id is not None and oauth_token.id is None:
        user.oauth_tokens.filter_by(service=name).delete(
            synchronize_session=False,
        )

    if oauth_token.user is None:
        user.oauth_tokens.append(oauth_token)

    elif oauth_token.user != user:
        raise OAuthRegError("An existing user is already associated "
                            "with that %s account" % name.title())

    DB.session.add(oauth_token)
    DB.session.add(user)
    DB.session.commit()

    flash(u"Connected to %s" % name.title(), 'success')
    login_user(user)

def get_oauth_token(name, response):
    """
    Retrieve an ``OAuthToken`` for the response. If a token exists with the
    same service name, and key then we update it with the new details
    """
    try:
        oauth_token = OAuthToken.query.filter_by(
            service=name,
            key=response['access_token'],
        ).one()

    except NoResultFound:
        return create_oauth_token(name, response)

    except MultipleResultsFound:
        raise OAuthRegError(
            "Multiple accounts associated with this "
            "%s account. This shouldn't happen" % name.title()
        )

    else:
        oauth_token.update_details_from(
            create_oauth_token(name, response)
        )
        return oauth_token

def create_oauth_token(name, response):
    """ Create a new ``OAuthToken`` from an OAuth response """
    return OAuthToken(
        service=name,
        key=response['access_token'],
        secret='',
        scope=OAUTH_APPS_SCOPE_SERIALIZERS[name](response['scope'])
    )

def user_from_oauth(name, response):
    """
    Given an OAuth response, extrapolate a ``User`` and an ``OAuthToken``.

    First, if a token exists, we get it's user. Otherwise, we look up the email
    address from the service and look for a matching user. If there is no user
    registered ith the email, we create a new user and return it (uncommitted)

    If the email can't be retrieved from the service, ``OAuthRegError`` raises

    If a user exists with the email from the service, ``OAuthRegError`` raises
    """
    oauth_app = OAUTH_APPS[name]
    oauth_token = get_oauth_token(name, response)
    oauth_token_tuple = (oauth_token.key, oauth_token.secret)

    if oauth_token.id is not None:
        return oauth_token.user, oauth_token

    user_data = oauth_app.get(
        USER_API_PATHS[name],
        token=oauth_token_tuple,
    ).data
    user_email = user_data['email']

    if user_email is None:
        raise OAuthRegError(
            "Couldn't get email address from %s" % name.title())

    try:
        user_by_email = User.query.filter_by(email=user_email).one()

    except NoResultFound:
        pass

    else:
        if user_by_email.oauth_tokens.filter_by(service=name).count() > 0:
            return user_by_email, oauth_token

        else:
            raise OAuthRegError("A user is already registered "
                                "with the email '%s'" % user_email)

    return User(
        email=user_email,
        active=True,
    ), oauth_token

def oauth_response(oauth_app):
    """
    Build an authorization for the given OAuth service, and return the response
    """
    callback_uri = ext_url_for(
        'oauth_authorized',
        name=oauth_app.name,
        next=request.args.get('next') or request.referrer or url_for('index_view'),
    )

    return oauth_app.authorize(callback=callback_uri)

def oauth_required(acceptable=None, force_name=None):
    """
    Wrap the view in oauth functionality to make sure there's an acceptable
    token, before moving on to the view
    """
    if force_name is not None and acceptable is not None:
        if force_name not in acceptable:
            raise ValueError(
                "Name has been forced to '%s', but this is never acceptable "
                "for set %s",
                force_name,
                acceptable,
            )

    def outer(func):
        """
        Wrap func in the inner check, and possibly force_name helper
        """
        @login_required
        @wraps(func)
        def inner_with_name(name, *args, **kwargs):
            """
            Wrap a view function to provide the redirect/next
            functionality on current user having no oauth
            """
            if name not in acceptable:
                return abort(404)

            oauth_app = OAUTH_APPS[name]
            if not get_token_for(oauth_app):
                resp = oauth_response(oauth_app)
                return Response(json.dumps({'redirect': resp.location}),
                                mimetype='application/json')

            else:
                if force_name is not None:
                    return func(*args, **kwargs)

                else:
                    return func(name, *args, **kwargs)

        @wraps(inner_with_name)
        def inner_forced_name(*args, **kwargs):
            """
            Wrap inner_with_name to add the name as the value given in the
            force_name param earlier
            """
            return inner_with_name(force_name, *args, **kwargs)

        if force_name is None:
            return inner_with_name

        else:
            return inner_forced_name

    return outer
<|endoftext|>


<commit_before><commit_msg>Fix undercloud validate password file<commit_after><|endoftext|>


<commit_before>import codecs
import os
import re
import string
import unicodedata

import normalizr.regex as regex

path = os.path.dirname(__file__)

class Normalizr:
    """
    This class offers methods for text normalization.

    Attributes:
        language (string): Language used for normalization.
    """
    __punctuation = set(string.punctuation)
    __stop_words = set()

    def __init__(self, language='en'):
        self._load_stop_words(language)

    def _load_stop_words(self, language):
        """
        Load stop words into __stop_words set.

        Stop words will be loaded according to the language code received during instantiation.

        Params:
            language (string): Language code.
        """
        with codecs.open(os.path.join(path, 'data/stop-' + language), 'r', 'UTF-8') as file:
            for line in file:
                fields = line.split('|')
                if fields:
                    for word in fields[0].split(): self.__stop_words.add(word.strip())

    def normalize(self, text, normalizations=None):
        """
        Normalize a given text applying all normalizations.

        Normalizations to apply can be specified through a list parameter and will be executed
        in the same order.

        Params:
            text (string): The text to be processed.
            normalizations (list): List of normalizations to apply.

        Returns:
            The text normalized.
        """
        if normalizations is None:
            normalizations = ['whitespaces', 'punctuation', 'symbols', 'stopwords']

        methods = {
            'accents': self.remove_accent_marks,
            'hyphens': self.replace_hyphens,
            'punctuation': self.remove_punctuation,
            'stopwords': self.remove_stop_words,
            'symbols': self.remove_symbols,
            'whitespaces': self.remove_extra_whitespaces
        }

        for normalization in normalizations:
            text = methods[normalization](text)

        return text

    def remove_accent_marks(self, text, format='NFKD', excluded=set()):
        """
        Remove accent marks from input text.

        Params:
            text (string): The text to be processed.
            format (string): Unicode format.
            excluded (set): Set of unicode characters to exclude.

        Returns:
            The text without accent marks.
        """
        return ''.join(c for c in unicodedata.normalize(format, text)
                       if unicodedata.category(c) != 'Mn' or c in excluded)

    def remove_extra_whitespaces(self, text):
        """
        Remove extra whitespaces from input text.

        This function removes whitespaces from the beginning and the end of
        the string, but also duplicated whitespaces between words.

        Params:
            text (string): The text to be processed.

        Returns:
            The text without extra whitespaces.
        """
        return ' '.join(text.strip().split())

    def remove_stop_words(self, text, ignore_case=True):
        """
        Remove stop words.

        Stop words are loaded on class instantiation according with the specified language.

        Params:
            text (string): The text to be processed.
            ignore_case (boolean): Whether or not ignore case.

        Returns:
            The text without stop words.
        """
        return ' '.join(
            word for word in text.split(' ') if (word.lower() if ignore_case else word) not in self.__stop_words)

    def replace_emojis(self, text, replacement=''):
        """
        Remove emojis from input text or replace them with a string if specified.

        Params:
            text (string): The text to be processed.
            replacement (string): New text that will replace emojis.

        Returns:
            The text without hyphens.
        """
        try:
            highpoints = re.compile(u'([\U00002600-\U000027BF])|([\U0001f300-\U0001f64F])|([\U0001f680-\U0001f6FF])')
        except re.error:
            highpoints = re.compile(u'([\u2600-\u27BF])|([\uD83C][\uDF00-\uDFFF])|([\uD83D][\uDC00-\uDE4F])|([\uD83D][\uDE80-\uDEFF])')

        return highpoints.sub(replacement, text)

    def replace_hyphens(self, text, replacement=' '):
        """
        Replace hyphens from input text with a whitespace or a string if specified.

        Params:
            text (string): The text to be processed.
            replacement (string): New text that will replace hyphens.

        Returns:
            The text without hyphens.
        """
        return text.replace('-', replacement)

    def replace_punctuation(self, text, excluded=set(), replacement='',):
        """
        Remove punctuation from input text or replace them with a string if specified.

        This function will remove characters from string.punctuation.

        Params:
            text (string): The text to be processed.
            excluded (set): Set of characters to exclude.
            replacement (string): New text that will replace punctuation.

        Returns:
            The text without punctuation.
        """
        return ''.join(c if c not in self.__punctuation or c in excluded else replacement for c in text)

    def replace_symbols(self, text, format='NFKD', excluded=set(), replacement=''):
        """
        Remove symbols from input text or replace them with a string if specified.

        Params:
            text (string): The text to be processed.
            format (string): Unicode format.
            excluded (set): Set of unicode characters to exclude.
            replacement (string): New text that will replace symbols.

        Returns:
            The text without symbols.
        """
        categories = set(['Mn', 'Sc', 'Sk', 'Sm', 'So'])
        return ''.join(c if unicodedata.category(c) not in categories or c in excluded else replacement
                       for c in unicodedata.normalize(format, text))

    def replace_urls(self, text, replacement=''):
        """
        Remove URLs from input text or replace them with a string if specified.

        Params:
            text (string): The text to be processed.
            replacement (string): New text that will replace URLs.

        Returns:
            The text without URLs.
        """
        return re.sub(regex.URL_REGEX, replacement, text)<commit_msg>Reimplement normalize function<commit_after>import codecs
import os
import re
import string
import unicodedata

import normalizr.regex as regex

path = os.path.dirname(__file__)

DEFAULT_NORMALIZATIONS = [
    'remove_extra_whitespaces', 'replace_punctuation', 'replace_symbols', 'remove_stop_words'
]

class Normalizr:
    """
    This class offers methods for text normalization.

    Attributes:
        language (string): Language used for normalization.
    """
    __punctuation = set(string.punctuation)
    __stop_words = set()

    def __init__(self, language='en'):
        self._load_stop_words(language)

    def _load_stop_words(self, language):
        """
        Load stop words into __stop_words set.

        Stop words will be loaded according to the language code received during instantiation.

        Params:
            language (string): Language code.
        """
        with codecs.open(os.path.join(path, 'data/stop-' + language), 'r', 'UTF-8') as file:
            for line in file:
                fields = line.split('|')
                if fields:
                    for word in fields[0].split(): self.__stop_words.add(word.strip())

    def normalize(self, text, normalizations=None):

        if normalizations is None:
            normalizations = ['whitespaces', 'punctuation', 'symbols', 'stopwords']

        methods = {
            'accents': self.remove_accent_marks,
            'hyphens': self.replace_hyphens,
            'punctuation': self.remove_punctuation,
            'stopwords': self.remove_stop_words,
            'symbols': self.remove_symbols,
            'whitespaces': self.remove_extra_whitespaces
        }

        for normalization in normalizations:
            text = methods[normalization](text)

        return text

    def _parse_normalizations(self, normalizations):
        for normalization in normalizations:
            if isinstance(normalization, str):
                kwargs = {}
            else:
                normalization, kwargs = normalization

            yield (normalization, kwargs)

    def normalize(self, text, normalizations=None):
        """
        Normalize a given text applying all normalizations.

        Normalizations to apply can be specified through a list parameter and will be executed
        in the same order.

        Params:
            text (string): The text to be processed.
            normalizations (list): List of normalizations to apply.

        Returns:
            The text normalized.
        """
        for normalization, kwargs in self._parse_normalizations(normalizations or DEFAULT_NORMALIZATIONS):
            text = getattr(self, normalization)(text, **kwargs)
        return text

    def remove_accent_marks(self, text, format='NFKD', excluded=set()):
        """
        Remove accent marks from input text.

        Params:
            text (string): The text to be processed.
            format (string): Unicode format.
            excluded (set): Set of unicode characters to exclude.

        Returns:
            The text without accent marks.
        """
        return ''.join(c for c in unicodedata.normalize(format, text)
                       if unicodedata.category(c) != 'Mn' or c in excluded)

    def remove_extra_whitespaces(self, text):
        """
        Remove extra whitespaces from input text.

        This function removes whitespaces from the beginning and the end of
        the string, but also duplicated whitespaces between words.

        Params:
            text (string): The text to be processed.

        Returns:
            The text without extra whitespaces.
        """
        return ' '.join(text.strip().split())

    def remove_stop_words(self, text, ignore_case=True):
        """
        Remove stop words.

        Stop words are loaded on class instantiation according with the specified language.

        Params:
            text (string): The text to be processed.
            ignore_case (boolean): Whether or not ignore case.

        Returns:
            The text without stop words.
        """
        return ' '.join(
            word for word in text.split(' ') if (word.lower() if ignore_case else word) not in self.__stop_words)

    def replace_emojis(self, text, replacement=''):
        """
        Remove emojis from input text or replace them with a string if specified.

        Params:
            text (string): The text to be processed.
            replacement (string): New text that will replace emojis.

        Returns:
            The text without hyphens.
        """
        try:
            highpoints = re.compile(u'([\U00002600-\U000027BF])|([\U0001f300-\U0001f64F])|([\U0001f680-\U0001f6FF])')
        except re.error:
            highpoints = re.compile(u'([\u2600-\u27BF])|([\uD83C][\uDF00-\uDFFF])|([\uD83D][\uDC00-\uDE4F])|([\uD83D][\uDE80-\uDEFF])')

        return highpoints.sub(replacement, text)

    def replace_hyphens(self, text, replacement=' '):
        """
        Replace hyphens from input text with a whitespace or a string if specified.

        Params:
            text (string): The text to be processed.
            replacement (string): New text that will replace hyphens.

        Returns:
            The text without hyphens.
        """
        return text.replace('-', replacement)

    def replace_punctuation(self, text, excluded=set(), replacement='',):
        """
        Remove punctuation from input text or replace them with a string if specified.

        This function will remove characters from string.punctuation.

        Params:
            text (string): The text to be processed.
            excluded (set): Set of characters to exclude.
            replacement (string): New text that will replace punctuation.

        Returns:
            The text without punctuation.
        """
        return ''.join(c if c not in self.__punctuation or c in excluded else replacement for c in text)

    def replace_symbols(self, text, format='NFKD', excluded=set(), replacement=''):
        """
        Remove symbols from input text or replace them with a string if specified.

        Params:
            text (string): The text to be processed.
            format (string): Unicode format.
            excluded (set): Set of unicode characters to exclude.
            replacement (string): New text that will replace symbols.

        Returns:
            The text without symbols.
        """
        categories = set(['Mn', 'Sc', 'Sk', 'Sm', 'So'])
        return ''.join(c if unicodedata.category(c) not in categories or c in excluded else replacement
                       for c in unicodedata.normalize(format, text))

    def replace_urls(self, text, replacement=''):
        """
        Remove URLs from input text or replace them with a string if specified.

        Params:
            text (string): The text to be processed.
            replacement (string): New text that will replace URLs.

        Returns:
            The text without URLs.
        """
        return re.sub(regex.URL_REGEX, replacement, text)<|endoftext|>


<commit_before>import os
import datetime
import subprocess
from warnings import warn
from nose.plugins import Plugin
from pkg_resources import resource_filename

__version__ = '0.2'

START_TITLE = 'Starting tests...'
START_ICON = resource_filename('nosegrowl2', 'start.png')

SUCCESS_TITLE = '%s tests run OK'
SUCCESS_MESSAGE = 'Completed in %s.%s seconds'
SUCCESS_ICON = resource_filename('nosegrowl2', 'success.png')

FAILURE_TITLE = '%s tests run. %s failures, %s errors'
FAILURE_ICON = resource_filename('nosegrowl2', 'failure.png')

def _check_growlnotify():
    """Check whether the growlnotify command-line too is installed"""
    try:
        subprocess.check_call(
            ['growlnotify', '--version'],
            stderr=subprocess.STDOUT,
            stdout=subprocess.PIPE
        )
        return True
    except:
        warn(
            "Plugin NoseGrowl2 requires the command-line tool growlnotify. "
            "Please install from http://growl.info/extras.php#growlnotify "
            "or with `brew install growlnotify`."
        )
        return False

def _growl(title, message, icon):
    args = [
        'growlnotify',
        '--title', title,
        '--message', message,
        '--image', icon,
        '--name', 'nosegrowl2',
    ]
    subprocess.call(args)

class NoseGrowl(Plugin):

    name = 'growl'

    def configure(self, options, conf):
        super(NoseGrowl, self).configure(options, conf)
        self.enabled = _check_growlnotify()

    def begin(self):
        self.start_time = datetime.datetime.now()
        _growl(START_TITLE, '', START_ICON)

    def finalize(self, result):
        self.end_time = datetime.datetime.now()

        if result.wasSuccessful():
            time_taken = self.end_time - self.start_time
            _growl(
                SUCCESS_TITLE % result.testsRun,
                SUCCESS_MESSAGE % (
                    time_taken.seconds,
                    time_taken.microseconds,
                ),
                SUCCESS_ICON
            )
        else:
            failures = '\n'.join(["Failures: %s" % name for name, ex in result.failures])
            errors = '\n'.join(["Errors: %s" % name for name, ex in result.errors])
            failure_message = '\n'.join([failures, errors])
            _growl(
                FAILURE_TITLE % (
                    result.testsRun,
                    len(result.failures),
                    len(result.errors),
                ),
                failure_message,
                FAILURE_ICON
            )
<commit_msg>Fix typo<commit_after>import os
import datetime
import subprocess
from warnings import warn
from nose.plugins import Plugin
from pkg_resources import resource_filename

__version__ = '0.2'

START_TITLE = 'Starting tests...'
START_ICON = resource_filename('nosegrowl2', 'start.png')

SUCCESS_TITLE = '%s tests run OK'
SUCCESS_MESSAGE = 'Completed in %s.%s seconds'
SUCCESS_ICON = resource_filename('nosegrowl2', 'success.png')

FAILURE_TITLE = '%s tests run. %s failures, %s errors'
FAILURE_ICON = resource_filename('nosegrowl2', 'failure.png')

def _check_growlnotify():
    """Check whether the growlnotify command-line tool is installed"""
    try:
        subprocess.check_call(
            ['growlnotify', '--version'],
            stderr=subprocess.STDOUT,
            stdout=subprocess.PIPE
        )
        return True
    except:
        warn(
            "Plugin NoseGrowl2 requires the command-line tool growlnotify. "
            "Please install from http://growl.info/extras.php#growlnotify "
            "or with `brew install growlnotify`."
        )
        return False

def _growl(title, message, icon):
    args = [
        'growlnotify',
        '--title', title,
        '--message', message,
        '--image', icon,
        '--name', 'nosegrowl2',
    ]
    subprocess.call(args)

class NoseGrowl(Plugin):

    name = 'growl'

    def configure(self, options, conf):
        super(NoseGrowl, self).configure(options, conf)
        self.enabled = _check_growlnotify()

    def begin(self):
        self.start_time = datetime.datetime.now()
        _growl(START_TITLE, '', START_ICON)

    def finalize(self, result):
        self.end_time = datetime.datetime.now()

        if result.wasSuccessful():
            time_taken = self.end_time - self.start_time
            _growl(
                SUCCESS_TITLE % result.testsRun,
                SUCCESS_MESSAGE % (
                    time_taken.seconds,
                    time_taken.microseconds,
                ),
                SUCCESS_ICON
            )
        else:
            failures = '\n'.join(["Failures: %s" % name for name, ex in result.failures])
            errors = '\n'.join(["Errors: %s" % name for name, ex in result.errors])
            failure_message = '\n'.join([failures, errors])
            _growl(
                FAILURE_TITLE % (
                    result.testsRun,
                    len(result.failures),
                    len(result.errors),
                ),
                failure_message,
                FAILURE_ICON
            )
<|endoftext|>


<commit_before><commit_msg>remember url's padded in even if download folder is not yet available<commit_after><|endoftext|>


<commit_before>from .logging import get_logger
from .modules.utils import get_ident_string

class ModuleWrapper(object):
    """Wraps a module. It is used to hold a reference to a loaded module in the
    Manager.
    """

    def __init__(self, module):
        super().__init__()
        self.module = module
        self.name = get_ident_string(module.__class__)
        self.logger = get_logger(str(self))

    def __repr__(self):
        return '%s: %s' % (self.__class__.__name__, self.module)

    def start(self):
        self.logger.debug('Start')
        self.module.start()

    def stop(self):
        self.logger.debug('Stop')
        self.module.stop()
<commit_msg>Removed an unnecessary call to the parent constructor<commit_after>from .logging import get_logger
from .modules.utils import get_ident_string

class ModuleWrapper(object):
    """Wraps a module. It is used to hold a reference to a loaded module in the
    Manager.
    """

    def __init__(self, module):
        self.module = module
        self.name = get_ident_string(module.__class__)
        self.logger = get_logger(str(self))

    def __repr__(self):
        return '%s: %s' % (self.__class__.__name__, self.module)

    def start(self):
        self.logger.debug('Start')
        self.module.start()

    def stop(self):
        self.logger.debug('Stop')
        self.module.stop()
<|endoftext|>


<commit_before>import json
import random
from utils import RangedNum, RangedInt, RangedJSONEncoder
from builder.network_builder import Network
import os
import math
import copy

class CandidateNN:

    # ---------------------- Static class attributes ----------------------

    OPTIMIZER_CHOICES = ('AdamOptimizer', 'AdadeltaOptimizer', 'AdagradOptimizer',
                         'FtrlOptimizer', 'ProximalGradientDescentOptimizer', 'ProximalAdagradOptimizer',
                         'RMSPropOptimizer', 'GradientDescentOptimizer')
    ACTIVATION_CHOICES = ('relu', 'relu6', 'sigmoid', 'tanh', 'elu', 'softplus', 'softsign')
    LAYER_TYPES = ("conv_layer", "maxpool_layer", "feedforward_layer")
    OPTIMIZING_PARMS = {
        'conv_layer':
        [
            {'parms':
                {'hierarchi': ['convolution', 'filter', 'height'],
                 'min': 1,
                 'max': 5,
                 'type': 'int'}
            },
            {'parms':
                {'hierarchi': ['convolution', 'filter', 'width'],
                 'min': 1,
                 'max': 5,
                 'type': 'int'}
            },
            {'parms':
                {'hierarchi': ['convolution', 'filter', 'outchannels'],
                 'min': 1,
                 'max': 64,
                 'type': 'int'}
            },
            {'parms':
                 {'hierarchi': ['convolution', 'strides', 'x'],
                  'min': 1,
                  'max': 2,
                  'type': 'int'}
            },
            {'parms':
                 {'hierarchi': ['convolution', 'strides', 'y'],
                  'min': 1,
                  'max': 2,
                  'type': 'int'}
             },
            {'parms':
                 {'hierarchi': ['convolution', 'strides', 'inchannels'],
                  'min': 1,
                  'max': 1,
                  'type': 'int'}
             },
            {'parms':
                 {'hierarchi': ['convolution', 'strides', 'batch'],
                  'min': 1,
                  'max': 1,
                  'type': 'int'}
             },
        ],
        'maxpool_layer':
         [
             {'parms':
                  {'hierarchi': ['kernel', 'height'],
                   'min': 1,
                   'max': 5,
                   'type': 'int'}
              },
             {'parms':
                  {'hierarchi': ['kernel', 'width'],
                   'min': 1,
                   'max': 5,
                   'type': 'int'}
              },
             {'parms':
                  {'hierarchi': ['kernel', 'inchannels'],
                   'min': 1,
                   'max': 1,
                   'type': 'int'}
              },
             {'parms':
                  {'hierarchi': ['kernel', 'outchannels'],
                   'min': 1,
                   'max': 1,
                   'type': 'int'}
              },
             {'parms':
                  {'hierarchi': ['strides', 'x'],
                   'min': 1,
                   'max': 5,
                   'type': 'int'}
              },
             {'parms':
                  {'hierarchi': ['strides', 'y'],
                   'min': 1,
                   'max': 5,
                   'type': 'int'}
              },
             {'parms':
                  {'hierarchi': ['strides', 'inchannels'],
                   'min': 1,
                   'max': 1,
                   'type': 'int'}
              },
             {'parms':
                  {'hierarchi': ['strides', 'batch'],
                   'min': 1,
                   'max': 1,
                   'type': 'int'}
              }
         ],
        'feedforward_layer':
        [
            {'parms':
                 {'hierarchi': ['size'],
                  'min': 256,
                  'max': 2048,
                  'type': 'int'}
             }
        ]
    }

    ACCURACY_WEIGHT = 20
    LAYER_CNT_WEIGHT = 2
    MAX_LAYERS = 3

    def __init__(self, candidate_id, start_time_str, runtime_spec, network_spec=None ):
        self.runtime_spec = copy.deepcopy(runtime_spec)

        self._base_logdir = os.path.join(self.runtime_spec['logdir'], str(start_time_str))

        self._candidate_id = candidate_id
        self._fitness = None
        if network_spec is None:
            network_spec = self._create_random_network()
        self.network_spec = network_spec

    def to_next_generation(self, generation):

        generation_dir = 'generation_{}/'.format(generation)
        id_dir = '{}/'.format(self._candidate_id)
        self.runtime_spec['logdir'] = os.path.join(self._base_logdir, generation_dir, id_dir)
        self.network_spec.update(self.runtime_spec)

    def crossover(self, crossover_parms, other_candidate):
        self._fitness = None

        if crossover_parms['strategy'] == 'uniform_crossover':
            self._crossover_uniform(crossver_rate=crossover_parms['rate'],
                                    other_candidate= other_candidate,
                                    uniform_method=crossover_parms['uniform_method'])
        else:
            raise ValueError('not implemented crossover strategy')

    def _crossover_uniform(self, crossver_rate, other_candidate, uniform_method):
        """Performs a unifrom Crossover between two Candidates"""
        if(uniform_method == 'swap'):
            min_layers = min(len(self.network_spec['layers']),len(other_candidate.network_spec['layers']))
            for layer_idx, layer in enumerate(self.network_spec['layers'][:min_layers]):
                layer_dict = layer
                other_layer_dict = other_candidate.network_spec['layers'][layer_idx]

                if (random.uniform(0, 1) <= crossver_rate):
                    layer_dict['activation_function'] = random.choice(self.ACTIVATION_CHOICES)

                if(layer_dict['type'] == other_layer_dict['type']):
                    self._swap_values(layer_dict,other_layer_dict, crossver_rate)

        else:
            raise ValueError('not implemented uniform_crossover_method')
    def _swap_values(self, dict, other_dict,rate):
        """Swaps Properties between two Layers of the same type with Propapility rate"""
        for parm in self.OPTIMIZING_PARMS[dict['type']]:
            if random.uniform(0,1)<=rate:
                parm_h = parm['parms']['hierarchi']
                if len(parm_h) == 1:
                    # Save old own
                    tmp = dict[parm_h[0]]
                    # own in other
                    dict[parm_h[0]] = \
                        other_dict[parm_h[0]]
                    # saved in own
                    other_dict[parm_h[0]] = tmp
                elif len(parm_h) == 2:
                    # Save old own
                    tmp = dict[parm_h[0]][parm_h[1]]
                    # own in other
                    dict[parm_h[0]][parm_h[1]] = \
                        other_dict[parm_h[0]][parm_h[1]]
                    # saved in own
                    other_dict[parm_h[0]][parm_h[1]] = tmp
                elif len(parm_h) == 3:
                    # Save old own
                    tmp = dict[parm_h[0]][parm_h[1]][parm_h[2]]
                    # own in other
                    dict[parm_h[0]][parm_h[1]][parm_h[2]] = \
                        other_dict[parm_h[0]][parm_h[1]][parm_h[2]]
                    # saved in own
                    other_dict[parm_h[0]][parm_h[1]][parm_h[2]] = tmp
                else:
                    raise ValueError('length of hierarchi must 1,2 or 3')

    def mutation(self, mutation_rate):
        # TODO: Check the mutation of a layer and the mutation of properties, layer mutation can hide value mutation
        """
        Mutate properties(layer-structure and layer-values of a Candidate)

        """
        self._fitness = None

        #Mutate layer
        for i, layer_spec in enumerate(self.network_spec['layers']):
            #Mutate complet layer
            if(random.uniform(0,1)<=(mutation_rate/10)):
                new_layer_type = random.choice(self.LAYER_TYPES)
                self.network_spec['layers'][i] = self._create_randomize_layer(layer_type=new_layer_type)
            else:#Only mutate Values if no new random layer
                self._mutate_layer_values(layer_dict=self.network_spec['layers'][i], mutation_rate=mutation_rate)

    def _mutate_layer_values(self, layer_dict, mutation_rate):
        """
        Mutate each value of a layer with a probability of mutation_rate
        """
        if(random.uniform(0,1) <= mutation_rate):
            layer_dict['activation_function'] = random.choice(self.ACTIVATION_CHOICES)
        for parms in self.OPTIMIZING_PARMS[layer_dict['type']]:
            if parms['parms']['max'] == parms['parms']['min']:
                break
            parm_h = parms['parms']['hierarchi']
            variance = (parms['parms']['max'] - parms['parms']['min']) / 2
            if variance == 0 :
                variance = 1
            if parms['parms']['type'] == 'int':
                variance = int(variance)

            if len(parm_h) == 1:
                layer_dict[parm_h[0]] = self._mutation_value_strategy(
                    old_value=layer_dict[parm_h[0]],
                    variance=variance)
            elif len(parm_h) == 2:
                layer_dict[parm_h[0]][parm_h[1]] = self._mutation_value_strategy(
                                                    old_value=layer_dict[parm_h[0]][parm_h[1]],
                                                    variance=variance)
            elif len(parm_h) == 3:
                layer_dict[parm_h[0]][parm_h[1]][parm_h[2]] = self._mutation_value_strategy(
                                                        old_value=layer_dict[parm_h[0]][parm_h[1]][parm_h[2]],
                                                        variance=variance)
            else:

                raise ValueError('length of hierarchi must 1,2 or 3')

    def _mutation_value_strategy(self, old_value, variance):
        """ sub/add a number between -variance and variance"""
        return old_value + old_value.__class__(-variance, variance).value

    def get_diversity(self, otherCandidate):

        return random.random()

    def get_fitness(self):
        """Get fitness of the candidate. If not yet tested, test the fitness based on the network specificaton."""
        if(self._fitness is None):
            network = Network(self._serialze_network_spec())
            extended_spec_json = network.evaluate()
            extended_spec = json.loads(extended_spec_json)
            result_spec = extended_spec['results']
            print(result_spec)
            self._fitness = self._fitness_function(result_spec)
            del network
        return self._fitness

    def _fitness_function(self, results):
        """Calculate the fitness based on the network evaluation."""
        # TODO: get the number of weights as penalty?
        return  1 / (- self.ACCURACY_WEIGHT * math.log(results['accuracy'])
                     + self.LAYER_CNT_WEIGHT * len(self.network_spec['layers']))

    def _create_random_network(self):
        """Construct a random network specification."""

        #TODO: should this be done in this class?
        # Finalize runtime specification.

        layer_cnt = RangedInt(1, self.MAX_LAYERS)

        network_spec = {
            'hyperparameters': {
                'learningrate': RangedNum(1e-4, 1e-3),
                'optimizer': random.choice(self.OPTIMIZER_CHOICES),
                'batchsize': 100  # Fixed batch size for comparability.
            },
            'layers': []
        }

        cnt_layer_conv = RangedInt(0, layer_cnt.value)
        cnt_layer_max_pool = RangedInt(0, layer_cnt.value - cnt_layer_conv.value)
        cnt_layer_ff = layer_cnt.value - cnt_layer_conv.value - cnt_layer_max_pool.value

        layer_types = ['conv_layer' for _ in range(cnt_layer_conv.value)]
        layer_types += ['maxpool_layer' for _ in range(cnt_layer_max_pool.value)]
        random.shuffle(layer_types)
        layer_types += ['feedforward_layer' for _ in range(cnt_layer_ff)]

        for layer_type in layer_types:
            layer_spec = self._create_randomize_layer(layer_type=layer_type)
            #layer_spec = self._generate_network_layer(type=layer_type)
            # Add layer to the network spec.
            network_spec['layers'].append(layer_spec)

        return network_spec

    def _create_randomize_layer(self, layer_type):
        """
        Create a layer based on layer_tape
        """
        if layer_type == 'conv_layer':
            layer_spec = self._create_conv_layer()
        elif layer_type == 'maxpool_layer':
            layer_spec = self._create_maxpool_layer()
        elif layer_type == 'feedforward_layer':
            layer_spec = self._create_ff_layer()
        return layer_spec

    def _create_ff_layer(self):
        """
        Create dict for a random initialized Feedforward network
        :return:
        """

        layer = {
                  'type': 'feedforward_layer',
                  'size': RangedInt(256, 2048),
                  'activation_function': random.choice(self.ACTIVATION_CHOICES)
                }
        return layer

    def _create_conv_layer(self):
        """
        Create dict for a random initialized convolutional Layer
        """
        layer = {
            'type': 'conv_layer',
            'convolution': {
                'filter': {
                    'height': RangedInt(1, 5),
                    'width': RangedInt(1, 5),
                    'outchannels': RangedInt(1, 64)
                },
                'strides': {
                    'x': RangedInt(1, 2),
                    'y': RangedInt(1, 2),
                    'inchannels': 1,  # Must be 1. See https://www.tensorflow.org/api_docs/python/tf/nn/conv2d
                    'batch': 1
                }

            },
            'activation_function': random.choice(self.ACTIVATION_CHOICES)
        }
        return layer

    def _create_maxpool_layer(self):
        """
        Create dict for a random initialized Maxpool-layer
        """
        layer = {
            'type': 'maxpool_layer',
            'kernel': {
                'height': RangedInt(1, 5),
                'width': RangedInt(1, 5),
                'inchannels': 1,
            # Must probably be 1 as well. See https://www.tensorflow.org/api_docs/python/tf/nn/conv2d
                'outchannels': 1,
            },
            'strides': {
                'y': RangedInt(1, 5),
                'x': RangedInt(1, 5),
                'inchannels': 1,
            # Must probably be 1 as well. See https://www.tensorflow.org/api_docs/python/tf/nn/conv2d
                'batch': 1
            }
        }
        return layer
    def _generate_network_layer(self, type):
        # TODO: Implement this!
        raise Exception("Not implemented")
        for parms in self.OPTIMIZING_PARMS[type]:
            parm = parms['parms']
            parm_h = parm['hierarchi']
            layer = dict()
            layer['type'] = type
            layer['activation_function'] = random.choice(self.ACTIVATION_CHOICES)
            if len(parm_h) == 1:
                layer[parm_h[0]] = RangedInt(min=parm['min'], max=parm['max'])
            elif len(parm_h) == 2:
                layer[parm_h[0]][parm_h[2]] = RangedInt(min=parm['min'], max=parm['max'])
            elif len(parm_h) == 3:
                #layer.update({parm_h[0]:{parm_h[1]:{parm_h[2]:RangedInt(min=parm['min'], max=parm['max'])}}})
                #layer[parm_h[0]][parm_h[1]][parm_h[2]] = RangedInt(min=parm['min'], max=parm['max'])
                raise Exception("Not implemented")
            else:
                raise ValueError('length of hierarchi must 1,2 or 3')
        return layer
    def _serialze_network_spec(self):

        return RangedJSONEncoder().encode(self.network_spec)
<commit_msg>FIX?!?!???<commit_after>import json
import random
from utils import RangedNum, RangedInt, RangedJSONEncoder
from builder.network_builder import Network
import os
import math
import copy

class CandidateNN:

    # ---------------------- Static class attributes ----------------------

    OPTIMIZER_CHOICES = ('AdamOptimizer', 'AdadeltaOptimizer', 'AdagradOptimizer',
                         'FtrlOptimizer', 'ProximalGradientDescentOptimizer', 'ProximalAdagradOptimizer',
                         'RMSPropOptimizer', 'GradientDescentOptimizer')
    ACTIVATION_CHOICES = ('relu', 'relu6', 'sigmoid', 'tanh', 'elu', 'softplus', 'softsign')
    LAYER_TYPES = ("conv_layer", "maxpool_layer", "feedforward_layer")
    OPTIMIZING_PARMS = {
        'conv_layer':
        [
            {'parms':
                {'hierarchi': ['convolution', 'filter', 'height'],
                 'min': 1,
                 'max': 5,
                 'type': 'int'}
            },
            {'parms':
                {'hierarchi': ['convolution', 'filter', 'width'],
                 'min': 1,
                 'max': 5,
                 'type': 'int'}
            },
            {'parms':
                {'hierarchi': ['convolution', 'filter', 'outchannels'],
                 'min': 1,
                 'max': 64,
                 'type': 'int'}
            },
            {'parms':
                 {'hierarchi': ['convolution', 'strides', 'x'],
                  'min': 1,
                  'max': 2,
                  'type': 'int'}
            },
            {'parms':
                 {'hierarchi': ['convolution', 'strides', 'y'],
                  'min': 1,
                  'max': 2,
                  'type': 'int'}
             },
            {'parms':
                 {'hierarchi': ['convolution', 'strides', 'inchannels'],
                  'min': 1,
                  'max': 1,
                  'type': 'int'}
             },
            {'parms':
                 {'hierarchi': ['convolution', 'strides', 'batch'],
                  'min': 1,
                  'max': 1,
                  'type': 'int'}
             },
        ],
        'maxpool_layer':
         [
             {'parms':
                  {'hierarchi': ['kernel', 'height'],
                   'min': 1,
                   'max': 5,
                   'type': 'int'}
              },
             {'parms':
                  {'hierarchi': ['kernel', 'width'],
                   'min': 1,
                   'max': 5,
                   'type': 'int'}
              },
             {'parms':
                  {'hierarchi': ['kernel', 'inchannels'],
                   'min': 1,
                   'max': 1,
                   'type': 'int'}
              },
             {'parms':
                  {'hierarchi': ['kernel', 'outchannels'],
                   'min': 1,
                   'max': 1,
                   'type': 'int'}
              },
             {'parms':
                  {'hierarchi': ['strides', 'x'],
                   'min': 1,
                   'max': 5,
                   'type': 'int'}
              },
             {'parms':
                  {'hierarchi': ['strides', 'y'],
                   'min': 1,
                   'max': 5,
                   'type': 'int'}
              },
             {'parms':
                  {'hierarchi': ['strides', 'inchannels'],
                   'min': 1,
                   'max': 1,
                   'type': 'int'}
              },
             {'parms':
                  {'hierarchi': ['strides', 'batch'],
                   'min': 1,
                   'max': 1,
                   'type': 'int'}
              }
         ],
        'feedforward_layer':
        [
            {'parms':
                 {'hierarchi': ['size'],
                  'min': 256,
                  'max': 2048,
                  'type': 'int'}
             }
        ]
    }

    ACCURACY_WEIGHT = 20
    LAYER_CNT_WEIGHT = 2
    MAX_LAYERS = 3

    def __init__(self, candidate_id, start_time_str, runtime_spec, network_spec=None ):
        self.runtime_spec = copy.deepcopy(runtime_spec)

        self._base_logdir = os.path.join(self.runtime_spec['logdir'], str(start_time_str))

        self._candidate_id = candidate_id
        self._fitness = None
        if network_spec is None:
            network_spec = self._create_random_network()
        self.network_spec = network_spec

    def to_next_generation(self, generation):

        generation_dir = 'generation_{}/'.format(generation)
        id_dir = '{}/'.format(self._candidate_id)
        self.runtime_spec['logdir'] = os.path.join(self._base_logdir, generation_dir, id_dir)
        self.network_spec.update(self.runtime_spec)

    def crossover(self, crossover_parms, other_candidate):
        self._fitness = None

        if crossover_parms['strategy'] == 'uniform_crossover':
            self._crossover_uniform(crossver_rate=crossover_parms['rate'],
                                    other_candidate= other_candidate,
                                    uniform_method=crossover_parms['uniform_method'])
        else:
            raise ValueError('not implemented crossover strategy')

    def _crossover_uniform(self, crossver_rate, other_candidate, uniform_method):
        """Performs a unifrom Crossover between two Candidates"""
        if(uniform_method == 'swap'):
            min_layers = min(len(self.network_spec['layers']),len(other_candidate.network_spec['layers']))
            for layer_idx, layer in enumerate(self.network_spec['layers'][:min_layers]):
                layer_dict = layer
                other_layer_dict = other_candidate.network_spec['layers'][layer_idx]

                if (random.uniform(0, 1) <= crossver_rate):
                    layer_dict['activation_function'] = random.choice(self.ACTIVATION_CHOICES)

                if(layer_dict['type'] == other_layer_dict['type']):
                    self._swap_values(layer_dict,other_layer_dict, crossver_rate)

        else:
            raise ValueError('not implemented uniform_crossover_method')
    def _swap_values(self, dict, other_dict,rate):
        """Swaps Properties between two Layers of the same type with Propapility rate"""
        for parm in self.OPTIMIZING_PARMS[dict['type']]:
            if random.uniform(0,1)<=rate:
                parm_h = parm['parms']['hierarchi']
                if len(parm_h) == 1:
                    # Save old own
                    tmp = dict[parm_h[0]]
                    # own in other
                    dict[parm_h[0]] = \
                        other_dict[parm_h[0]]
                    # saved in own
                    other_dict[parm_h[0]] = tmp
                elif len(parm_h) == 2:
                    # Save old own
                    tmp = dict[parm_h[0]][parm_h[1]]
                    # own in other
                    dict[parm_h[0]][parm_h[1]] = \
                        other_dict[parm_h[0]][parm_h[1]]
                    # saved in own
                    other_dict[parm_h[0]][parm_h[1]] = tmp
                elif len(parm_h) == 3:
                    # Save old own
                    tmp = dict[parm_h[0]][parm_h[1]][parm_h[2]]
                    # own in other
                    dict[parm_h[0]][parm_h[1]][parm_h[2]] = \
                        other_dict[parm_h[0]][parm_h[1]][parm_h[2]]
                    # saved in own
                    other_dict[parm_h[0]][parm_h[1]][parm_h[2]] = tmp
                else:
                    raise ValueError('length of hierarchi must 1,2 or 3')

    def mutation(self, mutation_rate):
        # TODO: Check the mutation of a layer and the mutation of properties, layer mutation can hide value mutation
        """
        Mutate properties(layer-structure and layer-values of a Candidate)

        """
        self._fitness = None

        #Mutate layer
        for i, layer_spec in enumerate(self.network_spec['layers']):
            #Mutate complet layer
            if(random.uniform(0,1)<=(mutation_rate/10)):
                new_layer_type = random.choice(self.LAYER_TYPES)
                self.network_spec['layers'][i] = self._create_randomize_layer(layer_type=new_layer_type)
            else:#Only mutate Values if no new random layer
                self._mutate_layer_values(layer_dict=self.network_spec['layers'][i], mutation_rate=mutation_rate)

    def _mutate_layer_values(self, layer_dict, mutation_rate):
        """
        Mutate each value of a layer with a probability of mutation_rate
        """
        if(random.uniform(0,1) <= mutation_rate):
            layer_dict['activation_function'] = random.choice(self.ACTIVATION_CHOICES)
        for parms in self.OPTIMIZING_PARMS[layer_dict['type']]:
            if parms['parms']['max'] != parms['parms']['min']:

                parm_h = parms['parms']['hierarchi']
                variance = (parms['parms']['max'] - parms['parms']['min']) / 2
                if variance == 0 :
                    variance = 1
                if parms['parms']['type'] == 'int':
                    variance = int(variance)

                if len(parm_h) == 1:
                    layer_dict[parm_h[0]] = self._mutation_value_strategy(
                        old_value=layer_dict[parm_h[0]],
                        variance=variance)
                elif len(parm_h) == 2:
                    layer_dict[parm_h[0]][parm_h[1]] = self._mutation_value_strategy(
                                                        old_value=layer_dict[parm_h[0]][parm_h[1]],
                                                        variance=variance)
                elif len(parm_h) == 3:
                    layer_dict[parm_h[0]][parm_h[1]][parm_h[2]] = self._mutation_value_strategy(
                                                            old_value=layer_dict[parm_h[0]][parm_h[1]][parm_h[2]],
                                                            variance=variance)
                else:

                    raise ValueError('length of hierarchi must 1,2 or 3')

    def _mutation_value_strategy(self, old_value, variance):
        """ sub/add a number between -variance and variance"""
        return old_value + old_value.__class__(-variance, variance).value

    def get_diversity(self, otherCandidate):

        return random.random()

    def get_fitness(self):
        """Get fitness of the candidate. If not yet tested, test the fitness based on the network specificaton."""
        if(self._fitness is None):
            network = Network(self._serialze_network_spec())
            extended_spec_json = network.evaluate()
            extended_spec = json.loads(extended_spec_json)
            result_spec = extended_spec['results']
            print(result_spec)
            self._fitness = self._fitness_function(result_spec)
            del network
        return self._fitness

    def _fitness_function(self, results):
        """Calculate the fitness based on the network evaluation."""
        # TODO: get the number of weights as penalty?
        return  1 / (- self.ACCURACY_WEIGHT * math.log(results['accuracy'])
                     + self.LAYER_CNT_WEIGHT * len(self.network_spec['layers']))

    def _create_random_network(self):
        """Construct a random network specification."""

        #TODO: should this be done in this class?
        # Finalize runtime specification.

        layer_cnt = RangedInt(1, self.MAX_LAYERS)

        network_spec = {
            'hyperparameters': {
                'learningrate': RangedNum(1e-4, 1e-3),
                'optimizer': random.choice(self.OPTIMIZER_CHOICES),
                'batchsize': 100  # Fixed batch size for comparability.
            },
            'layers': []
        }

        cnt_layer_conv = RangedInt(0, layer_cnt.value)
        cnt_layer_max_pool = RangedInt(0, layer_cnt.value - cnt_layer_conv.value)
        cnt_layer_ff = layer_cnt.value - cnt_layer_conv.value - cnt_layer_max_pool.value

        layer_types = ['conv_layer' for _ in range(cnt_layer_conv.value)]
        layer_types += ['maxpool_layer' for _ in range(cnt_layer_max_pool.value)]
        random.shuffle(layer_types)
        layer_types += ['feedforward_layer' for _ in range(cnt_layer_ff)]

        for layer_type in layer_types:
            layer_spec = self._create_randomize_layer(layer_type=layer_type)
            #layer_spec = self._generate_network_layer(type=layer_type)
            # Add layer to the network spec.
            network_spec['layers'].append(layer_spec)

        return network_spec

    def _create_randomize_layer(self, layer_type):
        """
        Create a layer based on layer_tape
        """
        if layer_type == 'conv_layer':
            layer_spec = self._create_conv_layer()
        elif layer_type == 'maxpool_layer':
            layer_spec = self._create_maxpool_layer()
        elif layer_type == 'feedforward_layer':
            layer_spec = self._create_ff_layer()
        return layer_spec

    def _create_ff_layer(self):
        """
        Create dict for a random initialized Feedforward network
        :return:
        """

        layer = {
                  'type': 'feedforward_layer',
                  'size': RangedInt(256, 2048),
                  'activation_function': random.choice(self.ACTIVATION_CHOICES)
                }
        return layer

    def _create_conv_layer(self):
        """
        Create dict for a random initialized convolutional Layer
        """
        layer = {
            'type': 'conv_layer',
            'convolution': {
                'filter': {
                    'height': RangedInt(1, 5),
                    'width': RangedInt(1, 5),
                    'outchannels': RangedInt(1, 64)
                },
                'strides': {
                    'x': RangedInt(1, 2),
                    'y': RangedInt(1, 2),
                    'inchannels': 1,  # Must be 1. See https://www.tensorflow.org/api_docs/python/tf/nn/conv2d
                    'batch': 1
                }

            },
            'activation_function': random.choice(self.ACTIVATION_CHOICES)
        }
        return layer

    def _create_maxpool_layer(self):
        """
        Create dict for a random initialized Maxpool-layer
        """
        layer = {
            'type': 'maxpool_layer',
            'kernel': {
                'height': RangedInt(1, 5),
                'width': RangedInt(1, 5),
                'inchannels': 1,
            # Must probably be 1 as well. See https://www.tensorflow.org/api_docs/python/tf/nn/conv2d
                'outchannels': 1,
            },
            'strides': {
                'y': RangedInt(1, 5),
                'x': RangedInt(1, 5),
                'inchannels': 1,
            # Must probably be 1 as well. See https://www.tensorflow.org/api_docs/python/tf/nn/conv2d
                'batch': 1
            }
        }
        return layer
    def _generate_network_layer(self, type):
        # TODO: Implement this!
        raise Exception("Not implemented")
        for parms in self.OPTIMIZING_PARMS[type]:
            parm = parms['parms']
            parm_h = parm['hierarchi']
            layer = dict()
            layer['type'] = type
            layer['activation_function'] = random.choice(self.ACTIVATION_CHOICES)
            if len(parm_h) == 1:
                layer[parm_h[0]] = RangedInt(min=parm['min'], max=parm['max'])
            elif len(parm_h) == 2:
                layer[parm_h[0]][parm_h[2]] = RangedInt(min=parm['min'], max=parm['max'])
            elif len(parm_h) == 3:
                #layer.update({parm_h[0]:{parm_h[1]:{parm_h[2]:RangedInt(min=parm['min'], max=parm['max'])}}})
                #layer[parm_h[0]][parm_h[1]][parm_h[2]] = RangedInt(min=parm['min'], max=parm['max'])
                raise Exception("Not implemented")
            else:
                raise ValueError('length of hierarchi must 1,2 or 3')
        return layer
    def _serialze_network_spec(self):

        return RangedJSONEncoder().encode(self.network_spec)
<|endoftext|>


<commit_before>#!/usr/bin/env python2.7

import argparse
import glob
import os

parser = argparse.ArgumentParser( description="Creates the configs for Tasks" );
parser.add_argument( "list_path", default="./config", help="creates the folder and stores all configs within")
parser.add_argument( "config_path", default="./config", help="creates the folder and stores all configs within")
parser.add_argument( "submit_path", default="./config", help="creates the folder and stores all configs within")
parser.add_argument( "exe_path", default="./config", help="creates the folder and stores all configs within")

args = parser.parse_args()

t_submit="""
#############################################
# Create PidHisto files for PID Rcp Analysis
#############################################

Executable    = {exe}
initialdir 		= {wd}
"""

t_arg = """
Arguments = {cfg} {list} {prefix}
Queue
"""

list_files = glob.glob( os.path.join( args.list_path, "list_*" ) )

for f in list_files :
	print f

<commit_msg>working on condor_submit.py<commit_after>#!/usr/bin/env python2.7

import argparse
import glob
import os

parser = argparse.ArgumentParser( description="Creates the configs for Tasks" );
parser.add_argument( "list_path", default="./config", help="creates the folder and stores all configs within")
parser.add_argument( "config_path", default="./config", help="creates the folder and stores all configs within")
parser.add_argument( "submit_path", default="./config", help="creates the folder and stores all configs within")
parser.add_argument( "exe_path", default="./config", help="creates the folder and stores all configs within")

args = parser.parse_args()

t_submit="""
#############################################
# Create PidHisto files for PID Rcp Analysis
#############################################

Executable    = {exe}
initialdir 		= {wd}
"""

t_arg = """
Arguments = {cfg} {list} {prefix}
Queue
"""

list_files = glob.glob( os.path.join( args.list_path, "list_*" ) )

for f in list_files :
	print f
	prefix = f.split( '_' )[-1]
	print prefix + "_config.xml"

<|endoftext|>


<commit_before><commit_msg>Add option to not run in MPI mode<commit_after><|endoftext|>


<commit_before>"""

        Run the Strongly Connected Components algorithm
         by Travis Riggs

    This runs a strongly connected component analysis algorithm
    on a given graph edge file.

"""

from graphs import DirectedGraph

if __name__ == '__main__':

    pass

<commit_msg>Added loading of raw graph file<commit_after>"""

        Run the Strongly Connected Components algorithm
         by Travis Riggs

    This runs a strongly connected component analysis algorithm
    on a given graph edge file.

"""

from graphs import DirectedGraph

if __name__ == '__main__':

    digraph = DirectedGraph()

    with open("data/SCC.txt", "r") as f:

        for line in f.readlines():

            head, tail = line.strip().split(" ")

            digraph.add_di_edge(head, tail)

    digraph.save_graph("data/bigDiGraph.dot")

<|endoftext|>