<commit_before>from __future__ import unicode_literals

import datetime
import logging
import random

import re
from six.moves.urllib.parse import quote, urlsplit

from bs4 import BeautifulSoup
from requests import Session

logger = logging.getLogger('incapsula')

class IncapBlocked(ValueError):

    def __init__(self, response, *args):
        self.response = response
        super(IncapBlocked, self).__init__(*args)

# A list of valid values which are tested in the incapsula test method.
# These values are pulled straight from my browser and should be enough to spoof
# the robot check when setting the cookie.
o = [
    ('navigator', 'true'),
    ('navigator.vendor', 'Google Inc.'),
    ('navigator.appName', 'Netscape'),
    ('navigator.plugins.length==0', 'false'),
    ('navigator.platform', 'Linux x86_64'),
    ('navigator.webdriver', 'undefined'),
    ('plugin_ext', 'no extention'),
    ('plugin_ext', 'so'),
    ('ActiveXObject', 'false'),
    ('webkitURL', 'true'),
    ('_phantom', 'false'),
    ('callPhantom', 'false'),
    ('chrome', 'true'),
    ('yandex', 'false'),
    ('opera', 'false'),
    ('opr', 'false'),
    ('safari', 'false'),
    ('awesomium', 'false'),
    ('puffinDevice', 'false'),
    ('__nightmare', 'false'),
    ('_Selenium_IDE_Recorder', 'false'),
    ('document.__webdriver_script_fn', 'false'),
    ('document.$cdc_asdjflasutopfhvcZLmcfl_', 'false'),
    ('process.version', 'false'),
    ('navigator.cpuClass', 'false'),
    ('navigator.oscpu', 'false'),
    ('navigator.connection', 'false'),
    ('window.outerWidth==0', 'false'),
    ('window.outerHeight==0', 'false'),
    ('window.WebGLRenderingContext', 'true'),
    ('document.documentMode', 'undefined'),
    ('eval.toString().length', '33')
]

def test():
    """
    Quote each value in the tuple list and return a comma delimited string of the parameters.

    This method is a shortened version of incapsulas test method. What the original method does is check
    for specific plugins in your browser and set a cookie based on which extensions you have installed.
    The list of the values is taken from my own browser after running the test method so they are all valid.

    This is just more of a shortcut method instead of trying to reverse engineer the entire code that they had.
    :return:
    """
    # safe param set to () for the single parameter with the key of "eval.toString().length".
    # This is needed to match the cookie value exactly with what is expected from incapsula.
    r = [quote('='.join(x), safe='()') for x in o]
    return ','.join(r)

def simple_digest(s):
    """
    Create a sum of the ordinal values of the characters passed in from s.

    Translated From:
    ```javascript
    function simpleDigest(mystr) {
        var res = 0;
        for (var i = 0; i < mystr.length; i++) {
            res += mystr.charCodeAt(i);
        }
        return res;
    }
    ```
    :param s: The string to calculate the digest from.
    :return:
    """
    res = 0
    for ch in s:
        res += ord(ch)
    return str(res)

class IncapSession(Session):
    # Max retries before giving up cracking incapsula.
    MAX_INCAP_RETRIES = 3
    # noinspection PyPep8
    default_useragent = 'IncapUnblockSession (sdscdeveloper@gmail.com | https://github.com/ziplokk1/incapsula-cracker-py3)'

    def __init__(self, **kwargs):
        # noinspection PyPep8
        """

        :param kwargs:
        :param user_agent: Change the default user agent when sending requests.
        :param cookie_domain: Use this param to change the domain which is set in the cookie.
            Sometimes the domain set for the cookie isn't the same as the actual host.
            i.e. .domain.com instead of www.domain.com.

        """
        user_agent = kwargs.pop('user_agent', self.default_useragent)
        self.cookie_domain = kwargs.pop('cookie_domain', '')
        super(IncapSession, self).__init__()
        self.headers['User-Agent'] = user_agent

    def get_session_cookies(self):
        """
        Get cookies from the session to use with set_incap_cookie.

        Needed to create the simple_digest for the final cookie later.

        Translated from:

        ```javascript
        function getSessionCookies() {
            var cookieArray = new Array();
            var cName = /^\s?incap_ses_/;
            var c = document.cookie.split(";");
            for (var i = 0; i < c.length; i++) {
                var key = c[i].substr(0, c[i].indexOf("="));
                var value = c[i].substr(c[i].indexOf("=") + 1, c[i].length);
                if (cName.test(key)) {
                    cookieArray[cookieArray.length] = value;
                }
            }
            return cookieArray;
        }
        ```
        :param self:
        :return:
        """
        a = []
        for cookie in self.cookies:
            if cookie.name.startswith('incap_ses_'):
                a.append(cookie.value)
        return a

    def create_cookie(self, name, value, seconds, domain=''):
        """
        Set the incapsula cookie in the session cookies.

        :param domain:
        :param name: Cookie name.
        :param value: Cookie value.
        :param seconds: Expiry seconds from the current time.
        :return:
        """
        expires = None
        if seconds:
            d = datetime.datetime.now()
            d += datetime.timedelta(seconds=seconds)
            expires = round((d - datetime.datetime.utcfromtimestamp(0)).total_seconds() * 1000)
        self.cookies.set(name, value, domain=domain, path='/', expires=expires)

    def set_incap_cookie(self, v_array, domain=''):
        """
        Calculate the final value for the cookie needed to bypass incapsula.

        Translated from:
        ```javascript
        function setIncapCookie(vArray) {
            var res;
            try {
                var cookies = getSessionCookies();
                var digests = new Array(cookies.length);
                for (var i = 0; i < cookies.length; i++) {
                    digests[i] = simpleDigest((vArray) + cookies[i]);
                }
                res = vArray + ",digest=" + (digests.join());
            } catch (e) {
                res = vArray + ",digest=" + (encodeURIComponent(e.toString()));
            }
            createCookie("___utmvc", res, 20);
        }
        ```
        :param domain:
        :param v_array:
        :return:
        """
        cookies = self.get_session_cookies()
        digests = []
        for cookie_val in cookies:
            digests.append(simple_digest(v_array + cookie_val))
        res = v_array + ',digest=' + ','.join(digests)
        logger.debug('setting ___utmvc cookie to {}'.format(res))
        self.create_cookie('___utmvc', res, 20, domain=domain)

    # noinspection PyMethodMayBeStatic
    def incap_blocked(self, content):
        """
        Check if the resource is blocked by incapsula.

        :param content: HTML string.
        :return:
        """
        # Check for the ROBOTS meta tag in the content.
        # This is a dead giveaway that the resource is blocked by incap.
        soup = BeautifulSoup(content, 'html.parser')
        # Not always reliable since some pages have a <meta name="robots"> even if its not blocked by incap.
        # To circumvent that, we also look for an iframe which shows the incapsula error message.
        robots_tag = soup.find('meta', {'name':  re.compile('^robots$', re.IGNORECASE)})
        incap_iframe = soup.find('iframe', {'src': re.compile('^/_Incapsula_Resource.*|^//content.incapsula.com.*')})
        return robots_tag and incap_iframe

    def html_is_recaptcha(self, html_string):
        """
        Check if the html contains recaptcha robot prevention.

        :param html_string:
        :return:
        """
        soup = BeautifulSoup(html_string, 'html.parser')
        return bool(soup.find('form', {'id': 'captcha-form'}) or soup.find('div', {'class': 'g-recaptcha'}))

    def incap_recaptcha_blocked(self, scheme, host, content):
        """
        Get the content from the iframe in a blocked request to determine if the block contains a captcha.

        :param scheme: http, ftp, https, etc.
        :param host: www.example.com.
        :param content: HTML content from a response.
        :return:
        """
        soup = BeautifulSoup(content, 'html.parser')
        # Get incap iframe
        incap_iframe = soup.find('iframe', {'src': re.compile('^/_Incapsula_Resource.*')})
        # If the iframe doesn't exist then it's not recaptcha blocked.
        if not incap_iframe:
            return False

        # Send request to get the content of the iframe.
        iframe_url = incap_iframe.get('src')
        resource = self.get(scheme + '://' + host + iframe_url, bypass_crack=True)

        # If the below method is true, then the iframe content is for a recaptcha and there's no way around that.
        return self.html_is_recaptcha(resource.content)

    def crack(self, resp, org=None, tries=0):
        # Use to hold the original request so that when attempting the new unblocked request, we have a reference
        # to the original url.
        org = org or resp

        # Return original response after too many tries to bypass incap.
        if tries >= self.MAX_INCAP_RETRIES:
            raise IncapBlocked(resp, 'max retries exceeded when attempting to crack incapsula.')

        if self.incap_blocked(resp.content):
            logger.debug('incap blocked. attempt={} url={}'.format(tries, resp.url))

            # Split the url so that no matter what site is being requested, we can figure out the host of
            # the incapsula resource.
            split = urlsplit(org.url)
            scheme = split.scheme
            host = split.netloc

            if self.incap_recaptcha_blocked(scheme, host, resp.content):
                raise IncapBlocked(resp, 'resource blocked by incapsula re-captcha method.')

            # Set cookie then send request to incap resource to "apply" cookie.
            self.set_incap_cookie(test(), self.cookie_domain or host)
            self.get(scheme + '://' + host + '/_Incapsula_Resource?SWKMTFSR=1&e={}'.format(random.random()))

            # Call crack() again since if the request isn't blocked after the above cookie-set and request,
            # then it will just return the unblocked resource.
            return self.crack(self.get(org.url, bypass_crack=True), org=org, tries=tries+1)
        return resp

    # noinspection PyIncorrectDocstring
    def get(self, url, **kwargs):
        """
        Sends a GET request. Returns :class:`Response` object.

        :param url: URL for the new :class:`Request` object.
        :param kwargs: Optional arguments that ``request`` takes.
        :param kwargs['bypass_crack']: Use when sending a request that you dont want to go through the incapsula crack.
            Used in this class so when sending a get request from this instance,
            we dont end up creating an infinate loop by calling .get() then .crack() which calls .get()
            and repeat x infinity. Also any requests made to get encapsula resources dont need to be cracked.
        :rtype: requests.Response
        """

        kwargs.setdefault('allow_redirects', True)

        # If the request is to get the incapsula resources, then we dont call crack().
        if kwargs.pop('bypass_crack', False):
            return self.request('GET', url, **kwargs)

        return self.crack(self.request('GET', url, **kwargs))
<commit_msg>Split logic into parsing and requesting.<commit_after>from __future__ import unicode_literals

import datetime
import logging
import random

import re
from six.moves.urllib.parse import quote, urlsplit

from bs4 import BeautifulSoup
from requests import Session

logger = logging.getLogger('incapsula')

class IncapBlocked(ValueError):

    def __init__(self, response, *args):
        self.response = response
        super(IncapBlocked, self).__init__(*args)

# A list of valid values which are tested in the incapsula test method.
# These values are pulled straight from my browser and should be enough to spoof
# the robot check when setting the cookie.
o = [
    ('navigator', 'true'),
    ('navigator.vendor', 'Google Inc.'),
    ('navigator.appName', 'Netscape'),
    ('navigator.plugins.length==0', 'false'),
    ('navigator.platform', 'Linux x86_64'),
    ('navigator.webdriver', 'undefined'),
    ('plugin_ext', 'no extention'),
    ('plugin_ext', 'so'),
    ('ActiveXObject', 'false'),
    ('webkitURL', 'true'),
    ('_phantom', 'false'),
    ('callPhantom', 'false'),
    ('chrome', 'true'),
    ('yandex', 'false'),
    ('opera', 'false'),
    ('opr', 'false'),
    ('safari', 'false'),
    ('awesomium', 'false'),
    ('puffinDevice', 'false'),
    ('__nightmare', 'false'),
    ('_Selenium_IDE_Recorder', 'false'),
    ('document.__webdriver_script_fn', 'false'),
    ('document.$cdc_asdjflasutopfhvcZLmcfl_', 'false'),
    ('process.version', 'false'),
    ('navigator.cpuClass', 'false'),
    ('navigator.oscpu', 'false'),
    ('navigator.connection', 'false'),
    ('window.outerWidth==0', 'false'),
    ('window.outerHeight==0', 'false'),
    ('window.WebGLRenderingContext', 'true'),
    ('document.documentMode', 'undefined'),
    ('eval.toString().length', '33')
]

def test():
    """
    Quote each value in the tuple list and return a comma delimited string of the parameters.

    This method is a shortened version of incapsulas test method. What the original method does is check
    for specific plugins in your browser and set a cookie based on which extensions you have installed.
    The list of the values is taken from my own browser after running the test method so they are all valid.

    This is just more of a shortcut method instead of trying to reverse engineer the entire code that they had.
    :return:
    """
    # safe param set to () for the single parameter with the key of "eval.toString().length".
    # This is needed to match the cookie value exactly with what is expected from incapsula.
    r = [quote('='.join(x), safe='()') for x in o]
    return ','.join(r)

def simple_digest(s):
    """
    Create a sum of the ordinal values of the characters passed in from s.

    Translated From:
    ```javascript
    function simpleDigest(mystr) {
        var res = 0;
        for (var i = 0; i < mystr.length; i++) {
            res += mystr.charCodeAt(i);
        }
        return res;
    }
    ```
    :param s: The string to calculate the digest from.
    :return:
    """
    res = 0
    for ch in s:
        res += ord(ch)
    return str(res)

class BaseResourceParser(object):
    """
    Superclass for all other parser objects.
    """

    def __init__(self, response):
        """
        :type response: requests.Response
        :param response:
        """
        self.response = response
        split = urlsplit(response.url)
        self.scheme = split.scheme
        self.host = split.netloc
        self.soup = BeautifulSoup(self.response.content, 'html.parser')

class IframeResourceParser(BaseResourceParser):
    """
    Parser object to obtain the contents of the incapsula iframe.
    """

    def __init__(self, response):
        """

        :param response: The response of the request sent to the incapsula iframe url.
        :type response: requests.Response
        """
        super(IframeResourceParser, self).__init__(response)
        self.recaptcha_elements = [
            ('form', {'id': 'captcha-form'}),
            ('div', {'class': 'g-recaptcha'})
        ]

    def is_recaptcha(self):
        """
        Determine whether the iframe contents is a google recaptcha.

        :return: True if the iframe contains a google recaptcha.
        """
        for element in self.recaptcha_elements:
            if self.soup.find(*element):
                return True
        return False

class WebsiteResourceParser(BaseResourceParser):
    """
    Parser object for the inital resource which was requested.
    """

    def __init__(self, response, extra_iframe_patterns=None):
        """

        :param response: The response of the request sent to the targeted host.
        :param extra_iframe_patterns: A list of compiled regex patterns which will match the incapsula iframe src attribute value.
            Use when there are issues detecting whether the resource is blocked by incapsula.
        :type extra_iframe_patterns: list[_sre.SRE_Pattern]
        """
        super(WebsiteResourceParser, self).__init__(response)
        # Common incapsula iframe src values.
        # Used to find the iframe which hosts the incapsula resource.
        self.incapsula_iframe_src_patterns = [
            re.compile('^/_Incapsula_Resource.*'),
            re.compile('^//content\.incapsula\.com.*')
        ]
        if extra_iframe_patterns:
            self.incapsula_iframe_src_patterns.extend(extra_iframe_patterns)

    @property
    def robots_meta(self):
        """
        The <meta name="ROBOTS"> tag which is so commonly found in incapsula blocked resources.

        :rtype: bs4.element.Tag
        :return:
        """
        return self.soup.find('meta', {'name': re.compile('^robots$', re.IGNORECASE)})

    @property
    def incapsula_iframe(self):
        """
        The iframe which contains the javascript code that runs on browser load.

        :rtype: bs4.element.Tag
        :return:
        """
        for pattern in self.incapsula_iframe_src_patterns:
            iframe = self.soup.find('iframe', {'src': pattern})
            if iframe:
                return iframe

    @property
    def incapsula_iframe_url(self):
        """
        The src attribute value of the incapsula iframe.

        :rtype: str
        :return:
        """
        if self.incapsula_iframe:
            uri = self.incapsula_iframe.get('src')
            # Case when uri isn't actually a uri, but an external resource.
            if uri.startswith('//'):
                return self.scheme + ':' + uri
            return self.scheme + '://' + self.host + uri

    def is_blocked(self):
        """
        Determine whether the resource is blocked by incapsula or not.

        If the resource has the <meta name="ROBOTS"> tag and the incapsula IFrame
        then we can assume the resource is blocked.
        :return:
        """
        return bool(self.robots_meta) and bool(self.incapsula_iframe)

class IncapSession(Session):

    def __init__(self, max_retries=3, user_agent=None, cookie_domain=''):
        """

        :param max_retries: The number of times to attempt to get the incapsula resource before giving up.
            Set this to `None` to never give up.
        :param user_agent: Change the default user agent when sending requests.
        :param cookie_domain: Use this param to change the domain which is set in the cookie.
            Sometimes the domain set for the cookie isn't the same as the actual host.
            i.e. .domain.com instead of www.domain.com.

        """
        super(IncapSession, self).__init__()

        default_useragent = 'IncapUnblockSession (https://github.com/ziplokk1/incapsula-cracker-py3)'
        user_agent = user_agent or default_useragent

        self.max_retries = max_retries
        self.cookie_domain = cookie_domain
        self.headers['User-Agent'] = user_agent

    def get_incapsula_resource_url(self, scheme, host):
        """
        Override this method to change the get request after the cookies are set.

        After the cookies are set, there is a GET request which must get sent to validate the session.
        Override this method to return a different url to send the GET request to.
        This method is more of a future proofing measure than anything.

        Reverse engineer from:
        ```javascript
        setIncapCookie(test(o));
        document.createElement("img").src = "/_Incapsula_Resource?SWKMTFSR=1&e=" + Math.random()
        ```
        """
        rdm = random.random()
        return scheme + '://' + host + '/_Incapsula_Resource?SWKMTFSR=1&e={}'.format(rdm)

    def get_session_cookies(self):
        """
        Get a list of cookies which start with 'incap_ses_'.

        Needed to create the simple_digest for the final cookie later.

        Translated from:

        ```javascript
        function getSessionCookies() {
            var cookieArray = new Array();
            var cName = /^\s?incap_ses_/;
            var c = document.cookie.split(";");
            for (var i = 0; i < c.length; i++) {
                var key = c[i].substr(0, c[i].indexOf("="));
                var value = c[i].substr(c[i].indexOf("=") + 1, c[i].length);
                if (cName.test(key)) {
                    cookieArray[cookieArray.length] = value;
                }
            }
            return cookieArray;
        }
        ```
        :param self:
        :return:
        """
        return [cookie.value for cookie in self.cookies if cookie.name.startswith('incap_ses_')]

    def create_cookie(self, name, value, seconds, domain=''):
        """
        Set the incapsula cookie in the session cookies.

        :param domain:
        :param name: Cookie name.
        :param value: Cookie value.
        :param seconds: Expiry seconds from the current time.
        :return:
        """
        expires = None
        if seconds:
            d = datetime.datetime.now()
            d += datetime.timedelta(seconds=seconds)
            expires = round((d - datetime.datetime.utcfromtimestamp(0)).total_seconds() * 1000)
        self.cookies.set(name, value, domain=domain, path='/', expires=expires)

    def set_incap_cookie(self, v_array, domain=''):
        """
        Calculate the final value for the cookie needed to bypass incapsula.

        Translated from:
        ```javascript
        function setIncapCookie(vArray) {
            var res;
            try {
                var cookies = getSessionCookies();
                var digests = new Array(cookies.length);
                for (var i = 0; i < cookies.length; i++) {
                    digests[i] = simpleDigest((vArray) + cookies[i]);
                }
                res = vArray + ",digest=" + (digests.join());
            } catch (e) {
                res = vArray + ",digest=" + (encodeURIComponent(e.toString()));
            }
            createCookie("___utmvc", res, 20);
        }
        ```
        :param domain:
        :param v_array:
        :return:
        """
        cookies = self.get_session_cookies()
        digests = []
        for cookie_val in cookies:
            digests.append(simple_digest(v_array + cookie_val))
        res = v_array + ',digest=' + ','.join(digests)
        logger.debug('setting ___utmvc cookie to {}'.format(res))
        self.create_cookie('___utmvc', res, 20, domain=domain)

    def raise_for_recaptcha(self, resource):
        """
        Raise an IncapBlocked exception if the iframe in the original resource contains a recaptcha.

        Send get request to iframe url to get the contents and raise if the contents contain a re-captcha.

        :param resource: Resource object from original request.
        :return:
        """
        # Get the content from the iframe.
        iframe_response = self.get(resource.incapsula_iframe_url, bypass_crack=True)
        iframe_resource = IframeResourceParser(iframe_response)

        if iframe_resource.is_recaptcha():
            raise IncapBlocked(iframe_response, 'resource blocked by re-captcha')

    def apply_cookies(self, original_url):
        """
        Set the session cookies and send the necessary GET request to "apply" the cookies.

        :param original_url: The url of the original request.
            Needed to determine the scheme and host of the domain to send the request to apply the cookies.
        :return:
        """
        # Split the url so that no matter what site is being requested, we can figure out the host of
        # the incapsula resource.
        split = urlsplit(original_url)
        scheme = split.scheme
        host = split.netloc

        # Set the cookie then send request to incap resource to "apply" cookie.
        self.set_incap_cookie(test(), self.cookie_domain or host)
        self.get(self.get_incapsula_resource_url(scheme, host), bypass_crack=True)

    def crack(self, resp, org=None, tries=0):
        """
        If the response is blocked by incapsula then set the necessary cookies and attempt to bypass it.

        :param resp: Response to check.
        :param org: Original response. Used only when called recursively.
        :param tries: Number of attempts. Used only when called recursively.
        :return:
        """
        # Use to hold the original request so that when attempting the new unblocked request, we have a reference
        # to the original url.
        org = org or resp

        # Return original response after too many tries to bypass incap.
        # If max_retries is None then this part will never get executed allowing a continuous retry.
        if self.max_retries is not None and tries >= self.max_retries:
            raise IncapBlocked(resp, 'max retries exceeded when attempting to crack incapsula')

        resource = WebsiteResourceParser(resp)
        if resource.is_blocked():
            logger.debug('Resource is blocked. attempt={} url={}'.format(tries, resp.url))
            # Raise if the response content's iframe contains a recaptcha.
            self.raise_for_recaptcha(resource)

            # Apply cookies and send GET request to apply them.
            self.apply_cookies(org.url)

            # Recursively call crack() again since if the request isn't blocked after the above cookie-set and request,
            # then it will just return the unblocked resource.
            return self.crack(self.get(org.url, bypass_crack=True), org=org, tries=tries+1)

        return resp

    def get(self, url, bypass_crack=False, **kwargs):
        """
        Sends a GET request. Returns :class:`Response` object.

        :param url: URL for the new :class:`Request` object.
        :param bypass_crack: Use when sending a request that you dont want to go through the incapsula crack.
        :param kwargs: Optional arguments that ``request`` takes.
            Used in this class so when sending a get request from this instance,
            we dont end up creating an infinate loop by calling .get() then .crack() which calls .get()
            and repeat x infinity. Also any requests made to get incapsula resources don't need to be cracked.
        :rtype: requests.Response
        """

        kwargs.setdefault('allow_redirects', True)

        # If the request is to get the incapsula resources, then we dont call crack().
        if bypass_crack:
            return self.request('GET', url, **kwargs)

        return self.crack(self.request('GET', url, **kwargs))
<|endoftext|>


<commit_before>from django.urls import reverse_lazy
from django.views.generic import CreateView, UpdateView, DetailView, ListView

from django.contrib import messages

from django.shortcuts import render, get_object_or_404, redirect
from django.template import RequestContext

from account.mixins import LoginRequiredMixin
from django.contrib.auth.mixins import PermissionRequiredMixin

from .forms import ProfileForm, ContactProfileForm, Profile_AffiliationForm
from .models import *
from account.decorators import login_required
from reversion.models import Version
from django.http import HttpResponseRedirect, Http404
from project.models import Get_User_Tasks, Get_Profile_Tasks

class ProfileEditView(LoginRequiredMixin, UpdateView):

    form_class = ProfileForm
    model = Profile

    def get_object(self):
        return self.request.user.profile

    def get_success_url(self):
        return reverse_lazy( my_profile_view )

    def form_valid(self, form):
        response = super(ProfileEditView, self).form_valid(form)
        messages.success(self.request, "You successfully updated your profile.")
        return response

class ProfileDetailView(DetailView):

    model = Profile
    slug_url_kwarg = "profile_id"
    slug_field = "id"
    context_object_name = "profile"

    def get_context_data(self, **kwargs):
        context = super(ProfileDetailView, self).get_context_data(**kwargs)
        current_profile = self.get_object()
        if current_profile.user == self.request.user:
            #
            context['user_repo_pw'] = current_profile.repo_pw

        context['main_profiles'] = current_profile.main_profiles()
        context['sub_profiles'] =  current_profile.sub_profiles()
        if ( current_profile.profile_type == PROFILE_TYPE_USER ) and ( not( current_profile.user is None ) ):
            context['controlled_profiles'] = Profile_Control_User.objects.filter( control_user = current_profile.user )
            context['user_tasks'] = Get_User_Tasks( current_profile.user )
        else:
            if ( current_profile.profile_type in PROFILE_TYPE_FOR_TASK ):
                context['profile_tasks'] = Get_Profile_Tasks( current_profile )

        context['controlled_by_user'] = Profile_Control_User.objects.filter( controlled_profile = current_profile )

        return context

@login_required
def my_profile_view(request):
    return redirect( 'profiles_detail', pk = request.user.profile.id )

class ProfileListView(ListView):

    model = Profile
    context_object_name = "profiles"

    def get_context_data(self, **kwargs):
        context = super(ProfileListView, self).get_context_data(**kwargs)
        # check if user has permission to create profile (or super user)
        context[ 'can_add_profile' ] = self.request.user.has_perm('profiles.add_profile')
        return context

class ProfileCreateView(LoginRequiredMixin, PermissionRequiredMixin, CreateView):
    model = Profile
    form_class = ContactProfileForm
    permission_required = 'profiles.add_profile'
    raise_exception = True

class ProfileUpdateView(LoginRequiredMixin, UpdateView):
    model = Profile
    form_class = ContactProfileForm

class ProfileCreateSubView(LoginRequiredMixin, CreateView):
    model = Profile_Affiliation
    form_class = Profile_AffiliationForm

    def get_initial(self):
        try:
            self.mp = get_object_or_404( Profile, pk = self.kwargs['pk'])
            return { 'main_profile': self.mp, }
        except:
            Http404()

    def form_valid(self, form):
        self.object = form.save(commit=False)
        self.object.main_profile = self.mp
        self.object.save()

        messages.success(self.request, "You successfully add the affiliation!")
        return HttpResponseRedirect( self.object.main_profile.get_absolute_url() )

from .models import Notification, GetUserNoticationsQ

@login_required
def notifications_view_prepare(request, arg_new):
    context = RequestContext(request)
    u = request.user
    if u is None:
        raise Http404

    OLD_NOTIFICATIONS_VIEW_LIMIT = 20
    notifications = GetUserNoticationsQ( u, arg_new )
    if arg_new:
        notifications.order_by('sender_user')
    else:
        notifications = notifications[:OLD_NOTIFICATIONS_VIEW_LIMIT]

    context_dict = { 'notifications' : notifications, 'filter_new' : arg_new, 'OLD_NOTIFICATIONS_VIEW_LIMIT' : OLD_NOTIFICATIONS_VIEW_LIMIT }
    return render( request, 'profiles/notifications.html', context_dict )

@login_required
def notifications_view_unread(request):
    return notifications_view_prepare( request, True )

@login_required
def notifications_view_read(request):
    return notifications_view_prepare( request, False )

@login_required
def notification_read( request, notification_id ):
    from django.http import HttpResponse
    #
    n = get_object_or_404( Notification, pk=notification_id )
    # ,   -
    if n.reciever_user == request.user:
        if n.get_unreaded:
            n.mark_readed()

        if ( n.msg_url is None ) or ( n.msg_url == '' ):
            return redirect( 'unread_notifications_view' )
        else:
            return redirect( n.msg_url )
    else:
        raise Http404()
<commit_msg>#110 Fix error in ordering of new notifications by sender<commit_after>from django.urls import reverse_lazy
from django.views.generic import CreateView, UpdateView, DetailView, ListView

from django.contrib import messages

from django.shortcuts import render, get_object_or_404, redirect
from django.template import RequestContext

from account.mixins import LoginRequiredMixin
from django.contrib.auth.mixins import PermissionRequiredMixin

from .forms import ProfileForm, ContactProfileForm, Profile_AffiliationForm
from .models import *
from account.decorators import login_required
from reversion.models import Version
from django.http import HttpResponseRedirect, Http404
from project.models import Get_User_Tasks, Get_Profile_Tasks

class ProfileEditView(LoginRequiredMixin, UpdateView):

    form_class = ProfileForm
    model = Profile

    def get_object(self):
        return self.request.user.profile

    def get_success_url(self):
        return reverse_lazy( my_profile_view )

    def form_valid(self, form):
        response = super(ProfileEditView, self).form_valid(form)
        messages.success(self.request, "You successfully updated your profile.")
        return response

class ProfileDetailView(DetailView):

    model = Profile
    slug_url_kwarg = "profile_id"
    slug_field = "id"
    context_object_name = "profile"

    def get_context_data(self, **kwargs):
        context = super(ProfileDetailView, self).get_context_data(**kwargs)
        current_profile = self.get_object()
        if current_profile.user == self.request.user:
            #
            context['user_repo_pw'] = current_profile.repo_pw

        context['main_profiles'] = current_profile.main_profiles()
        context['sub_profiles'] =  current_profile.sub_profiles()
        if ( current_profile.profile_type == PROFILE_TYPE_USER ) and ( not( current_profile.user is None ) ):
            context['controlled_profiles'] = Profile_Control_User.objects.filter( control_user = current_profile.user )
            context['user_tasks'] = Get_User_Tasks( current_profile.user )
        else:
            if ( current_profile.profile_type in PROFILE_TYPE_FOR_TASK ):
                context['profile_tasks'] = Get_Profile_Tasks( current_profile )

        context['controlled_by_user'] = Profile_Control_User.objects.filter( controlled_profile = current_profile )

        return context

@login_required
def my_profile_view(request):
    return redirect( 'profiles_detail', pk = request.user.profile.id )

class ProfileListView(ListView):

    model = Profile
    context_object_name = "profiles"

    def get_context_data(self, **kwargs):
        context = super(ProfileListView, self).get_context_data(**kwargs)
        # check if user has permission to create profile (or super user)
        context[ 'can_add_profile' ] = self.request.user.has_perm('profiles.add_profile')
        return context

class ProfileCreateView(LoginRequiredMixin, PermissionRequiredMixin, CreateView):
    model = Profile
    form_class = ContactProfileForm
    permission_required = 'profiles.add_profile'
    raise_exception = True

class ProfileUpdateView(LoginRequiredMixin, UpdateView):
    model = Profile
    form_class = ContactProfileForm

class ProfileCreateSubView(LoginRequiredMixin, CreateView):
    model = Profile_Affiliation
    form_class = Profile_AffiliationForm

    def get_initial(self):
        try:
            self.mp = get_object_or_404( Profile, pk = self.kwargs['pk'])
            return { 'main_profile': self.mp, }
        except:
            Http404()

    def form_valid(self, form):
        self.object = form.save(commit=False)
        self.object.main_profile = self.mp
        self.object.save()

        messages.success(self.request, "You successfully add the affiliation!")
        return HttpResponseRedirect( self.object.main_profile.get_absolute_url() )

from .models import Notification, GetUserNoticationsQ

@login_required
def notifications_view_prepare(request, arg_new):
    context = RequestContext(request)
    u = request.user
    if u is None:
        raise Http404

    OLD_NOTIFICATIONS_VIEW_LIMIT = 20
    notifications_q = GetUserNoticationsQ( u, arg_new )
    if arg_new:
        notifications = notifications_q.order_by('sender_user')
    else:
        notifications = notifications_q[:OLD_NOTIFICATIONS_VIEW_LIMIT]

    context_dict = { 'notifications' : notifications, 'filter_new' : arg_new, 'OLD_NOTIFICATIONS_VIEW_LIMIT' : OLD_NOTIFICATIONS_VIEW_LIMIT }
    return render( request, 'profiles/notifications.html', context_dict )

@login_required
def notifications_view_unread(request):
    return notifications_view_prepare( request, True )

@login_required
def notifications_view_read(request):
    return notifications_view_prepare( request, False )

@login_required
def notification_read( request, notification_id ):
    from django.http import HttpResponse
    #
    n = get_object_or_404( Notification, pk=notification_id )
    # ,   -
    if n.reciever_user == request.user:
        if n.get_unreaded:
            n.mark_readed()

        if ( n.msg_url is None ) or ( n.msg_url == '' ):
            return redirect( 'unread_notifications_view' )
        else:
            return redirect( n.msg_url )
    else:
        raise Http404()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# tests app_config
#
# Copyright 2016-2017 Canonical, Ltd.

import json
import tempfile
import unittest
from unittest.mock import MagicMock

from kv import KV
from ubuntui.widgets.input import StringEditor

from conjureup.app_config import AppConfig
from conjureup.models.provider import AWS, Field, Form

from .helpers import AsyncMock, test_loop

class AppConfigTestCase(unittest.TestCase):

    def setUp(self):
        self.expected_keys = [
            'config',
            'provider'
        ]
        self.app = AppConfig()
        self.db_file = tempfile.NamedTemporaryFile()
        self.app.state = KV(self.db_file.name)
        self.app.provider = AWS()
        self.app.provider.controller = "fake-tester-controller"
        self.app.provider.model = "fake-tester-model"
        self.app.provider.cloud_type = "ec2"
        self.app.config = {'spell': 'kubernetes-core'}

        self.app.juju.client = AsyncMock()
        self.app.log = MagicMock()

    def tearDown(self):
        self.db_file.close()

    @unittest.skip("This requires our app_config.restore/save "
                   "to serialize/deserialize app.provider class")
    def test_config_state_save(self):
        "app_config.test_config_state_save"
        self.app.juju.authenticated = False
        with test_loop() as loop:
            loop.run_until_complete(self.app.save())

        results = self.app.state.get(self.app._internal_state_key)
        assert set(json.loads(results.decode('utf8')).keys()) == set(
            self.expected_keys)

    def test_provider_form_query_key(self):
        "app_config.provider_form_widget_query_key"
        self.app.provider.form = Form(
            [Field(label='test widget',
                   widget=StringEditor(default='hai2u'),
                   key='test-key')])
        assert self.app.provider.form.field('test-key').value == 'hai2u'

    def test_config_juju_model_save(self):
        "app_config.test_config_juju_model_save"

        self.app.juju.authenticated = True
        with test_loop() as loop:
            loop.run_until_complete(self.app.save())

        assert self.app.juju.client.set_config.called

    def test_config_restore(self):
        "app_config.test_config_restore"
        self.app.juju.authenticated = False

        with test_loop() as loop:
            yield self.app.save()
            loop.run_until_complete(self.app.restore())

        results_json = self.app.state.get(self.app._internal_state_key)
        results = json.loads(results_json.decode('utf8'))

        assert self.app.app.controller == results['controller']

    @unittest.skip("Also need serialize/deserialize during "
                   "save/restore of app.provider class")
    def test_config_juju_restore(self):
        "app_config.test_config_juju_restore"
        class FakeExtraInfo:
            def __init__(self):
                self.value = b'{"controller": "moo"}'

        self.app.juju.authenticated = True
        self.app.juju.client.get_config.return_value = {
            "extra-info": FakeExtraInfo()}
        with test_loop() as loop:
            loop.run_until_complete(self.app.restore())
        assert self.app.provider.controller == 'moo'

    def test_config_guard_unknown_attribute(self):
        "app_config.test_config_guard_unknown_attribute"
        with self.assertRaises(Exception):
            self.app.chimichanga = "Yum"
<commit_msg>Fix test to replicate bug<commit_after>#!/usr/bin/env python
#
# tests app_config
#
# Copyright 2016-2017 Canonical, Ltd.

import json
import tempfile
import unittest
from unittest.mock import MagicMock

from kv import KV
from ubuntui.widgets.input import StringEditor

from conjureup.app_config import AppConfig
from conjureup.models.provider import AWS, Field, Form

from .helpers import AsyncMock, test_loop

class AppConfigTestCase(unittest.TestCase):

    def setUp(self):
        self.expected_keys = [
            'config',
            'provider'
        ]
        self.app = AppConfig()
        self.db_file = tempfile.NamedTemporaryFile()
        self.app.state = KV(self.db_file.name)
        self.app.provider = AWS()
        self.app.provider.controller = "fake-tester-controller"
        self.app.provider.model = "fake-tester-model"
        self.app.provider.cloud_type = "ec2"
        self.app.config = {'spell': 'kubernetes-core'}

        self.app.juju.client = AsyncMock()
        self.app.log = MagicMock()

    def tearDown(self):
        self.db_file.close()

    @unittest.skip("This requires our app_config.restore/save "
                   "to serialize/deserialize app.provider class")
    def test_config_state_save(self):
        "app_config.test_config_state_save"
        self.app.juju.authenticated = False
        with test_loop() as loop:
            loop.run_until_complete(self.app.save())

        results = self.app.state.get(self.app._internal_state_key)
        assert set(json.loads(results.decode('utf8')).keys()) == set(
            self.expected_keys)

    def test_provider_form_query_key(self):
        "app_config.provider_form_widget_query_key"
        self.app.provider.form = Form(
            [Field(label='test widget',
                   widget=StringEditor(default='hai2u'),
                   key='test-key')])
        assert self.app.provider.form.field('test-key').value == 'hai2u'

    def test_config_juju_model_save(self):
        "app_config.test_config_juju_model_save"

        self.app.juju.authenticated = True
        with test_loop() as loop:
            loop.run_until_complete(self.app.save())

        assert self.app.juju.client.set_config.called

    def test_config_restore(self):
        "app_config.test_config_restore"
        self.app.juju.authenticated = False

        with test_loop() as loop:
            loop.run_until_complete(self.app.save())
            loop.run_until_complete(self.app.restore())

        results_json = self.app.state.get(self.app._internal_state_key)
        results = json.loads(results_json.decode('utf8'))

        assert self.app.app.controller == results['controller']

    @unittest.skip("Also need serialize/deserialize during "
                   "save/restore of app.provider class")
    def test_config_juju_restore(self):
        "app_config.test_config_juju_restore"
        class FakeExtraInfo:
            def __init__(self):
                self.value = b'{"controller": "moo"}'

        self.app.juju.authenticated = True
        self.app.juju.client.get_config.return_value = {
            "extra-info": FakeExtraInfo()}
        with test_loop() as loop:
            loop.run_until_complete(self.app.restore())
        assert self.app.provider.controller == 'moo'

    def test_config_guard_unknown_attribute(self):
        "app_config.test_config_guard_unknown_attribute"
        with self.assertRaises(Exception):
            self.app.chimichanga = "Yum"
<|endoftext|>


<commit_before>from urllib import urlencode

from django.http import HttpResponse
from django.conf import settings

def redirect_to_facebook_authorization(redirect_uri):
  """
  Redirect the user to authorize the application.

  Redirection is done by rendering a JavaScript snippet that redirects the parent
  window to the authorization URI, since Facebook will not allow this inside an iframe.
  """
  request_variables = {
    'client_id': settings.FACEBOOK_APPLICATION_ID,
    'redirect_uri': redirect_uri
  }

  if hasattr(settings, 'FACEBOOK_APPLICATION_INITIAL_PERMISSIONS'):
    request_variables['scope'] = ', '.join(settings.FACEBOOK_APPLICATION_INITIAL_PERMISSIONS)

  urlencoded_request_variables = urlencode(request_variables)

  html = """
    <!DOCTYPE html>

          You must authorize the application in order to continue.

  """ % (urlencoded_request_variables, urlencoded_request_variables)

  return HttpResponse(html)<commit_msg>Fix indentation<commit_after>from urllib import urlencode

from django.http import HttpResponse
from django.conf import settings

def redirect_to_facebook_authorization(redirect_uri):
    """
    Redirect the user to authorize the application.

    Redirection is done by rendering a JavaScript snippet that redirects the parent
    window to the authorization URI, since Facebook will not allow this inside an iframe.
    """
    request_variables = {
        'client_id': settings.FACEBOOK_APPLICATION_ID,
        'redirect_uri': redirect_uri
    }

    if hasattr(settings, 'FACEBOOK_APPLICATION_INITIAL_PERMISSIONS'):
        request_variables['scope'] = ', '.join(settings.FACEBOOK_APPLICATION_INITIAL_PERMISSIONS)

    urlencoded_request_variables = urlencode(request_variables)

    html = """
        <!DOCTYPE html>

                    You must authorize the application in order to continue.

    """ % (urlencoded_request_variables, urlencoded_request_variables)

    return HttpResponse(html)<|endoftext|>


<commit_before>import unittest
from bsi import BsiString

class TestBsiString(unittest.TestCase):
    def setUp(self):
        self.bsi_string = BsiString('undead')

    def test_to_string(self):
        self.assertEqual(str(self.bsi_string), '"undead"')
<commit_msg>string equality test<commit_after>import unittest
from bsi import BsiString

class TestBsiString(unittest.TestCase):
    def setUp(self):
        self.bsi_string = BsiString('undead')

    def test_to_string(self):
        self.assertEqual(str(self.bsi_string), '"undead"')

    def test_eq(self):
        a = 'abc'
        b = 'bbc'

        self.assertEqual(BsiString(a), BsiString(a))
        self.assertNotEqual(BsiString(a), BsiString(b))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from . import convs, widgets, fields
from ..utils import N_

class PasswordConv(convs.Char):

    error_mismatch = N_('password and confirm mismatch')
    error_required = N_('password required')

    def from_python(self, value):
        return dict([(field.name, None) for field in self.field.fields])

    def get_initial(self):
        return ''

    def to_python(self, value):
        etalon = value[list(value)[0]]
        for field in self.field.fields:
            self.assert_(value[field.name] == etalon,
                         self.error_mismatch)
        self.assert_(etalon not in (None, '')  or self.required,
                     self.error_required)
        return etalon

def PasswordSet(name='password',
                 min_length=3, max_length=200, required=False,
                 password_label=None, confirm_label='confirmmm',
                 **kwargs):
        # class implementation has problem with Fieldset copying:
        # it requires to save all kwargs in object's __dict__
        char = convs.Char(convs.length(min_length, max_length),
                          required=required)
        items = (('pass', password_label), ('conf', confirm_label))

        kwargs['fields'] = [fields.Field(subfieldname,
                                         conv=char,
                                         label=label,
                                         widget=widgets.PasswordInput)
                            for subfieldname, label in items]
        kwargs.setdefault('conv', PasswordConv(required=required))
        kwargs.setdefault('template', 'fieldset-line')

        return fields.FieldSet(name, get_initial=lambda: '', **kwargs)

<commit_msg>fix forms.shortcut.PasswordSet - no more required error if required=False<commit_after># -*- coding: utf-8 -*-

from . import convs, widgets, fields
from ..utils import N_

class PasswordConv(convs.Char):

    error_mismatch = N_('password and confirm mismatch')
    error_required = N_('password required')

    def from_python(self, value):
        return dict([(field.name, None) for field in self.field.fields])

    def get_initial(self):
        return ''

    def to_python(self, value):
        etalon = value[list(value)[0]]
        for field in self.field.fields:
            self.assert_(value[field.name] == etalon,
                         self.error_mismatch)
        if self.required:
            self.assert_(etalon not in (None, ''), self.error_required)
        elif etalon in (None, ''):
            return None
        return etalon

def PasswordSet(name='password',
                 min_length=3, max_length=200, required=False,
                 password_label=None, confirm_label='confirmmm',
                 **kwargs):
        # class implementation has problem with Fieldset copying:
        # it requires to save all kwargs in object's __dict__
        char = convs.Char(convs.length(min_length, max_length),
                          required=required)
        items = (('pass', password_label), ('conf', confirm_label))
        kwargs['fields'] = [fields.Field(subfieldname,
                                         conv=char,
                                         label=label,
                                         widget=widgets.PasswordInput)
                            for subfieldname, label in items]
        kwargs.setdefault('conv', PasswordConv(required=required))
        kwargs.setdefault('template', 'fieldset-line')

        return fields.FieldSet(name, get_initial=lambda: '', **kwargs)

<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Copyright (c) 2015-2016:
#   Frederic Mohier, frederic.mohier@gmail.com
#
# This file is part of (WebUI).
#
# (WebUI) is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# (WebUI) is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with (WebUI).  If not, see <http://www.gnu.org/licenses/>.
"""
    Application configuration and settings module
"""
from __future__ import print_function

import os
import traceback
from collections import OrderedDict
from configparser import ConfigParser

# Do not use logger in this module ... or it may fail!

class Settings(OrderedDict):
    """
    Class used to manage configuration file and application configuration
    """

    def __init__(self, filename=None):
        """
        Initialize configuration

        If a filename is provided it will be used instead of the default list.
        `filename` may be a list of files to use.

        :param filename: file to use for loading configuration
        """
        super(Settings, self).__init__()

        self.filename = filename

    def read(self, app_name):
        # pylint: disable=too-many-nested-blocks
        """ Read configuration file

        Tries to load a configuration from the following files:

        - /usr/local/etc/alignak_webui/settings.cfg (FreeBSD)
        - /etc/alignak_webui/settings.cfg (Debian)
        - ~/alignak_webui/settings.cfg (Current user home directory)
        - ./etc/settings.cfg (Current working directory etc sub directory)
        - ./settings.cfg (Current working directory)
        - cfg_file parameter

        This list of file is used by the Python ConfigParser to build the application configuration.

        The parameters found in the sections of the configuration are stored in the global
        ``settings`` dictionary. A variable named *var* in the section *section* is stored with the
        key *section.var* of the ``settings`` dictionary. As of it, parameters of the *ui* section
        are all prefixed with *ui.* ...

        Returns None if no configuration file could be found, else returns ConfigParser object

        :param app_name: application name (to build configuration file name)

        """
        if not app_name:
            return None

        if self.filename:
            if not isinstance(self.filename, list):
                settings_filenames = [
                    os.path.abspath(self.filename)
                ]
            else:
                settings_filenames = self.filename
        else:
            settings_filenames = [
                '/usr/local/etc/%s/settings.cfg' % app_name.lower(),
                '/etc/%s/settings.cfg' % app_name.lower(),
                '~/%s/settings.cfg' % app_name.lower(),
                os.path.abspath('../etc/settings.cfg'),
                os.path.abspath('../%s/etc/settings.cfg' % app_name.lower()),
                './settings.cfg'
            ]

        try:
            config = ConfigParser()
            found_cfg_file = config.read(settings_filenames)
            if found_cfg_file:
                # Build settings dictionnary for application parameters
                for section in config.sections():
                    for option in config.options(section):
                        if app_name in section:
                            self[option] = config.get(section, option)
                            if self[option] in ['True', 'true']:
                                self[option] = True
                            if self[option] in ['False', 'false']:
                                self[option] = False
                        else:
                            self[section + '.' + option] = config.get(section, option)
                            if self[section + '.' + option] in ['True', 'true']:
                                self[section + '.' + option] = True
                            if self[section + '.' + option] in ['False', 'false']:
                                self[section + '.' + option] = False
            else:  # pragma: no cover - should never happen ...
                print("No configuration file found in %s." % settings_filenames)

            return found_cfg_file
        except Exception as e:
            print("Bad formed configuration file.")
            print("Exception: %s" % str(e))
            print("Traceback: %s" % traceback.format_exc())
            return None
<commit_msg>Fix #58<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Copyright (c) 2015-2016:
#   Frederic Mohier, frederic.mohier@gmail.com
#
# This file is part of (WebUI).
#
# (WebUI) is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# (WebUI) is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with (WebUI).  If not, see <http://www.gnu.org/licenses/>.
"""
    Application configuration and settings module
"""
from __future__ import print_function

import os
import traceback
from collections import OrderedDict
from configparser import ConfigParser

# Do not use logger in this module ... or it may fail!

class Settings(OrderedDict):
    """
    Class used to manage configuration file and application configuration
    """

    def __init__(self, filename=None):
        """
        Initialize configuration

        If a filename is provided it will be used instead of the default list.
        `filename` may be a list of files to use.

        :param filename: file to use for loading configuration
        """
        super(Settings, self).__init__()

        self.filename = filename

    def read(self, app_name):
        # pylint: disable=too-many-nested-blocks
        """ Read configuration file

        Tries to load a configuration from the following files:

        - /usr/local/etc/alignak_webui/settings.cfg (FreeBSD)
        - /etc/alignak_webui/settings.cfg (Debian)
        - ~/alignak_webui/settings.cfg (Current user home directory)
        - ./etc/settings.cfg (Current working directory etc sub directory)
        - ./settings.cfg (Current working directory)
        - cfg_file parameter

        This list of file is used by the Python ConfigParser to build the application configuration.

        The parameters found in the sections of the configuration are stored in the global
        ``settings`` dictionary. A variable named *var* in the section *section* is stored with the
        key *section.var* of the ``settings`` dictionary. As of it, parameters of the *ui* section
        are all prefixed with *ui.* ...

        Returns None if no configuration file could be found, else returns ConfigParser object

        :param app_name: application name (to build configuration file name)

        """
        if not app_name:
            return None

        if self.filename:
            if not isinstance(self.filename, list):
                settings_filenames = [
                    os.path.abspath(self.filename)
                ]
            else:
                settings_filenames = self.filename
        else:
            settings_filenames = [
                '/usr/local/etc/%s/settings.cfg' % app_name.lower(),
                '/etc/%s/settings.cfg' % app_name.lower(),
                '~/%s/settings.cfg' % app_name.lower(),
                os.path.abspath('../etc/settings.cfg'),
                os.path.abspath('../%s/etc/settings.cfg' % app_name.lower()),
                './settings.cfg'
            ]

        try:
            config = ConfigParser()
            found_cfg_file = config.read(settings_filenames)
            if found_cfg_file:
                # Build settings dictionnary for application parameters
                for section in config.sections():
                    for option in config.options(section):
                        if app_name == section:
                            self[option] = config.get(section, option)
                            if self[option] in ['True', 'true']:
                                self[option] = True
                            if self[option] in ['False', 'false']:
                                self[option] = False
                        else:
                            self[section + '.' + option] = config.get(section, option)
                            if self[section + '.' + option] in ['True', 'true']:
                                self[section + '.' + option] = True
                            if self[section + '.' + option] in ['False', 'false']:
                                self[section + '.' + option] = False
            else:  # pragma: no cover - should never happen ...
                print("No configuration file found in %s." % settings_filenames)

            return found_cfg_file
        except Exception as e:
            print("Bad formed configuration file.")
            print("Exception: %s" % str(e))
            print("Traceback: %s" % traceback.format_exc())
            return None
<|endoftext|>


<commit_before>import re
import logging
import os
import subprocess

import nbformat
from nbconvert.preprocessors import Preprocessor

from nb_utils.macro_processing import MacroProcessor

DEBUG = 0
if DEBUG:
    logging.basicConfig(level=logging.DEBUG)

IGNORE_UNKNOWN_MACROS = 0

class UnrecognizedMacroException(Exception):
    pass

def get_git_branch():
    """Return the current git branch as a string"""
    return subprocess.check_output(["git", "symbolic-ref", "--short", "HEAD"])\
            .decode('utf8').strip()

class LearnLessonPreprocessor(Preprocessor):

    # NB: This is the only overridden Preprocessor method. All other methods are custom.
    def preprocess(self, nb, resources):
        # See render.py for how resources as populated.
        self.track = resources['track_meta']

        # May be None for notebooks with type='extra'
        self.lesson = resources['lesson']
        # Corresponds to track_config.yaml
        track_cfg = resources['track_cfg']
        self.cfg = track_cfg
        self.nb_meta = resources['nb_meta']

        macroer = MacroProcessor(track_cfg)

        new_cells = []
        for cell in nb.cells:
            # XXX: Kind of hacky implementation.
            src = cell['source']
            if src.strip() == '#%%RM_BELOW%%':
                break
            c = self.process_cell(cell)
            c = c and macroer.process_cell(c)
            if c is not None:
                new_cells.append(c)

        # Add header and footer for self-paced courses, but not for daily-email notebooks (where it'd be confusing)
        if not self.cfg.get('daily'):
            self.add_header_and_footer(new_cells)

        nb.cells = new_cells
        # NB: There may be some cases where we need to access learntools in a tutorial
        # or ancillary notebook as well. We encode this in track_meta.
        if track_cfg.get('development', False) and self.nb_meta.type == 'exercise':
            self.pip_install_lt_hack(nb)
        return nb, resources

    def add_header_and_footer(self, cells):
        """Inserts header cell at front of cells and appends footer cell to end. Both new cells have course links"""
        course_link ="""**[{} Home Page]({})**\n\n""".format(self.track.course_name,
        self.track.course_url)
        horizontal_line_break = "---\n"
        header_content = course_link + horizontal_line_break
        footer_content = horizontal_line_break + course_link

        forum_cta = """\n\n\n\n*Have questions or comments? Visit the [Learn Discussion forum]({}) to chat with other Learners.*""".format(self.track.course_forum_url)
        footer_content += forum_cta
        header_cell = self.make_cell(cell_type='markdown', source=header_content)
        footer_cell = self.make_cell(cell_type='markdown', source=footer_content)
        cells.insert(0, header_cell)
        cells.append(footer_cell)

    def pip_install_lt_hack(self, nb):
        """pip install learntools @ the present branch when running on Kernels"""
        try:
            branch = get_git_branch()
        except subprocess.CalledProcessError:
            # This call fails in CI. The reason is unclear, but it's benign enough
            # that we can just ignore it and forge on.
            logging.warn("Failed to capture current git branch. Falling back to master.")
            branch = 'master'
        pkg = 'git+https://github.com/Kaggle/learntools.git@{}'.format(branch)
        self.pip_install_hack(nb, [pkg])

    def pip_install_hack(self, nb, pkgs):
        """Insert some cells at the top of this notebook that pip install the given
        packages to /kaggle/working, then add that directory to sys.path.
        """
        if not pkgs:
            return
        extra_cells = []
        for pkg in pkgs:
            extra_cells.append(self.pip_install_cell(pkg))

        # NB: Workaround for 'read-only file sytem' issue when pip installing.
        # Hopefully at some point this becomes easier.
        syspath_lines = [
                'import sys\n',
                "sys.path.append('/kaggle/working')",
        ]
        syspath_cell = self.make_cell(cell_type='code', source=syspath_lines)
        extra_cells.append(syspath_cell)
        nb.cells = extra_cells + nb.cells

    @classmethod
    def pip_install_cell(cls, pkg_spec):
        cmd = '!pip install -U -t /kaggle/working/ {}'.format(pkg_spec)
        return cls.make_cell(cell_type='code', source=[cmd])

    @staticmethod
    def make_cell(cell_type, **kwargs):
        """Returns a NotebookNode object populated with kwargs. cell_type should be either 'markdown' or 'code'"""
        defaults = dict(
                cell_type=cell_type,
                metadata={},
                source=[],
                )
        if cell_type == "code":
            defaults['execution_count'] = None
            defaults['outputs'] = []

        defaults.update(kwargs)
        return nbformat.from_dict(defaults)

    def process_cell(self, cell):
        # Find all things that look like macros
        pattern = r'#\$([^$]+)\$'
        # This is one big string. It occurs to me that this is actually kind of weird
        # given that inspecting the ipynb file format, source seems to be a list of
        # strings, 1 per line (this is also what's expected by nbformat.from_dict).
        # I have no idea why this disagreement exists.
        src = cell['source']
        macros = re.finditer(pattern, src)
        newsrc = ''
        i = 0
        for match in macros:
            logging.debug(match)
            a, b = match.span()
            macro = match.group(1)
            try:
                expansion = self.expand_macro(macro, cell)
            except UnrecognizedMacroException as e:
                if IGNORE_UNKNOWN_MACROS:
                    expansion = None
                    logging.warn("Unrecognized macro: {}".format(macro))
                else:
                    raise e
            except Exception as e:
                print("Error parsing macro match {}".format(match))
                raise e
            newsrc += src[i:a]
            # Some macros might actually expand to nothing (e.g. if their purpose is to mutate cell metadata)
            if expansion:
                newsrc += expansion
            i = b
        newsrc += src[i:]
        cell['source'] = newsrc
        if cell['cell_type'] == 'markdown':
            cell['metadata']['KAGGLE_FREEZE'] = True
        return cell

    def expand_macro(self, macro, cell):
        """Expand the given macro string (or apply it to the given cell, if
        it's side-effecty), by looking up and calling the corresponding
        LessonPreprocessor method.
        """
        args = []
        if macro.endswith(')'):
            macro, argstr = macro[:-1].split('(')
            args = [argstr.strip()] if argstr.strip() else []
        macro_method = getattr(self, macro, None)
        if macro_method is None:
            raise UnrecognizedMacroException("Don't know how to handle the macro with name: {}".format(macro))
        return macro_method(*args, cell=cell)

    # TODO: Should consider separating macro logic out into separate module/class.
    def EXERCISE_FORKING_URL(self, lesson_num=None, **kwargs):
        if lesson_num is None:
            return self.lesson.exercise.forking_url
        else:
            lesson_idx = int(lesson_num) - 1
            lesson = self.track.lessons[lesson_idx]
            return lesson.exercise.forking_url

    def HIDE_INPUT(self, cell):
        cell['metadata']['_kg_hide-input'] = True

    def HIDE_OUTPUT(self, cell):
        cell['metadata']['_kg_hide-output'] = True

    def HIDE(self, cell):
        self.HIDE_INPUT(cell)
        self.HIDE_OUTPUT(cell)

    def YOURTURN(self, **kwargs):
        """Some boilerplate text to be used at the end of a tutorial notebook, to
        lead into the corresponding exercise.
        """
        return """# Your Turn

Try the [hands-on exercise]({}) with {}""".format(
        self.lesson.exercise.forking_url, self.lesson.topic
        )

    def TUTORIAL_URL(self, lesson_num=None, **kwargs):
        if lesson_num is None:
            lesson = self.lesson
        else:
            lesson_idx = int(lesson_num) - 1
            lesson = self.track.lessons[lesson_idx]
        return lesson.tutorial.url

    def EXERCISE_URL(self, lesson_num, **kwargs):
        # TODO: unify this + EXERCISE_FORKING_URL (have that be this with default arg)
        lesson_idx = int(lesson_num) - 1
        lesson = self.track.lessons[lesson_idx]
        return lesson.exercise.forking_url

    def NEXT_NOTEBOOK_URL(self, **kwargs):
        '''
        Return link to the next notebook. Use forking link when link to exercise. Use regular/view link for tutorials
        '''
        if (self.nb_meta.type == 'tutorial') and hasattr(self.lesson, 'exercise'):
            return self.lesson.exercise.forking_url
        else:
            next_lesson = self.lesson.next
            if hasattr(self.lesson, 'tutorial'):
                return next_lesson.tutorial.url
            else:
                return next_lesson.exercise.forking_url

    def KEEP_GOING(self, **kwargs):

        # In "daily challenge" mode, the end of the exercise should not point to
        # the next lesson (they have to wait a day to start the next lesson)
        if self.cfg.get('daily'):
            return "\n**You'll get another email tomorrow so you can keep learning. See you then.**"
        # Don't use this macro for the very last exercise
        next_lesson = self.lesson.next
        if hasattr(next_lesson, 'tutorial'):
            next_url = next_lesson.tutorial.url
        else:
            next_url = next_lesson.exercise.forking_url

        res = """# Keep Going

You are ready for **[{}]({}).**
""".format(next_lesson.topic, next_url)

        return res

        # Alternative formulation (used on days 5 and 6 of Python challenge):
        # Want feedback on your code? To share it with others or ask for help, you'll need to make it public. Save a version of your notebook that shows your current work by hitting the "Commit & Run" button. Once your notebook is finished running, go to the Settings tab in the panel to the left (you may have to expand it by hitting the [<] button next to the "Commit & Run" button) and set the "Visibility" dropdown to "Public".

    def SUBMIT_TO_COMP(self, **kwargs):
        return ("1. Begin by clicking on the blue **Save Version** button in the top right corner of the window.  This will generate a pop-up window.  \n"
"2. Ensure that the **Save and Run All** option is selected, and then click on the blue **Save** button.\n"
"3. This generates a window in the bottom left corner of the notebook.  After it has finished running, click on the number to the right of the **Save Version** button.  This pulls up a list of versions on the right of the screen.  Click on the ellipsis **(...)** to the right of the most recent version, and select **Open in Viewer**.  This brings you into view mode of the same page. You will need to scroll down to get back to these instructions.\n"
"4. Click on the **Output** tab on the right of the screen.  Then, click on the blue **Submit** button to submit your results to the leaderboard.\n"
"\n"
"You have now successfully submitted to the competition!\n"
"\n"
"If you want to keep working to improve your performance, select the blue **Edit** button in the top right of the screen. Then you can change your code and repeat the process. There's a lot of room to improve, and you will climb up the leaderboard as you work.\n")

<commit_msg>edits to preprocessor<commit_after>import re
import logging
import os
import subprocess

import nbformat
from nbconvert.preprocessors import Preprocessor

from nb_utils.macro_processing import MacroProcessor

DEBUG = 0
if DEBUG:
    logging.basicConfig(level=logging.DEBUG)

IGNORE_UNKNOWN_MACROS = 0

class UnrecognizedMacroException(Exception):
    pass

def get_git_branch():
    """Return the current git branch as a string"""
    return subprocess.check_output(["git", "symbolic-ref", "--short", "HEAD"])\
            .decode('utf8').strip()

class LearnLessonPreprocessor(Preprocessor):

    # NB: This is the only overridden Preprocessor method. All other methods are custom.
    def preprocess(self, nb, resources):
        # See render.py for how resources as populated.
        self.track = resources['track_meta']

        # May be None for notebooks with type='extra'
        self.lesson = resources['lesson']
        # Corresponds to track_config.yaml
        track_cfg = resources['track_cfg']
        self.cfg = track_cfg
        self.nb_meta = resources['nb_meta']

        macroer = MacroProcessor(track_cfg)

        new_cells = []
        for cell in nb.cells:
            # XXX: Kind of hacky implementation.
            src = cell['source']
            if src.strip() == '#%%RM_BELOW%%':
                break
            c = self.process_cell(cell)
            c = c and macroer.process_cell(c)
            if c is not None:
                new_cells.append(c)

        # Add header and footer for self-paced courses, but not for daily-email notebooks (where it'd be confusing)
        if not self.cfg.get('daily'):
            self.add_header_and_footer(new_cells)

        nb.cells = new_cells
        # NB: There may be some cases where we need to access learntools in a tutorial
        # or ancillary notebook as well. We encode this in track_meta.
        if track_cfg.get('development', False) and self.nb_meta.type == 'exercise':
            self.pip_install_lt_hack(nb)
        return nb, resources

    def EXERCISE_HEADER(self, **kwargs):
        exercise_header = course_link ="""**This notebook is an exercise in the [{}]({}) course.  You can reference the tutorial at [this link]({}).**\n\n""".format(self.track.course_name,
            self.track.course_url, self.lesson.tutorial.url)
        return res

    def add_header_and_footer(self, cells):
        """Inserts header cell at front of cells (if exercise notebook) and appends footer cell to end."""
        exercise_header = course_link ="""**This notebook is an exercise in the [{}]({}) course.  You can reference the tutorial at [this link]({}).**\n\n""".format(self.track.course_name,
            self.track.course_url, self.lesson.tutorial.url)
        horizontal_line_break = "---\n"

        header_content = exercise_header + horizontal_line_break
        footer_content = horizontal_line_break #+ course_link

        forum_cta = """\n\n\n\n*Have questions or comments? Visit the [Learn Discussion forum]({}) to chat with other Learners.*""".format(self.track.course_forum_url)
        footer_content += forum_cta

        header_cell = self.make_cell(cell_type='markdown', source=header_content)
        footer_cell = self.make_cell(cell_type='markdown', source=footer_content)
        if self.nb_meta.type == 'exercise':
            cells.insert(0, header_cell)
        cells.append(footer_cell)

    def pip_install_lt_hack(self, nb):
        """pip install learntools @ the present branch when running on Kernels"""
        try:
            branch = get_git_branch()
        except subprocess.CalledProcessError:
            # This call fails in CI. The reason is unclear, but it's benign enough
            # that we can just ignore it and forge on.
            logging.warn("Failed to capture current git branch. Falling back to master.")
            branch = 'master'
        pkg = 'git+https://github.com/Kaggle/learntools.git@{}'.format(branch)
        self.pip_install_hack(nb, [pkg])

    def pip_install_hack(self, nb, pkgs):
        """Insert some cells at the top of this notebook that pip install the given
        packages to /kaggle/working, then add that directory to sys.path.
        """
        if not pkgs:
            return
        extra_cells = []
        for pkg in pkgs:
            extra_cells.append(self.pip_install_cell(pkg))

        # NB: Workaround for 'read-only file sytem' issue when pip installing.
        # Hopefully at some point this becomes easier.
        syspath_lines = [
                'import sys\n',
                "sys.path.append('/kaggle/working')",
        ]
        syspath_cell = self.make_cell(cell_type='code', source=syspath_lines)
        extra_cells.append(syspath_cell)
        nb.cells = extra_cells + nb.cells

    @classmethod
    def pip_install_cell(cls, pkg_spec):
        cmd = '!pip install -U -t /kaggle/working/ {}'.format(pkg_spec)
        return cls.make_cell(cell_type='code', source=[cmd])

    @staticmethod
    def make_cell(cell_type, **kwargs):
        """Returns a NotebookNode object populated with kwargs. cell_type should be either 'markdown' or 'code'"""
        defaults = dict(
                cell_type=cell_type,
                metadata={},
                source=[],
                )
        if cell_type == "code":
            defaults['execution_count'] = None
            defaults['outputs'] = []

        defaults.update(kwargs)
        return nbformat.from_dict(defaults)

    def process_cell(self, cell):
        # Find all things that look like macros
        pattern = r'#\$([^$]+)\$'
        # This is one big string. It occurs to me that this is actually kind of weird
        # given that inspecting the ipynb file format, source seems to be a list of
        # strings, 1 per line (this is also what's expected by nbformat.from_dict).
        # I have no idea why this disagreement exists.
        src = cell['source']
        macros = re.finditer(pattern, src)
        newsrc = ''
        i = 0
        for match in macros:
            logging.debug(match)
            a, b = match.span()
            macro = match.group(1)
            try:
                expansion = self.expand_macro(macro, cell)
            except UnrecognizedMacroException as e:
                if IGNORE_UNKNOWN_MACROS:
                    expansion = None
                    logging.warn("Unrecognized macro: {}".format(macro))
                else:
                    raise e
            except Exception as e:
                print("Error parsing macro match {}".format(match))
                raise e
            newsrc += src[i:a]
            # Some macros might actually expand to nothing (e.g. if their purpose is to mutate cell metadata)
            if expansion:
                newsrc += expansion
            i = b
        newsrc += src[i:]
        cell['source'] = newsrc
        if cell['cell_type'] == 'markdown':
            cell['metadata']['KAGGLE_FREEZE'] = True
        return cell

    def expand_macro(self, macro, cell):
        """Expand the given macro string (or apply it to the given cell, if
        it's side-effecty), by looking up and calling the corresponding
        LessonPreprocessor method.
        """
        args = []
        if macro.endswith(')'):
            macro, argstr = macro[:-1].split('(')
            args = [argstr.strip()] if argstr.strip() else []
        macro_method = getattr(self, macro, None)
        if macro_method is None:
            raise UnrecognizedMacroException("Don't know how to handle the macro with name: {}".format(macro))
        return macro_method(*args, cell=cell)

    # TODO: Should consider separating macro logic out into separate module/class.
    def EXERCISE_FORKING_URL(self, lesson_num=None, **kwargs):
        if lesson_num is None:
            return self.lesson.exercise.forking_url
        else:
            lesson_idx = int(lesson_num) - 1
            lesson = self.track.lessons[lesson_idx]
            return lesson.exercise.forking_url

    def HIDE_INPUT(self, cell):
        cell['metadata']['_kg_hide-input'] = True

    def HIDE_OUTPUT(self, cell):
        cell['metadata']['_kg_hide-output'] = True

    def HIDE(self, cell):
        self.HIDE_INPUT(cell)
        self.HIDE_OUTPUT(cell)

    def YOURTURN(self, **kwargs):
        """Some boilerplate text to be used at the end of a tutorial notebook, to
        lead into the corresponding exercise.
        """
        return """# Your Turn

Try the [hands-on exercise]({}) with {}""".format(
        self.lesson.exercise.forking_url, self.lesson.topic
        )

    def TUTORIAL_URL(self, lesson_num=None, **kwargs):
        if lesson_num is None:
            lesson = self.lesson
        else:
            lesson_idx = int(lesson_num) - 1
            lesson = self.track.lessons[lesson_idx]
        return lesson.tutorial.url

    def EXERCISE_URL(self, lesson_num, **kwargs):
        # TODO: unify this + EXERCISE_FORKING_URL (have that be this with default arg)
        lesson_idx = int(lesson_num) - 1
        lesson = self.track.lessons[lesson_idx]
        return lesson.exercise.forking_url

    def NEXT_NOTEBOOK_URL(self, **kwargs):
        '''
        Return link to the next notebook. Use forking link when link to exercise. Use regular/view link for tutorials
        '''
        if (self.nb_meta.type == 'tutorial') and hasattr(self.lesson, 'exercise'):
            return self.lesson.exercise.forking_url
        else:
            next_lesson = self.lesson.next
            if hasattr(self.lesson, 'tutorial'):
                return next_lesson.tutorial.url
            else:
                return next_lesson.exercise.forking_url

    def KEEP_GOING(self, **kwargs):

        # In "daily challenge" mode, the end of the exercise should not point to
        # the next lesson (they have to wait a day to start the next lesson)
        if self.cfg.get('daily'):
            return "\n**You'll get another email tomorrow so you can keep learning. See you then.**"
        # Don't use this macro for the very last exercise
        next_lesson = self.lesson.next
        if hasattr(next_lesson, 'tutorial'):
            next_url = next_lesson.tutorial.url
        else:
            next_url = next_lesson.exercise.forking_url

        res = """# Keep Going

You are ready for **[{}]({}).**
""".format(next_lesson.topic, next_url)

        return res

        # Alternative formulation (used on days 5 and 6 of Python challenge):
        # Want feedback on your code? To share it with others or ask for help, you'll need to make it public. Save a version of your notebook that shows your current work by hitting the "Commit & Run" button. Once your notebook is finished running, go to the Settings tab in the panel to the left (you may have to expand it by hitting the [<] button next to the "Commit & Run" button) and set the "Visibility" dropdown to "Public".

    def SUBMIT_TO_COMP(self, **kwargs):
        return ("1. Begin by clicking on the blue **Save Version** button in the top right corner of the window.  This will generate a pop-up window.  \n"
"2. Ensure that the **Save and Run All** option is selected, and then click on the blue **Save** button.\n"
"3. This generates a window in the bottom left corner of the notebook.  After it has finished running, click on the number to the right of the **Save Version** button.  This pulls up a list of versions on the right of the screen.  Click on the ellipsis **(...)** to the right of the most recent version, and select **Open in Viewer**.  This brings you into view mode of the same page. You will need to scroll down to get back to these instructions.\n"
"4. Click on the **Output** tab on the right of the screen.  Then, click on the blue **Submit** button to submit your results to the leaderboard.\n"
"\n"
"You have now successfully submitted to the competition!\n"
"\n"
"If you want to keep working to improve your performance, select the blue **Edit** button in the top right of the screen. Then you can change your code and repeat the process. There's a lot of room to improve, and you will climb up the leaderboard as you work.\n")

<|endoftext|>


<commit_before># Futures #

# Built-in modules #

# Internal modules #

# Third party modules #
from ftputil import FTPHost

###############################################################################
class RefSeq(object):
    """We need to download the raw sequences of the refseq protein database."""

    ftp_url = "ftp://ftp.ncbi.nlm.nih.gov/refseq/release/complete/"

    all_paths = """
    /lorem
    """

    def __init__(self, base_dir=None, seq_type='prot'):
        # Attributes #
        self.seq_type = seq_type
        self.base_dir = base_dir
        self.p = AutoPaths(self.base_dir, self.all_paths)

    def download(self):
        """Retrieve all files from the FTP site"""
        self.ftp = FTPHost(self.ftp_url)
        self.ftp_files = self.ftp.listdir()
        files_we_want = files<commit_msg>Added the refseq prot db<commit_after># Futures #

# Built-in modules #
import fnmatch

# Internal modules #
from plumbing.autopaths import AutoPaths

# Third party modules #
from ftputil import FTPHost
from tqdm import tqdm

# Constants #
directory = "/glob/lucass/"

###############################################################################
class RefSeqBacteriaProtNR(object):
    """the RefSeq sequences for only bacteria, only protein, and only the
    non-redundant version. We will download the raw sequences by FTP
    and format them as a blast data base."""

    short_name = "refseq_bact_prot_nr"
    ftp_url    = "ftp.ncbi.nlm.nih.gov"
    ftp_dir    = "/refseq/release/bacteria/"

    all_paths = """
    /raw/
    /blast_db/
    """

    def __init__(self):
        # Attributes #
        self.seq_type = 'prot'
        self.base_dir = directory + self.short_name
        self.p = AutoPaths(self.base_dir, self.all_paths)

    def download(self):
        """Retrieve all files from the FTP site"""
        self.ftp = FTPHost(self.ftp_url, "anonymous")
        self.ftp.chdir(self.ftp_dir)
        self.ftp_files = self.ftp.listdir(self.ftp.curdir)
        self.pattern = 'bacteria.nonredundant_protein.*.protein.faa.gz'
        self.files_we_want = [f for f in self.ftp_files if fnmatch.fnmatch(f, self.pattern)]
        for f in tqdm(self.files_we_want): self.ftp.download(f, self.p.raw_dir + f)

###############################################################################
refseq_bact_prot_nr = RefSeqBacteriaProtNR()<|endoftext|>