<commit_before>import urlparse
import os.path
import lasso
import saml2utils
import saml11utils
from django.conf import settings
from django.http import HttpResponseRedirect, Http404, HttpResponse
from django.shortcuts import render_to_response
from django.utils.translation import ugettext as _
from models import *

SAML_PRIVATE_KEY = settings.SAML_PRIVATE_KEY

def get_soap_message(request, on_error_raise = True):
    '''Verify that POST content looks like a SOAP message and returns it'''
    if request.method != 'POST' or \
            request.META['CONTENT_TYPE'] != 'text/xml':
       if on_error_raise:
           raise Http404(_('Only SOAP messages here'))
       else:
           return None
    return request.raw_post_data

def get_http_binding(request):
    if request.method == 'GET':
        return 'GET'
    elif request.method == 'POST':
        # disambiguate SOAP and form POST
        if request.META.get('CONTENT_TYPE') == 'text/xml':
            return 'SOAP'
        else:
            return 'POST'

# SAMLv2 methods

def get_base_path(request, metadata):
    '''Get endpoints base path given metadata path
    '''
    path = urlparse.urlparse(metadata).path
    return request.build_absolute_uri(os.path.dirname(path))

def get_entity_id(request, metadata):
    '''Return the EntityID, given metadata absolute path
    '''
    return request.build_absolute_uri(metadata)

def get_saml2_metadata(request, metadata):
    metagen = saml2utils.Saml2Metadata(get_entity_id(request, metadata),
            url_prefix = get_base_path(request, metadata))
    synchronous_bindings = [ lasso.SAML2_METADATA_BINDING_REDIRECT,
                    lasso.SAML2_METADATA_BINDING_POST ]
    map = (('SingleSignOnService', synchronous_bindings , '/sso'),)
    options = { 'signing_key': SAML_PRIVATE_KEY }
    metagen.add_idp_descriptor(map, options)
    return str(metagen)

def create_saml2_server(request, metadata):
    '''Create a lasso Server object for using with a profile'''
    server = lasso.Server.newFromBuffers(get_saml2_metadata(request, metadata),
            SAML_PRIVATE_KEY)
    if not server:
        raise Exception('Cannot create LassoServer object')
    return server

def get_saml2_post_request(request):
    '''Extract the SAMLRequest field from the POST'''
    return request.POST.get(lasso.SAML2_FIELD_REQUEST, '')

def get_saml2_query_request(request):
    return request.META.get('QUERY_STRING', '')

def get_saml2_soap_request(request):
    return request.raw_post_data

def get_saml2_request_message(request):
    '''Return SAMLv2 message whatever the HTTP binding used'''
    binding = get_http_binding(request)
    if binding == 'GET':
        return get_saml2_query_request(request)
    elif binding == 'POST':
        return get_saml2_post_request(request)
    elif binding == 'SOAP':
        return get_saml2_soap_request(request)

def return_saml2_response(profile, title = ''):
    '''Finish your SAMLv2 views with this method to return a SAML
    response'''
    return return_saml2(profile, lasso.SAML2_FIELD_RESPONSE, title)

def return_saml2_request(profile, title = ''):
    '''Finish your SAMLv2 views with this method to return a SAML
    request'''
    return return_saml2(profile, lasso.SAML2_FIELD_REQUEST, title)

def return_saml2(profile, field_name, title = ''):
    '''Helper to handle SAMLv2 bindings to emit request and responses'''
    if profile.msgBody:
        if profile.msgUrl:
            render_to_response('saml/post_form.html',{
                        'title': title,
                        'url': profile.msgUrl,
                        'fieldname': field_name,
                        'body': profile.msgBody,
                        'relay_state': profile.msgRelayState })
        return HttpResponse(profile.msgBody, mimetype = 'text/xml')
    elif profile.msgUrl:
        return HttpResponseRedirect(profile.msgUrl)
    else:
        return TypeError('profile do not contain a response')

# ID-FF 1.2 methods
def get_idff12_metadata(request, metadata):
    '''Produce SAMLv1.1 metadata for our ID-FF 1.2 IdP

      This method only works with request pointing to an endpoint'''

    metagen = saml11utils.Saml11Metadata(get_entity_id(request, metadata),
            url_prefix = get_base_path(request, metadata),
            protocol_support_enumeration = [ lasso.LIB_HREF ])
    sso_protocol_profiles = [
        lasso.LIB_PROTOCOL_PROFILE_BRWS_ART,
        lasso.LIB_PROTOCOL_PROFILE_BRWS_POST,
        lasso.LIB_PROTOCOL_PROFILE_BRWS_LECP ]
    map = {
            'SoapEndpoint': '/soap',
            'SingleSignOn': (('/sso',), sso_protocol_profiles)
          }
    options = { 'signing_key': SAML_PRIVATE_KEY }
    metagen.add_idp_descriptor(map, options)
    return str(metagen)

def create_idff12_server(request, metadata):
    server = lasso.Server.newFromBuffers(get_idff12_metadata(request, metadata),
            SAML_PRIVATE_KEY)
    if not server:
        raise Exception('Cannot create LassoServer object')
    return server

def get_idff12_post_request(request):
    '''Return a SAML 1.1 request transmitted through a POST request'''
    return request.POST.get('LAREQ')

get_idff12_query_request = get_saml2_query_request

def get_idff12_request_message(request):
    binding = get_http_binding(request)
    if binding == 'GET':
        return get_idff12_query_request(request)
    elif binding == 'POST':
        return get_idff12_post_request(request)

def return_saml_soap_response(profile):
    return HttpResponse(profile.msgBody, mimetype = 'text/xml')

def return_idff12_response(profile, title = ''):
    '''Finish your ID-FFv1.2 views with this method to return a SAML
    response'''
    return return_saml2(profile, 'LARES', title)

def return_idff12_request(profile, title = ''):
    '''Finish your SAMLv2 views with this method to return a SAML
    request'''
    return return_saml2(profile, 'LAREQ', title)

def return_idff12(profile, field_name, title = ''):
    '''Helper to handle SAMLv2 bindings to emit request and responses'''
    if profile.msgBody:
        if profile.msgUrl:
            render_to_response('saml/post_form.html',{
                        'title': title,
                        'url': profile.msgUrl,
                        'fieldname': field_name,
                        'body': profile.msgBody,
                        'relay_state': profile.msgRelayState })
        return return_saml_soap_response(profile)
    elif profile.msgUrl:
        return HttpResponseRedirect(profile.msgUrl)
    else:
        return TypeError('profile do not contain a response')

# Helper method to handle profiles endpoints
# In the future we should move away from monolithic object (LassoIdentity and
# LassoSession) holding all the datas, to manipulate them at row Level with
# LibertyFederation and LibertyAssertion objects.

def load_federation(request, login):
    '''Load an identity dump from the database'''
    q = LibertyIdentityDump.objects.filter(user = request.user)
    if not q:
        return
    login.setIdentityFromDump(q[0].identity_dump)

def load_session(request, login, session_key = None):
    '''Load a session dump from the database'''
    if not session_key:
        session_key = request.session.session_key
    q = LibertySessionDump.objects.filter(django_session_key = session_key)
    if not q:
        return
    login.setSessionFromDump(q[0].session_dump)

def save_federation(request, login):
    '''Save identity dump to database'''
    if login.isIdentityDirty:
        q = LibertyIdentityDump.objects.filter(user = request.user)
        if q:
            if login.identity:
                q[0].identity_dump = login.identity.dump()
            else:
                q[0].identity_dump = None
            q[0].save()
        elif login.identity:
            LibertyIdentityDump(user = request.user,
                    identity_dump = login.identity.dump()).save()

def save_session(request, login, session_key = None):
    '''Save session dump to database'''
    if not session_key:
        session_key = request.session.session_key
    if login.isSessionDirty:
        q = LibertySessionDump.objects.filter(
                django_session_key = session_key)
        if q:
            if login.session:
                q[0].session_dump = login.session.dump()
            else:
                q[0].session_dump = None
            q[0].save()
        elif login.session:
            LibertySessionDump(django_session_key = request.session.session_key,
                    session_dump = login.session.dump()).save()

# TODO: handle autoloading of metadatas
def load_provider(request, login, provider_id, sp_or_idp = 'sp'):
    '''Look up a provider in the database, and verify it handles wanted
       role be it sp or idp.
    '''
    liberty_provider = LibertyProvider.objects.get(entity_id = provider_id)
    if not liberty_provider:
        return False
    if sp_or_idp == 'sp':
        try:
            service_provider = liberty_provider.service_provider
        except LibertyServiceProvider.DoesNotExist:
            return False
        if not service_provider.enabled:
            return False
        login.server.addProviderFromBuffer(lasso.PROVIDER_ROLE_SP,
                liberty_provider.metadata.read())
    elif sp_or_idp == 'idp':
        try:
            identity_provider = liberty_provider.identity_provider
        except LibertyIdentityProvider.DoesNotExist:
            return False
        if not identity_provider.enabled:
            return False
        login.server.addProviderFromBuffer(lasso.PROVIDER_ROLE_IDP,
                liberty_provider.metadata.read())
    else:
        raise Exception('unsupported option sp_or_idp = %r' % sp_or_idp)

    return liberty_provider
<commit_msg>SAML common: add possibility to load federation from another user<commit_after>import urlparse
import os.path
import lasso
import saml2utils
import saml11utils
from django.conf import settings
from django.http import HttpResponseRedirect, Http404, HttpResponse
from django.shortcuts import render_to_response
from django.utils.translation import ugettext as _
from models import *

SAML_PRIVATE_KEY = settings.SAML_PRIVATE_KEY

def get_soap_message(request, on_error_raise = True):
    '''Verify that POST content looks like a SOAP message and returns it'''
    if request.method != 'POST' or \
            request.META['CONTENT_TYPE'] != 'text/xml':
       if on_error_raise:
           raise Http404(_('Only SOAP messages here'))
       else:
           return None
    return request.raw_post_data

def get_http_binding(request):
    if request.method == 'GET':
        return 'GET'
    elif request.method == 'POST':
        # disambiguate SOAP and form POST
        if request.META.get('CONTENT_TYPE') == 'text/xml':
            return 'SOAP'
        else:
            return 'POST'

# SAMLv2 methods

def get_base_path(request, metadata):
    '''Get endpoints base path given metadata path
    '''
    path = urlparse.urlparse(metadata).path
    return request.build_absolute_uri(os.path.dirname(path))

def get_entity_id(request, metadata):
    '''Return the EntityID, given metadata absolute path
    '''
    return request.build_absolute_uri(metadata)

def get_saml2_metadata(request, metadata):
    metagen = saml2utils.Saml2Metadata(get_entity_id(request, metadata),
            url_prefix = get_base_path(request, metadata))
    synchronous_bindings = [ lasso.SAML2_METADATA_BINDING_REDIRECT,
                    lasso.SAML2_METADATA_BINDING_POST ]
    map = (('SingleSignOnService', synchronous_bindings , '/sso'),)
    options = { 'signing_key': SAML_PRIVATE_KEY }
    metagen.add_idp_descriptor(map, options)
    return str(metagen)

def create_saml2_server(request, metadata):
    '''Create a lasso Server object for using with a profile'''
    server = lasso.Server.newFromBuffers(get_saml2_metadata(request, metadata),
            SAML_PRIVATE_KEY)
    if not server:
        raise Exception('Cannot create LassoServer object')
    return server

def get_saml2_post_request(request):
    '''Extract the SAMLRequest field from the POST'''
    return request.POST.get(lasso.SAML2_FIELD_REQUEST, '')

def get_saml2_query_request(request):
    return request.META.get('QUERY_STRING', '')

def get_saml2_soap_request(request):
    return request.raw_post_data

def get_saml2_request_message(request):
    '''Return SAMLv2 message whatever the HTTP binding used'''
    binding = get_http_binding(request)
    if binding == 'GET':
        return get_saml2_query_request(request)
    elif binding == 'POST':
        return get_saml2_post_request(request)
    elif binding == 'SOAP':
        return get_saml2_soap_request(request)

def return_saml2_response(profile, title = ''):
    '''Finish your SAMLv2 views with this method to return a SAML
    response'''
    return return_saml2(profile, lasso.SAML2_FIELD_RESPONSE, title)

def return_saml2_request(profile, title = ''):
    '''Finish your SAMLv2 views with this method to return a SAML
    request'''
    return return_saml2(profile, lasso.SAML2_FIELD_REQUEST, title)

def return_saml2(profile, field_name, title = ''):
    '''Helper to handle SAMLv2 bindings to emit request and responses'''
    if profile.msgBody:
        if profile.msgUrl:
            render_to_response('saml/post_form.html',{
                        'title': title,
                        'url': profile.msgUrl,
                        'fieldname': field_name,
                        'body': profile.msgBody,
                        'relay_state': profile.msgRelayState })
        return HttpResponse(profile.msgBody, mimetype = 'text/xml')
    elif profile.msgUrl:
        return HttpResponseRedirect(profile.msgUrl)
    else:
        return TypeError('profile do not contain a response')

# ID-FF 1.2 methods
def get_idff12_metadata(request, metadata):
    '''Produce SAMLv1.1 metadata for our ID-FF 1.2 IdP

      This method only works with request pointing to an endpoint'''

    metagen = saml11utils.Saml11Metadata(get_entity_id(request, metadata),
            url_prefix = get_base_path(request, metadata),
            protocol_support_enumeration = [ lasso.LIB_HREF ])
    sso_protocol_profiles = [
        lasso.LIB_PROTOCOL_PROFILE_BRWS_ART,
        lasso.LIB_PROTOCOL_PROFILE_BRWS_POST,
        lasso.LIB_PROTOCOL_PROFILE_BRWS_LECP ]
    map = {
            'SoapEndpoint': '/soap',
            'SingleSignOn': (('/sso',), sso_protocol_profiles)
          }
    options = { 'signing_key': SAML_PRIVATE_KEY }
    metagen.add_idp_descriptor(map, options)
    return str(metagen)

def create_idff12_server(request, metadata):
    server = lasso.Server.newFromBuffers(get_idff12_metadata(request, metadata),
            SAML_PRIVATE_KEY)
    if not server:
        raise Exception('Cannot create LassoServer object')
    return server

def get_idff12_post_request(request):
    '''Return a SAML 1.1 request transmitted through a POST request'''
    return request.POST.get('LAREQ')

get_idff12_query_request = get_saml2_query_request

def get_idff12_request_message(request):
    binding = get_http_binding(request)
    if binding == 'GET':
        return get_idff12_query_request(request)
    elif binding == 'POST':
        return get_idff12_post_request(request)

def return_saml_soap_response(profile):
    return HttpResponse(profile.msgBody, mimetype = 'text/xml')

def return_idff12_response(profile, title = ''):
    '''Finish your ID-FFv1.2 views with this method to return a SAML
    response'''
    return return_saml2(profile, 'LARES', title)

def return_idff12_request(profile, title = ''):
    '''Finish your SAMLv2 views with this method to return a SAML
    request'''
    return return_saml2(profile, 'LAREQ', title)

def return_idff12(profile, field_name, title = ''):
    '''Helper to handle SAMLv2 bindings to emit request and responses'''
    if profile.msgBody:
        if profile.msgUrl:
            render_to_response('saml/post_form.html',{
                        'title': title,
                        'url': profile.msgUrl,
                        'fieldname': field_name,
                        'body': profile.msgBody,
                        'relay_state': profile.msgRelayState })
        return return_saml_soap_response(profile)
    elif profile.msgUrl:
        return HttpResponseRedirect(profile.msgUrl)
    else:
        return TypeError('profile do not contain a response')

# Helper method to handle profiles endpoints
# In the future we should move away from monolithic object (LassoIdentity and
# LassoSession) holding all the datas, to manipulate them at row Level with
# LibertyFederation and LibertyAssertion objects.

def load_federation(request, login, user = None):
    '''Load an identity dump from the database'''
    if not user:
        user = request.user
    q = LibertyIdentityDump.objects.filter(user = user)
    if not q:
        return
    login.setIdentityFromDump(q[0].identity_dump)

def load_session(request, login, session_key = None):
    '''Load a session dump from the database'''
    if not session_key:
        session_key = request.session.session_key
    q = LibertySessionDump.objects.filter(django_session_key = session_key)
    if not q:
        return
    login.setSessionFromDump(q[0].session_dump)

def save_federation(request, login, user = None):
    '''Save identity dump to database'''
    if not user:
        user = request.user
    if login.isIdentityDirty:
        q = LibertyIdentityDump.objects.filter(user = user)
        if q:
            if login.identity:
                q[0].identity_dump = login.identity.dump()
            else:
                q[0].identity_dump = None
            q[0].save()
        elif login.identity:
            LibertyIdentityDump(user = request.user,
                    identity_dump = login.identity.dump()).save()

def save_session(request, login, session_key = None):
    '''Save session dump to database'''
    if not session_key:
        session_key = request.session.session_key
    if login.isSessionDirty:
        q = LibertySessionDump.objects.filter(
                django_session_key = session_key)
        if q:
            if login.session:
                q[0].session_dump = login.session.dump()
            else:
                q[0].session_dump = None
            q[0].save()
        elif login.session:
            LibertySessionDump(django_session_key = request.session.session_key,
                    session_dump = login.session.dump()).save()

# TODO: handle autoloading of metadatas
def load_provider(request, login, provider_id, sp_or_idp = 'sp'):
    '''Look up a provider in the database, and verify it handles wanted
       role be it sp or idp.
    '''
    liberty_provider = LibertyProvider.objects.get(entity_id = provider_id)
    if not liberty_provider:
        return False
    if sp_or_idp == 'sp':
        try:
            service_provider = liberty_provider.service_provider
        except LibertyServiceProvider.DoesNotExist:
            return False
        if not service_provider.enabled:
            return False
        login.server.addProviderFromBuffer(lasso.PROVIDER_ROLE_SP,
                liberty_provider.metadata.read())
    elif sp_or_idp == 'idp':
        try:
            identity_provider = liberty_provider.identity_provider
        except LibertyIdentityProvider.DoesNotExist:
            return False
        if not identity_provider.enabled:
            return False
        login.server.addProviderFromBuffer(lasso.PROVIDER_ROLE_IDP,
                liberty_provider.metadata.read())
    else:
        raise Exception('unsupported option sp_or_idp = %r' % sp_or_idp)

    return liberty_provider
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# This file is part of easydms.
# Copyright (c) 2015 Peter Kessen
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject
# to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#
# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4

"""Test core funtionalities."""

from _common import TestCase, TestCaseCommandline
import tempfile
import os
import sys
import shutil

import easydms.config
import easydms.cli

class TestConfig(TestCase):
    def setUp(self):
        super(TestConfig, self).setUp()
        self.createConfigDir()
        self.io.install()

    def test_nonexisting_config(self):
        """Check behaviour of not existing configuration file"""
        try:
            tempdir = tempfile.mkdtemp()
            filename = os.path.join(tempdir, "Config.yaml")
            with self.assertRaises(easydms.config.ErrorNoConfiguration):
                easydms.config.Config(filename)

            with self.assertRaises(SystemExit) as cm:
                sys.argv = ["prog", "-c", filename]
                easydms.cli.main()
            self.assertNotEqual(cm.exception.code, 0)

        finally:
            shutil.rmtree(tempdir)

    def test_dump_config(self):
        """Check dump of config"""
        config = easydms.config.Config()
        print(config)
        self.io.stdout.clear()
        sys.argv = ["prog", "config", "dump"]
        with self.assertRaises(SystemExit):
            easydms.cli.main()

    def test_get_key(self):
        """Check getkey method of config"""
        pairs = {'library': '~/home/documents/dms',
                 'key': 'value',
                 }
        try:
            tempconfig = tempfile.NamedTemporaryFile(
                suffix=".yaml", delete=False)
            for key, value in pairs.iteritems():
                tempconfig.write("{0}: {1}\n".format(key, value))
            tempconfig.close()
            config = easydms.config.Config(tempconfig.name)

            for key, value in pairs.iteritems():
                self.assertEqual(config.getKey(key, "Spam"), value)
            for key, value in pairs.iteritems():
                self.assertEqual(config.getRequiredKey(key), value)
        finally:
            os.remove(tempconfig.name)

    def test_get_invalid_key(self):
        """Check getkey method of config with not existing key"""
        pairs = {'library': '~/home/documents/dms',
                 'key': 'value',
                 }
        exceptionKeys = ['Hello', 'spam']
        try:
            tempconfig = tempfile.NamedTemporaryFile(
                suffix=".yaml", delete=False)
            tempconfig.write('ham: eggs')
            tempconfig.close()
            config = easydms.config.Config(tempconfig.name)

            for key, value in pairs.iteritems():
                self.assertEqual(config.getKey(key, value), value)

            for key in exceptionKeys:
                with self.assertRaises(easydms.config.ErrorConfigKeyNotFound):
                    config.getRequiredKey(key)
        finally:
            os.remove(tempconfig.name)

class TestConfigCmd(TestCaseCommandline):
    def setUp(self):
        super(TestConfigCmd, self).setUp()
        self.createConfigDir()

    def test_nonexisting_config_cmd(self):
        """Check behaviour of not existing configuration file"""
        try:
            tempdir = tempfile.mkdtemp()
            filename = os.path.join(tempdir, "Config.yaml")
            ret = self.call("easydms", ["-c {0}".format(filename)])
            self.assertNotEqual(ret, 0)
        finally:
            shutil.rmtree(tempdir)

    def test_dump_config_cmd(self):
        """Check dump of config by commandline invoke"""
        ret = self.call("easydms", ["config", "dump"])
        self.assertEqual(ret, 0)
<commit_msg>Added test for creation of library directory<commit_after># -*- coding: utf-8 -*-
#
# This file is part of easydms.
# Copyright (c) 2015 Peter Kessen
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject
# to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#
# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4

"""Test core funtionalities."""

from _common import TestCase, TestCaseCommandline
import tempfile
import os
import sys
import shutil

import easydms.config
import easydms.cli

class TestConfig(TestCase):
    def setUp(self):
        super(TestConfig, self).setUp()
        self.createConfigDir()
        self.io.install()

    def test_nonexisting_config(self):
        """Check behaviour of not existing configuration file"""
        try:
            tempdir = tempfile.mkdtemp()
            filename = os.path.join(tempdir, "Config.yaml")
            with self.assertRaises(easydms.config.ErrorNoConfiguration):
                easydms.config.Config(filename)

            with self.assertRaises(SystemExit) as cm:
                sys.argv = ["prog", "-c", filename]
                easydms.cli.main()
            self.assertNotEqual(cm.exception.code, 0)

        finally:
            shutil.rmtree(tempdir)

    def test_dump_config(self):
        """Check dump of config"""
        config = easydms.config.Config()
        print(config)
        self.io.stdout.clear()
        sys.argv = ["prog", "config", "dump"]
        with self.assertRaises(SystemExit):
            easydms.cli.main()

    def test_get_key(self):
        """Check getkey method of config"""
        pairs = {'library': '~/home/documents/dms',
                 'key': 'value',
                 }
        try:
            tempconfig = tempfile.NamedTemporaryFile(
                suffix=".yaml", delete=False)
            for key, value in pairs.iteritems():
                tempconfig.write("{0}: {1}\n".format(key, value))
            tempconfig.close()
            config = easydms.config.Config(tempconfig.name)

            for key, value in pairs.iteritems():
                self.assertEqual(config.getKey(key, "Spam"), value)
            for key, value in pairs.iteritems():
                self.assertEqual(config.getRequiredKey(key), value)
        finally:
            os.remove(tempconfig.name)

    def test_get_invalid_key(self):
        """Check getkey method of config with not existing key"""
        pairs = {'library': '~/home/documents/dms',
                 'key': 'value',
                 }
        exceptionKeys = ['Hello', 'spam']
        try:
            tempconfig = tempfile.NamedTemporaryFile(
                suffix=".yaml", delete=False)
            tempconfig.write('ham: eggs')
            tempconfig.close()
            config = easydms.config.Config(tempconfig.name)

            for key, value in pairs.iteritems():
                self.assertEqual(config.getKey(key, value), value)

            for key in exceptionKeys:
                with self.assertRaises(easydms.config.ErrorConfigKeyNotFound):
                    config.getRequiredKey(key)
        finally:
            os.remove(tempconfig.name)

    def test_create_dir(self):
        dmsdir = '~/documents/dms'
        configfilePath = os.path.expanduser("~/.config/easydms.yaml")
        configfile = open(configfilePath, mode="w")
        configfile.write('library: easydms.db\n')
        configfile.write('directory: {0}\n'.format(dmsdir))
        configfile.close()

        with self.assertRaises(SystemExit):
            self.io.addinput("n")
            sys.argv = ["prog"]
            easydms.cli.main()
        self.assertNotExists(os.path.expanduser(dmsdir))
        self.io.addinput("y")
        sys.argv = ["prog"]
        easydms.cli.main()
        self.assertExists(os.path.expanduser(dmsdir))
        sys.argv = ["prog"]
        easydms.cli.main()
        self.assertExists(os.path.expanduser(dmsdir))

class TestConfigCmd(TestCaseCommandline):
    def setUp(self):
        super(TestConfigCmd, self).setUp()
        self.createConfigDir()

    def test_nonexisting_config_cmd(self):
        """Check behaviour of not existing configuration file"""
        try:
            tempdir = tempfile.mkdtemp()
            filename = os.path.join(tempdir, "Config.yaml")
            ret = self.call("easydms", ["-c {0}".format(filename)])
            self.assertNotEqual(ret, 0)
        finally:
            shutil.rmtree(tempdir)

    def test_dump_config_cmd(self):
        """Check dump of config by commandline invoke"""
        ret = self.call("easydms", ["config", "dump"])
        self.assertEqual(ret, 0)
<|endoftext|>


<commit_before># coding: utf-8

import logging
import collections
from datetime import datetime
from flask import request, redirect, url_for  # render_template
from flask.views import MethodView
from flask.ext.mongoengine.wtf import model_form
from quokka.core.models import Channel, Content, Comment
from quokka.core.templates import render_template

logger = logging.getLogger()

class ContentList(MethodView):

    def get(self, long_slug):
        now = datetime.now()
        path = long_slug.split('/')
        mpath = ",".join(path)
        mpath = ",{0},".format(mpath)

        channel = Channel.objects.get_or_404(mpath=mpath)

        filters = {
            'published': True,
            'available_at__lte': now,
            'show_on_channel': True
        }

        if not channel.is_homepage:
            filters['__raw__'] = {'mpath': {'$regex': mpath}}

        contents = Content.objects(**filters)

        return render_template('content/list.html', contents=contents)

class ContentDetail(MethodView):
    form = model_form(
        Comment,
        exclude=['created_at', 'created_by', 'published']
    )

    def get_context(self, long_slug):
        now = datetime.now()
        homepage = Channel.objects.get(is_homepage=True)

        if long_slug.startswith(homepage.slug) and \
                len(long_slug.split('/')) < 3:
            slug = long_slug.split('/')[-1]
            return redirect(url_for('detail', long_slug=slug))

        filters = {
            'published': True,
            'available_at__lte': now
        }

        try:
            content = Content.objects.get(
                long_slug=long_slug,
                **filters
            )
        except Content.DoesNotExist:
            content = Content.objects.get_or_404(
                channel=homepage,
                slug=long_slug,
                **filters
            )

        form = self.form(request.form)

        context = {
            "content": content,
            "form": form
        }
        return context

    def get(self, long_slug):
        context = self.get_context(long_slug)
        if isinstance(context, collections.Callable):
            return context
        return render_template('content/detail.html', **context)

    def post(self, long_slug):
        context = self.get_context(long_slug)
        form = context.get('form')

        if form.validate():
            comment = Comment()
            form.populate_obj(comment)

            content = context.get('content')
            content.comments.append(comment)
            content.save()

            return redirect(url_for('.detail', long_slug=long_slug))

        return render_template('content/detail.html', **context)

class ContentFeed(MethodView):
    pass

class ChannelFeed(ContentFeed):
    pass
<commit_msg>whitespace cleanup<commit_after># coding: utf-8

import logging
import collections
from datetime import datetime
from flask import request, redirect, url_for  # render_template
from flask.views import MethodView
from flask.ext.mongoengine.wtf import model_form
from quokka.core.models import Channel, Content, Comment
from quokka.core.templates import render_template

logger = logging.getLogger()

class ContentList(MethodView):

    def get(self, long_slug):
        now = datetime.now()
        path = long_slug.split('/')
        mpath = ",".join(path)
        mpath = ",{0},".format(mpath)

        channel = Channel.objects.get_or_404(mpath=mpath)

        filters = {
            'published': True,
            'available_at__lte': now,
            'show_on_channel': True
        }

        if not channel.is_homepage:
            filters['__raw__'] = {'mpath': {'$regex': mpath}}

        contents = Content.objects(**filters)

        return render_template('content/list.html', contents=contents)

class ContentDetail(MethodView):
    form = model_form(
        Comment,
        exclude=['created_at', 'created_by', 'published']
    )

    def get_context(self, long_slug):
        now = datetime.now()
        homepage = Channel.objects.get(is_homepage=True)

        if long_slug.startswith(homepage.slug) and \
                len(long_slug.split('/')) < 3:
            slug = long_slug.split('/')[-1]
            return redirect(url_for('detail', long_slug=slug))

        filters = {
            'published': True,
            'available_at__lte': now
        }

        try:
            content = Content.objects.get(
                long_slug=long_slug,
                **filters
            )
        except Content.DoesNotExist:
            content = Content.objects.get_or_404(
                channel=homepage,
                slug=long_slug,
                **filters
            )

        form = self.form(request.form)

        context = {
            "content": content,
            "form": form
        }
        return context

    def get(self, long_slug):
        context = self.get_context(long_slug)
        if isinstance(context, collections.Callable):
            return context
        return render_template('content/detail.html', **context)

    def post(self, long_slug):
        context = self.get_context(long_slug)
        form = context.get('form')

        if form.validate():
            comment = Comment()
            form.populate_obj(comment)

            content = context.get('content')
            content.comments.append(comment)
            content.save()

            return redirect(url_for('.detail', long_slug=long_slug))

        return render_template('content/detail.html', **context)

class ContentFeed(MethodView):
    pass

class ChannelFeed(ContentFeed):
    pass
<|endoftext|>


<commit_before><commit_msg>added template core file<commit_after>import numpy as np
import pandas as pd
import sklearn as sk
import matplotlib.pyplot as plt

z = pd.read_json('http://moatsearch-data.s3.amazonaws.com/homework/ad_classification_hw_dataset.json')
desc = z.describe
print desc

<|endoftext|>


<commit_before>import boto3
import botocore
import copy
import threading

from collections import deque

class FileObj(object):
    def __init__(self, data=None, id_=None):
        if isinstance(data, bytes):
            self.data = data
        else:
            self.data = ''.encode('utf-8')
        self.id_ = id_
        self.index = 0

    def read(self, size=None, *args, **kwargs):
        # returns bytes, reads the data
        if size is None:
            size = len(self.data)
        size = int(size)
        # print('read %s from %s' % (self.data, self.id_,))
        if self.index + size >= len(self.data):
            old_index = self.index + 0
            self.index = len(self.data)
            print('read %s from %s' % (self.data[old_index:], self.id_,))
            return self.data[old_index:]
        else:
            old_index = self.index + 0
            self.index += size
            print('read %s from %s' % (self.data[old_index:self.index], self.id_,))
            return self.data[old_index:self.index]

    def write(self, bytes_, *args, **kwargs):
        # print('write %s to %s' % (bytes_, self.id_,))
        print('write %s bytes to %s' % (bytes_, self.id_,))
        self.data = bytes_

class WriteOnceFileObj(FileObj):
    def __init__(self, data, id_):
        super(WriteOnceFileObj, self).__init__(data, id_)
        self.written = False

    def write(self, bytes_):
        if self.written:
            print('already wrote to this file object...')
        else:
            self.written = True
            super(WriteOnceFileObj, self).write(bytes_)
            print(self.data)

class AbstractRegion(object):
    def __init__(self, region_id):
        if self.validate_region_id(region_id):
            self.region_id = region_id
        else:
            self.region_id = None

    def validate_region_id(self, region_id):
        return True

class AbstractProvider(object):
    def list(self):
        raise NotYetImplemented()

    def create(self, bucket_name, region):
        raise NotYetImplemented()

    def delete(self, bucket_name):
        raise NotYetImplemented()

    def delete_all(self):
        raise NotYetImplemented()

    def upload(self, bucket_name, file_key, file_obj):
        raise NotYetImplemented()

    def download(self, bucket_name, file_key, file_obj):
        raise NotYetImplemented()

class S3(AbstractProvider):
    def __init__(self):
        self.s3 = boto3.resource('s3')
        self.s3_client = boto3.client('s3')

    class Region(AbstractRegion):
        def validate_region_id(self, region_id):
            return region_id in {
                'ap-northeast-1',
                'ap-northeast-2',
                'ap-south-1',
                'ap-southeast-1',
                'ap-southeast-2',
                'ca-central-1',
                'eu-central-1',
                'eu-east-1',
                'eu-west-1',
                'eu-west-2',
                'sa-east-1',
                'us-east-1',
                'us-east-2',
                'us-west-1',
                'us-west-2',
            }

    def list(self):
        for bucket in self.s3_client.list_buckets()['Buckets']:
            yield bucket

    def create(self, bucket_name, region):
        if not isinstance(region, S3.Region):
            return False
        try:
            self.s3.Bucket(bucket_name).create(
                CreateBucketConfiguration = {
                    'LocationConstraint': region.region_id,
                })
            return True
        except botocore.exceptions.ClientError:
            return False

    def delete(self, bucket_name):
        # All objects (including all object versions and Delete Markers) in the
        #  bucket must be deleted before the bucket itself can be deleted.
        # print('begin delete %s' % (bucket_name,))
        self.s3.Bucket(bucket_name).objects.all().delete()
        self.s3.Bucket(bucket_name).delete()
        # print('end delete %s' % (bucket_name,))

    def delete_all(self):
        for bucket in self.list():
            self.delete(bucket['Name'])

    def upload(self, bucket_name, file_key, file_obj):
        self.s3.Bucket(bucket_name).upload_fileobj(Fileobj=file_obj, Key=file_key)

    def download(self, bucket_name, file_key, file_obj):
        self.s3.Bucket(bucket_name).download_fileobj(Key=file_key, Fileobj=file_obj)

class R4(AbstractProvider):
    def __init__(self):
        pass

    class Region(AbstractRegion):
        def validate_region_id(self, region_id):
            # accept 'filesystem' and '<portnum>.LocalR4'
            if region_id == 'filesystem':
                return True
            else:
                if '.' in region_id:
                    splits = region_id.split('.', 2)
                    if len(splits) == 2:
                        try:
                            num = int(splits[0])
                        except ValueError:
                            return False
                        return 1024 <= num <= 49151 and splits[1] == 'LocalR4'
                    return False

SUPPORTED_SERVICES = [
    'Amazon Web Services S3',
    'R4 Filesystem',
    'R4 Local Server',
    ]

class Client(AbstractProvider):
    '''
    A Client is an AbstractProvider that accepts any and all regions and
    distributes its actions around all of the regions. It abstracts away needing
    to deal with each individual service and region while also automatically
    increasing performance through parallelization and increasing reliability by
    eliminating a single point of failure
    '''
    default_regions = [
        S3.Region('us-east-1'),
        S3.Region('us-east-2'),
    ]
    def __init__(self, regions):
        self.clients = {}
        if regions is None:
            self.regions = default_regions
        else:
            self.regions = regions
        self._setup_regions()

    def _setup_regions(self):
        for region in self.regions:
            if isinstance(region, S3.Region):
                if 's3.'+region.region_id not in self.clients:
                    self.clients['s3.'+region.region_id] = S3()
            elif isinstance(region, R4.Region):
                if 'r4' not in self.clients:
                    self.clients['r4'] = R4()
            else:
                print('Unsupported Region %s' % (region,))

    def list(self):
        '''
        list all of the buckets that are available across all regions
        '''
        # TODO parallelize this with threads so that all the buckets are
        #  returned as soon as possible, not blocking for one long call

        for region in self.regions:
            if isinstance(region, S3.Region):
                client = 's3.'+region.region_id
            elif isinstance(region, R4.Region):
                client = 'r4'
            else:
                client = None
            if client is not None:
                for bucket in self.clients[client].list():
                    yield bucket

    def create(self, bucket_name):

        threads = []

        for region in self.regions:
            if isinstance(region, S3.Region):
                client = 's3.'+region.region_id
            elif isinstance(region, R4.Region):
                client = 'r4'
            else:
                client = None
            if client is not None:
                threads.append(threading.Thread(target=self.clients[client].create, args=(region.region_id + '.' + bucket_name, region,)))

        for t in threads:
            t.start()

        for t in threads:
            t.join()

    def delete(self, bucket_name):
        threads = []

        for region in self.regions:
            if isinstance(region, S3.Region):
                client = 's3.'+region.region_id
            elif isinstance(region, R4.Region):
                client = 'r4'
            else:
                client = None
            if client is not None:
                threads.append(threading.Thread(target=self.clients[client].delete, args=(bucket_name,)))

        for t in threads:
            t.start()

        for t in threads:
            t.join()

    def delete_all(self):
        threads = []

        for region in self.regions:
            if isinstance(region, S3.Region):
                client = 's3.'+region.region_id
            elif isinstance(region, R4.Region):
                client = 'r4'
            else:
                client = None
            if client is not None:
                threads.append(threading.Thread(target=self.clients[client].delete_all))

        for t in threads:
            t.start()

        for t in threads:
            t.join()

    def upload(self, bucket_name, file_key, file_obj):
        threads = []

        for region in self.regions:
            if isinstance(region, S3.Region):
                client = 's3.'+region.region_id
            elif isinstance(region, R4.Region):
                client = 'r4'
            else:
                client = None
            if client is not None:
                threads.append(threading.Thread(target=self.clients[client].upload, args=(bucket_name, file_key, copy.deepcopy(file_obj),)))

        for t in threads:
            t.start()

        for t in threads:
            t.join()

    def download(self, bucket_name, file_key, file_obj):
        threads = []

        for region in self.regions:
            if isinstance(region, S3.Region):
                client = 's3.'+region.region_id
            elif isinstance(region, R4.Region):
                client = 'r4'
            else:
                client = None
            if client is not None:
                threads.append(threading.Thread(target=self.clients[client].download, args=(bucket_name, file_key, file_obj,)))

        for t in threads:
            t.start()

        for t in threads:
            t.join()

if __name__ == '__main__':
    s3 = Client(regions=[S3.Region('us-east-1'), S3.Region('us-east-2')])
    bucket_found = False
    for bucket in s3.list():
        # print('Bucket %s' % bucket['Name'])
        bucket_found = True
        break

    if not bucket_found:
        print('No buckets found')

    upload_this = FileObj('Hello AWS World again'.encode('utf-8'), 'upload_this')
    key = 'test_file'
    bucket_name = 'io.r4.username03'

    s3.create(bucket_name)

    s3.upload(bucket_name, key, upload_this)

    print('uploaded file %s to bucket %s' % (key, bucket_name))

    download_here = WriteOnceFileObj(None, 'once download_here')

    s3.download(bucket_name, key, download_here)

    print('downloaded file %s from bucket %s\n%s' % (key, bucket_name, download_here.data))

    # s3.delete_all_buckets()
<commit_msg>Get bucket creation in multiple regions working<commit_after>import boto3
import botocore
import copy
import logging
import threading

from collections import deque

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
print = logger.info

class FileObj(object):
    def __init__(self, data=None, id_=None):
        if isinstance(data, bytes):
            self.data = data
        else:
            self.data = ''.encode('utf-8')
        self.id_ = id_
        self.index = 0

    def read(self, size=None, *args, **kwargs):
        # returns bytes, reads the data
        if size is None:
            size = len(self.data)
        size = int(size)

        if self.index + size >= len(self.data):
            old_index = self.index + 0
            self.index = len(self.data)
            logger.info('read %s from %s' % (self.data[old_index:], self.id_,))
            return self.data[old_index:]
        else:
            old_index = self.index + 0
            self.index += size
            logger.info('read %s from %s' % (self.data[old_index:self.index], self.id_,))
            return self.data[old_index:self.index]

    def write(self, bytes_, *args, **kwargs):
        # print('write %s to %s' % (bytes_, self.id_,))
        print('write %s bytes to %s' % (bytes_, self.id_,))
        self.data = bytes_

class WriteOnceFileObj(FileObj):
    def __init__(self, data, id_):
        super(WriteOnceFileObj, self).__init__(data, id_)
        self.written = False

    def write(self, bytes_):
        if self.written:
            print('already wrote to this file object...')
        else:
            self.written = True
            super(WriteOnceFileObj, self).write(bytes_)
            print(self.data)

class AbstractRegion(object):
    def __init__(self, region_id):
        if self.validate_region_id(region_id):
            self.region_id = region_id
        else:
            self.region_id = None

    def __str__(self):
        return 'Region("%s")' % (self.region_id,)

    def __repr__(self):
        return 'Region("%s")' % (self.region_id,)

    def validate_region_id(self, region_id):
        return True

class AbstractProvider(object):
    def __str__(self):
        raise NotYetImplemented()

    def list(self):
        raise NotYetImplemented()

    def create(self, bucket_name, region):
        raise NotYetImplemented()

    def delete(self, bucket_name):
        raise NotYetImplemented()

    def delete_all(self):
        raise NotYetImplemented()

    def upload(self, bucket_name, file_key, file_obj):
        raise NotYetImplemented()

    def download(self, bucket_name, file_key, file_obj):
        raise NotYetImplemented()

class S3(AbstractProvider):
    def __init__(self, region):
        if isinstance(region, S3.Region):
            self.region = region
            self.s3 = boto3.resource('s3', region_name=region.region_id)
            self.s3_client = boto3.client('s3', region_name=region.region_id)

    class Region(AbstractRegion):
        def validate_region_id(self, region_id):
            return region_id in {
                'ap-northeast-1',
                'ap-northeast-2',
                'ap-south-1',
                'ap-southeast-1',
                'ap-southeast-2',
                'ca-central-1',
                'eu-central-1',
                'eu-east-1',
                'eu-west-1',
                'eu-west-2',
                'sa-east-1',
                'us-east-1',
                'us-east-2',
                'us-west-1',
                'us-west-2',
            }

    def list(self):
        for bucket in self.s3_client.list_buckets()['Buckets']:
            yield bucket

    def create(self, bucket_name):
        try:
            if self.region.region_id == 'us-east-1':
                self.s3.Bucket(bucket_name).create()
            else:
                self.s3.Bucket(bucket_name).create(
                    CreateBucketConfiguration = {
                        'LocationConstraint': self.region.region_id,
                    })
            return True
        except botocore.exceptions.ClientError as e:
            if '%s' % (type(e),) == 'botocore.errorfactory.BucketAlreadyOwnedByYou':
                print('bucket %s already exists' % (bucket_name,))
            raise
            return False

    def delete(self, bucket_name):
        # All objects (including all object versions and Delete Markers) in the
        #  bucket must be deleted before the bucket itself can be deleted.
        # print('begin delete %s' % (bucket_name,))
        self.s3.Bucket(bucket_name).objects.all().delete()
        self.s3.Bucket(bucket_name).delete()
        # print('end delete %s' % (bucket_name,))

    def delete_all(self):
        for bucket in self.list():
            self.delete(bucket['Name'])

    def upload(self, bucket_name, file_key, file_obj):
        self.s3.Bucket(bucket_name).upload_fileobj(Fileobj=file_obj, Key=file_key)

    def download(self, bucket_name, file_key, file_obj):
        self.s3.Bucket(bucket_name).download_fileobj(Key=file_key, Fileobj=file_obj)

class R4(AbstractProvider):
    def __init__(self):
        pass

    class Region(AbstractRegion):
        def validate_region_id(self, region_id):
            # accept 'filesystem' and '<portnum>.LocalR4'
            if region_id == 'filesystem':
                return True
            else:
                if '.' in region_id:
                    splits = region_id.split('.', 2)
                    if len(splits) == 2:
                        try:
                            num = int(splits[0])
                        except ValueError:
                            return False
                        return 1024 <= num <= 49151 and splits[1] == 'LocalR4'
                    return False

SUPPORTED_SERVICES = [
    'Amazon Web Services S3',
    'R4 Filesystem',
    'R4 Local Server',
    ]

class Client(AbstractProvider):
    '''
    A Client is an AbstractProvider that accepts any and all regions and
    distributes its actions around all of the regions. It abstracts away needing
    to deal with each individual service and region while also automatically
    increasing performance through parallelization and increasing reliability by
    eliminating a single point of failure
    '''
    default_regions = [
        S3.Region('us-east-1'),
        S3.Region('us-east-2'),
    ]
    def __init__(self, regions):
        self.clients = {}
        if regions is None:
            self.regions = default_regions
        else:
            self.regions = regions
        self._setup_regions()

    def _setup_regions(self):
        for region in self.regions:
            if isinstance(region, S3.Region):
                if 's3.'+region.region_id not in self.clients:
                    self.clients['s3.'+region.region_id] = S3(region)
            elif isinstance(region, R4.Region):
                if 'r4' not in self.clients:
                    self.clients['r4'] = R4()
            else:
                print('Unsupported Region %s' % (region,))

    def list(self):
        '''
        list all of the buckets that are available across all regions
        '''
        # TODO parallelize this with threads so that all the buckets are
        #  returned as soon as possible, not blocking for one long call

        for region in self.regions:
            if isinstance(region, S3.Region):
                client = 's3.'+region.region_id
            elif isinstance(region, R4.Region):
                client = 'r4'
            else:
                client = None
            if client is not None:
                for bucket in self.clients[client].list():
                    yield bucket

    def create(self, bucket_name):

        threads = []

        for region in self.regions:
            if isinstance(region, S3.Region):
                client = 's3.'+region.region_id
            elif isinstance(region, R4.Region):
                client = 'r4'
            else:
                client = None
            if client is not None:
                args = (region.region_id + '.' + bucket_name,)
                print('create %s' % (args,))
                threads.append(threading.Thread(target=self.clients[client].create, args=args))

        for t in threads:
            t.start()

        for t in threads:
            t.join()

    def delete(self, bucket_name):
        threads = []

        for region in self.regions:
            if isinstance(region, S3.Region):
                client = 's3.'+region.region_id
            elif isinstance(region, R4.Region):
                client = 'r4'
            else:
                client = None
            if client is not None:
                threads.append(threading.Thread(target=self.clients[client].delete, args=(region.region_id + '.' + bucket_name,)))

        for t in threads:
            t.start()

        for t in threads:
            t.join()

    def delete_all(self):
        threads = []

        for region in self.regions:
            if isinstance(region, S3.Region):
                client = 's3.'+region.region_id
            elif isinstance(region, R4.Region):
                client = 'r4'
            else:
                client = None
            if client is not None:
                threads.append(threading.Thread(target=self.clients[client].delete_all))

        for t in threads:
            t.start()

        for t in threads:
            t.join()

    def upload(self, bucket_name, file_key, file_obj):
        threads = []

        for region in self.regions:
            if isinstance(region, S3.Region):
                client = 's3.'+region.region_id
            elif isinstance(region, R4.Region):
                client = 'r4'
            else:
                client = None
            if client is not None:
                threads.append(threading.Thread(target=self.clients[client].upload, args=(region.region_id + '.' + bucket_name, file_key, copy.deepcopy(file_obj),)))

        for t in threads:
            t.start()

        for t in threads:
            t.join()

    def download(self, bucket_name, file_key, file_obj):
        threads = []

        for region in self.regions:
            print('region: %s' % (region,))
            if isinstance(region, S3.Region):
                client = 's3.'+region.region_id
            elif isinstance(region, R4.Region):
                client = 'r4'
            else:
                client = None
            if client is not None:
                threads.append(threading.Thread(target=self.clients[client].download, args=(region.region_id + '.' + bucket_name, file_key, file_obj,)))

        for t in threads:
            t.start()

        for t in threads:
            t.join()

if __name__ == '__main__':

    s3 = Client(regions=[
        S3.Region('us-east-1'),
        # S3.Region('us-east-2'),
        ])

    # print('enumerate buckets')

    # bucket_found = False
    # for bucket in s3.list():
    #     print('Bucket %s' % bucket['Name'])
    #     bucket_found = True

    # if not bucket_found:
    #     print('No buckets found')
    # else:
    #     print('Found at least one bucket')

    # upload_this = FileObj('Hello AWS World again'.encode('utf-8'), 'upload_this')
    # key = 'test_file'
    bucket_name = 'io.r4.username03'

    print('Client.create("%s")' % (bucket_name,))
    s3.create(bucket_name)

    for bucket in s3.list():
        print('Bucket? %s' % bucket['Name'])

    # s3.upload(bucket_name, key, upload_this)

    # print('uploaded file %s to bucket %s' % (key, bucket_name))

    # download_here = WriteOnceFileObj(None, 'once download_here')

    # s3.download(bucket_name, key, download_here)

    # print('downloaded file %s from bucket %s\n%s' % (key, bucket_name, download_here.data))

    # s3.delete_all_buckets()
<|endoftext|>


<commit_before>import operator
import six
import logging
import os
import glob
import copy

# noinspection PyUnresolvedReferences
from six.moves import reduce
from avro.schema import UnionSchema, Schema

if six.PY2:
    from avro.io import validate
else:
    from avro.io import Validate as validate

from rabix.common.util import sec_files_naming_conv, wrap_in_list, to_abspath
from rabix.expressions import Evaluator

log = logging.getLogger(__name__)

class AdapterEvaluator(object):

    def __init__(self, job):
        self.job = job
        self.ev = Evaluator()

    def evaluate(self, expr_object, context=None, job=None):
        if isinstance(expr_object, dict):
            return self.ev.evaluate(
                expr_object.get('engine', 'javascript'),
                expr_object.get('script'),
                job.to_dict() if job else self.job.to_dict(),
                context
            )
        else:
            return self.ev.evaluate('javascript', expr_object,
                                    job.to_dict() if job else
                                    self.job.to_dict(), context)

    def resolve(self, val, context=None, job=None):
        if not isinstance(val, dict):
            return val
        if 'engine' in val and 'script' in val:
            v = val.get('script')
            return self.evaluate(v, context, job=job)
        return val

def intersect_dicts(d1, d2):
    return {k: v for k, v in six.iteritems(d1) if v == d2.get(k)}

def meta(path, inputs, eval, outputBinding):
    meta, result = outputBinding.get('metadata', {}), {}
    inherit = meta.pop('__inherit__', None)
    if inherit:
        src = inputs.get(inherit)
        if isinstance(src, list):
            result = reduce(intersect_dicts, [x.meta for x in src]) \
                if len(src) > 1 else src[0].meta
        else:
            result = src.meta
    result.update(**meta)
    for k, v in six.iteritems(result):
        result[k] = eval.resolve(v, context=path)
    return result

def secondary_files(p, outputBinding, evaluator):
    secondaryFiles = []
    secFiles = wrap_in_list(evaluator.resolve(outputBinding.get('secondaryFiles', [])))
    for s in secFiles:
        path = sec_files_naming_conv(p, s)
        secondaryFiles.append({'path': to_abspath(path)})
    return secondaryFiles

class InputAdapter(object):
    def __init__(self, value, evaluator, schema, input_binding=None, key=''):
        self.evaluator = evaluator
        self.schema = schema
        self.adapter = input_binding or (
            isinstance(self.schema, Schema) and
            self.schema.props.get('inputBinding')
        )
        self.has_adapter = self.adapter is not None
        self.adapter = self.adapter or {}
        self.key = key
        if isinstance(self.schema, UnionSchema):
            for opt in self.schema.schemas:
                if validate(opt, value):
                    self.schema = opt
                    break
        self.value = evaluator.resolve(value)

    __str__ = lambda self: six.text_type(self.value)
    __repr__ = lambda self: 'InputAdapter(%s)' % self
    position = property(lambda self: (self.adapter.get('position', 9999999), self.key))
    prefix = property(lambda self: self.adapter.get('prefix'))
    item_separator = property(lambda self: self.adapter.get('itemSeparator', ','))
    separate = property(lambda self: self.adapter.get('separate', True))

    def arg_list(self):
        if isinstance(self.value, dict):
            return self.as_dict()
        if isinstance(self.value, list):
            return self.as_list()
        return self.as_primitive()

    def as_primitive(self):
        if self.value is None or self.value is False:
            return []
        if self.value is True:
            if not self.prefix:
                raise ValueError('Boolean arguments must have a prefix.')
            return [self.prefix]
        if not self.prefix:
            return [six.text_type(self.value)]
        if self.separate:
            return [self.prefix, self.value]
        return [self.prefix + six.text_type(self.value)]

    def as_dict(self):
        sch = lambda key: next(field for field in self.schema.fields
                               if field.name == key)
        adapters = [InputAdapter(v, self.evaluator, sch(k), key=k)
                    for k, v in six.iteritems(self.value)]
        adapters = [adp for adp in adapters if adp.has_adapter]
        res = reduce(
            operator.add,
            [a.arg_list() for a in sorted(adapters, key=lambda x: x.position)],
            []
        )
        return res

    def as_toplevel(self, mix_with):
        sch = lambda key: next(inp for inp in self.schema
                               if inp.id == key.split('.')[-1])
        adapters = mix_with + [
            InputAdapter(v, self.evaluator, sch(k).validator,
                         sch(k).input_binding, key=k)
            for k, v in six.iteritems(self.value)
            ]

        res = reduce(
            operator.add,
            [a.arg_list() for a in sorted(adapters, key=lambda x: x.position)],
            []
        )
        return res

    def as_list(self):
        items = [InputAdapter(item, self.evaluator, self.schema.items)
                 for item in self.value]

        if not self.prefix:
            return reduce(operator.add, [a.arg_list() for a in items], [])

        if self.separate and self.item_separator is None:
            return reduce(operator.add, [[self.prefix] + a.arg_list()
                                         for a in items], [])

        if not self.separate and self.item_separator is None:
            return [self.prefix + a.list_item()
                    for a in items if a.list_item() is not None]

        joined = self.item_separator.join(
            filter(None, [a.list_item() for a in items])
        )

        if self.separate and self.item_separator is not None:
            return [self.prefix, joined]
        return [self.prefix + joined]

    def list_item(self):
        as_arg_list = self.arg_list()
        if not as_arg_list:
            return None
        if len(as_arg_list) > 1 or isinstance(as_arg_list[0], (dict, list)):
            raise ValueError('Only lists of primitive values can use itemSeparator.')
        return six.text_type(as_arg_list[0])

class CLIJob(object):
    def __init__(self, job):
        self.job = job
        self.app = job.app
        self._stdin = self.app.stdin
        self._stdout = self.app.stdout
        self.base_cmd = self.app.base_command
        if isinstance(self.base_cmd, six.string_types):
            self.base_cmd = self.base_cmd.split(' ')
        self.args = self.app.arguments
        self.eval = AdapterEvaluator(job)

    @property
    def stdin(self):
        if self._stdin:
            return self.eval.resolve(self._stdin)

    @property
    def stdout(self):
        if self._stdout:
            return self.eval.resolve(self._stdout)

    def make_arg_list(self):
        adapters = [InputAdapter(a.get('valueFrom'), self.eval, {}, a)
                    for a in self.args]
        ia = InputAdapter(self.job.inputs, self.eval, self.app.inputs)
        args = ia.as_toplevel(adapters)
        base_cmd = [self.eval.resolve(item) for item in self.base_cmd]

        return [six.text_type(arg) for arg in base_cmd + args]

    def cmd_line(self):
        a = self.make_arg_list()

        if self._stdin:
            a += ['<', self.stdin]
        if self._stdout:
            a += ['>', self.eval.resolve(self.stdout)]
        return ' '.join(a)  # TODO: escape

    def get_outputs(self, job_dir, job):
        result, outs = {}, self.app.outputs
        for out in outs:
            out_binding = out.output_binding
            ret = os.getcwd()
            os.chdir(job_dir)
            pattern = self.eval.resolve(out_binding.get('glob'), job=job) or ""
            files = glob.glob(pattern)
            result[out.id] = [{'path': os.path.abspath(p),
                          'metadata': meta(p, job.inputs, self.eval, out_binding),
                          'secondaryFiles': secondary_files(p, out_binding, self.eval)} for p in files]
            os.chdir(ret)
            if out.depth == 0:
                result[out.id] = result[out.id][0] if result[out.id] else None
        return result
<commit_msg>hack for top-lvel list arguments<commit_after>import operator
import six
import logging
import os
import glob
import copy

# noinspection PyUnresolvedReferences
from six.moves import reduce
from avro.schema import UnionSchema, Schema, ArraySchema

if six.PY2:
    from avro.io import validate
else:
    from avro.io import Validate as validate

from rabix.common.util import sec_files_naming_conv, wrap_in_list, to_abspath
from rabix.expressions import Evaluator

log = logging.getLogger(__name__)

class AdapterEvaluator(object):

    def __init__(self, job):
        self.job = job
        self.ev = Evaluator()

    def evaluate(self, expr_object, context=None, job=None):
        if isinstance(expr_object, dict):
            return self.ev.evaluate(
                expr_object.get('engine', 'javascript'),
                expr_object.get('script'),
                job.to_dict() if job else self.job.to_dict(),
                context
            )
        else:
            return self.ev.evaluate('javascript', expr_object,
                                    job.to_dict() if job else
                                    self.job.to_dict(), context)

    def resolve(self, val, context=None, job=None):
        if not isinstance(val, dict):
            return val
        if 'engine' in val and 'script' in val:
            v = val.get('script')
            return self.evaluate(v, context, job=job)
        return val

def intersect_dicts(d1, d2):
    return {k: v for k, v in six.iteritems(d1) if v == d2.get(k)}

def meta(path, inputs, eval, outputBinding):
    meta, result = outputBinding.get('metadata', {}), {}
    inherit = meta.pop('__inherit__', None)
    if inherit:
        src = inputs.get(inherit)
        if isinstance(src, list):
            result = reduce(intersect_dicts, [x.meta for x in src]) \
                if len(src) > 1 else src[0].meta
        else:
            result = src.meta
    result.update(**meta)
    for k, v in six.iteritems(result):
        result[k] = eval.resolve(v, context=path)
    return result

def secondary_files(p, outputBinding, evaluator):
    secondaryFiles = []
    secFiles = wrap_in_list(evaluator.resolve(outputBinding.get('secondaryFiles', [])))
    for s in secFiles:
        path = sec_files_naming_conv(p, s)
        secondaryFiles.append({'path': to_abspath(path)})
    return secondaryFiles

class InputAdapter(object):
    def __init__(self, value, evaluator, schema, input_binding=None, key=''):
        self.evaluator = evaluator
        self.schema = schema
        self.adapter = input_binding or (
            isinstance(self.schema, Schema) and
            self.schema.props.get('inputBinding')
        )
        self.has_adapter = self.adapter is not None
        self.adapter = self.adapter or {}
        self.key = key
        if isinstance(self.schema, UnionSchema):
            for opt in self.schema.schemas:
                if validate(opt, value):
                    self.schema = opt
                    break
        self.value = evaluator.resolve(value)

    __str__ = lambda self: six.text_type(self.value)
    __repr__ = lambda self: 'InputAdapter(%s)' % self
    position = property(lambda self: (self.adapter.get('position', 9999999), self.key))
    prefix = property(lambda self: self.adapter.get('prefix'))
    item_separator = property(lambda self: self.adapter.get('itemSeparator', ','))
    separate = property(lambda self: self.adapter.get('separate', True))

    def arg_list(self):
        if isinstance(self.value, dict):
            return self.as_dict()
        if isinstance(self.value, list):
            return self.as_list()
        return self.as_primitive()

    def as_primitive(self):
        if self.value is None or self.value is False:
            return []
        if self.value is True:
            if not self.prefix:
                raise ValueError('Boolean arguments must have a prefix.')
            return [self.prefix]
        if not self.prefix:
            return [six.text_type(self.value)]
        if self.separate:
            return [self.prefix, self.value]
        return [self.prefix + six.text_type(self.value)]

    def as_dict(self):
        sch = lambda key: next(field for field in self.schema.fields
                               if field.name == key)
        adapters = [InputAdapter(v, self.evaluator, sch(k), key=k)
                    for k, v in six.iteritems(self.value)]
        adapters = [adp for adp in adapters if adp.has_adapter]
        res = reduce(
            operator.add,
            [a.arg_list() for a in sorted(adapters, key=lambda x: x.position)],
            []
        )
        return res

    def as_toplevel(self, mix_with):
        sch = lambda key: next(inp for inp in self.schema
                               if inp.id == key.split('.')[-1])
        adapters = mix_with + [
            InputAdapter(v, self.evaluator, sch(k).validator,
                         sch(k).input_binding, key=k)
            for k, v in six.iteritems(self.value)
            ]

        res = reduce(
            operator.add,
            [a.arg_list() for a in sorted(adapters, key=lambda x: x.position)],
            []
        )
        return res

    def as_list(self):
        # on top-level, I have type and depth, for easier parallelism,
        # but this is hacky as hell
        schema = (
            self.schema.items if isinstance(self.schema, ArraySchema)
            else self.schema
        )
        items = [InputAdapter(item, self.evaluator, schema)
                 for item in self.value]

        if not self.prefix:
            return reduce(operator.add, [a.arg_list() for a in items], [])

        if self.separate and self.item_separator is None:
            return reduce(operator.add, [[self.prefix] + a.arg_list()
                                         for a in items], [])

        if not self.separate and self.item_separator is None:
            return [self.prefix + a.list_item()
                    for a in items if a.list_item() is not None]

        joined = self.item_separator.join(
            filter(None, [a.list_item() for a in items])
        )

        if self.separate and self.item_separator is not None:
            return [self.prefix, joined]
        return [self.prefix + joined]

    def list_item(self):
        as_arg_list = self.arg_list()
        if not as_arg_list:
            return None
        if len(as_arg_list) > 1 or isinstance(as_arg_list[0], (dict, list)):
            raise ValueError('Only lists of primitive values can use itemSeparator.')
        return six.text_type(as_arg_list[0])

class CLIJob(object):
    def __init__(self, job):
        self.job = job
        self.app = job.app
        self._stdin = self.app.stdin
        self._stdout = self.app.stdout
        self.base_cmd = self.app.base_command
        if isinstance(self.base_cmd, six.string_types):
            self.base_cmd = self.base_cmd.split(' ')
        self.args = self.app.arguments
        self.eval = AdapterEvaluator(job)

    @property
    def stdin(self):
        if self._stdin:
            return self.eval.resolve(self._stdin)

    @property
    def stdout(self):
        if self._stdout:
            return self.eval.resolve(self._stdout)

    def make_arg_list(self):
        adapters = [InputAdapter(a.get('valueFrom'), self.eval, {}, a)
                    for a in self.args]
        ia = InputAdapter(self.job.inputs, self.eval, self.app.inputs)
        args = ia.as_toplevel(adapters)
        base_cmd = [self.eval.resolve(item) for item in self.base_cmd]

        return [six.text_type(arg) for arg in base_cmd + args]

    def cmd_line(self):
        a = self.make_arg_list()

        if self._stdin:
            a += ['<', self.stdin]
        if self._stdout:
            a += ['>', self.eval.resolve(self.stdout)]
        return ' '.join(a)  # TODO: escape

    def get_outputs(self, job_dir, job):
        result, outs = {}, self.app.outputs
        for out in outs:
            out_binding = out.output_binding
            ret = os.getcwd()
            os.chdir(job_dir)
            pattern = self.eval.resolve(out_binding.get('glob'), job=job) or ""
            files = glob.glob(pattern)
            result[out.id] = [{'path': os.path.abspath(p),
                          'metadata': meta(p, job.inputs, self.eval, out_binding),
                          'secondaryFiles': secondary_files(p, out_binding, self.eval)} for p in files]
            os.chdir(ret)
            if out.depth == 0:
                result[out.id] = result[out.id][0] if result[out.id] else None
        return result
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright (c) 2013 Corey Goldberg
#
#   This file is part of: sauceclient
#   https://github.com/cgoldberg/sauceclient
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   ----------------
#
#   Sauce Labs REST API documentation:
#     http://saucelabs.com/docs/rest

import base64
import sys
import json

__version__ = '0.2.1'

is_py2 = sys.version_info.major is 2

if is_py2:
    import httplib as http_client
else:
    import http.client as http_client

def json_loads(json_data):
    if not is_py2:
        json_data = json_data.decode(encoding='UTF-8')
    return json.loads(json_data)

class SauceException(Exception):
    def __init__(self, *args, **kwargs):
        super(SauceException, self).__init__(*args, **kwargs)
        self.response = kwargs.get("response")

class SauceClient(object):
    def __init__(self, sauce_username=None, sauce_access_key=None):
        self.sauce_username = sauce_username
        self.sauce_access_key = sauce_access_key
        self.headers = self.make_headers()
        self.information = Information(self)
        self.jobs = Jobs(self)
        self.provisioning = Provisioning(self)
        self.usage = Usage(self)

    def make_headers(self):
        base64string = self.get_encoded_auth_string()
        headers = {
            'Authorization': 'Basic %s' % base64string,
            'Content-Type': 'application/json',
        }
        return headers

    def request(self, method, url, body=None):
        connection = http_client.HTTPSConnection('saucelabs.com')
        connection.request(method, url, body, headers=self.headers)
        response = connection.getresponse()
        json_data = response.read()
        connection.close()
        if response.status != 200:
            raise SauceException('%s: %s.\nSauce Status NOT OK' %
                                 (response.status, response.reason), response=response)
        return json_data

    def get_encoded_auth_string(self):
        auth_info = '%s:%s' % (self.sauce_username, self.sauce_access_key)
        if is_py2:
            base64string = base64.encodestring(auth_info)[:-1]
        else:
            base64string = base64.b64encode(auth_info.encode(encoding='UTF-8')).decode(encoding='UTF-8')
        return base64string

class Jobs(object):
    def __init__(self, client):
        self.client = client

    def get_job_ids(self):
        """List all jobs id's belonging to the user."""
        method = 'GET'
        url = '/rest/v1/%s/jobs' % self.client.sauce_username
        json_data = self.client.request(method, url)
        jobs = json_loads(json_data)
        job_ids = [attr['id'] for attr in jobs]
        return job_ids

    def get_jobs(self):
        """List all jobs belonging to the user."""
        method = 'GET'
        url = '/rest/v1/%s/jobs?full=true' % self.client.sauce_username
        json_data = self.client.request(method, url)
        jobs = json_loads(json_data)
        return jobs

    def get_job_attributes(self, job_id):
        """Get information for the specified job."""
        method = 'GET'
        url = '/rest/v1/%s/jobs/%s' % (self.client.sauce_username, job_id)
        json_data = self.client.request(method, url)
        attributes = json_loads(json_data)
        return attributes

    def update_job(self, job_id, build_num=None, custom_data=None,
                   name=None, passed=None, public=None, tags=None):
        """Update attributes for the specified job."""
        content = {}
        if build_num is not None:
            content['build'] = build_num
        if custom_data is not None:
            content['custom-data'] = custom_data
        if name is not None:
            content['name'] = name
        if passed is not None:
            content['passed'] = passed
        if public is not None:
            content['public'] = public
        if tags is not None:
            content['tags'] = tags
        body = json.dumps(content)
        method = 'PUT'
        url = '/rest/v1/%s/jobs/%s' % (self.client.sauce_username, job_id)
        json_data = self.client.request(method, url, body=body)
        attributes = json_loads(json_data)
        return attributes

    def get_job_assets(self, job_id):
        """Get the list of assets for the specified job."""
        method = 'GET'
        url = '/rest/v1/%s/jobs/%s/assets' % (self.client.sauce_username,
                                              job_id)
        json_data = self.client.request(method, url)
        assets = json.loads(json_data)
        return assets

class Provisioning(object):
    def __init__(self, client):
        self.client = client

    def get_account_details(self):
        """Access account details."""
        method = 'GET'
        url = '/rest/v1/users/%s' % self.client.sauce_username
        json_data = self.client.request(method, url)
        attributes = json_loads(json_data)
        return attributes

    def get_account_limits(self):
        """Access account limits."""
        method = 'GET'
        url = '/rest/v1/%s/limits' % self.client.sauce_username
        json_data = self.client.request(method, url)
        attributes = json_loads(json_data)
        return attributes

class Information(object):
    def __init__(self, client):
        self.client = client

    def get_status(self):
        """Access the current status of Sauce Labs' services."""
        method = 'GET'
        url = '/rest/v1/info/status'
        json_data = self.client.request(method, url)
        status = json_loads(json_data)
        return status

    def get_browsers(self):
        """Get details of all browsers currently supported on Sauce Labs."""
        method = 'GET'
        url = '/rest/v1/info/browsers'
        json_data = self.client.request(method, url)
        browsers = json_loads(json_data)
        return browsers

    def get_count(self):
        """Get number of test executed so far on Sauce Labs."""
        method = 'GET'
        url = '/rest/v1/info/counter'
        json_data = self.client.request(method, url)
        count = json_loads(json_data)
        return count

class Usage(object):
    def __init__(self, client):
        self.client = client

    def get_current_activity(self):
        """Access current account activity.

        Returns active job counts broken down by job status and subaccount.
        """
        method = 'GET'
        url = '/rest/v1/%s/activity' % self.client.sauce_username
        json_data = self.client.request(method, url)
        activity = json_loads(json_data)
        return activity

    def get_historical_usage(self):
        """Access historical account usage."""
        method = 'GET'
        url = '/rest/v1/users/%s/usage' % self.client.sauce_username
        json_data = self.client.request(method, url)
        historical_usage = json_loads(json_data)
        return historical_usage
<commit_msg>fixed exception<commit_after>#!/usr/bin/env python
#
# Copyright (c) 2013 Corey Goldberg
#
#   This file is part of: sauceclient
#   https://github.com/cgoldberg/sauceclient
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   ----------------
#
#   Sauce Labs REST API documentation:
#     http://saucelabs.com/docs/rest

import base64
import sys
import json

__version__ = '0.2.1'

is_py2 = sys.version_info.major is 2

if is_py2:
    import httplib as http_client
else:
    import http.client as http_client

def json_loads(json_data):
    if not is_py2:
        json_data = json_data.decode(encoding='UTF-8')
    return json.loads(json_data)

class SauceException(Exception):
    def __init__(self, *args, **kwargs):
        super(SauceException, self).__init__(*args)
        self.response = kwargs.get('response')

class SauceClient(object):
    def __init__(self, sauce_username=None, sauce_access_key=None):
        self.sauce_username = sauce_username
        self.sauce_access_key = sauce_access_key
        self.headers = self.make_headers()
        self.information = Information(self)
        self.jobs = Jobs(self)
        self.provisioning = Provisioning(self)
        self.usage = Usage(self)

    def make_headers(self):
        base64string = self.get_encoded_auth_string()
        headers = {
            'Authorization': 'Basic %s' % base64string,
            'Content-Type': 'application/json',
        }
        return headers

    def request(self, method, url, body=None):
        connection = http_client.HTTPSConnection('saucelabs.com')
        connection.request(method, url, body, headers=self.headers)
        response = connection.getresponse()
        json_data = response.read()
        connection.close()
        if response.status != 200:
            raise SauceException('%s: %s.\nSauce Status NOT OK' %
                                 (response.status, response.reason), response=response)
        return json_data

    def get_encoded_auth_string(self):
        auth_info = '%s:%s' % (self.sauce_username, self.sauce_access_key)
        if is_py2:
            base64string = base64.encodestring(auth_info)[:-1]
        else:
            base64string = base64.b64encode(auth_info.encode(encoding='UTF-8')).decode(encoding='UTF-8')
        return base64string

class Jobs(object):
    def __init__(self, client):
        self.client = client

    def get_job_ids(self):
        """List all jobs id's belonging to the user."""
        method = 'GET'
        url = '/rest/v1/%s/jobs' % self.client.sauce_username
        json_data = self.client.request(method, url)
        jobs = json_loads(json_data)
        job_ids = [attr['id'] for attr in jobs]
        return job_ids

    def get_jobs(self):
        """List all jobs belonging to the user."""
        method = 'GET'
        url = '/rest/v1/%s/jobs?full=true' % self.client.sauce_username
        json_data = self.client.request(method, url)
        jobs = json_loads(json_data)
        return jobs

    def get_job_attributes(self, job_id):
        """Get information for the specified job."""
        method = 'GET'
        url = '/rest/v1/%s/jobs/%s' % (self.client.sauce_username, job_id)
        json_data = self.client.request(method, url)
        attributes = json_loads(json_data)
        return attributes

    def update_job(self, job_id, build_num=None, custom_data=None,
                   name=None, passed=None, public=None, tags=None):
        """Update attributes for the specified job."""
        content = {}
        if build_num is not None:
            content['build'] = build_num
        if custom_data is not None:
            content['custom-data'] = custom_data
        if name is not None:
            content['name'] = name
        if passed is not None:
            content['passed'] = passed
        if public is not None:
            content['public'] = public
        if tags is not None:
            content['tags'] = tags
        body = json.dumps(content)
        method = 'PUT'
        url = '/rest/v1/%s/jobs/%s' % (self.client.sauce_username, job_id)
        json_data = self.client.request(method, url, body=body)
        attributes = json_loads(json_data)
        return attributes

    def get_job_assets(self, job_id):
        """Get the list of assets for the specified job."""
        method = 'GET'
        url = '/rest/v1/%s/jobs/%s/assets' % (self.client.sauce_username,
                                              job_id)
        json_data = self.client.request(method, url)
        assets = json.loads(json_data)
        return assets

class Provisioning(object):
    def __init__(self, client):
        self.client = client

    def get_account_details(self):
        """Access account details."""
        method = 'GET'
        url = '/rest/v1/users/%s' % self.client.sauce_username
        json_data = self.client.request(method, url)
        attributes = json_loads(json_data)
        return attributes

    def get_account_limits(self):
        """Access account limits."""
        method = 'GET'
        url = '/rest/v1/%s/limits' % self.client.sauce_username
        json_data = self.client.request(method, url)
        attributes = json_loads(json_data)
        return attributes

class Information(object):
    def __init__(self, client):
        self.client = client

    def get_status(self):
        """Access the current status of Sauce Labs' services."""
        method = 'GET'
        url = '/rest/v1/info/status'
        json_data = self.client.request(method, url)
        status = json_loads(json_data)
        return status

    def get_browsers(self):
        """Get details of all browsers currently supported on Sauce Labs."""
        method = 'GET'
        url = '/rest/v1/info/browsers'
        json_data = self.client.request(method, url)
        browsers = json_loads(json_data)
        return browsers

    def get_count(self):
        """Get number of test executed so far on Sauce Labs."""
        method = 'GET'
        url = '/rest/v1/info/counter'
        json_data = self.client.request(method, url)
        count = json_loads(json_data)
        return count

class Usage(object):
    def __init__(self, client):
        self.client = client

    def get_current_activity(self):
        """Access current account activity.

        Returns active job counts broken down by job status and subaccount.
        """
        method = 'GET'
        url = '/rest/v1/%s/activity' % self.client.sauce_username
        json_data = self.client.request(method, url)
        activity = json_loads(json_data)
        return activity

    def get_historical_usage(self):
        """Access historical account usage."""
        method = 'GET'
        url = '/rest/v1/users/%s/usage' % self.client.sauce_username
        json_data = self.client.request(method, url)
        historical_usage = json_loads(json_data)
        return historical_usage
<|endoftext|>


<commit_before>from scapy.all import *

packetCount = 0
message = False
messageend = False
characterlist = []

def customAction(packet):
	global packetCount
	global message
	global messageend
	global characterlist
	#print "Packet #" + str(packetCount) + ": " +packet[0][1].src + "==>" + packet[0][1].dst + ": " + str(packet[0].SUFFIX)
	suffixnum = packet[0].SUFFIX
	if(suffixnum == 23130):
		message = True
		#print message
		messageend = False
		#print messageend
		packetCount += 1
	if(suffixnum == 23130 and packetCount > 1):
		messageend = True
		#print messageend
	if(messageend == True and suffixnum == 23130):
		message = False
		packetCount = 0
		print characterlist
		#print message
	asciivalue = hex(suffixnum)[4:]
	asciivalue = int(asciivalue, 16)
	character = chr(asciivalue)
	if(message == True and packetCount != 1):
		characterlist.append(character)
		packetCount += 1

sniff(filter="port 137", prn=customAction)
<commit_msg>update<commit_after>from scapy.all import *

packetCount = 0
message = False
messageend = False
characterlist = []

def customAction(packet):
	global packetCount
	global message
	global messageend
	global characterlist
	#print "Packet #" + str(packetCount) + ": " +packet[0][1].src + "==>" + packet[0][1].dst + ": " + str(packet[0].SUFFIX)
	suffixnum = packet[0].SUFFIX
	if(suffixnum == 23130):
		message = True
		#print message
		messageend = False
		#print messageend
		packetCount += 1
	if(suffixnum == 23130 and packetCount > 1):
		messageend = True
		#print messageend
	if(messageend == True and suffixnum == 23130):
		message = False
		packetCount = 0
		characterlist = characterlist[1:]
		print characterlist
		#print message
	asciivalue = hex(suffixnum)[4:]
	asciivalue = int(asciivalue, 16)
	character = chr(asciivalue)
	if(message == True):
		characterlist.append(character)
		packetCount += 1

sniff(filter="port 137", prn=customAction)
<|endoftext|>