<commit_before>"""

Handlers for the Roles service.

This is a set of mongodb backed handlers for a REST based authorization
service. The authorization service simply serves up JSON docs that specify
a set of permissions, and the users who have that set of permissions. Here
is a sample JSON object:

{
    "role_owner": "sychan",
    "role_id": "kbase_users",
    "description": "List of user ids who are considered KBase users",
    "members": [
        "sychan",
        "kbasetest",
        "kbauthorz"
    ],
    "role_updater": [
        "sychan",
        "kbauthorz"
    ],
    "read": [],
    "create": [],
    "modify": [],
    "impersonate": [],
    "delete": []
}

   Here are the semantics of the fields:

   'role_id' : 'Unique human readable identifer for role (required)',
   'description' : 'Description of the role (required)',
   'role_owner' : 'Owner(creator) of this role',
   'role_updater' : 'User_ids that can update this role',
   'members' : 'A list of the user_ids who are members of this group',
   'read' : 'List of kbase object ids (strings) that this role allows read privs',
   'modify' : 'List of kbase object ids (strings) that this role allows modify privs',
   'delete' : 'List of kbase object ids (strings) that this role allows delete privs',
   'impersonate' : 'List of kbase user_ids (strings) that this role allows impersonate privs',
   'grant' : 'List of kbase authz role_ids (strings) that this role allows grant privs',
   'create' : 'Boolean value - does this role provide the create privilege'

   The service is typically mounted under /Roles. Here are the authentication requirements
for each HTTP method:

   GET : requires valid token and membership in role_id in settings.kbase_users
   PUT : requires valid token and user in role_owner, or in the role_updater field for object
   POST : requires valid token and membership in role_id in settings.kbase_users
   DELETE : requires valid token and request must come from role_owner of target object

   The GET method supports the MongoDB options for filter and fields. For queries and filter
parameters the filter parameter is passed as the first parameter and the fields parameter is
the second parameter passed into the pymongo collection.find() method, see:

http://api.mongodb.org/python/current/api/pymongo/collection.html

   As an example, the following query returns all role objects that have sychan as a member:
http://authorization_host/Roles?filter={ "members" : "sychan"}

   To pull up only the role_id fields:

http://authorization_host/Roles?filter={ "members" : "sychan"}&fields={"role_id" : "1"}

   To pull up the role_id fields for roles with "test" as part of their name (PCRE regex):

http://authorization_host/Roles?filter={ "role_id" : { "$regex" : ".*test.*" }}&fields={ "role_id" : "1" }

"""

from piston.handler import BaseHandler
from piston.utils import rc
import pprint
import datetime
import json
from pymongo import Connection
from piston.resource import Resource
from django.conf import settings

pp = pprint.PrettyPrinter(indent=4)

# Convert QuerySet into a dictionary keyed on the field named in 2nd parameter
def dictify(objs,key):
    results = {}
    for x in range(len(objs)):
        results[objs[x][key]] = objs[x]

    return results

class RoleHandler( BaseHandler):
    allowed_methods = ('GET','POST','PUT','DELETE')
    fields = ('role_id','description','members','read','modify','delete',
              'impersonate','grant','create','role_owner','role_updater')
    exclude = ( '_id', )
    # We need to define the appropriate settings and set them here
    try:
        conn = Connection(settings.MONGODB_CONN)
    except AttributeError as e:
        print "No connection settings specified: %s\n" % e
        conn = Connection(['mongodb.kbase.us'])
    except Exception as e:
        print "Generic exception %s: %s\n" % (type(e),e)
        conn = Connection()
    db = conn.authorization
    roles = db.roles
    # Set the role_id to require for updates to the roles db
    try:
        kbase_users = settings.kbase_users
    except AttributeError as e:
        kbase_users = 'kbase_users'
    # Help object when queries go to the top level with no search specification
    help_json = { 'id' : 'KBase Authorization',
                  'documentation' : 'https://docs.google.com/document/d/1CTkthDUPwNzMF22maLyNIktI1sHdWPwtd3lJk0aFb20/edit',
                  'resources' : { 'role_id' : 'Unique human readable identifer for role (required)',
                                  'description' : 'Description of the role (required)',
                                  'role_owner' : 'Owner(creator) of this role',
                                  'role_updater' : 'User_ids that can update this role',
                                  'members' : 'A list of the user_ids who are members of this group',
                                  'read' : 'List of kbase object ids (strings) that this role allows read privs',
                                  'modify' : 'List of kbase object ids (strings) that this role allows modify privs',
                                  'delete' : 'List of kbase object ids (strings) that this role allows delete privs',
                                  'impersonate' : 'List of kbase user_ids (strings) that this role allows impersonate privs',
                                  'grant' : 'List of kbase authz role_ids (strings) that this role allows grant privs',
                                  'create' : 'Boolean value - does this role provide the create privilege'
                                  },
                  'contact' : { 'email' : 'sychan@lbl.gov'},
                  'usage'   : 'This is a standard REST service. Note that read handler takes\n' +
                  'MongoDB filtering and JSON field selection options passed as\n' +
                  'URL parameters \'filter\' and \'fields\' respectively.\n' +
                  'Please look at MongoDB pymongo collection documentation for details.\n' +
                  'Read and Create are currently open to all authenticated users in role "%s", but\n' % kbase_users +
                  'delete requires ownership of the document (in field role_owner),\n' +
                  'update requires ownership or membership in the target document\'s role_updaters list\n'
                  }

    # Check mongodb to see if the user is in kbase_user role, necessary
    # before they can perform any kinds of updates
    # Note that possessing a Globus Online ID is not sufficient
    def check_kbase_user(self, user_id):
        try:
            res = self.roles.find_one( { 'role_id' : self.kbase_users,
                                          'members' : user_id })
            print "Results are %s" % pp.pformat( res)
            return res is not None
        except:
            return False

    # Get all role_ids associated with a user_id, returns a
    # an array of role_ids (strings)
    def get_user_roles(self, user_id):
        try:
            roles= self.roles.find( { 'members' : user_id },
                                    { 'role_id' : '1' })
            return [ roles[x]['role_id'] for x in range( roles.count())]
        except Exception as e:
            print "Error while fetching roles for user %s: %s" % ( user_id, e)
            return []

    def read(self, request, role_id=None):
        try:
            if not request.user.username or not self.check_kbase_user( request.user.username):
                res = rc.FORBIDDEN
                res.write(' request not from a member of %s' % self.kbase_users)
            else:
                if role_id == None and 'role_id' in request.GET:
                    role_id = request.GET.get('role_id')
                filter = request.GET.get('filter', None)
                fields = request.GET.get('fields', None)
                if role_id == None and filter == None:
                    res = self.help_json
                elif role_id != None:
                    res = self.roles.find_one( { 'role_id': role_id })
                    if res != None:
                        for excl in self.exclude:
                            if excl in res:
                                del res[excl]
                else:
#                       print "Filter = %s\n" % pp.pformat(filter)
                    filter = json.loads(filter)
                    if fields:
                        fields = json.loads(fields)
                        match = self.roles.find(filter, fields)
                    else:
                        match = self.roles.find( filter )
                    res = [ match[x] for x in range( match.count())]
                    for x in res:
                        for excl in self.exclude:
                            if excl in x:
                                del x[excl]
        except Exception as e:
            res = rc.BAD_REQUEST
            res.write(' error: %s' % e )
        return(res)

    def create(self, request):
        r = request.data
#        print pp.pformat( r)
        try:
            if not request.user.username:
                res = rc.FORBIDDEN
                res.write(' request does not have username ')
            elif not self.check_kbase_user( request.user.username):
                res = rc.FORBIDDEN
                res.write(' request not from a member of %s' % self.kbase_users)
            elif self.roles.find( { 'role_id': r['role_id'] }).count() == 0:
                new = { x : r.get(x, []) for x in ('read','modify','delete','impersonate','grant','create') }
                new['role_id'] = r['role_id']
                new['description'] = r['description']
                new['role_owner'] = request.user.username
                new['role_updater'] = [request.user.username]
                self.roles.insert( new)
                res = rc.CREATED
            else:
                res = rc.DUPLICATE_ENTRY
        except KeyError as e:
            res = rc.BAD_REQUEST
            res.write(' required fields: %s' % e )
        except Exception as e:
            res = rc.BAD_REQUEST
            res.write(' error: %s' % e )
        return(res)
    def update(self, request, role_id=None):
        r = request.data
#        print pp.pformat( r)
        try:
            if not request.user.username:
                res = rc.FORBIDDEN
                res.write(' request does not have username ')
            elif not self.check_kbase_user( request.user.username):
                res = rc.FORBIDDEN
                res.write(' request not from a member of %s' % self.kbase_users)
            elif role_id == None:
                role_id = request.data['role_id']
            old = self.roles.find_one( { 'role_id': role_id })
            if old != None:
                if request.user.username == old['role_owner'] or request.user.username in old['role_updater'] :
                    old.update(r)
                    self.roles.save( old)
                    res = rc.CREATED
                else:
                    res = rc.FORBIDDEN
                    res.write( " %s is owned by %s and updated by %s, but request is from user %s" %
                               (old['role_id'],old['role_owner'], pp.pformat(old['role_updater']), request.user.username))
            else:
                res = rc.NOT_HERE
        except KeyError as e:
            res = rc.BAD_REQUEST
            res.write(' required fields: %s' % e )
        except Exception as e:
            res = rc.BAD_REQUEST
            res.write(' error: %s' % e )
        return(res)
    def delete(self, request, role_id = None):
        try:
            if not request.user.username:
                res = rc.FORBIDDEN
                res.write(' request does not have username ')
            elif not self.check_kbase_user( request.user.username):
                res = rc.FORBIDDEN
                res.write(' request not from a member of %s' % self.kbase_users)
            elif role_id is None:
                raise KeyError('No role_id specified')
            old = self.roles.find_one( { 'role_id': role_id })
            if old != None:
                if request.user.username == old['role_owner']:
                    self.roles.remove( { '_id' : r['_id'] }, safe=True)
                    res = rc.DELETED
                else:
                    res = rc.FORBIDDEN
                    res.write( " %s is owned by %s, but request is from user %s" %
                               (old['role_id'],old['role_owner'], request.user.username))
            else:
                res = rc.NOT_HERE
        except KeyError as e:
            res = rc.BAD_REQUEST
            res.write(' role_id must be specified')
        except Exception as e:
            res = rc.BAD_REQUEST
            res.write(' error: %s' % e)
        return(res)

# Handlers for piston API
# sychan 9/6/2012

role_handler = Resource( RoleHandler)
<commit_msg>   Fixed delete issues<commit_after>"""

Handlers for the Roles service.

This is a set of mongodb backed handlers for a REST based authorization
service. The authorization service simply serves up JSON docs that specify
a set of permissions, and the users who have that set of permissions. Here
is a sample JSON object:

{
    "role_owner": "sychan",
    "role_id": "kbase_users",
    "description": "List of user ids who are considered KBase users",
    "members": [
        "sychan",
        "kbasetest",
        "kbauthorz"
    ],
    "role_updater": [
        "sychan",
        "kbauthorz"
    ],
    "read": [],
    "create": [],
    "modify": [],
    "impersonate": [],
    "delete": []
}

   Here are the semantics of the fields:

   'role_id' : 'Unique human readable identifer for role (required)',
   'description' : 'Description of the role (required)',
   'role_owner' : 'Owner(creator) of this role',
   'role_updater' : 'User_ids that can update this role',
   'members' : 'A list of the user_ids who are members of this group',
   'read' : 'List of kbase object ids (strings) that this role allows read privs',
   'modify' : 'List of kbase object ids (strings) that this role allows modify privs',
   'delete' : 'List of kbase object ids (strings) that this role allows delete privs',
   'impersonate' : 'List of kbase user_ids (strings) that this role allows impersonate privs',
   'grant' : 'List of kbase authz role_ids (strings) that this role allows grant privs',
   'create' : 'Boolean value - does this role provide the create privilege'

   The service is typically mounted under /Roles. Here are the authentication requirements
for each HTTP method:

   GET : requires valid token and membership in role_id in settings.kbase_users
   PUT : requires valid token and user in role_owner, or in the role_updater field for object
   POST : requires valid token and membership in role_id in settings.kbase_users
   DELETE : requires valid token and request must come from role_owner of target object

   The GET method supports the MongoDB options for filter and fields. For queries and filter
parameters the filter parameter is passed as the first parameter and the fields parameter is
the second parameter passed into the pymongo collection.find() method, see:

http://api.mongodb.org/python/current/api/pymongo/collection.html

   As an example, the following query returns all role objects that have sychan as a member:
http://authorization_host/Roles?filter={ "members" : "sychan"}

   To pull up only the role_id fields:

http://authorization_host/Roles?filter={ "members" : "sychan"}&fields={"role_id" : "1"}

   To pull up the role_id fields for roles with "test" as part of their name (PCRE regex):

http://authorization_host/Roles?filter={ "role_id" : { "$regex" : ".*test.*" }}&fields={ "role_id" : "1" }

"""

from piston.handler import BaseHandler
from piston.utils import rc
import pprint
import datetime
import json
from pymongo import Connection
from piston.resource import Resource
from django.conf import settings

pp = pprint.PrettyPrinter(indent=4)

# Convert QuerySet into a dictionary keyed on the field named in 2nd parameter
def dictify(objs,key):
    results = {}
    for x in range(len(objs)):
        results[objs[x][key]] = objs[x]

    return results

class RoleHandler( BaseHandler):
    allowed_methods = ('GET','POST','PUT','DELETE')
    fields = ('role_id','description','members','read','modify','delete',
              'impersonate','grant','create','role_owner','role_updater')
    exclude = ( '_id', )
    # We need to define the appropriate settings and set them here
    try:
        conn = Connection(settings.MONGODB_CONN)
    except AttributeError as e:
        print "No connection settings specified: %s\n" % e
        conn = Connection()
    except Exception as e:
        print "Generic exception %s: %s\n" % (type(e),e)
        conn = Connection()
    db = conn.authorization
    roles = db.roles
    # Set the role_id to require for updates to the roles db
    try:
        kbase_users = settings.kbase_users
    except AttributeError as e:
        kbase_users = 'kbase_users'
    # Help object when queries go to the top level with no search specification
    help_json = { 'id' : 'KBase Authorization',
                  'documentation' : 'https://docs.google.com/document/d/1CTkthDUPwNzMF22maLyNIktI1sHdWPwtd3lJk0aFb20/edit',
                  'resources' : { 'role_id' : 'Unique human readable identifer for role (required)',
                                  'description' : 'Description of the role (required)',
                                  'role_owner' : 'Owner(creator) of this role',
                                  'role_updater' : 'User_ids that can update this role',
                                  'members' : 'A list of the user_ids who are members of this group',
                                  'read' : 'List of kbase object ids (strings) that this role allows read privs',
                                  'modify' : 'List of kbase object ids (strings) that this role allows modify privs',
                                  'delete' : 'List of kbase object ids (strings) that this role allows delete privs',
                                  'impersonate' : 'List of kbase user_ids (strings) that this role allows impersonate privs',
                                  'grant' : 'List of kbase authz role_ids (strings) that this role allows grant privs',
                                  'create' : 'Boolean value - does this role provide the create privilege'
                                  },
                  'contact' : { 'email' : 'sychan@lbl.gov'},
                  'usage'   : 'This is a standard REST service. Note that read handler takes\n' +
                  'MongoDB filtering and JSON field selection options passed as\n' +
                  'URL parameters \'filter\' and \'fields\' respectively.\n' +
                  'Please look at MongoDB pymongo collection documentation for details.\n' +
                  'Read and Create are currently open to all authenticated users in role "%s", but\n' % kbase_users +
                  'delete requires ownership of the document (in field role_owner),\n' +
                  'update requires ownership or membership in the target document\'s role_updaters list\n'
                  }

    # Check mongodb to see if the user is in kbase_user role, necessary
    # before they can perform any kinds of updates
    # Note that possessing a Globus Online ID is not sufficient
    def check_kbase_user(self, user_id):
        try:
            res = self.roles.find_one( { 'role_id' : self.kbase_users,
                                          'members' : user_id })
            return res is not None
        except:
            return False

    # Get all role_ids associated with a user_id, returns a
    # an array of role_ids (strings)
    def get_user_roles(self, user_id):
        try:
            roles= self.roles.find( { 'members' : user_id },
                                    { 'role_id' : '1' })
            return [ roles[x]['role_id'] for x in range( roles.count())]
        except Exception as e:
            print "Error while fetching roles for user %s: %s" % ( user_id, e)
            return []

    def read(self, request, role_id=None):
        try:
            if not request.user.username or not self.check_kbase_user( request.user.username):
                res = rc.FORBIDDEN
                res.write(' request not from a member of %s' % self.kbase_users)
            else:
                if role_id == None and 'role_id' in request.GET:
                    role_id = request.GET.get('role_id')
                filter = request.GET.get('filter', None)
                fields = request.GET.get('fields', None)
                if role_id == None and filter == None:
                    res = self.help_json
                elif role_id != None:
                    res = self.roles.find_one( { 'role_id': role_id })
                    if res != None:
                        for excl in self.exclude:
                            if excl in res:
                                del res[excl]
                else:
#                       print "Filter = %s\n" % pp.pformat(filter)
                    filter = json.loads(filter)
                    if fields:
                        fields = json.loads(fields)
                        match = self.roles.find(filter, fields)
                    else:
                        match = self.roles.find( filter )
                    res = [ match[x] for x in range( match.count())]
                    for x in res:
                        for excl in self.exclude:
                            if excl in x:
                                del x[excl]
        except Exception as e:
            res = rc.BAD_REQUEST
            res.write(' error: %s' % e )
        return(res)

    def create(self, request):
        r = request.data
        try:
            if not request.user.username:
                res = rc.FORBIDDEN
                res.write(' request does not have username ')
            elif not self.check_kbase_user( request.user.username):
                res = rc.FORBIDDEN
                res.write(' request not from a member of %s' % self.kbase_users)
            elif self.roles.find( { 'role_id': r['role_id'] }).count() == 0:
                new = { x : r.getlist(x) for x in ('read','modify','delete',
                                                   'impersonate','grant','create','members','role_updater') }
                new['role_id'] = r['role_id']
                new['description'] = r['description']
                new['role_owner'] = request.user.username
                self.roles.insert( new)
                res = rc.CREATED
            else:
                res = rc.DUPLICATE_ENTRY
        except KeyError as e:
            res = rc.BAD_REQUEST
            res.write(' required fields: %s' % e )
        except Exception as e:
            res = rc.BAD_REQUEST
            res.write(' error: %s' % e )
        return(res)
    def update(self, request, role_id=None):
        r = request.data
#        print pp.pformat( r)
        try:
            if not request.user.username:
                res = rc.FORBIDDEN
                res.write(' request does not have username ')
            elif not self.check_kbase_user( request.user.username):
                res = rc.FORBIDDEN
                res.write(' request not from a member of %s' % self.kbase_users)
            elif role_id == None:
                role_id = request.data['role_id']
            old = self.roles.find_one( { 'role_id': role_id })
            if old != None:
                if request.user.username == old['role_owner'] or request.user.username in old['role_updater'] :
                    old.update(r)
                    self.roles.save( old)
                    res = rc.CREATED
                else:
                    res = rc.FORBIDDEN
                    res.write( " %s is owned by %s and updated by %s, but request is from user %s" %
                               (old['role_id'],old['role_owner'], pp.pformat(old['role_updater']), request.user.username))
            else:
                res = rc.NOT_HERE
        except KeyError as e:
            res = rc.BAD_REQUEST
            res.write(' required fields: %s' % e )
        except Exception as e:
            res = rc.BAD_REQUEST
            res.write(' error: %s' % e )
        return(res)
    def delete(self, request, role_id = None):
        try:
            if not request.user.username:
                res = rc.FORBIDDEN
                res.write(' request does not have username ')
            elif not self.check_kbase_user( request.user.username):
                res = rc.FORBIDDEN
                res.write(' request not from a member of %s' % self.kbase_users)
            elif role_id is None:
                raise KeyError('No role_id specified')
            old = self.roles.find_one( { 'role_id': role_id })
            if old != None:
                if request.user.username == old['role_owner']:
                    self.roles.remove( { '_id' : old['_id'] }, safe=True)
                    res = rc.DELETED
                else:
                    res = rc.FORBIDDEN
                    res.write( " %s is owned by %s, but request is from user %s" %
                               (old['role_id'],old['role_owner'], request.user.username))
            else:
                res = rc.NOT_HERE
        except KeyError as e:
            res = rc.BAD_REQUEST
            res.write(' role_id must be specified')
        except Exception as e:
            res = rc.BAD_REQUEST
            res.write(' error: %s' % e)
        return(res)

# Handlers for piston API
# sychan 9/6/2012

role_handler = Resource( RoleHandler)
<|endoftext|>


<commit_before>from flask import Flask
from flask_uploads import UploadSet, configure_uploads

import logging
import modes
import os
import re

def configure_logger(app, mode):
  logger = logging.getLogger(__name__)

  if mode is modes.PRODUCTION:
    logger.setLevel(logging.INFO)
  else:
    logger.setLevel(logging.DEBUG)

  formatter = logging.Formatter(
    fmt = '{asctime} {name:12} {levelname:8} {message}',
    datefmt = '%y-%m-%d %H:%M',
    style = '{')

  log_file_path = prepare_log_file(app, mode)
  file_handler = logging.FileHandler(log_file_path)
  file_handler.setFormatter(formatter)
  logger.addHandler(file_handler)

  if mode is not modes.TEST:
    console_handler = logging.StreamHandler()
    console_handler.setFormatter(formatter)
    logger.addHandler(console_handler)

  logger.info("Logging configured.")
  logger.info("Application mode is set to: '%s'", mode)

  return logger

def prepare_log_file(app, mode):
  instance_path = app.instance_path
  logs_dir = os.path.join(instance_path, mode, "logs")
  log_file_path = os.path.join(logs_dir, "server.log")
  os.makedirs(logs_dir, exist_ok = True)

  return log_file_path

def configure_log_messages(app):
  messages = {
    "SCHEMATIC_UPLOAD_SUCCESS":               "Schematic upload successful: '%s'",
    "SCHEMATIC_UPLOAD_FAILURE":               "Schematic upload failed: '%s', Exception: '%s'",
    "SCHEMATIC_UPLOAD_USERNAME_EMPTY":        "Schematic upload failed. Username is empty.",
    "SCHEMATIC_UPLOAD_USERNAME_WHITESPACE":   "Schematic upload failed. Username contains whitespace: '%s'",
    "SCHEMATIC_UPLOAD_NO_FILES":              "Schematic upload failed. No files specified.",
    "SCHEMATIC_UPLOAD_TOO_MANY_FILES":        "Schematic upload failed. Too many files.",
    "SCHEMATIC_UPLOAD_FILE_TOO_LARGE":        "Schematic upload failed. File too large: '%s'",
    "SCHEMATIC_UPLOAD_FILE_EXISTS":           "Schematic upload failed. File already exists: '%s'",
    "SCHEMATIC_UPLOAD_FILENAME_WHITESPACE":   "Schematic upload failed. Filename contains whitespace: '%s'",
    "SCHEMATIC_UPLOAD_FILENAME_EXTENSION":    "Schematic upload failed. Filename has invalid extension: '%s'",
    "SCHEMATIC_DOWNLOAD_SUCCESS":             "Schematic download initiated: '%s'",
    "SCHEMATIC_DOWNLOAD_FILENAME_EMPTY":      "Schematic download failed. Filename is empty.",
    "SCHEMATIC_DOWNLOAD_FILENAME_WHITESPACE": "Schematic download failed. Filename contains whitespace: '%s'",
    "SCHEMATIC_DOWNLOAD_FILENAME_NOT_UNIQUE": "Schematic download failed. Multiple files with this filename exist: '%s'",
    "SCHEMATIC_DOWNLOAD_FILE_NOT_FOUND":      "Schematic download failed. File does not exist: '%s'",
    "WORLD_DOWNLOAD_SUCCESS":                 "World download initiated: '%s'",
  }

  app.config['LOG_MESSAGES'] = messages
  logger.info("Log messages configured.")

def configure_flash_messages(app):
  messages = {
    "SCHEMATIC_UPLOAD_SUCCESS":               "Upload Successful!",
    "SCHEMATIC_UPLOAD_FAILURE":               "Upload Failed! Please contact the admins for assistance.",
    "SCHEMATIC_UPLOAD_USERNAME_EMPTY":        "Upload Failed! Username must not be empty.",
    "SCHEMATIC_UPLOAD_USERNAME_WHITESPACE":   "Upload Failed! Username must not contain spaces.",
    "SCHEMATIC_UPLOAD_NO_FILES":              "Upload Failed! No files selected.",
    "SCHEMATIC_UPLOAD_TOO_MANY_FILES":        "Upload Failed! A maximum of {} files can be uploaded at one time.".format( \
                                               app.config['MAX_NUMBER_OF_UPLOAD_FILES']),
    "SCHEMATIC_UPLOAD_FILE_TOO_LARGE":        "Upload Failed! File size is larger than allowed maximum of {} bytes.".format( \
                                               app.config['MAX_UPLOAD_FILE_SIZE']),
    "SCHEMATIC_UPLOAD_FILE_EXISTS":           "Upload Failed! File with same name already exists on the server.",
    "SCHEMATIC_UPLOAD_FILENAME_WHITESPACE":   "Upload Failed! File name must not contain spaces.",
    "SCHEMATIC_UPLOAD_FILENAME_EXTENSION":    "Upload Failed! File must end with the .schematic or .schem extension.",
    "SCHEMATIC_DOWNLOAD_FILENAME_EMPTY":      "Download Failed! Filename must not be empty.",
    "SCHEMATIC_DOWNLOAD_FILENAME_WHITESPACE": "Download Failed! Filename must not contain spaces.",
    "SCHEMATIC_DOWNLOAD_FILENAME_NOT_UNIQUE": "Download Failed! Multiple files with this filename exist. Please contact an admin for further assistance.",
    "SCHEMATIC_DOWNLOAD_FILE_NOT_FOUND":      "Download Failed! File does not exist."
  }

  app.config['FLASH_MESSAGES'] = messages
  logger.info("Flash messages configured.")

def load_environment_config(app, mode):
  instance_path = app.instance_path
  config_file_path = os.path.join(instance_path, mode, "config.py")

  if os.path.isfile(config_file_path):
    app.config.from_pyfile(config_file_path)
    logger.info("Environment config loaded from: '%s'", config_file_path)
  else:
    logger.info("Environment config not found.")

def configure_instance_folders(app, mode):
  instance_path = app.instance_path
  mode_dir = os.path.join(instance_path, mode)

  downloads_dir = os.path.join(mode_dir, "downloads")
  world_downloads_dir = os.path.join(downloads_dir, "worlds")
  schematic_downloads_dir = os.path.join(downloads_dir, "schematics")

  uploads_dir = os.path.join(mode_dir, "uploads")
  schematic_uploads_dir = os.path.join(uploads_dir, "schematics")

  os.makedirs(world_downloads_dir, exist_ok = True)
  os.makedirs(schematic_downloads_dir, exist_ok = True)
  os.makedirs(schematic_uploads_dir, exist_ok = True)

  set_config_variable('DOWNLOADS_DIR', downloads_dir)
  set_config_variable('WORLD_DOWNLOADS_DIR', world_downloads_dir)
  set_config_variable('SCHEMATIC_DOWNLOADS_DIR', schematic_downloads_dir)
  set_config_variable('UPLOADS_DIR', uploads_dir)
  set_config_variable('SCHEMATIC_UPLOADS_DIR', schematic_uploads_dir)

  # Used by Flask-Uploads to determine where to upload schematics
  set_config_variable('UPLOADED_SCHEMATICS_DEST', schematic_uploads_dir)

def set_config_variable(name, value):
  app.config[name] = value
  logger.info("Config variable '%s' set to: '%s'", name, value)

def configure_schematic_uploads(app):
  schematics = UploadSet('schematics', extensions = ['schematic'])
  configure_uploads(app, schematics)
  logger.info("Schematic uploads configured.")
  return schematics

app = Flask(__name__, instance_relative_config = True)
app.config.from_object('mrt_file_server.default_config')

mode = os.environ.get(modes.ENVIRONMENT_VARIABLE, modes.DEVELOPMENT)

logger = configure_logger(app, mode)
configure_log_messages(app)
configure_flash_messages(app)
load_environment_config(app, mode)
configure_instance_folders(app, mode)
schematics = configure_schematic_uploads(app)

import mrt_file_server.views

if __name__ == "__main__":
  app.run()

logger.info("Application ready.")<commit_msg>Added 'schem' to the list of permitted extensions.<commit_after>from flask import Flask
from flask_uploads import UploadSet, configure_uploads

import logging
import modes
import os
import re

def configure_logger(app, mode):
  logger = logging.getLogger(__name__)

  if mode is modes.PRODUCTION:
    logger.setLevel(logging.INFO)
  else:
    logger.setLevel(logging.DEBUG)

  formatter = logging.Formatter(
    fmt = '{asctime} {name:12} {levelname:8} {message}',
    datefmt = '%y-%m-%d %H:%M',
    style = '{')

  log_file_path = prepare_log_file(app, mode)
  file_handler = logging.FileHandler(log_file_path)
  file_handler.setFormatter(formatter)
  logger.addHandler(file_handler)

  if mode is not modes.TEST:
    console_handler = logging.StreamHandler()
    console_handler.setFormatter(formatter)
    logger.addHandler(console_handler)

  logger.info("Logging configured.")
  logger.info("Application mode is set to: '%s'", mode)

  return logger

def prepare_log_file(app, mode):
  instance_path = app.instance_path
  logs_dir = os.path.join(instance_path, mode, "logs")
  log_file_path = os.path.join(logs_dir, "server.log")
  os.makedirs(logs_dir, exist_ok = True)

  return log_file_path

def configure_log_messages(app):
  messages = {
    "SCHEMATIC_UPLOAD_SUCCESS":               "Schematic upload successful: '%s'",
    "SCHEMATIC_UPLOAD_FAILURE":               "Schematic upload failed: '%s', Exception: '%s'",
    "SCHEMATIC_UPLOAD_USERNAME_EMPTY":        "Schematic upload failed. Username is empty.",
    "SCHEMATIC_UPLOAD_USERNAME_WHITESPACE":   "Schematic upload failed. Username contains whitespace: '%s'",
    "SCHEMATIC_UPLOAD_NO_FILES":              "Schematic upload failed. No files specified.",
    "SCHEMATIC_UPLOAD_TOO_MANY_FILES":        "Schematic upload failed. Too many files.",
    "SCHEMATIC_UPLOAD_FILE_TOO_LARGE":        "Schematic upload failed. File too large: '%s'",
    "SCHEMATIC_UPLOAD_FILE_EXISTS":           "Schematic upload failed. File already exists: '%s'",
    "SCHEMATIC_UPLOAD_FILENAME_WHITESPACE":   "Schematic upload failed. Filename contains whitespace: '%s'",
    "SCHEMATIC_UPLOAD_FILENAME_EXTENSION":    "Schematic upload failed. Filename has invalid extension: '%s'",
    "SCHEMATIC_DOWNLOAD_SUCCESS":             "Schematic download initiated: '%s'",
    "SCHEMATIC_DOWNLOAD_FILENAME_EMPTY":      "Schematic download failed. Filename is empty.",
    "SCHEMATIC_DOWNLOAD_FILENAME_WHITESPACE": "Schematic download failed. Filename contains whitespace: '%s'",
    "SCHEMATIC_DOWNLOAD_FILENAME_NOT_UNIQUE": "Schematic download failed. Multiple files with this filename exist: '%s'",
    "SCHEMATIC_DOWNLOAD_FILE_NOT_FOUND":      "Schematic download failed. File does not exist: '%s'",
    "WORLD_DOWNLOAD_SUCCESS":                 "World download initiated: '%s'",
  }

  app.config['LOG_MESSAGES'] = messages
  logger.info("Log messages configured.")

def configure_flash_messages(app):
  messages = {
    "SCHEMATIC_UPLOAD_SUCCESS":               "Upload Successful!",
    "SCHEMATIC_UPLOAD_FAILURE":               "Upload Failed! Please contact the admins for assistance.",
    "SCHEMATIC_UPLOAD_USERNAME_EMPTY":        "Upload Failed! Username must not be empty.",
    "SCHEMATIC_UPLOAD_USERNAME_WHITESPACE":   "Upload Failed! Username must not contain spaces.",
    "SCHEMATIC_UPLOAD_NO_FILES":              "Upload Failed! No files selected.",
    "SCHEMATIC_UPLOAD_TOO_MANY_FILES":        "Upload Failed! A maximum of {} files can be uploaded at one time.".format( \
                                               app.config['MAX_NUMBER_OF_UPLOAD_FILES']),
    "SCHEMATIC_UPLOAD_FILE_TOO_LARGE":        "Upload Failed! File size is larger than allowed maximum of {} bytes.".format( \
                                               app.config['MAX_UPLOAD_FILE_SIZE']),
    "SCHEMATIC_UPLOAD_FILE_EXISTS":           "Upload Failed! File with same name already exists on the server.",
    "SCHEMATIC_UPLOAD_FILENAME_WHITESPACE":   "Upload Failed! File name must not contain spaces.",
    "SCHEMATIC_UPLOAD_FILENAME_EXTENSION":    "Upload Failed! File must end with the .schematic or .schem extension.",
    "SCHEMATIC_DOWNLOAD_FILENAME_EMPTY":      "Download Failed! Filename must not be empty.",
    "SCHEMATIC_DOWNLOAD_FILENAME_WHITESPACE": "Download Failed! Filename must not contain spaces.",
    "SCHEMATIC_DOWNLOAD_FILENAME_NOT_UNIQUE": "Download Failed! Multiple files with this filename exist. Please contact an admin for further assistance.",
    "SCHEMATIC_DOWNLOAD_FILE_NOT_FOUND":      "Download Failed! File does not exist."
  }

  app.config['FLASH_MESSAGES'] = messages
  logger.info("Flash messages configured.")

def load_environment_config(app, mode):
  instance_path = app.instance_path
  config_file_path = os.path.join(instance_path, mode, "config.py")

  if os.path.isfile(config_file_path):
    app.config.from_pyfile(config_file_path)
    logger.info("Environment config loaded from: '%s'", config_file_path)
  else:
    logger.info("Environment config not found.")

def configure_instance_folders(app, mode):
  instance_path = app.instance_path
  mode_dir = os.path.join(instance_path, mode)

  downloads_dir = os.path.join(mode_dir, "downloads")
  world_downloads_dir = os.path.join(downloads_dir, "worlds")
  schematic_downloads_dir = os.path.join(downloads_dir, "schematics")

  uploads_dir = os.path.join(mode_dir, "uploads")
  schematic_uploads_dir = os.path.join(uploads_dir, "schematics")

  os.makedirs(world_downloads_dir, exist_ok = True)
  os.makedirs(schematic_downloads_dir, exist_ok = True)
  os.makedirs(schematic_uploads_dir, exist_ok = True)

  set_config_variable('DOWNLOADS_DIR', downloads_dir)
  set_config_variable('WORLD_DOWNLOADS_DIR', world_downloads_dir)
  set_config_variable('SCHEMATIC_DOWNLOADS_DIR', schematic_downloads_dir)
  set_config_variable('UPLOADS_DIR', uploads_dir)
  set_config_variable('SCHEMATIC_UPLOADS_DIR', schematic_uploads_dir)

  # Used by Flask-Uploads to determine where to upload schematics
  set_config_variable('UPLOADED_SCHEMATICS_DEST', schematic_uploads_dir)

def set_config_variable(name, value):
  app.config[name] = value
  logger.info("Config variable '%s' set to: '%s'", name, value)

def configure_schematic_uploads(app):
  schematics = UploadSet('schematics', extensions = ['schematic', 'schem'])
  configure_uploads(app, schematics)
  logger.info("Schematic uploads configured.")
  return schematics

app = Flask(__name__, instance_relative_config = True)
app.config.from_object('mrt_file_server.default_config')

mode = os.environ.get(modes.ENVIRONMENT_VARIABLE, modes.DEVELOPMENT)

logger = configure_logger(app, mode)
configure_log_messages(app)
configure_flash_messages(app)
load_environment_config(app, mode)
configure_instance_folders(app, mode)
schematics = configure_schematic_uploads(app)

import mrt_file_server.views

if __name__ == "__main__":
  app.run()

logger.info("Application ready.")<|endoftext|>


<commit_before>"""
Main migration logic.
"""

import sys

from django.core.exceptions import ImproperlyConfigured

import south.db
from south import exceptions
from south.models import MigrationHistory
from south.db import db, DEFAULT_DB_ALIAS
from south.migration.migrators import (Backwards, Forwards,
                                       DryRunMigrator, FakeMigrator,
                                       LoadInitialDataMigrator)
from south.migration.base import Migration, Migrations
from south.migration.utils import SortedSet
from south.migration.base import all_migrations
from south.signals import pre_migrate, post_migrate

def to_apply(forwards, done):
    return [m for m in forwards if m not in done]

def to_unapply(backwards, done):
    return [m for m in backwards if m in done]

def problems(pending, done):
    last = None
    if not pending:
        raise StopIteration()
    for migration in pending:
        if migration in done:
            last = migration
            continue
        if last and migration not in done:
            yield last, migration

def problems(pending, done):
    last = None
    if not pending:
        raise StopIteration()
    for migration in pending:
        if migration in done:
            last = migration
            continue
        if last and migration not in done:
            yield last, migration

def forwards_problems(pending, done, verbosity):
    """
    Takes the list of linearised pending migrations, and the set of done ones,
    and returns the list of problems, if any.
    """
    result = []
    for last, migration in problems(reversed(pending), done):
        missing = [m for m in last.forwards_plan()[:-1]
                   if m not in done]
        if verbosity:
            m = ", ".join(str(m) for m in missing)
            print (" ! Migration %s should not have been applied "
                   "before %s but was." % (last, m))
        result.append((last, missing))
    return result

def backwards_problems(pending, done, verbosity):
    result = []
    for last, migration in problems(pending, done):
        missing = [m for m in migration.backwards_plan()[:-1]
                   if m in done]
        if verbosity:
            m = ", ".join(str(m) for m in missing)
            print " ! Migration %s should have been applied before %s but wasn't." % (migration, m)
        result.append((migration, missing))
    return result

def check_migration_histories(histories, delete_ghosts=False):
    "Checks that there's no 'ghost' migrations in the database."
    exists = SortedSet()
    ghosts = []
    for h in histories:
        try:
            m = h.get_migration()
            m.migration()
        except exceptions.UnknownMigration:
            ghosts.append(h)
        except ImproperlyConfigured:
            pass                        # Ignore missing applications
        exists.add(m)
    if ghosts:
        # They may want us to delete ghosts.
        if delete_ghosts:
            for h in ghosts:
                h.delete()
        else:
            raise exceptions.GhostMigrations(ghosts)
    return exists

def get_dependencies(target, migrations):
    forwards = list
    backwards = list
    if target is None:
        backwards = migrations[0].backwards_plan
    else:
        forwards = target.forwards_plan
        # When migrating backwards we want to remove up to and
        # including the next migration up in this app (not the next
        # one, that includes other apps)
        migration_before_here = target.next()
        if migration_before_here:
            backwards = migration_before_here.backwards_plan
    return forwards, backwards

def get_direction(target, applied, migrations, verbosity):
    # Get the forwards and reverse dependencies for this target
    forwards, backwards = get_dependencies(target, migrations)
    # Is the whole forward branch applied?
    problems = None
    forwards = forwards()
    workplan = to_apply(forwards, applied)
    if not workplan:
        # If they're all applied, we only know it's not backwards
        direction = None
    else:
        # If the remaining migrations are strictly a right segment of
        # the forwards trace, we just need to go forwards to our
        # target (and check for badness)
        problems = forwards_problems(forwards, applied, verbosity)
        direction = Forwards(verbosity=verbosity)
    if not problems:
        # What about the whole backward trace then?
        backwards = backwards()
        missing_backwards = to_apply(backwards, applied)
        if missing_backwards != backwards:
            # If what's missing is a strict left segment of backwards (i.e.
            # all the higher migrations) then we need to go backwards
            workplan = to_unapply(backwards, applied)
            problems = backwards_problems(backwards, applied, verbosity)
            direction = Backwards(verbosity=verbosity)
    return direction, problems, workplan

def get_migrator(direction, db_dry_run, fake, load_initial_data):
    if not direction:
        return direction
    if db_dry_run:
        direction = DryRunMigrator(migrator=direction, ignore_fail=False)
    elif fake:
        direction = FakeMigrator(migrator=direction)
    elif load_initial_data:
        direction = LoadInitialDataMigrator(migrator=direction)
    return direction

def migrate_app(migrations, target_name=None, merge=False, fake=False, db_dry_run=False, yes=False, verbosity=0, load_initial_data=False, skip=False, database=DEFAULT_DB_ALIAS, delete_ghosts=False):
    app_label = migrations.app_label()

    verbosity = int(verbosity)
    db.debug = (verbosity > 1)
    # Fire off the pre-migrate signal
    pre_migrate.send(None, app=app_label)

    # If there aren't any, quit quizically
    if not migrations:
        print "? You have no migrations for the '%s' app. You might want some." % app_label
        return

    # Load the entire dependency graph
    Migrations.calculate_dependencies()

    # Check there's no strange ones in the database
    applied = MigrationHistory.objects.filter(applied__isnull=False)
    # If we're using a different database, use that
    if database != DEFAULT_DB_ALIAS:
        applied = applied.using(database)
        south.db.db = south.db.dbs[database]
    applied = check_migration_histories(applied, delete_ghosts)

    # Guess the target_name
    target = migrations.guess_migration(target_name)
    if verbosity:
        if target_name not in ('zero', None) and target.name() != target_name:
            print " - Soft matched migration %s to %s." % (target_name,
                                                           target.name())
        print "Running migrations for %s:" % app_label

    # Get the forwards and reverse dependencies for this target
    direction, problems, workplan = get_direction(target, applied,
                                                  migrations, verbosity)
    if problems and not (merge or skip):
        raise exceptions.InconsistentMigrationHistory(problems)

    # Perform the migration
    migrator = get_migrator(direction, db_dry_run, fake, load_initial_data)
    if migrator:
        migrator.print_title(target)
        success = migrator.migrate_many(target, workplan)
        # Finally, fire off the post-migrate signal
        if success:
            post_migrate.send(None, app=app_label)
    elif verbosity:
        print '- Nothing to migrate.'
        post_migrate.send(None, app=app_label)
<commit_msg>Fix migration error checkers to work on a graph, not a linear history.<commit_after>"""
Main migration logic.
"""

import sys

from django.core.exceptions import ImproperlyConfigured

import south.db
from south import exceptions
from south.models import MigrationHistory
from south.db import db, DEFAULT_DB_ALIAS
from south.migration.migrators import (Backwards, Forwards,
                                       DryRunMigrator, FakeMigrator,
                                       LoadInitialDataMigrator)
from south.migration.base import Migration, Migrations
from south.migration.utils import SortedSet
from south.migration.base import all_migrations
from south.signals import pre_migrate, post_migrate

def to_apply(forwards, done):
    return [m for m in forwards if m not in done]

def to_unapply(backwards, done):
    return [m for m in backwards if m in done]

def problems(pending, done):
    last = None
    if not pending:
        raise StopIteration()
    for migration in pending:
        if migration in done:
            last = migration
            continue
        if last and migration not in done:
            yield last, migration

def problems(pending, done):
    last = None
    if not pending:
        raise StopIteration()
    for migration in pending:
        if migration in done:
            last = migration
            continue
        if last and migration not in done:
            yield last, migration

def forwards_problems(pending, done, verbosity):
    """
    Takes the list of linearised pending migrations, and the set of done ones,
    and returns the list of problems, if any.
    """
    return inner_problem_check(problems(reversed(pending), done), done, verbosity)

def backwards_problems(pending, done, verbosity):
    return inner_problem_check(problems(pending, done), done, verbosity)

def inner_problem_check(problems, done, verbosity):
    "Takes a set of possible problems and gets the actual issues out of it."
    result = []
    for last, migration in problems:
        # 'Last' is the last applied migration. Step back from it until we
        # either find nothing wrong, or we find something.
        to_check = list(last.dependencies)
        while to_check:
            checking = to_check.pop()
            if checking not in done:
                # That's bad. Error.
                if verbosity:
                    print (" ! Migration %s should not have been applied "
                           "before %s but was." % (last, checking))
                result.append((last, checking))
            else:
                to_check.extend(checking.dependencies)
    return result

def check_migration_histories(histories, delete_ghosts=False):
    "Checks that there's no 'ghost' migrations in the database."
    exists = SortedSet()
    ghosts = []
    for h in histories:
        try:
            m = h.get_migration()
            m.migration()
        except exceptions.UnknownMigration:
            ghosts.append(h)
        except ImproperlyConfigured:
            pass                        # Ignore missing applications
        exists.add(m)
    if ghosts:
        # They may want us to delete ghosts.
        if delete_ghosts:
            for h in ghosts:
                h.delete()
        else:
            raise exceptions.GhostMigrations(ghosts)
    return exists

def get_dependencies(target, migrations):
    forwards = list
    backwards = list
    if target is None:
        backwards = migrations[0].backwards_plan
    else:
        forwards = target.forwards_plan
        # When migrating backwards we want to remove up to and
        # including the next migration up in this app (not the next
        # one, that includes other apps)
        migration_before_here = target.next()
        if migration_before_here:
            backwards = migration_before_here.backwards_plan
    return forwards, backwards

def get_direction(target, applied, migrations, verbosity):
    # Get the forwards and reverse dependencies for this target
    forwards, backwards = get_dependencies(target, migrations)
    # Is the whole forward branch applied?
    problems = None
    forwards = forwards()
    workplan = to_apply(forwards, applied)
    if not workplan:
        # If they're all applied, we only know it's not backwards
        direction = None
    else:
        # If the remaining migrations are strictly a right segment of
        # the forwards trace, we just need to go forwards to our
        # target (and check for badness)
        problems = forwards_problems(forwards, applied, verbosity)
        direction = Forwards(verbosity=verbosity)
    if not problems:
        # What about the whole backward trace then?
        backwards = backwards()
        missing_backwards = to_apply(backwards, applied)
        if missing_backwards != backwards:
            # If what's missing is a strict left segment of backwards (i.e.
            # all the higher migrations) then we need to go backwards
            workplan = to_unapply(backwards, applied)
            problems = backwards_problems(backwards, applied, verbosity)
            direction = Backwards(verbosity=verbosity)
    return direction, problems, workplan

def get_migrator(direction, db_dry_run, fake, load_initial_data):
    if not direction:
        return direction
    if db_dry_run:
        direction = DryRunMigrator(migrator=direction, ignore_fail=False)
    elif fake:
        direction = FakeMigrator(migrator=direction)
    elif load_initial_data:
        direction = LoadInitialDataMigrator(migrator=direction)
    return direction

def migrate_app(migrations, target_name=None, merge=False, fake=False, db_dry_run=False, yes=False, verbosity=0, load_initial_data=False, skip=False, database=DEFAULT_DB_ALIAS, delete_ghosts=False):
    app_label = migrations.app_label()

    verbosity = int(verbosity)
    db.debug = (verbosity > 1)
    # Fire off the pre-migrate signal
    pre_migrate.send(None, app=app_label)

    # If there aren't any, quit quizically
    if not migrations:
        print "? You have no migrations for the '%s' app. You might want some." % app_label
        return

    # Load the entire dependency graph
    Migrations.calculate_dependencies()

    # Check there's no strange ones in the database
    applied = MigrationHistory.objects.filter(applied__isnull=False)
    # If we're using a different database, use that
    if database != DEFAULT_DB_ALIAS:
        applied = applied.using(database)
        south.db.db = south.db.dbs[database]
    applied = check_migration_histories(applied, delete_ghosts)

    # Guess the target_name
    target = migrations.guess_migration(target_name)
    if verbosity:
        if target_name not in ('zero', None) and target.name() != target_name:
            print " - Soft matched migration %s to %s." % (target_name,
                                                           target.name())
        print "Running migrations for %s:" % app_label

    # Get the forwards and reverse dependencies for this target
    direction, problems, workplan = get_direction(target, applied,
                                                  migrations, verbosity)
    if problems and not (merge or skip):
        raise exceptions.InconsistentMigrationHistory(problems)

    # Perform the migration
    migrator = get_migrator(direction, db_dry_run, fake, load_initial_data)
    if migrator:
        migrator.print_title(target)
        success = migrator.migrate_many(target, workplan)
        # Finally, fire off the post-migrate signal
        if success:
            post_migrate.send(None, app=app_label)
    elif verbosity:
        print '- Nothing to migrate.'
        post_migrate.send(None, app=app_label)
<|endoftext|>


<commit_before>from twisted.trial.unittest import TestCase

from vumi.service import Worker
from vumi.tests.utils import get_stubbed_worker, FakeRedis
from vumi.tests.fake_amqp import FakeAMQClient

class ToyWorker(Worker):
    def poke(self):
        return "poke"

class UtilsTestCase(TestCase):

    def test_get_stubbed_worker(self):
        worker = get_stubbed_worker(ToyWorker)
        self.assertEqual("poke", worker.poke())
        self.assertTrue(isinstance(worker._amqp_client, FakeAMQClient))

    def test_get_stubbed_worker_with_config(self):
        options = {'key': 'value'}
        worker = get_stubbed_worker(ToyWorker, options)
        self.assertEqual({}, worker._amqp_client.vumi_options)
        self.assertEqual(options, worker.config)

class FakeRedisIncrTestCase(TestCase):

    def test_delete(self):
        self.r_server = FakeRedis()
        self.r_server.set("delete_me", 1)
        self.assertEqual(True, self.r_server.delete("delete_me"))
        self.assertEqual(False, self.r_server.delete("delete_me"))

    def test_incr(self):
        self.r_server = FakeRedis()
        self.r_server.set("inc", 1)
        self.assertEqual('1', self.r_server.get("inc"))
        self.assertEqual(2, self.r_server.incr("inc"))
        self.assertEqual(3, self.r_server.incr("inc"))
        self.assertEqual('3', self.r_server.get("inc"))

    def test_incrby(self):
        self.r_server = FakeRedis()
        self.r_server.set("inc", 1)
        self.assertEqual('1', self.r_server.get("inc"))
        self.assertEqual(2, self.r_server.incrby("inc", 1))
        self.assertEqual(4, self.r_server.incrby("inc", 2))
        self.assertEqual(7, self.r_server.incrby("inc", 3))
        self.assertEqual(11, self.r_server.incrby("inc", 4))
        self.assertEqual(111, self.r_server.incrby("inc", 100))
        self.assertEqual('111', self.r_server.get("inc"))
<commit_msg>minor name fix<commit_after>from twisted.trial.unittest import TestCase

from vumi.service import Worker
from vumi.tests.utils import get_stubbed_worker, FakeRedis
from vumi.tests.fake_amqp import FakeAMQClient

class ToyWorker(Worker):
    def poke(self):
        return "poke"

class UtilsTestCase(TestCase):

    def test_get_stubbed_worker(self):
        worker = get_stubbed_worker(ToyWorker)
        self.assertEqual("poke", worker.poke())
        self.assertTrue(isinstance(worker._amqp_client, FakeAMQClient))

    def test_get_stubbed_worker_with_config(self):
        options = {'key': 'value'}
        worker = get_stubbed_worker(ToyWorker, options)
        self.assertEqual({}, worker._amqp_client.vumi_options)
        self.assertEqual(options, worker.config)

class FakeRedisTestCase(TestCase):

    def test_delete(self):
        self.r_server = FakeRedis()
        self.r_server.set("delete_me", 1)
        self.assertEqual(True, self.r_server.delete("delete_me"))
        self.assertEqual(False, self.r_server.delete("delete_me"))

    def test_incr(self):
        self.r_server = FakeRedis()
        self.r_server.set("inc", 1)
        self.assertEqual('1', self.r_server.get("inc"))
        self.assertEqual(2, self.r_server.incr("inc"))
        self.assertEqual(3, self.r_server.incr("inc"))
        self.assertEqual('3', self.r_server.get("inc"))

    def test_incrby(self):
        self.r_server = FakeRedis()
        self.r_server.set("inc", 1)
        self.assertEqual('1', self.r_server.get("inc"))
        self.assertEqual(2, self.r_server.incrby("inc", 1))
        self.assertEqual(4, self.r_server.incrby("inc", 2))
        self.assertEqual(7, self.r_server.incrby("inc", 3))
        self.assertEqual(11, self.r_server.incrby("inc", 4))
        self.assertEqual(111, self.r_server.incrby("inc", 100))
        self.assertEqual('111', self.r_server.get("inc"))
<|endoftext|>


<commit_before>"""
Provide an easy interface for loading data into L{PRDD}s for Spark.
"""
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from sparklingpandas.utils import add_pyspark_path, run_tests

add_pyspark_path()
import pandas
from StringIO import StringIO
from pyspark.context import SparkContext
from sparklingpandas.prdd import PRDD

class PSparkContext():

    """
    This is a thin wrapper around SparkContext from
    PySpark which makes it easy to load data into L{PRDD}s.
    """

    def __init__(self, sparkcontext):
        self.sc = sparkcontext

    @classmethod
    def simple(cls, *args, **kwargs):
        """
        Takes the same arguments as SparkContext and constructs a PSparkContext
        """
        return PSparkContext(SparkContext(*args, **kwargs))

    def read_csv(self, name, use_whole_file=False, names=None, skiprows=0,
                 *args, **kwargs):
        """Read a CSV file in and parse it into Pandas DataFrames.
        If no names is provided we use the first row for the names.
        header=0 is the default unless names is provided in which case
        header=None is the default.
        skiprows indicates how many rows of input to skip. This will
        only be applied to the first partition of the data (so if
        #skiprows > #row in first partition this will not work). Generally
        this shouldn't be an issue for small values of skiprows.
        No other values of header is supported.
        All additional parameters are passed to the read_csv function.
        """
        def csv_file(partitionNumber, files):
            file_count = 0
            for filename, contents in files:
                # Only skip lines on the first file
                if partitionNumber == 0 and file_count == 0 and _skiprows > 0:
                    yield pandas.read_csv(StringIO(contents), *args,
                                          header=None,
                                          names=mynames,
                                          skiprows=_skiprows, **kwargs)
                else:
                    file_count += 1
                    yield pandas.read_csv(StringIO(contents), *args,
                                          header=None,
                                          names=mynames,
                                          **kwargs)

        def csv_rows(partitionNumber, rows):
            rowCount = 0
            inputStr = "\n".join(rows)
            if partitionNumber == 0:
                return pandas.read_csv(StringIO(row), *args, header=None,
                                      names=mynames, skiprows=_skiprows, **kwargs)
            else:
                return pandas.read_csv(StringIO(row), *args, header=None,
                                      names=mynames, **kwargs)

        # If we need to peak at the first partition and determine the column
        # names
        mynames = None
        _skiprows = skiprows
        if names:
            mynames = names
        else:
            # In the future we could avoid this expensive call.
            first_line = self.sc.textFile(name).first()
            frame = pandas.read_csv(StringIO(first_line))
            mynames = list(frame.columns.values)
            _skiprows += 1

        # Do the actual load
        if use_whole_file:
            return PRDD.fromRDD(
                self.sc.wholeTextFiles(name).mapPartitionsWithIndex(csv_file))
        else:
            return PRDD.fromRDD(
                self.sc.textFile(name).mapPartitionsWithIndex(csv_rows))

    def DataFrame(self, elements, *args, **kwargs):
        """
        Wraps the pandas.DataFrame operation.
        """
        return PRDD.fromRDD(self.sc.parallelize(elements).map(
            lambda element: pandas.DataFrame(data=[element], *args, **kwargs)))

    def stop(self):
        """
        Stop the underlying SparkContext
        """
        self.sc.stop()

if __name__ == "__main__":
    run_tests()
<commit_msg>Fix pep8 issues<commit_after>"""
Provide an easy interface for loading data into L{PRDD}s for Spark.
"""
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from sparklingpandas.utils import add_pyspark_path, run_tests

add_pyspark_path()
import pandas
from StringIO import StringIO
from pyspark.context import SparkContext
from sparklingpandas.prdd import PRDD

class PSparkContext():

    """
    This is a thin wrapper around SparkContext from
    PySpark which makes it easy to load data into L{PRDD}s.
    """

    def __init__(self, sparkcontext):
        self.sc = sparkcontext

    @classmethod
    def simple(cls, *args, **kwargs):
        """
        Takes the same arguments as SparkContext and constructs a PSparkContext
        """
        return PSparkContext(SparkContext(*args, **kwargs))

    def read_csv(self, name, use_whole_file=False, names=None, skiprows=0,
                 *args, **kwargs):
        """Read a CSV file in and parse it into Pandas DataFrames.
        If no names is provided we use the first row for the names.
        header=0 is the default unless names is provided in which case
        header=None is the default.
        skiprows indicates how many rows of input to skip. This will
        only be applied to the first partition of the data (so if
        #skiprows > #row in first partition this will not work). Generally
        this shouldn't be an issue for small values of skiprows.
        No other values of header is supported.
        All additional parameters are passed to the read_csv function.
        """
        def csv_file(partitionNumber, files):
            file_count = 0
            for filename, contents in files:
                # Only skip lines on the first file
                if partitionNumber == 0 and file_count == 0 and _skiprows > 0:
                    yield pandas.read_csv(StringIO(contents), *args,
                                          header=None,
                                          names=mynames,
                                          skiprows=_skiprows, **kwargs)
                else:
                    file_count += 1
                    yield pandas.read_csv(StringIO(contents), *args,
                                          header=None,
                                          names=mynames,
                                          **kwargs)

        def csv_rows(partitionNumber, rows):
            rowCount = 0
            inputStr = "\n".join(rows)
            if partitionNumber == 0:
                return pandas.read_csv(StringIO(row), *args, header=None,
                                       names=mynames, skiprows=_skiprows,
                                       **kwargs)
            else:
                return pandas.read_csv(StringIO(row), *args, header=None,
                                       names=mynames, **kwargs)

        # If we need to peak at the first partition and determine the column
        # names
        mynames = None
        _skiprows = skiprows
        if names:
            mynames = names
        else:
            # In the future we could avoid this expensive call.
            first_line = self.sc.textFile(name).first()
            frame = pandas.read_csv(StringIO(first_line))
            mynames = list(frame.columns.values)
            _skiprows += 1

        # Do the actual load
        if use_whole_file:
            return PRDD.fromRDD(
                self.sc.wholeTextFiles(name).mapPartitionsWithIndex(csv_file))
        else:
            return PRDD.fromRDD(
                self.sc.textFile(name).mapPartitionsWithIndex(csv_rows))

    def DataFrame(self, elements, *args, **kwargs):
        """
        Wraps the pandas.DataFrame operation.
        """
        return PRDD.fromRDD(self.sc.parallelize(elements).map(
            lambda element: pandas.DataFrame(data=[element], *args, **kwargs)))

    def stop(self):
        """
        Stop the underlying SparkContext
        """
        self.sc.stop()

if __name__ == "__main__":
    run_tests()
<|endoftext|>


<commit_before>from dataservice.models import *
from django.contrib import admin

admin.site.register(HostingContainer)
admin.site.register(DataService)
admin.site.register(MFile)
admin.site.register(MFolder)
admin.site.register(Usage)
admin.site.register(Auth)
admin.site.register(Role)
admin.site.register(BackupFile)
admin.site.register(ClientConsumer)
admin.site.register(MServeProfile)

from piston.models import Token
from piston.models import Consumer

admin.site.register(Token)
admin.site.register(Consumer)<commit_msg>Add MFileOAuthToken to Admin<commit_after>from dataservice.models import *
from django.contrib import admin

admin.site.register(HostingContainer)
admin.site.register(DataService)
admin.site.register(MFile)
admin.site.register(MFolder)
admin.site.register(Usage)
admin.site.register(Auth)
admin.site.register(Role)
admin.site.register(BackupFile)
admin.site.register(ClientConsumer)
admin.site.register(MServeProfile)
admin.site.register(MFileOAuthToken)

from piston.models import Token
from piston.models import Consumer

admin.site.register(Token)
admin.site.register(Consumer)<|endoftext|>


<commit_before>"""
Load a shared library.
"""
import os
import sys
import ctypes
import ctypes.util
import subprocess

from . import (
    utils,
    DEFAULT_EXTENSION,
    IS_PYTHON2,
)

_encoding = sys.getfilesystemencoding()

class LoadLibrary(object):

    def __init__(self, path, libtype=None, **kwargs):
        """Load a shared library.

        For example, a C/C++, FORTRAN, C#, Java, Delphi, LabVIEW, ActiveX, ... library.

        Based on the value of `libtype` this class will load the shared library as a:

            * :class:`~ctypes.CDLL` if `libtype` is ``'cdll'``,
            * :class:`~ctypes.WinDLL` if `libtype` is ``'windll'``,
            * :class:`~ctypes.OleDLL` if `libtype` is ``'oledll'``,
            * `System.Reflection.Assembly <Assembly_>`_ if `libtype` is ``'net'`` or ``'clr'`` ,
            * :class:`~.py4j.java_gateway.JavaGateway` if `libtype` is ``'java'``, or
            * comtypes.CreateObject_ if `libtype` is ``'com'``.

        .. versionchanged:: 0.4
           Added support for Java archives

        .. versionchanged:: 0.5
           Added support for COM_ libraries

        .. _Assembly: https://msdn.microsoft.com/en-us/library/system.reflection.assembly(v=vs.110).aspx
        .. _comtypes.CreateObject: https://pythonhosted.org/comtypes/#creating-and-accessing-com-objects
        .. _COM: https://en.wikipedia.org/wiki/Component_Object_Model

        Parameters
        ----------
        path : :class:`str`
            The path to the shared library.

            The search order for finding the shared library is:

                1. assume that a full or a relative (to the current working directory)
                   path is specified,
                2. use :func:`ctypes.util.find_library` to find the shared library file,
                3. search :data:`sys.path`, then
                4. search :data:`os.environ['PATH'] <os.environ>` to find the shared library.

            If loading a COM_ library then `path` represents the `progid <comtypes.CreateObject_>`_
            argument.

        libtype : :class:`str`, optional
            The library type. The following values are currently supported:

            * ``'cdll'`` -- for a library that uses the __cdecl calling convention
            * ``'windll'`` or ``'oledll'`` -- for a __stdcall calling convention
            * ``'net'`` or ``'clr'`` -- for Microsoft's .NET Framework (Common Language Runtime)
            * ``'java'`` -- for a Java archive, ``.jar``, or Java byte code, ``.class``, file
            * ``'com'`` -- for a COM_ library.

            Default is ``'cdll'``.

            .. tip::
               Since the ``.jar`` or ``.class`` extension uniquely defines a Java library,
               the `libtype` will be automatically set to ``'java'`` if `path` ends with
               ``.jar`` or ``.class``. If `path` starts with ``'{'`` and ends with ``'}'``
               then this uniquely defines the Class ID for a COM_ library and so `libtype`
               will be automatically set to ``'com'``.

        **kwargs
            Keyword arguments that are passed to the object that loads the library.

        Raises
        ------
        IOError
            If the shared library cannot be loaded.
        TypeError
            If `libtype` is not a supported library type.
        """
        # a reference to the shared library
        self._lib = None

        # a reference to the .NET Runtime Assembly
        self._assembly = None

        # a reference to the Py4J JavaGateway
        self._gateway = None

        if not path:
            raise ValueError('You must specify the path, got {!r}'.format(path))

        # fixes Issue #8, if `path` is a <class 'pathlib.Path'> object
        if hasattr(path, 'as_posix'):
            path = path.as_posix()

        # try to automatically determine the libtype
        if libtype is None:
            if path.startswith('{') and path.endswith('}'):
                libtype = 'com'
            elif path.endswith('.jar') or path.endswith('.class'):
                libtype = 'java'
            else:
                libtype = 'cdll'
        else:
            libtype = libtype.lower()

        # create a new reference to `path` just in case the
        # DEFAULT_EXTENSION is appended below so that the
        # ctypes.util.find_library function call will use the
        # unmodified value of `path`
        _path = path

        # assume a default extension if no extension was provided
        ext = os.path.splitext(path)[1]
        if libtype != 'java' and libtype != 'com' and not ext:
            _path += DEFAULT_EXTENSION

        if IS_PYTHON2:
            _path = _path.encode(_encoding)

        if libtype != 'com':
            self._path = os.path.abspath(_path)
            if not os.path.isfile(self._path):
                # for find_library use the original 'path' value since it may be a library name
                # without any prefix like 'lib', suffix like '.so', '.dylib' or version number
                self._path = ctypes.util.find_library(path)
                if self._path is None:  # then search sys.path and os.environ['PATH']
                    success = False
                    search_dirs = sys.path + os.environ['PATH'].split(os.pathsep)
                    for directory in search_dirs:
                        p = os.path.join(directory, _path)
                        if os.path.isfile(p):
                            self._path = p
                            success = True
                            break
                    if not success:
                        raise IOError('Cannot find the shared library {!r}'.format(path))
        else:
            self._path = _path

        if libtype == 'cdll':
            self._lib = ctypes.CDLL(self._path, **kwargs)
        elif libtype == 'windll':
            self._lib = ctypes.WinDLL(self._path, **kwargs)
        elif libtype == 'oledll':
            self._lib = ctypes.OleDLL(self._path, **kwargs)
        elif libtype == 'com':
            if not utils.is_comtypes_installed():
                raise IOError('Cannot load a COM library because comtypes is not installed.\n'
                              'To install comtypes run: pip install comtypes')
            from comtypes.client import CreateObject
            self._lib = CreateObject(self._path, **kwargs)
        elif libtype == 'java':
            if not utils.is_py4j_installed():
                raise IOError('Cannot load a Java file because Py4J is not installed.\n'
                              'To install Py4J run: pip install py4j')

            from py4j.version import __version__
            from py4j.java_gateway import JavaGateway, GatewayParameters

            # the address and port to use to host the py4j.GatewayServer
            address = kwargs.pop('address', '127.0.0.1')
            port = kwargs.pop('port', utils.get_available_port())

            # find the py4j JAR (needed to import py4j.GatewayServer on the Java side)
            root = os.path.dirname(sys.executable)
            filename = 'py4j'+__version__+'.jar'
            py4j_jar = os.path.join(root, 'share', 'py4j', filename)
            if not os.path.isfile(py4j_jar):
                root = os.path.dirname(root)  # then check one folder up (for unix or venv)
                py4j_jar = os.path.join(root, 'share', 'py4j', filename)
                if not os.path.isfile(py4j_jar):
                    py4j_jar = os.environ.get('PY4J_JAR', '')  # then check the environment variable
                    if not os.path.isfile(py4j_jar):
                        raise IOError('Cannot find {0}\nCreate a PY4J_JAR environment '
                                      'variable to be equal to the full path to {0}'.format(filename))

            # build the java command
            wrapper = os.path.join(os.path.dirname(__file__), 'py4j-wrapper.jar')
            cmd = ['java', '-cp', py4j_jar + os.pathsep + wrapper, 'Py4JWrapper', str(port)]

            # from the URLClassLoader documentation:
            #   Any URL that ends with a '/' is assumed to refer to a directory. Otherwise, the URL
            #   is assumed to refer to a JAR file which will be downloaded and opened as needed.
            if ext == '.jar':
                cmd.append(self._path)
            else:  # it is a .class file
                cmd.append(os.path.dirname(self._path) + '/')

            try:
                # start the py4j.GatewayServer
                subprocess.Popen(cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
                err = None
            except IOError as e:
                err = str(e) + '\nYou must have a Java Runtime Environment installed and available on PATH'

            if err:
                raise IOError(err)

            utils.wait_for_server(address, port, 5.0)

            self._gateway = JavaGateway(
                gateway_parameters=GatewayParameters(address=address, port=port, **kwargs)
            )

            self._lib = self._gateway.jvm

        elif libtype == 'net' or libtype == 'clr':
            if not utils.is_pythonnet_installed():
                raise IOError('Cannot load a .NET Assembly because pythonnet is not installed.\n'
                              'To install pythonnet run: pip install pythonnet')

            import clr

            # the shared library must be available in sys.path
            head, tail = os.path.split(self._path)
            sys.path.insert(0, head)

            # don't include the library extension
            clr.AddReference(os.path.splitext(tail)[0])

            import System
            dotnet = {'System': System}

            try:
                # By default, pythonnet can only load libraries that are for .NET 4.0+.3.9
                #
                # In order to allow pythonnet to load a library from .NET <4.0 the
                # useLegacyV2RuntimeActivationPolicy property needs to be enabled
                # in a <python-executable>.config file. If the following statement
                # raises a FileLoadException then attempt to create the configuration
                # file that has the property enabled and then notify the user why
                # loading the library failed and ask them to re-run their Python
                # script to load the .NET library.
                self._assembly = System.Reflection.Assembly.LoadFile(self._path)

            except System.IO.FileLoadException as err:
                # Example error message that can occur if the library is for .NET <4.0,
                # and the useLegacyV2RuntimeActivationPolicy is not enabled:
                #
                # " Mixed mode assembly is built against version 'v2.0.50727' of the
                #  runtime and cannot be loaded in the 4.0 runtime without additional
                #  configuration information. "
                if str(err).startswith('Mixed mode assembly'):
                    status, msg = utils.check_dot_net_config(sys.executable)
                    if not status == 0:
                        raise IOError(msg)
                    else:
                        update_msg = 'Checking .NET config returned "{}" '.format(msg)
                        update_msg += 'and still cannot load library.\n'
                        update_msg += str(err)
                        raise IOError(update_msg)
                raise IOError('The above "System.IO.FileLoadException" is not handled.\n')

            try:
                types = self._assembly.GetTypes()
            except Exception as e:
                utils.logger.error(e)
                utils.logger.error('The LoaderExceptions are:')
                for item in e.LoaderExceptions:
                    utils.logger.error('  ' + item.Message)
            else:
                for t in types:
                    try:
                        if t.Namespace:
                            obj = __import__(t.Namespace)
                        else:
                            obj = getattr(__import__('clr'), t.FullName)
                    except:
                        obj = t
                        obj.__name__ = t.FullName

                    if obj.__name__ not in dotnet:
                        dotnet[obj.__name__] = obj

            self._lib = DotNet(dotnet, self._path)

        else:
            raise TypeError('Cannot load libtype={}'.format(libtype))

        if IS_PYTHON2:
            self._path = self._path.decode(_encoding)

        utils.logger.debug('Loaded ' + self._path)

    def __repr__(self):
        path = self._path.encode(_encoding) if IS_PYTHON2 else self._path
        return '<LoadLibrary libtype={} path={}>'.format(self._lib.__class__.__name__, path)

    def __del__(self):
        if self._gateway is not None:
            self._gateway.shutdown()
            utils.logger.debug('shutdown py4j.GatewayServer')

    @property
    def assembly(self):
        """
        Returns a reference to the `.NET Runtime Assembly <Assembly_>`_ object, only if
        the shared library is a .NET Framework, otherwise returns :data:`None`.

        .. tip::
           The `JetBrains dotPeek`_ program can be used to reliably decompile any
           .NET Assembly in to the equivalent source code.

        .. _JetBrains dotPeek: https://www.jetbrains.com/decompiler/
        """
        return self._assembly

    @property
    def gateway(self):
        """
        Returns the :class:`~py4j.java_gateway.JavaGateway` object, only if
        the shared library is a Java archive, otherwise returns :data:`None`.
        """
        return self._gateway

    @property
    def lib(self):
        """Returns the reference to the loaded library object.

        For example, if `libtype` is

            * ``'cdll'`` then a :class:`~ctypes.CDLL` object
            * ``'windll'`` then a :class:`~ctypes.WinDLL` object
            * ``'oledll'`` then a :class:`~ctypes.OleDLL` object
            * ``'net'`` or ``'clr'`` then a :class:`~.load_library.DotNet` object
            * ``'java'`` then a :class:`~py4j.java_gateway.JVMView` object
            * ``'com'`` then the interface pointer returned by comtypes.CreateObject_
        """
        return self._lib

    @property
    def path(self):
        """:class:`str`: The path to the shared library file."""
        return self._path

class DotNet(object):

    def __init__(self, dot_net_dict, path):
        """Contains the namespace_ modules, classes and `System.Type`_ objects of a .NET Assembly.

        Do not instantiate this class directly.

        .. _namespace: https://msdn.microsoft.com/en-us/library/z2kcy19k.aspx
        .. _System.Type: https://msdn.microsoft.com/en-us/library/system.type(v=vs.110).aspx
        """
        self.__dict__.update(dot_net_dict)
        self._path = path

    def __repr__(self):
        return '<{} path={}>'.format(self.__class__.__name__, self._path)
<commit_msg>add period to docstring of LoadLibrary<commit_after>"""
Load a shared library.
"""
import os
import sys
import ctypes
import ctypes.util
import subprocess

from . import (
    utils,
    DEFAULT_EXTENSION,
    IS_PYTHON2,
)

_encoding = sys.getfilesystemencoding()

class LoadLibrary(object):

    def __init__(self, path, libtype=None, **kwargs):
        """Load a shared library.

        For example, a C/C++, FORTRAN, C#, Java, Delphi, LabVIEW, ActiveX, ... library.

        Based on the value of `libtype` this class will load the shared library as a:

            * :class:`~ctypes.CDLL` if `libtype` is ``'cdll'``,
            * :class:`~ctypes.WinDLL` if `libtype` is ``'windll'``,
            * :class:`~ctypes.OleDLL` if `libtype` is ``'oledll'``,
            * `System.Reflection.Assembly <Assembly_>`_ if `libtype` is ``'net'`` or ``'clr'`` ,
            * :class:`~.py4j.java_gateway.JavaGateway` if `libtype` is ``'java'``, or
            * comtypes.CreateObject_ if `libtype` is ``'com'``.

        .. versionchanged:: 0.4
           Added support for Java archives.

        .. versionchanged:: 0.5
           Added support for COM_ libraries.

        .. _Assembly: https://msdn.microsoft.com/en-us/library/system.reflection.assembly(v=vs.110).aspx
        .. _comtypes.CreateObject: https://pythonhosted.org/comtypes/#creating-and-accessing-com-objects
        .. _COM: https://en.wikipedia.org/wiki/Component_Object_Model

        Parameters
        ----------
        path : :class:`str`
            The path to the shared library.

            The search order for finding the shared library is:

                1. assume that a full or a relative (to the current working directory)
                   path is specified,
                2. use :func:`ctypes.util.find_library` to find the shared library file,
                3. search :data:`sys.path`, then
                4. search :data:`os.environ['PATH'] <os.environ>` to find the shared library.

            If loading a COM_ library then `path` represents the `progid <comtypes.CreateObject_>`_
            argument.

        libtype : :class:`str`, optional
            The library type. The following values are currently supported:

            * ``'cdll'`` -- for a library that uses the __cdecl calling convention
            * ``'windll'`` or ``'oledll'`` -- for a __stdcall calling convention
            * ``'net'`` or ``'clr'`` -- for Microsoft's .NET Framework (Common Language Runtime)
            * ``'java'`` -- for a Java archive, ``.jar``, or Java byte code, ``.class``, file
            * ``'com'`` -- for a COM_ library.

            Default is ``'cdll'``.

            .. tip::
               Since the ``.jar`` or ``.class`` extension uniquely defines a Java library,
               the `libtype` will be automatically set to ``'java'`` if `path` ends with
               ``.jar`` or ``.class``. If `path` starts with ``'{'`` and ends with ``'}'``
               then this uniquely defines the Class ID for a COM_ library and so `libtype`
               will be automatically set to ``'com'``.

        **kwargs
            Keyword arguments that are passed to the object that loads the library.

        Raises
        ------
        IOError
            If the shared library cannot be loaded.
        TypeError
            If `libtype` is not a supported library type.
        """
        # a reference to the shared library
        self._lib = None

        # a reference to the .NET Runtime Assembly
        self._assembly = None

        # a reference to the Py4J JavaGateway
        self._gateway = None

        if not path:
            raise ValueError('You must specify the path, got {!r}'.format(path))

        # fixes Issue #8, if `path` is a <class 'pathlib.Path'> object
        if hasattr(path, 'as_posix'):
            path = path.as_posix()

        # try to automatically determine the libtype
        if libtype is None:
            if path.startswith('{') and path.endswith('}'):
                libtype = 'com'
            elif path.endswith('.jar') or path.endswith('.class'):
                libtype = 'java'
            else:
                libtype = 'cdll'
        else:
            libtype = libtype.lower()

        # create a new reference to `path` just in case the
        # DEFAULT_EXTENSION is appended below so that the
        # ctypes.util.find_library function call will use the
        # unmodified value of `path`
        _path = path

        # assume a default extension if no extension was provided
        ext = os.path.splitext(path)[1]
        if libtype != 'java' and libtype != 'com' and not ext:
            _path += DEFAULT_EXTENSION

        if IS_PYTHON2:
            _path = _path.encode(_encoding)

        if libtype != 'com':
            self._path = os.path.abspath(_path)
            if not os.path.isfile(self._path):
                # for find_library use the original 'path' value since it may be a library name
                # without any prefix like 'lib', suffix like '.so', '.dylib' or version number
                self._path = ctypes.util.find_library(path)
                if self._path is None:  # then search sys.path and os.environ['PATH']
                    success = False
                    search_dirs = sys.path + os.environ['PATH'].split(os.pathsep)
                    for directory in search_dirs:
                        p = os.path.join(directory, _path)
                        if os.path.isfile(p):
                            self._path = p
                            success = True
                            break
                    if not success:
                        raise IOError('Cannot find the shared library {!r}'.format(path))
        else:
            self._path = _path

        if libtype == 'cdll':
            self._lib = ctypes.CDLL(self._path, **kwargs)
        elif libtype == 'windll':
            self._lib = ctypes.WinDLL(self._path, **kwargs)
        elif libtype == 'oledll':
            self._lib = ctypes.OleDLL(self._path, **kwargs)
        elif libtype == 'com':
            if not utils.is_comtypes_installed():
                raise IOError('Cannot load a COM library because comtypes is not installed.\n'
                              'To install comtypes run: pip install comtypes')
            from comtypes.client import CreateObject
            self._lib = CreateObject(self._path, **kwargs)
        elif libtype == 'java':
            if not utils.is_py4j_installed():
                raise IOError('Cannot load a Java file because Py4J is not installed.\n'
                              'To install Py4J run: pip install py4j')

            from py4j.version import __version__
            from py4j.java_gateway import JavaGateway, GatewayParameters

            # the address and port to use to host the py4j.GatewayServer
            address = kwargs.pop('address', '127.0.0.1')
            port = kwargs.pop('port', utils.get_available_port())

            # find the py4j JAR (needed to import py4j.GatewayServer on the Java side)
            root = os.path.dirname(sys.executable)
            filename = 'py4j'+__version__+'.jar'
            py4j_jar = os.path.join(root, 'share', 'py4j', filename)
            if not os.path.isfile(py4j_jar):
                root = os.path.dirname(root)  # then check one folder up (for unix or venv)
                py4j_jar = os.path.join(root, 'share', 'py4j', filename)
                if not os.path.isfile(py4j_jar):
                    py4j_jar = os.environ.get('PY4J_JAR', '')  # then check the environment variable
                    if not os.path.isfile(py4j_jar):
                        raise IOError('Cannot find {0}\nCreate a PY4J_JAR environment '
                                      'variable to be equal to the full path to {0}'.format(filename))

            # build the java command
            wrapper = os.path.join(os.path.dirname(__file__), 'py4j-wrapper.jar')
            cmd = ['java', '-cp', py4j_jar + os.pathsep + wrapper, 'Py4JWrapper', str(port)]

            # from the URLClassLoader documentation:
            #   Any URL that ends with a '/' is assumed to refer to a directory. Otherwise, the URL
            #   is assumed to refer to a JAR file which will be downloaded and opened as needed.
            if ext == '.jar':
                cmd.append(self._path)
            else:  # it is a .class file
                cmd.append(os.path.dirname(self._path) + '/')

            try:
                # start the py4j.GatewayServer
                subprocess.Popen(cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
                err = None
            except IOError as e:
                err = str(e) + '\nYou must have a Java Runtime Environment installed and available on PATH'

            if err:
                raise IOError(err)

            utils.wait_for_server(address, port, 5.0)

            self._gateway = JavaGateway(
                gateway_parameters=GatewayParameters(address=address, port=port, **kwargs)
            )

            self._lib = self._gateway.jvm

        elif libtype == 'net' or libtype == 'clr':
            if not utils.is_pythonnet_installed():
                raise IOError('Cannot load a .NET Assembly because pythonnet is not installed.\n'
                              'To install pythonnet run: pip install pythonnet')

            import clr

            # the shared library must be available in sys.path
            head, tail = os.path.split(self._path)
            sys.path.insert(0, head)

            # don't include the library extension
            clr.AddReference(os.path.splitext(tail)[0])

            import System
            dotnet = {'System': System}

            try:
                # By default, pythonnet can only load libraries that are for .NET 4.0+.3.9
                #
                # In order to allow pythonnet to load a library from .NET <4.0 the
                # useLegacyV2RuntimeActivationPolicy property needs to be enabled
                # in a <python-executable>.config file. If the following statement
                # raises a FileLoadException then attempt to create the configuration
                # file that has the property enabled and then notify the user why
                # loading the library failed and ask them to re-run their Python
                # script to load the .NET library.
                self._assembly = System.Reflection.Assembly.LoadFile(self._path)

            except System.IO.FileLoadException as err:
                # Example error message that can occur if the library is for .NET <4.0,
                # and the useLegacyV2RuntimeActivationPolicy is not enabled:
                #
                # " Mixed mode assembly is built against version 'v2.0.50727' of the
                #  runtime and cannot be loaded in the 4.0 runtime without additional
                #  configuration information. "
                if str(err).startswith('Mixed mode assembly'):
                    status, msg = utils.check_dot_net_config(sys.executable)
                    if not status == 0:
                        raise IOError(msg)
                    else:
                        update_msg = 'Checking .NET config returned "{}" '.format(msg)
                        update_msg += 'and still cannot load library.\n'
                        update_msg += str(err)
                        raise IOError(update_msg)
                raise IOError('The above "System.IO.FileLoadException" is not handled.\n')

            try:
                types = self._assembly.GetTypes()
            except Exception as e:
                utils.logger.error(e)
                utils.logger.error('The LoaderExceptions are:')
                for item in e.LoaderExceptions:
                    utils.logger.error('  ' + item.Message)
            else:
                for t in types:
                    try:
                        if t.Namespace:
                            obj = __import__(t.Namespace)
                        else:
                            obj = getattr(__import__('clr'), t.FullName)
                    except:
                        obj = t
                        obj.__name__ = t.FullName

                    if obj.__name__ not in dotnet:
                        dotnet[obj.__name__] = obj

            self._lib = DotNet(dotnet, self._path)

        else:
            raise TypeError('Cannot load libtype={}'.format(libtype))

        if IS_PYTHON2:
            self._path = self._path.decode(_encoding)

        utils.logger.debug('Loaded ' + self._path)

    def __repr__(self):
        path = self._path.encode(_encoding) if IS_PYTHON2 else self._path
        return '<LoadLibrary libtype={} path={}>'.format(self._lib.__class__.__name__, path)

    def __del__(self):
        if self._gateway is not None:
            self._gateway.shutdown()
            utils.logger.debug('shutdown py4j.GatewayServer')

    @property
    def assembly(self):
        """
        Returns a reference to the `.NET Runtime Assembly <Assembly_>`_ object, only if
        the shared library is a .NET Framework, otherwise returns :data:`None`.

        .. tip::
           The `JetBrains dotPeek`_ program can be used to reliably decompile any
           .NET Assembly in to the equivalent source code.

        .. _JetBrains dotPeek: https://www.jetbrains.com/decompiler/
        """
        return self._assembly

    @property
    def gateway(self):
        """
        Returns the :class:`~py4j.java_gateway.JavaGateway` object, only if
        the shared library is a Java archive, otherwise returns :data:`None`.
        """
        return self._gateway

    @property
    def lib(self):
        """Returns the reference to the loaded library object.

        For example, if `libtype` is

            * ``'cdll'`` then a :class:`~ctypes.CDLL` object
            * ``'windll'`` then a :class:`~ctypes.WinDLL` object
            * ``'oledll'`` then a :class:`~ctypes.OleDLL` object
            * ``'net'`` or ``'clr'`` then a :class:`~.load_library.DotNet` object
            * ``'java'`` then a :class:`~py4j.java_gateway.JVMView` object
            * ``'com'`` then the interface pointer returned by comtypes.CreateObject_
        """
        return self._lib

    @property
    def path(self):
        """:class:`str`: The path to the shared library file."""
        return self._path

class DotNet(object):

    def __init__(self, dot_net_dict, path):
        """Contains the namespace_ modules, classes and `System.Type`_ objects of a .NET Assembly.

        Do not instantiate this class directly.

        .. _namespace: https://msdn.microsoft.com/en-us/library/z2kcy19k.aspx
        .. _System.Type: https://msdn.microsoft.com/en-us/library/system.type(v=vs.110).aspx
        """
        self.__dict__.update(dot_net_dict)
        self._path = path

    def __repr__(self):
        return '<{} path={}>'.format(self.__class__.__name__, self._path)
<|endoftext|>