<commit_before># -*- coding: utf-8 -*-

import logging
from functools import wraps
from flask import session, request
from sqlalchemy.exc import IntegrityError
from werkzeug.security import generate_password_hash
from werkzeug.security import check_password_hash
from .core import db, bcrypt

class Perm(object):

    class Denied(Exception):
        pass

    def __init__(self, app=None):
        self.app = app
        self.user_callback = None
        self.current_user_callback = None
        self.users_callback = None
        self.users_count_callback = None
        self._admin_logger = logging.getLogger('flask_perm.admin')
        self.registered_permissions = set()
        if app is not None:
            self.init_app(app)

    @property
    def admin_logger(self):
        return self._admin_logger

    @admin_logger.setter
    def admin_logger(self, logger):
        self._admin_logger = logger

    def init_app(self, app):
        """Initialize Perm object.
        """
        if not hasattr(app, 'extensions'):
            app.extensions = {}
        app.extensions['perm'] = self

        db.app = app
        db.init_app(app)

        bcrypt.app = app
        bcrypt.init_app(app)

        app.config.setdefault('PERM_ADMIN_PREFIX', '/perm-admin')
        app.config.setdefault('PERM_ADMIN_ECHO', False)

        from . import models
        db.create_all()

        from .api import bp as api_bp
        app.register_blueprint(api_bp, url_prefix=app.config.get('PERM_ADMIN_PREFIX') + '/api')

        from .admin import bp as admin_bp
        app.register_blueprint(admin_bp, url_prefix=app.config.get('PERM_ADMIN_PREFIX'))

    def log_admin_action(self, msg):
        '''Log msg to `flask.admin` logger.'''
        if self.app.config.get('PERM_ADMIN_ECHO'):
            self.admin_logger.info(msg)

    def require_perm_admin(self, f):
        '''A decorator that can protect function from unauthorized request.

        Used in perm admin dashboard.'''
        @wraps(f)
        def _(*args, **kwargs):
            if not session.get('perm_admin_id'):
                raise self.Denied
            return f(*args, **kwargs)
        return _

    def create_super_admin(self, email, password):
        """Create superadmin / Reset password."""
        from .services import SuperAdminService
        try:
            superadmin = SuperAdminService.create(email, password)
        except IntegrityError:
            superadmin = SuperAdminService.get_by_email(email)
            superadmin = SuperAdminService.reset_password(superadmin.id, password)
        return SuperAdminService.to_dict(superadmin)

    def login_perm_admin(self, super_admin_id):
        """Get authorization to access perm admin dashboard."""
        session['perm_admin_id'] = super_admin_id

    def logout_perm_admin(self):
        """Revoke authorization from accessing perm admin dashboard."""
        session.pop('perm_admin_id', None)

    def get_perm_admin_id_from_session(self):
        from .services import SuperAdminService
        admin_id = session.get('perm_admin_id')
        super_admin = admin_id and SuperAdminService.get(admin_id)
        return super_admin and super_admin.id

    def get_perm_admin_id_by_auth(self, email, password):
        from .services import SuperAdminService
        if SuperAdminService.verify_password(email, password):
            super_admin = SuperAdminService.get_by_email(email)
            return super_admin and super_admin.id

    def get_perm_admin_id(self):
        """Get super admin id. Both basic authorization and cookie are support."""
        if request.authorization:
            auth = request.authorization
            return self.get_perm_admin_id_by_auth(auth.username, auth.password)
        return self.get_perm_admin_id_from_session()

    def has_perm_admin_logined(self):
        """"""
        return bool(self.get_perm_admin_id())

    def user_loader(self, callback):
        """Define user loader.

        Required if you plan to use perm admin dashboard.
        The callback will be used to render user basic information in dashboard.

        Callback must take `user_id` integer parameter.
        """
        self.user_callback = callback
        return callback

    def current_user_loader(self, callback):
        """Define current user loader.

        Required if you plan to use decorator to protect your function.
        The callback will be used in deciding whether current user has authority.

        Callback takes no parameters.
        """
        self.current_user_callback = callback
        return callback

    def users_loader(self, callback):
        """Define users loader.

        Required if you plan to use perm admin dashboard.
        The callback will be used to render whole user list in dashboard.

        Callback must take 5 parameters:
        * filter_by={},
        * sort_field='created_at',
        * sort_dir='desc',
        * offset=0,
        * limit=20
        """
        self.users_callback = callback
        return callback

    def users_count_loader(self, callback):
        """Define users count loader.

        Required if you plan to use perm admin dashboard.
        The callback will be used in paginating user list.

        Callback takes no parameters."""
        self.users_count_callback = callback
        return callback

    def load_user(self, user_id):
        if self.user_callback is None:
            raise NotImplementedError('You must register user_loader!')
        return self.user_callback(user_id)

    def load_current_user(self):
        if self.current_user_callback is None:
            raise NotImplementedError('You must register current_user_loader!')
        return self.current_user_callback()

    def load_users(self, filter_by={}, sort_field='created_at', sort_dir='desc', offset=0, limit=20):
        if self.users_callback is None:
            raise NotImplementedError('You must register users_loader!')
        return self.users_callback(**dict(
            filter_by=filter_by,
            sort_field=sort_field,
            sort_dir=sort_dir,
            offset=offset,
            limit=limit,
        ))

    def load_users_count(self):
        if self.users_count_callback is None:
            raise NotImplementedError('You must register users_count_loader')
        return self.users_count_callback()

    def has_permission(self, user_id, code):
        """Decide whether a user has a permission identified by `codes`.

        Code is defined in perm admin dashboard."""
        from .services import VerificationService, PermissionService
        permission = PermissionService.get_by_code(code)
        if not permission:
            return False
        return VerificationService.has_permission(user_id, permission.id)

    def has_permissions(self, user_id, *codes):
        """Decide whether a user has permissions identified by `codes`.

        Codes are defined in perm admin dashboard."""
        if not codes:
            return True
        if '*' in codes:
            return any(
                self.has_permission(user_id, code)
                for code in self.get_all_permission_codes()
            )
        else:
            return any(self.has_permission(user_id, code) for code in codes)

    def get_user_permissions(self, user_id):
        """Define all permission codes that authorized to a user.

        Codes are defined in perm admin dashboard."""
        from .services import VerificationService, PermissionService
        permission_ids = VerificationService.get_user_permissions(user_id)
        permissions = map(PermissionService.get, permission_ids)
        permissions = filter(None, permissions)
        permissions = map(PermissionService.rest, permissions)
        return permissions

    def get_all_permission_codes(self):
        """Get all permission codes.

        WARNING: this might have performance issue."""
        from .services import PermissionService
        permissions = PermissionService.get_permissions()
        return [permission.code for permission in permissions]

    def is_user_in_groups(self, user_id, *groups):
        """Decide whether a user is in groups.

        Groups are defined in perm admin dashboard."""
        from .services import UserGroupService, UserGroupMemberService
        if not groups:
            return False
        if '*' in groups:
            user_group_ids = UserGroupService.get_all_user_group_ids()
        else:
            user_groups = UserGroupService.get_user_groups_by_codes(groups)
            user_group_ids = [user_group.id for user_group in user_groups]
        if not user_group_ids:
            return False
        return UserGroupMemberService.is_user_in_groups(user_id, user_group_ids)

    def require_group(self, *groups):
        """A decorator that can decide whether current user is in listed groups.

        Groups are defined in perm admin dashboard."""
        from .services import UserGroupService, UserGroupMemberService

        if self.current_user_callback is None:
            raise NotImplementedError('You must register current_user_loader!')

        def deco(func):
            @wraps(func)
            def _(*args, **kwargs):
                current_user = self.current_user_callback()

                if not current_user:
                    raise self.Denied

                current_user_id = current_user.id
                is_allowed = self.is_user_in_groups(current_user_id, *groups)

                if is_allowed:
                    return func(*args, **kwargs)
                else:
                    raise self.Denied
            return _
        return deco

    def require_permission(self, *codes):
        """A decorator that can decide whether current user has listed permission codes.

        Codes are defined in perm admin dashboard."""
        if self.current_user_callback is None:
            raise NotImplementedError('You must register current_user_loader!')

        for code in codes:
            self.registered_permissions.add(code)

        def deco(func):
            @wraps(func)
            def _(*args, **kwargs):

                current_user = self.current_user_callback()
                if not current_user:
                    raise self.Denied

                is_allowed = self.has_permissions(current_user.id, *codes)

                if is_allowed:
                    return func(*args, **kwargs)
                else:
                    raise self.Denied

            return _
        return deco

    def register_commands(self, flask_script_manager):
        """Register several convinient Flask-Script commands.

        WARNING: make sure you have installed Flask-Script.

        :param flask_script_manager: a flask.ext.script.Manager object.
        """
        from .script import perm_manager
        flask_script_manager.add_command('perm', perm_manager)
<commit_msg>Add require_permission in template.<commit_after># -*- coding: utf-8 -*-

import logging
from functools import wraps
from flask import session, request
from sqlalchemy.exc import IntegrityError
from werkzeug.security import generate_password_hash
from werkzeug.security import check_password_hash
from .core import db, bcrypt

class Perm(object):

    class Denied(Exception):
        pass

    def __init__(self, app=None):
        self.app = app
        self.user_callback = None
        self.current_user_callback = None
        self.users_callback = None
        self.users_count_callback = None
        self._admin_logger = logging.getLogger('flask_perm.admin')
        self.registered_permissions = set()
        if app is not None:
            self.init_app(app)

    @property
    def admin_logger(self):
        return self._admin_logger

    @admin_logger.setter
    def admin_logger(self, logger):
        self._admin_logger = logger

    def init_app(self, app):
        """Initialize Perm object.
        """
        if not hasattr(app, 'extensions'):
            app.extensions = {}
        app.extensions['perm'] = self

        db.app = app
        db.init_app(app)

        bcrypt.app = app
        bcrypt.init_app(app)

        app.config.setdefault('PERM_ADMIN_PREFIX', '/perm-admin')
        app.config.setdefault('PERM_ADMIN_ECHO', False)

        from . import models
        db.create_all()

        from .api import bp as api_bp
        app.register_blueprint(api_bp, url_prefix=app.config.get('PERM_ADMIN_PREFIX') + '/api')

        from .admin import bp as admin_bp
        app.register_blueprint(admin_bp, url_prefix=app.config.get('PERM_ADMIN_PREFIX'))

        self.register_context_processors(app)

    def log_admin_action(self, msg):
        '''Log msg to `flask.admin` logger.'''
        if self.app.config.get('PERM_ADMIN_ECHO'):
            self.admin_logger.info(msg)

    def require_perm_admin(self, f):
        '''A decorator that can protect function from unauthorized request.

        Used in perm admin dashboard.'''
        @wraps(f)
        def _(*args, **kwargs):
            if not session.get('perm_admin_id'):
                raise self.Denied
            return f(*args, **kwargs)
        return _

    def create_super_admin(self, email, password):
        """Create superadmin / Reset password."""
        from .services import SuperAdminService
        try:
            superadmin = SuperAdminService.create(email, password)
        except IntegrityError:
            superadmin = SuperAdminService.get_by_email(email)
            superadmin = SuperAdminService.reset_password(superadmin.id, password)
        return SuperAdminService.to_dict(superadmin)

    def login_perm_admin(self, super_admin_id):
        """Get authorization to access perm admin dashboard."""
        session['perm_admin_id'] = super_admin_id

    def logout_perm_admin(self):
        """Revoke authorization from accessing perm admin dashboard."""
        session.pop('perm_admin_id', None)

    def get_perm_admin_id_from_session(self):
        from .services import SuperAdminService
        admin_id = session.get('perm_admin_id')
        super_admin = admin_id and SuperAdminService.get(admin_id)
        return super_admin and super_admin.id

    def get_perm_admin_id_by_auth(self, email, password):
        from .services import SuperAdminService
        if SuperAdminService.verify_password(email, password):
            super_admin = SuperAdminService.get_by_email(email)
            return super_admin and super_admin.id

    def get_perm_admin_id(self):
        """Get super admin id. Both basic authorization and cookie are support."""
        if request.authorization:
            auth = request.authorization
            return self.get_perm_admin_id_by_auth(auth.username, auth.password)
        return self.get_perm_admin_id_from_session()

    def has_perm_admin_logined(self):
        """"""
        return bool(self.get_perm_admin_id())

    def user_loader(self, callback):
        """Define user loader.

        Required if you plan to use perm admin dashboard.
        The callback will be used to render user basic information in dashboard.

        Callback must take `user_id` integer parameter.
        """
        self.user_callback = callback
        return callback

    def current_user_loader(self, callback):
        """Define current user loader.

        Required if you plan to use decorator to protect your function.
        The callback will be used in deciding whether current user has authority.

        Callback takes no parameters.
        """
        self.current_user_callback = callback
        return callback

    def users_loader(self, callback):
        """Define users loader.

        Required if you plan to use perm admin dashboard.
        The callback will be used to render whole user list in dashboard.

        Callback must take 5 parameters:
        * filter_by={},
        * sort_field='created_at',
        * sort_dir='desc',
        * offset=0,
        * limit=20
        """
        self.users_callback = callback
        return callback

    def users_count_loader(self, callback):
        """Define users count loader.

        Required if you plan to use perm admin dashboard.
        The callback will be used in paginating user list.

        Callback takes no parameters."""
        self.users_count_callback = callback
        return callback

    def load_user(self, user_id):
        if self.user_callback is None:
            raise NotImplementedError('You must register user_loader!')
        return self.user_callback(user_id)

    def load_current_user(self):
        if self.current_user_callback is None:
            raise NotImplementedError('You must register current_user_loader!')
        return self.current_user_callback()

    def load_users(self, filter_by={}, sort_field='created_at', sort_dir='desc', offset=0, limit=20):
        if self.users_callback is None:
            raise NotImplementedError('You must register users_loader!')
        return self.users_callback(**dict(
            filter_by=filter_by,
            sort_field=sort_field,
            sort_dir=sort_dir,
            offset=offset,
            limit=limit,
        ))

    def load_users_count(self):
        if self.users_count_callback is None:
            raise NotImplementedError('You must register users_count_loader')
        return self.users_count_callback()

    def has_permission(self, user_id, code):
        """Decide whether a user has a permission identified by `codes`.

        Code is defined in perm admin dashboard."""
        from .services import VerificationService, PermissionService
        permission = PermissionService.get_by_code(code)
        if not permission:
            return False
        return VerificationService.has_permission(user_id, permission.id)

    def has_permissions(self, user_id, *codes):
        """Decide whether a user has permissions identified by `codes`.

        Codes are defined in perm admin dashboard."""
        if not codes:
            return True
        if '*' in codes:
            return any(
                self.has_permission(user_id, code)
                for code in self.get_all_permission_codes()
            )
        else:
            return any(self.has_permission(user_id, code) for code in codes)

    def get_user_permissions(self, user_id):
        """Define all permission codes that authorized to a user.

        Codes are defined in perm admin dashboard."""
        from .services import VerificationService, PermissionService
        permission_ids = VerificationService.get_user_permissions(user_id)
        permissions = map(PermissionService.get, permission_ids)
        permissions = filter(None, permissions)
        permissions = map(PermissionService.rest, permissions)
        return permissions

    def get_all_permission_codes(self):
        """Get all permission codes.

        WARNING: this might have performance issue."""
        from .services import PermissionService
        permissions = PermissionService.get_permissions()
        return [permission.code for permission in permissions]

    def is_user_in_groups(self, user_id, *groups):
        """Decide whether a user is in groups.

        Groups are defined in perm admin dashboard."""
        from .services import UserGroupService, UserGroupMemberService
        if not groups:
            return False
        if '*' in groups:
            user_group_ids = UserGroupService.get_all_user_group_ids()
        else:
            user_groups = UserGroupService.get_user_groups_by_codes(groups)
            user_group_ids = [user_group.id for user_group in user_groups]
        if not user_group_ids:
            return False
        return UserGroupMemberService.is_user_in_groups(user_id, user_group_ids)

    def require_group(self, *groups):
        """A decorator that can decide whether current user is in listed groups.

        Groups are defined in perm admin dashboard."""
        from .services import UserGroupService, UserGroupMemberService

        if self.current_user_callback is None:
            raise NotImplementedError('You must register current_user_loader!')

        def deco(func):
            @wraps(func)
            def _(*args, **kwargs):
                current_user = self.current_user_callback()

                if not current_user:
                    raise self.Denied

                current_user_id = current_user.id
                is_allowed = self.is_user_in_groups(current_user_id, *groups)

                if is_allowed:
                    return func(*args, **kwargs)
                else:
                    raise self.Denied
            return _
        return deco

    def require_permission(self, *codes):
        """A decorator that can decide whether current user has listed permission codes.

        Codes are defined in perm admin dashboard."""
        if self.current_user_callback is None:
            raise NotImplementedError('You must register current_user_loader!')

        for code in codes:
            self.registered_permissions.add(code)

        def deco(func):
            @wraps(func)
            def _(*args, **kwargs):

                current_user = self.current_user_callback()
                if not current_user:
                    raise self.Denied

                is_allowed = self.has_permissions(current_user.id, *codes)

                if is_allowed:
                    return func(*args, **kwargs)
                else:
                    raise self.Denied

            return _
        return deco

    def require_permission_in_template(self, *codes):
        """Require permission in template."""
        if self.current_user_callback is None:
            raise NotImplementedError('You must register current_user_loader!')

        for code in codes:
            self.registered_permissions.add(code)

        current_user = self.current_user_callback()

        if not current_user:
            return False

        return self.has_permissions(current_user.id, *codes)

    def default_context_processors(self):
        return {
            'require_permission': self.require_permission_in_template,
        }

    def register_commands(self, flask_script_manager):
        """Register several convinient Flask-Script commands.

        WARNING: make sure you have installed Flask-Script.

        :param flask_script_manager: a flask.ext.script.Manager object.
        """
        from .script import perm_manager
        flask_script_manager.add_command('perm', perm_manager)

    def register_context_processors(self, app):
        """Register default context processors to app.
        """
        app.context_processor(self.default_context_processors)
<|endoftext|>


<commit_before>#!/usr/bin/python

import subprocess
import os
from install_printers_config import printers

predicate_installer = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'predicate_installer.py') # in this directory
if not os.path.exists(predicate_installer):
	predicate_installer = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'predicate_installer.py') # in parent directory

# restart CUPS and make sure it's running
subprocess.call(['/bin/launchctl', 'unload', '/System/Library/LaunchDaemons/org.cups.cupsd.plist'])
subprocess.call(['/bin/launchctl', 'load', '/System/Library/LaunchDaemons/org.cups.cupsd.plist'])
subprocess.check_call(['/bin/launchctl', 'start', 'org.cups.cupsd'])

for printer in printers:
	try:
		if 'ppd' in printer and not os.path.exists(printer['ppd']) and 'predicate' in printer:
			# mark printer as connected
			subprocess.check_call(['/usr/libexec/PlistBuddy', '-c', "Add InstalledPrinters: string " + printer['model'], '/Library/Printers/InstalledPrinters.plist'])
			# install printer driver
			subprocess.call(['/usr/bin/python', predicate_installer, printer['predicate']])
		elif 'ppd' in printer and 'download' in printer:
			from distutils.version import StrictVersion
			try:
				if not os.path.exists(printer['ppd']):
					raise Exception()
				installed_version = subprocess.check_output(['/usr/bin/zgrep', '^*FileVersion:', printer['ppd']]).split(':')[1].strip().replace('"','').replace("'",'')
			except:
				installed_version = '0.0'
			if not 'version' in printer:
				printer['version'] = '0.0'
			if not os.path.exists(printer['ppd']) or StrictVersion(printer['version']) > StrictVersion(installed_version):
				dmg = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.path.basename(printer['download']))
				subprocess.check_call(['/usr/bin/curl', '-L', '-o', dmg, printer['download']])
				mount = subprocess.check_output(['/usr/bin/hdiutil', 'attach', '-plist', dmg]).split('\n')
				mountpoint = None
				for i in range(len(mount)):
					if 'mount-point' in mount[i]:
						mountpoint = mount[i+1].strip().replace('<string>','').replace('</string>','')
						break
				if mountpoint is None:
					raise Exception('Could not mount DMG')
				pkgpath = os.path.join(mountpoint, printer['pkgpath'])
				print "Installing %s" % pkgpath
				subprocess.check_call(['/usr/sbin/installer', '-package', pkgpath, '-target', '/'])
				subprocess.check_call(['/usr/bin/hdiutil', 'detach', mountpoint])
		elif 'ppd' not in printer and 'download' in printer and printer['download'].endswith('.ppd'):
			printer['ppd'] = os.path.join('/usr/share/cups/model', os.path.basename(printer['download']))
			print "Downloading %s" % os.path.basename(printer['ppd'])
			subprocess.check_call(['/usr/bin/curl', '-L', '-o', printer['ppd'], printer['download']])
		if not os.path.exists(printer['ppd']):
			raise Exception('PPD not found')
		if printer['ppd'].startswith('/usr/share/cups/model'):
			printer['ppd'] = os.path.basename(printer['ppd']) # relative path required

		# delete print queue if it already exists
		DEVNULL = open(os.devnull, 'w')
		if subprocess.call(['/usr/bin/lpstat', '-p', printer['name']], stdout=DEVNULL, stderr=DEVNULL) == 0: # printer exists
			subprocess.call(['/usr/sbin/lpadmin', '-x', printer['name']])
		DEVNULL.close()

		# add print queue
		subprocess.check_call(['/usr/sbin/lpadmin', '-p', printer['name'], '-v', printer['uri'], '-m', printer['ppd'], '-L', printer['location'], '-E', '-o', 'printer-is-shared=false'])

		# set printer options
		if 'options' in printer and len(printer['options']) > 0:
			optionstrings = []
			for k,v in printer['options'].iteritems():
				optionstrings.append('-o')
				optionstrings.append('%s=%s' % (k,v))

			subprocess.check_call(['/usr/sbin/lpadmin', '-p', printer['name']] + optionstrings)

	except Exception as e:
		print "Could not add printer %s: %s" % (printer['name'], e)

subprocess.check_call(['/bin/launchctl', 'unload', '/System/Library/LaunchDaemons/org.cups.cupsd.plist'])
with open('/etc/cups/printers.conf') as f:
	pr = f.readlines()

for printer in printers:
	ppd_path = os.path.join('/etc/cups/ppd', printer['name'] + '.ppd')
	if 'remote' in printer and printer['remote'] == True:
		section = False
		for i,line in enumerate(pr):
			if line.startswith('<Printer ' + printer['name'] + '>'):
				section = True
			elif line.startswith('</Printer>'):
				section = False

			# set CUPS_PRINTER_REMOTE in Type
			if section and line.startswith('Type '):
				pr[i] = 'Type %d\n' % (int(line[5:-1]) | 2)

		# process the job on the server instead of on the client
		f_ppd = open(ppd_path, 'a')
		f_ppd.write('\n\n')
		for mimetype in ["application/pdf", "image/*", "application/postscript", "application/vnd.cups-postscript", "application/vnd.cups-command"]:
			f_ppd.write('*cupsFilter: "%s 0 -"\n' % mimetype)
		f_ppd.close()

	if 'editppd' in printer:
		import re
		f_ppd = open(ppd_path, 'r+')
		ppd = f_ppd.read()
		for o,n in printer['editppd'].iteritems():
			ppd = re.sub(o,n, ppd)
		f_ppd.seek(0)
		f_ppd.write(ppd)
		f_ppd.close()

with open('/etc/cups/printers.conf', 'w') as f:
	f.writelines(pr)
subprocess.check_call(['/bin/launchctl', 'load', '/System/Library/LaunchDaemons/org.cups.cupsd.plist'])
<commit_msg>Printers example: clear PPD cache<commit_after>#!/usr/bin/python

import subprocess
import os
from install_printers_config import printers

predicate_installer = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'predicate_installer.py') # in this directory
if not os.path.exists(predicate_installer):
	predicate_installer = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'predicate_installer.py') # in parent directory

# restart CUPS and make sure it's running
subprocess.call(['/bin/launchctl', 'unload', '/System/Library/LaunchDaemons/org.cups.cupsd.plist'])
subprocess.call(['/bin/launchctl', 'load', '/System/Library/LaunchDaemons/org.cups.cupsd.plist'])
subprocess.check_call(['/bin/launchctl', 'start', 'org.cups.cupsd'])

for printer in printers:
	try:
		if 'ppd' in printer and not os.path.exists(printer['ppd']) and 'predicate' in printer:
			# mark printer as connected
			subprocess.check_call(['/usr/libexec/PlistBuddy', '-c', "Add InstalledPrinters: string " + printer['model'], '/Library/Printers/InstalledPrinters.plist'])
			# install printer driver
			subprocess.call(['/usr/bin/python', predicate_installer, printer['predicate']])
		elif 'ppd' in printer and 'download' in printer:
			from distutils.version import StrictVersion
			try:
				if not os.path.exists(printer['ppd']):
					raise Exception()
				installed_version = subprocess.check_output(['/usr/bin/zgrep', '^*FileVersion:', printer['ppd']]).split(':')[1].strip().replace('"','').replace("'",'')
			except:
				installed_version = '0.0'
			if not 'version' in printer:
				printer['version'] = '0.0'
			if not os.path.exists(printer['ppd']) or StrictVersion(printer['version']) > StrictVersion(installed_version):
				dmg = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.path.basename(printer['download']))
				subprocess.check_call(['/usr/bin/curl', '-L', '-o', dmg, printer['download']])
				mount = subprocess.check_output(['/usr/bin/hdiutil', 'attach', '-plist', dmg]).split('\n')
				mountpoint = None
				for i in range(len(mount)):
					if 'mount-point' in mount[i]:
						mountpoint = mount[i+1].strip().replace('<string>','').replace('</string>','')
						break
				if mountpoint is None:
					raise Exception('Could not mount DMG')
				pkgpath = os.path.join(mountpoint, printer['pkgpath'])
				print "Installing %s" % pkgpath
				subprocess.check_call(['/usr/sbin/installer', '-package', pkgpath, '-target', '/'])
				subprocess.check_call(['/usr/bin/hdiutil', 'detach', mountpoint])
		elif 'ppd' not in printer and 'download' in printer and printer['download'].endswith('.ppd'):
			printer['ppd'] = os.path.join('/usr/share/cups/model', os.path.basename(printer['download']))
			print "Downloading %s" % os.path.basename(printer['ppd'])
			subprocess.check_call(['/usr/bin/curl', '-L', '-o', printer['ppd'], printer['download']])
		if not os.path.exists(printer['ppd']):
			raise Exception('PPD not found')
		if printer['ppd'].startswith('/usr/share/cups/model'):
			printer['ppd'] = os.path.basename(printer['ppd']) # relative path required

		# delete print queue if it already exists
		DEVNULL = open(os.devnull, 'w')
		if subprocess.call(['/usr/bin/lpstat', '-p', printer['name']], stdout=DEVNULL, stderr=DEVNULL) == 0: # printer exists
			subprocess.call(['/usr/sbin/lpadmin', '-x', printer['name']])
		DEVNULL.close()

		# add print queue
		subprocess.check_call(['/usr/sbin/lpadmin', '-p', printer['name'], '-v', printer['uri'], '-m', printer['ppd'], '-L', printer['location'], '-E', '-o', 'printer-is-shared=false'])

		# set printer options
		if 'options' in printer and len(printer['options']) > 0:
			optionstrings = []
			for k,v in printer['options'].iteritems():
				optionstrings.append('-o')
				optionstrings.append('%s=%s' % (k,v))

			subprocess.check_call(['/usr/sbin/lpadmin', '-p', printer['name']] + optionstrings)

	except Exception as e:
		print "Could not add printer %s: %s" % (printer['name'], e)

subprocess.check_call(['/bin/launchctl', 'unload', '/System/Library/LaunchDaemons/org.cups.cupsd.plist'])
with open('/etc/cups/printers.conf') as f:
	pr = f.readlines()

for printer in printers:
	cache_dirty = False
	ppd_path = os.path.join('/etc/cups/ppd', printer['name'] + '.ppd')
	if 'remote' in printer and printer['remote'] == True:
		section = False
		for i,line in enumerate(pr):
			if line.startswith('<Printer ' + printer['name'] + '>'):
				section = True
			elif line.startswith('</Printer>'):
				section = False

			# set CUPS_PRINTER_REMOTE in Type
			if section and line.startswith('Type '):
				pr[i] = 'Type %d\n' % (int(line[5:-1]) | 2)

		# process the job on the server instead of on the client
		f_ppd = open(ppd_path, 'a')
		f_ppd.write('\n\n')
		for mimetype in ["application/pdf", "image/*", "application/postscript", "application/vnd.cups-postscript", "application/vnd.cups-command"]:
			f_ppd.write('*cupsFilter: "%s 0 -"\n' % mimetype)
		f_ppd.close()
		cache_dirty = True

	if 'editppd' in printer:
		import re
		f_ppd = open(ppd_path, 'r+')
		ppd = f_ppd.read()
		for o,n in printer['editppd'].iteritems():
			ppd = re.sub(o,n, ppd)
		f_ppd.seek(0)
		f_ppd.write(ppd)
		f_ppd.close()
		cache_dirty = True

	if cache_dirty: # need to delete the PPD cache. OS X 10.8 appears to have done that automatically, but 10.9 keeps using the old cupsFilter list if we don't do this
		os.unlink('/var/spool/cups/cache/' + printer['name'] + '.data')

with open('/etc/cups/printers.conf', 'w') as f:
	f.writelines(pr)
subprocess.check_call(['/bin/launchctl', 'load', '/System/Library/LaunchDaemons/org.cups.cupsd.plist'])
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Hive Netius System
# Copyright (c) 2008-2015 Hive Solutions Lda.
#
# This file is part of Hive Netius System.
#
# Hive Netius System is free software: you can redistribute it and/or modify
# it under the terms of the Apache License as published by the Apache
# Foundation, either version 2.0 of the License, or (at your option) any
# later version.
#
# Hive Netius System is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Apache License for more details.
#
# You should have received a copy of the Apache License along with
# Hive Netius System. If not, see <http://www.apache.org/licenses/>.

__author__ = "João Magalhães <joamag@hive.pt>"
""" The author(s) of the module """

__version__ = "1.0.0"
""" The version of the module """

__revision__ = "$LastChangedRevision$"
""" The revision number of the module """

__date__ = "$LastChangedDate$"
""" The last change date of the module """

__copyright__ = "Copyright (c) 2008-2015 Hive Solutions Lda."
""" The copyright for the module """

__license__ = "Apache License, Version 2.0"
""" The license for the module """

import re
import time

import netius.common
import netius.servers

class ReverseProxyServer(netius.servers.ProxyServer):
    """
    Reverse HTTP proxy implementation based on the more generalized
    infra-structure of proxy server.

    Supports multiple scheduling strategies, being the "smart" one
    the most efficient for most of the possible scenarios.

    Conditional re-usage of connections based on boolean flag is possible,
    but care should be taken when using re-usage so that no multiple
    rules are applied for the same connection (eg: https://host.com/hive,
    https://host.com/colony as different rules), this would pose serious
    problems if the back-end servers are different for each rule or if
    the way the final back-end url is created is different for each rule.
    """

    def __init__(
        self,
        config = "proxy.json",
        regex = {},
        hosts = {},
        auth = {},
        strategy = "smart",
        reuse = True,
        *args,
        **kwargs
    ):
        netius.servers.ProxyServer.__init__(self, *args, **kwargs)
        if type(regex) == dict: regex = regex.items()
        if not type(hosts) == dict: hosts = dict(hosts)
        self.load_config(
            path = config,
            regex = regex,
            hosts = hosts,
            auth = auth,
            strategy = strategy,
            reuse = reuse,
            robin = dict(),
            smart = netius.common.PriorityDict()
        )
        self.busy_conn = 0
        self.balancer_m = getattr(self, "balancer_" + self.strategy)
        self.acquirer_m = getattr(self, "acquirer_" + self.strategy)
        self.releaser_m = getattr(self, "releaser_" + self.strategy)

    def info_dict(self, full = False):
        info = netius.servers.ProxyServer.info_dict(self, full = full)
        info.update(
            reuse = self.reuse,
            strategy = self.strategy,
            busy_conn = self.busy_conn
        )
        return info

    def on_headers(self, connection, parser):
        netius.servers.ProxyServer.on_headers(self, connection, parser)

        # retrieves the various parts/configuration values of the parser, that
        # are going to be used in the processing/routing of the proxy request,
        # note that some of these values need to be transformed to be used
        method = parser.method.upper()
        path = parser.path_s
        headers = parser.headers
        version_s = parser.version_s
        is_secure = connection.ssl
        host = headers.get("host", None)
        protocol = headers.get("x-forwarded-proto", None)
        print(headers)
        print(protocol)
        print("---------------")
        protocol = protocol or ("https" if is_secure else "http")
        print(protocol)
        print("###################")

        # tries to extract the various attributes of the current connection
        # that are going to be used for the routing operation, this attributes
        # should avoid a new rule setting operation (expensive) and provide
        # connection, prefix and state re-usage whenever possible, note that
        # under some situation the prefix may change for the same connection
        # and so the connection re-usage is not possible
        prefix = connection.prefix if hasattr(connection, "prefix") else None
        state = connection.state if hasattr(connection, "state") else None
        reusable = hasattr(connection, "proxy_c")

        # constructs the url that is going to be used by the rule engine and
        # then "forwards" both the url and the parser object to the rule engine
        # in order to obtain the possible prefix value for url reconstruction,
        # a state value is also retrieved, this value will be used latter for
        # the acquiring and releasing parts of the balancing strategy operation
        url = "%s://%s%s" % (protocol, host, path)
        if not self.reuse or not reusable: prefix, state = self.rules(url, parser)

        # in case no prefix is defined at this stage there's no matching
        # against the currently defined rules and so an error must be raised
        # and propagated to the client connection (end user notification)
        if not prefix:
            self.debug("No valid proxy endpoint found")
            connection.send_response(
                data = "No valid proxy endpoint found",
                headers = dict(
                    connection = "close"
                ),
                version = version_s,
                code = 404,
                code_s = "Not Found",
                apply = True,
                callback = self._prx_close
            )
            return

        # verifies if the current host requires come kind of special authorization
        # process using the default basic http authorization process
        auth = self.auth.get(host.split(":", 1)[0], None)
        if auth:
            # tries to retrieve the authorization header for the current request
            # and in case it's not defined "raises" a non authorized response
            authorization = headers.get("authorization", None)
            if not authorization:
                connection.send_response(
                    data = "Not authorized",
                    headers = {
                        "connection" : "close",
                        "wWW-authenticate" : "Basic realm=\"insert realm\""
                    },
                    version = version_s,
                    code = 401,
                    code_s = "Not Authorized",
                    apply = True,
                    callback = self._prx_close
                )
                return

            # runs the authorization process for the requested auth value (file path)
            # and using the current parser for the connection handling, this should
            # return a valid response in case the authorization is valid and an invalid
            # value otherwise, in case there's a failure a not authorized value is returned
            result = self.authorize(auth, parser)
            if not result:
                connection.send_response(
                    data = "Not authorized",
                    headers = {
                        "connection" : "close",
                        "wWW-authenticate" : "Basic realm=\"insert realm\""
                    },
                    version = version_s,
                    code = 401,
                    code_s = "Not Authorized",
                    apply = True,
                    callback = self._prx_close
                )
                return

        # runs the acquire operation for the current state, this should
        # update the current scheduling algorithm internal structures so
        # that they properly handle the new handle operation, an inverse
        # release operation should be performed at the end of the handling
        self.acquirer(state)

        # re-calculates the url for the reverse connection based on the
        # prefix value that has just been "resolved" using the rule engine
        # this value should be constructed based on the original path
        url = prefix + path

        # updates the various headers that are relates with the reverse
        # proxy operation this is required so that the request gets
        # properly handled by the reverse server, otherwise some problems
        # would occur in the operation of handling the request
        headers["x-forwarded-proto"] = protocol

        # verifies if the current connection already contains a valid
        # a proxy connection if that's the case that must be unset from
        # the connection and from the connection map internal structures
        # at least until the http client returns from the method call
        proxy_c = hasattr(connection, "proxy_c") and connection.proxy_c
        proxy_c = proxy_c or None
        connection.proxy_c = None
        if proxy_c in self.conn_map: del self.conn_map[proxy_c]

        # calls the proper (http) method in the client this should acquire
        # a new connection and start the process of sending the request
        # to the associated http server (request handling)
        _connection = self.http_client.method(
            method,
            url,
            headers = headers,
            version = version_s,
            encodings = None,
            connection = proxy_c
        )

        # sets the state attribute in the connection so that it's possible
        # to retrieve it latter for tagging evaluation, this is required for
        # advanced load balancing techniques to be performed
        _connection.state = state

        # sets the current connection as busy, as it's waiting for a message
        # to be returned from the the back-end side, then increments the number
        # of currently busy connections (as expected)
        _connection.busy = _connection.busy if hasattr(_connection, "busy") else 0
        _connection.busy += 1
        self.busy_conn += 1

        # prints a debug message about the connection becoming a waiting
        # connection meaning that the connection with the client host has
        # not been yet established (no data has been  received)
        self.debug("Setting connection as waiting, proxy connection loading ...")

        # sets the current http back-end client connection as waiting and then
        # maps it as the proxy connection in the connection and also creates
        # the reverse mapping using the connection map of the current server
        _connection.waiting = True
        connection.proxy_c = _connection
        connection.prefix = prefix
        connection.state = state
        self.conn_map[_connection] = connection

    def rules(self, url, parser):
        resolved = self.rules_regex(url, parser)
        if resolved[0]: return resolved
        resolved = self.rules_host(url, parser)
        if resolved[0]: return resolved
        return None, None

    def rules_regex(self, url, parser):
        # sets the prefix and state values initially for the invalid/unset value,
        # these values are going to be populated once a valid match is found
        prefix = None
        state = None

        # iterates over the complete set of defined regex values to try
        # to find a valid match and apply the groups value for format
        # if the complete chain of regex is processed but there's no
        # valid match the prefix value is considered to be unset
        for regex, _prefix in self.regex:
            match = regex.match(url)
            if not match: continue
            _prefix, _state = self.balancer(_prefix)
            groups = match.groups()
            if groups: _prefix = _prefix.format(*groups)
            prefix = _prefix
            state = _state
            break

        # returns the prefix and state values that have just been resolved
        # through regex based validation, this value may be unset for a mismatch
        return prefix, state

    def rules_host(self, url, parser):
        # retrieves the reference to the headers map from the parser so
        # that it may be used to retrieve the current host value and try
        # to match it against any of the currently defined rules
        headers = parser.headers

        # retrieves the host header from the received set of headers,
        # removing the port definition part of the host and then verifies
        # the complete set of defined hosts in order to check for the
        # presence of such rule, in case there's a match the defined
        # url prefix is going to be used instead, the balancer operation
        # is then used to "resolve" the final prefix value from sequence
        host = headers.get("host", None)
        if host: host = host.split(":", 1)[0]
        prefix = self.hosts.get(host, None)
        resolved = self.balancer(prefix)

        # returns the final "resolved" prefix value (in case there's any)
        # to the caller method, this should be used for url reconstruction
        # note that the state value is also returned and should be store in
        # the current handling connection so that it may latter be used
        return resolved

    def balancer(self, values):
        is_sequence = type(values) in (list, tuple)
        if not is_sequence: return values, None
        return self.balancer_m(values)

    def balancer_robin(self, values):
        index = self.robin.get(values, 0)
        prefix = values[index]
        next = 0 if index + 1 == len(values) else index + 1
        self.robin[values] = next
        return prefix, None

    def balancer_smart(self, values):
        queue = self.smart.get(values, None)
        if not queue:
            queue = netius.common.PriorityDict()
            for value in values: queue[value] = [0, 0]
            self.smart[values] = queue

        prefix = queue.smallest()

        return prefix, (prefix, queue)

    def acquirer(self, state):
        self.acquirer_m(state)

    def acquirer_robin(self, state):
        pass

    def acquirer_smart(self, state):
        if not state: return
        prefix, queue = state
        sorter = queue[prefix]
        sorter[0] += 1
        queue[prefix] = sorter

    def releaser(self, state):
        self.releaser_m(state)

    def releaser_robin(self, state):
        pass

    def releaser_smart(self, state):
        if not state: return
        prefix, queue = state
        sorter = queue[prefix]
        sorter[0] -= 1
        if sorter[0] == 0: sorter[1] = time.time() * -1
        queue[prefix] = sorter

    def _on_prx_message(self, client, parser, message):
        _connection = parser.owner
        busy = _connection.busy if hasattr(_connection, "busy") else 0
        state = _connection.state if hasattr(_connection, "state") else None
        if busy: self.busy_conn -= 1; _connection.busy -= 1
        if state: self.releaser(state); _connection.state = None
        netius.servers.ProxyServer._on_prx_message(self, client, parser, message)

    def _on_prx_close(self, client, _connection):
        busy = _connection.busy if hasattr(_connection, "busy") else 0
        state = _connection.state if hasattr(_connection, "state") else None
        if busy: self.busy_conn -= busy; _connection.busy -= busy
        if state: self.releaser(state); _connection.state = None
        netius.servers.ProxyServer._on_prx_close(self, client, _connection)

if __name__ == "__main__":
    import logging
    regex = (
        (re.compile(r"https://host\.com"), "http://localhost"),
        (re.compile(r"https://([a-zA-Z]*)\.host\.com"), "http://localhost/{0}")
    )
    hosts = {
        "host.com" : "http://localhost"
    }
    auth = {
        "host.com" : netius.PasswdAuth("extras/htpasswd")
    }
    server = ReverseProxyServer(
        regex = regex,
        hosts = hosts,
        level = logging.INFO
    )
    server.serve(env = True)
<commit_msg>removed extra prints<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Hive Netius System
# Copyright (c) 2008-2015 Hive Solutions Lda.
#
# This file is part of Hive Netius System.
#
# Hive Netius System is free software: you can redistribute it and/or modify
# it under the terms of the Apache License as published by the Apache
# Foundation, either version 2.0 of the License, or (at your option) any
# later version.
#
# Hive Netius System is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Apache License for more details.
#
# You should have received a copy of the Apache License along with
# Hive Netius System. If not, see <http://www.apache.org/licenses/>.

__author__ = "João Magalhães <joamag@hive.pt>"
""" The author(s) of the module """

__version__ = "1.0.0"
""" The version of the module """

__revision__ = "$LastChangedRevision$"
""" The revision number of the module """

__date__ = "$LastChangedDate$"
""" The last change date of the module """

__copyright__ = "Copyright (c) 2008-2015 Hive Solutions Lda."
""" The copyright for the module """

__license__ = "Apache License, Version 2.0"
""" The license for the module """

import re
import time

import netius.common
import netius.servers

class ReverseProxyServer(netius.servers.ProxyServer):
    """
    Reverse HTTP proxy implementation based on the more generalized
    infra-structure of proxy server.

    Supports multiple scheduling strategies, being the "smart" one
    the most efficient for most of the possible scenarios.

    Conditional re-usage of connections based on boolean flag is possible,
    but care should be taken when using re-usage so that no multiple
    rules are applied for the same connection (eg: https://host.com/hive,
    https://host.com/colony as different rules), this would pose serious
    problems if the back-end servers are different for each rule or if
    the way the final back-end url is created is different for each rule.
    """

    def __init__(
        self,
        config = "proxy.json",
        regex = {},
        hosts = {},
        auth = {},
        strategy = "smart",
        reuse = True,
        *args,
        **kwargs
    ):
        netius.servers.ProxyServer.__init__(self, *args, **kwargs)
        if type(regex) == dict: regex = regex.items()
        if not type(hosts) == dict: hosts = dict(hosts)
        self.load_config(
            path = config,
            regex = regex,
            hosts = hosts,
            auth = auth,
            strategy = strategy,
            reuse = reuse,
            robin = dict(),
            smart = netius.common.PriorityDict()
        )
        self.busy_conn = 0
        self.balancer_m = getattr(self, "balancer_" + self.strategy)
        self.acquirer_m = getattr(self, "acquirer_" + self.strategy)
        self.releaser_m = getattr(self, "releaser_" + self.strategy)

    def info_dict(self, full = False):
        info = netius.servers.ProxyServer.info_dict(self, full = full)
        info.update(
            reuse = self.reuse,
            strategy = self.strategy,
            busy_conn = self.busy_conn
        )
        return info

    def on_headers(self, connection, parser):
        netius.servers.ProxyServer.on_headers(self, connection, parser)

        # retrieves the various parts/configuration values of the parser, that
        # are going to be used in the processing/routing of the proxy request,
        # note that some of these values need to be transformed to be used
        method = parser.method.upper()
        path = parser.path_s
        headers = parser.headers
        version_s = parser.version_s
        is_secure = connection.ssl
        host = headers.get("host", None)
        protocol = headers.get("x-forwarded-proto", None)
        protocol = protocol or ("https" if is_secure else "http")

        # tries to extract the various attributes of the current connection
        # that are going to be used for the routing operation, this attributes
        # should avoid a new rule setting operation (expensive) and provide
        # connection, prefix and state re-usage whenever possible, note that
        # under some situation the prefix may change for the same connection
        # and so the connection re-usage is not possible
        prefix = connection.prefix if hasattr(connection, "prefix") else None
        state = connection.state if hasattr(connection, "state") else None
        reusable = hasattr(connection, "proxy_c")

        # constructs the url that is going to be used by the rule engine and
        # then "forwards" both the url and the parser object to the rule engine
        # in order to obtain the possible prefix value for url reconstruction,
        # a state value is also retrieved, this value will be used latter for
        # the acquiring and releasing parts of the balancing strategy operation
        url = "%s://%s%s" % (protocol, host, path)
        if not self.reuse or not reusable: prefix, state = self.rules(url, parser)

        # in case no prefix is defined at this stage there's no matching
        # against the currently defined rules and so an error must be raised
        # and propagated to the client connection (end user notification)
        if not prefix:
            self.debug("No valid proxy endpoint found")
            connection.send_response(
                data = "No valid proxy endpoint found",
                headers = dict(
                    connection = "close"
                ),
                version = version_s,
                code = 404,
                code_s = "Not Found",
                apply = True,
                callback = self._prx_close
            )
            return

        # verifies if the current host requires come kind of special authorization
        # process using the default basic http authorization process
        auth = self.auth.get(host.split(":", 1)[0], None)
        if auth:
            # tries to retrieve the authorization header for the current request
            # and in case it's not defined "raises" a non authorized response
            authorization = headers.get("authorization", None)
            if not authorization:
                connection.send_response(
                    data = "Not authorized",
                    headers = {
                        "connection" : "close",
                        "wWW-authenticate" : "Basic realm=\"insert realm\""
                    },
                    version = version_s,
                    code = 401,
                    code_s = "Not Authorized",
                    apply = True,
                    callback = self._prx_close
                )
                return

            # runs the authorization process for the requested auth value (file path)
            # and using the current parser for the connection handling, this should
            # return a valid response in case the authorization is valid and an invalid
            # value otherwise, in case there's a failure a not authorized value is returned
            result = self.authorize(auth, parser)
            if not result:
                connection.send_response(
                    data = "Not authorized",
                    headers = {
                        "connection" : "close",
                        "wWW-authenticate" : "Basic realm=\"insert realm\""
                    },
                    version = version_s,
                    code = 401,
                    code_s = "Not Authorized",
                    apply = True,
                    callback = self._prx_close
                )
                return

        # runs the acquire operation for the current state, this should
        # update the current scheduling algorithm internal structures so
        # that they properly handle the new handle operation, an inverse
        # release operation should be performed at the end of the handling
        self.acquirer(state)

        # re-calculates the url for the reverse connection based on the
        # prefix value that has just been "resolved" using the rule engine
        # this value should be constructed based on the original path
        url = prefix + path

        # updates the various headers that are relates with the reverse
        # proxy operation this is required so that the request gets
        # properly handled by the reverse server, otherwise some problems
        # would occur in the operation of handling the request
        headers["x-forwarded-proto"] = protocol

        # verifies if the current connection already contains a valid
        # a proxy connection if that's the case that must be unset from
        # the connection and from the connection map internal structures
        # at least until the http client returns from the method call
        proxy_c = hasattr(connection, "proxy_c") and connection.proxy_c
        proxy_c = proxy_c or None
        connection.proxy_c = None
        if proxy_c in self.conn_map: del self.conn_map[proxy_c]

        # calls the proper (http) method in the client this should acquire
        # a new connection and start the process of sending the request
        # to the associated http server (request handling)
        _connection = self.http_client.method(
            method,
            url,
            headers = headers,
            version = version_s,
            encodings = None,
            connection = proxy_c
        )

        # sets the state attribute in the connection so that it's possible
        # to retrieve it latter for tagging evaluation, this is required for
        # advanced load balancing techniques to be performed
        _connection.state = state

        # sets the current connection as busy, as it's waiting for a message
        # to be returned from the the back-end side, then increments the number
        # of currently busy connections (as expected)
        _connection.busy = _connection.busy if hasattr(_connection, "busy") else 0
        _connection.busy += 1
        self.busy_conn += 1

        # prints a debug message about the connection becoming a waiting
        # connection meaning that the connection with the client host has
        # not been yet established (no data has been  received)
        self.debug("Setting connection as waiting, proxy connection loading ...")

        # sets the current http back-end client connection as waiting and then
        # maps it as the proxy connection in the connection and also creates
        # the reverse mapping using the connection map of the current server
        _connection.waiting = True
        connection.proxy_c = _connection
        connection.prefix = prefix
        connection.state = state
        self.conn_map[_connection] = connection

    def rules(self, url, parser):
        resolved = self.rules_regex(url, parser)
        if resolved[0]: return resolved
        resolved = self.rules_host(url, parser)
        if resolved[0]: return resolved
        return None, None

    def rules_regex(self, url, parser):
        # sets the prefix and state values initially for the invalid/unset value,
        # these values are going to be populated once a valid match is found
        prefix = None
        state = None

        # iterates over the complete set of defined regex values to try
        # to find a valid match and apply the groups value for format
        # if the complete chain of regex is processed but there's no
        # valid match the prefix value is considered to be unset
        for regex, _prefix in self.regex:
            match = regex.match(url)
            if not match: continue
            _prefix, _state = self.balancer(_prefix)
            groups = match.groups()
            if groups: _prefix = _prefix.format(*groups)
            prefix = _prefix
            state = _state
            break

        # returns the prefix and state values that have just been resolved
        # through regex based validation, this value may be unset for a mismatch
        return prefix, state

    def rules_host(self, url, parser):
        # retrieves the reference to the headers map from the parser so
        # that it may be used to retrieve the current host value and try
        # to match it against any of the currently defined rules
        headers = parser.headers

        # retrieves the host header from the received set of headers,
        # removing the port definition part of the host and then verifies
        # the complete set of defined hosts in order to check for the
        # presence of such rule, in case there's a match the defined
        # url prefix is going to be used instead, the balancer operation
        # is then used to "resolve" the final prefix value from sequence
        host = headers.get("host", None)
        if host: host = host.split(":", 1)[0]
        prefix = self.hosts.get(host, None)
        resolved = self.balancer(prefix)

        # returns the final "resolved" prefix value (in case there's any)
        # to the caller method, this should be used for url reconstruction
        # note that the state value is also returned and should be store in
        # the current handling connection so that it may latter be used
        return resolved

    def balancer(self, values):
        is_sequence = type(values) in (list, tuple)
        if not is_sequence: return values, None
        return self.balancer_m(values)

    def balancer_robin(self, values):
        index = self.robin.get(values, 0)
        prefix = values[index]
        next = 0 if index + 1 == len(values) else index + 1
        self.robin[values] = next
        return prefix, None

    def balancer_smart(self, values):
        queue = self.smart.get(values, None)
        if not queue:
            queue = netius.common.PriorityDict()
            for value in values: queue[value] = [0, 0]
            self.smart[values] = queue

        prefix = queue.smallest()

        return prefix, (prefix, queue)

    def acquirer(self, state):
        self.acquirer_m(state)

    def acquirer_robin(self, state):
        pass

    def acquirer_smart(self, state):
        if not state: return
        prefix, queue = state
        sorter = queue[prefix]
        sorter[0] += 1
        queue[prefix] = sorter

    def releaser(self, state):
        self.releaser_m(state)

    def releaser_robin(self, state):
        pass

    def releaser_smart(self, state):
        if not state: return
        prefix, queue = state
        sorter = queue[prefix]
        sorter[0] -= 1
        if sorter[0] == 0: sorter[1] = time.time() * -1
        queue[prefix] = sorter

    def _on_prx_message(self, client, parser, message):
        _connection = parser.owner
        busy = _connection.busy if hasattr(_connection, "busy") else 0
        state = _connection.state if hasattr(_connection, "state") else None
        if busy: self.busy_conn -= 1; _connection.busy -= 1
        if state: self.releaser(state); _connection.state = None
        netius.servers.ProxyServer._on_prx_message(self, client, parser, message)

    def _on_prx_close(self, client, _connection):
        busy = _connection.busy if hasattr(_connection, "busy") else 0
        state = _connection.state if hasattr(_connection, "state") else None
        if busy: self.busy_conn -= busy; _connection.busy -= busy
        if state: self.releaser(state); _connection.state = None
        netius.servers.ProxyServer._on_prx_close(self, client, _connection)

if __name__ == "__main__":
    import logging
    regex = (
        (re.compile(r"https://host\.com"), "http://localhost"),
        (re.compile(r"https://([a-zA-Z]*)\.host\.com"), "http://localhost/{0}")
    )
    hosts = {
        "host.com" : "http://localhost"
    }
    auth = {
        "host.com" : netius.PasswdAuth("extras/htpasswd")
    }
    server = ReverseProxyServer(
        regex = regex,
        hosts = hosts,
        level = logging.INFO
    )
    server.serve(env = True)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os.path

from flask import Response, g, jsonify, redirect, render_template

from coaster.views import ClassView, jsonp, load_model, render_with, route

from .. import app, funnelapp, pages
from ..forms import SavedProjectForm
from ..models import Project, Proposal, db
from .project import project_data

@route('/')
class IndexView(ClassView):
    current_section = 'home'

    @render_with('index.html.jinja2')
    def home(self):
        g.profile = None
        projects = Project.all_unsorted(legacy=False)
        # TODO: Move these queries into the Project class
        all_projects = (
            projects.filter(
                Project.state.PUBLISHED,
                db.or_(Project.schedule_state.LIVE, Project.schedule_state.UPCOMING),
            )
            .order_by(Project.schedule_start_at.asc())
            .all()
        )
        upcoming_projects = all_projects[:3]
        all_projects = all_projects[3:]
        featured_project = (
            projects.filter(
                Project.state.PUBLISHED,
                db.or_(Project.schedule_state.LIVE, Project.schedule_state.UPCOMING),
                Project.featured.is_(True),
            )
            .order_by(Project.schedule_start_at.asc())
            .limit(1)
            .first()
        )
        if featured_project in upcoming_projects:
            upcoming_projects.remove(featured_project)
            upcoming_projects.insert(all_projects.pop(0))
        open_cfp_projects = (
            projects.filter(Project.state.PUBLISHED, Project.cfp_state.OPEN)
            .order_by(Project.schedule_start_at.asc())
            .all()
        )
        past_projects = (
            projects.filter(Project.state.PUBLISHED, Project.schedule_state.PAST)
            .order_by(Project.schedule_start_at.desc())
            .all()
        )

        return {
            'all_projects': [p.current_access() for p in all_projects],
            'upcoming_projects': [p.current_access() for p in upcoming_projects],
            'open_cfp_projects': [p.current_access() for p in open_cfp_projects],
            'featured_project': (
                featured_project.current_access() if featured_project else None
            ),
            'past_projects': [p.current_access() for p in past_projects],
            'project_save_form': SavedProjectForm(),
        }

@route('/')
class FunnelIndexView(ClassView):
    @render_with('funnelindex.html.jinja2')
    def home(self):
        g.profile = None
        projects = Project.fetch_sorted(legacy=True).all()  # NOQA
        return {'projects': projects}

IndexView.add_route_for('home', '', endpoint='index')
IndexView.init_app(app)
FunnelIndexView.add_route_for('home', '', endpoint='index')
FunnelIndexView.init_app(funnelapp)

@app.route('/api/whoami')
@funnelapp.route('/api/whoami')
def whoami():
    if g.user:
        return jsonify(message="Hey {0}!".format(g.user.fullname), code=200)
    else:
        return jsonify(message="Hmm, so who _are_ you?", code=401)

@app.route('/json')
def all_projects_json():
    g.profile = None
    projects = Project.fetch_sorted(legacy=False).all()  # NOQA
    return jsonp(
        projects=list(map(project_data, projects)),
        spaces=list(map(project_data, projects)),
    )  # FIXME: Remove when the native app switches over

@funnelapp.route('/json')
def funnelapp_all_projects_json():
    g.profile = None
    projects = Project.fetch_sorted().all()  # NOQA
    return jsonp(
        projects=list(map(project_data, projects)),
        spaces=list(map(project_data, projects)),
    )  # FIXME: Remove when the native app switches over

@funnelapp.route('/<project>/<int:id>-<name>')
@funnelapp.route('/<project>/<int:id>')
@load_model(Proposal, {'id': 'id'}, 'proposal')
def proposal_redirect(proposal):
    return redirect(proposal.url_for())

@app.route('/about/', defaults={'path': 'index'})
@app.route('/about/policy/', defaults={'path': 'policy/index'})
@app.route('/about/<path:path>')
def about(path):
    return render_template(
        'about.html.jinja2', page=pages.get_or_404(os.path.join('about', path))
    )

@app.route('/api/1/template/offline')
def offline():
    return render_template('offline.html.jinja2')

@app.route('/service-worker.js', methods=['GET'])
def sw():
    return app.send_static_file('service-worker.js')

@app.route('/manifest.json', methods=['GET'])
def manifest():
    return Response(
        render_template('manifest.json.jinja2'), mimetype='application/json'
    )

@app.route('/opensearch.xml')
def opensearch():
    return render_template('opensearch.xml.jinja2')
<commit_msg>fix upcoming project list<commit_after># -*- coding: utf-8 -*-

import os.path

from flask import Response, g, jsonify, redirect, render_template

from coaster.views import ClassView, jsonp, load_model, render_with, route

from .. import app, funnelapp, pages
from ..forms import SavedProjectForm
from ..models import Project, Proposal, db
from .project import project_data

@route('/')
class IndexView(ClassView):
    current_section = 'home'

    @render_with('index.html.jinja2')
    def home(self):
        g.profile = None
        projects = Project.all_unsorted(legacy=False)
        # TODO: Move these queries into the Project class
        all_projects = (
            projects.filter(
                Project.state.PUBLISHED,
                db.or_(Project.schedule_state.LIVE, Project.schedule_state.UPCOMING),
            )
            .order_by(Project.schedule_start_at.asc())
            .all()
        )
        upcoming_projects = all_projects[:3]
        all_projects = all_projects[3:]
        featured_project = (
            projects.filter(
                Project.state.PUBLISHED,
                db.or_(Project.schedule_state.LIVE, Project.schedule_state.UPCOMING),
                Project.featured.is_(True),
            )
            .order_by(Project.schedule_start_at.asc())
            .limit(1)
            .first()
        )
        if featured_project in upcoming_projects:
            upcoming_projects.remove(featured_project)
            upcoming_projects.append(all_projects.pop(0))
        open_cfp_projects = (
            projects.filter(Project.state.PUBLISHED, Project.cfp_state.OPEN)
            .order_by(Project.schedule_start_at.asc())
            .all()
        )
        past_projects = (
            projects.filter(Project.state.PUBLISHED, Project.schedule_state.PAST)
            .order_by(Project.schedule_start_at.desc())
            .all()
        )

        return {
            'all_projects': [p.current_access() for p in all_projects],
            'upcoming_projects': [p.current_access() for p in upcoming_projects],
            'open_cfp_projects': [p.current_access() for p in open_cfp_projects],
            'featured_project': (
                featured_project.current_access() if featured_project else None
            ),
            'past_projects': [p.current_access() for p in past_projects],
            'project_save_form': SavedProjectForm(),
        }

@route('/')
class FunnelIndexView(ClassView):
    @render_with('funnelindex.html.jinja2')
    def home(self):
        g.profile = None
        projects = Project.fetch_sorted(legacy=True).all()  # NOQA
        return {'projects': projects}

IndexView.add_route_for('home', '', endpoint='index')
IndexView.init_app(app)
FunnelIndexView.add_route_for('home', '', endpoint='index')
FunnelIndexView.init_app(funnelapp)

@app.route('/api/whoami')
@funnelapp.route('/api/whoami')
def whoami():
    if g.user:
        return jsonify(message="Hey {0}!".format(g.user.fullname), code=200)
    else:
        return jsonify(message="Hmm, so who _are_ you?", code=401)

@app.route('/json')
def all_projects_json():
    g.profile = None
    projects = Project.fetch_sorted(legacy=False).all()  # NOQA
    return jsonp(
        projects=list(map(project_data, projects)),
        spaces=list(map(project_data, projects)),
    )  # FIXME: Remove when the native app switches over

@funnelapp.route('/json')
def funnelapp_all_projects_json():
    g.profile = None
    projects = Project.fetch_sorted().all()  # NOQA
    return jsonp(
        projects=list(map(project_data, projects)),
        spaces=list(map(project_data, projects)),
    )  # FIXME: Remove when the native app switches over

@funnelapp.route('/<project>/<int:id>-<name>')
@funnelapp.route('/<project>/<int:id>')
@load_model(Proposal, {'id': 'id'}, 'proposal')
def proposal_redirect(proposal):
    return redirect(proposal.url_for())

@app.route('/about/', defaults={'path': 'index'})
@app.route('/about/policy/', defaults={'path': 'policy/index'})
@app.route('/about/<path:path>')
def about(path):
    return render_template(
        'about.html.jinja2', page=pages.get_or_404(os.path.join('about', path))
    )

@app.route('/api/1/template/offline')
def offline():
    return render_template('offline.html.jinja2')

@app.route('/service-worker.js', methods=['GET'])
def sw():
    return app.send_static_file('service-worker.js')

@app.route('/manifest.json', methods=['GET'])
def manifest():
    return Response(
        render_template('manifest.json.jinja2'), mimetype='application/json'
    )

@app.route('/opensearch.xml')
def opensearch():
    return render_template('opensearch.xml.jinja2')
<|endoftext|>


<commit_before>import json
import requests
from application import app
from application.utils import get_mp_data, constituency_extent, constituency_collection
from flask import render_template, request
from operator import itemgetter

@app.route('/', methods=["GET"])
def index():
    return render_template('index.html')

@app.route('/petitions', methods=["GET"])
def petitions():
    args = request.args.items()
    url = 'https://petition.parliament.uk/petitions.json?' + ''.join("%s=%s&" % tup for tup in args)
    page = request.args.get('page')
    r = requests.get(url)
    data = json.loads(r.text)
    return render_template('petitions.html', data=data, page=page, args=args)

@app.route('/petitions/<id>', methods=["GET"])
def petition(id):
    url = 'https://petition.parliament.uk/petitions/' + id + '.json'
    r = requests.get(url)
    data = json.loads(r.text)

    countries = data['data']['attributes']['signatures_by_country']
    sorted_countries = sorted(countries, key=itemgetter('signature_count'), reverse=True)

    constituencies = data['data']['attributes']['signatures_by_constituency']
    sorted_constituencies = sorted(constituencies, key=itemgetter('signature_count'), reverse=True)

    extents = constituency_collection(sorted_constituencies)

    return render_template('petition.html',
        data=data,
        countries=sorted_countries,
        constituencies=sorted_constituencies,
        extents=extents)

@app.route('/petitions/<id>/map', methods=["GET"])
def map(id):
    url = 'https://petition.parliament.uk/petitions/' + id + '.json'
    r = requests.get(url)
    data = json.loads(r.text)

    constituencies = data['data']['attributes']['signatures_by_constituency']
    sorted_constituencies = sorted(constituencies, key=itemgetter('signature_count'), reverse=True)

    extents = constituency_collection(sorted_constituencies)

    return render_template('map.html',
        extents=extents)
<commit_msg>mp data routing<commit_after>import json
import requests
from application import app
from application.utils import get_mp, constituency_extent, constituency_collection
from flask import render_template, request
from operator import itemgetter

@app.route('/', methods=["GET"])
def index():
    return render_template('index.html')

@app.route('/petitions', methods=["GET"])
def petitions():
    args = request.args.items()
    url = 'https://petition.parliament.uk/petitions.json?' + ''.join("%s=%s&" % tup for tup in args)
    page = request.args.get('page')
    r = requests.get(url)
    data = json.loads(r.text)
    return render_template('petitions.html', data=data, page=page, args=args)

@app.route('/petitions/<id>', methods=["GET"])
def petition(id):
    url = 'https://petition.parliament.uk/petitions/' + id + '.json'
    r = requests.get(url)
    data = json.loads(r.text)

    countries = data['data']['attributes']['signatures_by_country']
    sorted_countries = sorted(countries, key=itemgetter('signature_count'), reverse=True)

    constituencies = data['data']['attributes']['signatures_by_constituency']
    sorted_constituencies = sorted(constituencies, key=itemgetter('signature_count'), reverse=True)

    for constituency in sorted_constituencies [:10]:
        mp = get_mp(constituency['name'])
        constituency['party'] = mp['party']
        constituency['url'] = mp['url']

    extents = constituency_collection(sorted_constituencies)

    print sorted_countries
    return render_template('petition.html',
        data=data,
        countries=sorted_countries,
        constituencies=sorted_constituencies,
        extents=extents)

@app.route('/petitions/<id>/map', methods=["GET"])
def map(id):
    url = 'https://petition.parliament.uk/petitions/' + id + '.json'
    r = requests.get(url)
    data = json.loads(r.text)

    constituencies = data['data']['attributes']['signatures_by_constituency']
    sorted_constituencies = sorted(constituencies, key=itemgetter('signature_count'), reverse=True)

    for constituency in sorted_constituencies [:10]:
        mp = get_mp(constituency['name'])
        constituency['party'] = mp['party']
        constituency['url'] = mp['url']

    extents = constituency_collection(sorted_constituencies)

    return render_template('map.html',
        extents=extents)
<|endoftext|>


<commit_before>"""Integration tests for setuptools that focus on building packages via pip.

The idea behind these tests is not to exhaustively check all the possible
combinations of packages, operating systems, supporting libraries, etc, but
rather check a limited number of popular packages and how they interact with
the exposed public API. This way if any change in API is introduced, we hope to
identify backward compatibility problems before publishing a release.

The number of tested packages is purposefully kept small, to minimise duration
and the associated maintenance cost (changes in the way these packages define
their build process may require changes in the tests).
"""
import json
import os
import shutil
import sys
from enum import Enum
from glob import glob
from hashlib import md5
from urllib.request import urlopen

import pytest
from packaging.requirements import Requirement

from .helpers import Archive, run

pytestmark = pytest.mark.integration

LATEST, = list(Enum("v", "LATEST"))
"""Default version to be checked"""
# There are positive and negative aspects of checking the latest version of the
# packages.
# The main positive aspect is that the latest version might have already
# removed the use of APIs deprecated in previous releases of setuptools.

# Packages to be tested:
# (Please notice the test environment cannot support EVERY library required for
# compiling binary extensions. In Ubuntu/Debian nomenclature, we only assume
# that `build-essential`, `gfortran` and `libopenblas-dev` are installed,
# due to their relevance to the numerical/scientific programming ecosystem)
EXAMPLES = [
    ("pandas", LATEST),  # cython + custom build_ext
    ("sphinx", LATEST),  # custom setup.py
    ("pip", LATEST),  # just in case...
    ("pytest", LATEST),  # uses setuptools_scm
    ("mypy", LATEST),  # custom build_py + ext_modules

    # --- Popular packages: https://hugovk.github.io/top-pypi-packages/ ---
    ("botocore", LATEST),
    ("kiwisolver", "1.3.2"),  # build_ext, version pinned due to setup_requires
    ("brotli", LATEST),  # not in the list but used by urllib3

    # When adding packages to this list, make sure they expose a `__version__`
    # attribute, or modify the tests below
]

# Some packages have "optional" dependencies that modify their build behaviour
# and are not listed in pyproject.toml, others still use `setup_requires`
EXTRA_BUILD_DEPS = {
    "sphinx": ("babel>=1.3",),
    "kiwisolver": ("cppy>=1.1.0",)
}

VIRTUALENV = (sys.executable, "-m", "virtualenv")

# By default, pip will try to build packages in isolation (PEP 517), which
# means it will download the previous stable version of setuptools.
# `pip` flags can avoid that (the version of setuptools under test
# should be the one to be used)
SDIST_OPTIONS = (
    "--ignore-installed",
    "--no-build-isolation",
    # We don't need "--no-binary :all:" since we specify the path to the sdist.
    # It also helps with performance, since dependencies can come from wheels.
)
# The downside of `--no-build-isolation` is that pip will not download build
# dependencies. The test script will have to also handle that.

@pytest.fixture
def venv_python(tmp_path):
    run([*VIRTUALENV, str(tmp_path / ".venv")])
    possible_path = (str(p.parent) for p in tmp_path.glob(".venv/*/python*"))
    return shutil.which("python", path=os.pathsep.join(possible_path))

@pytest.fixture(autouse=True)
def _prepare(tmp_path, venv_python, monkeypatch, request):
    download_path = os.getenv("DOWNLOAD_PATH", str(tmp_path))
    os.makedirs(download_path, exist_ok=True)

    # Environment vars used for building some of the packages
    monkeypatch.setenv("USE_MYPYC", "1")

    def _debug_info():
        # Let's provide the maximum amount of information possible in the case
        # it is necessary to debug the tests directly from the CI logs.
        print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
        print("Temporary directory:")
        map(print, tmp_path.glob("*"))
        print("Virtual environment:")
        run([venv_python, "-m", "pip", "freeze"])
    request.addfinalizer(_debug_info)

ALREADY_LOADED = ("pytest", "mypy")  # loaded by pytest/pytest-enabler

@pytest.mark.parametrize('package, version', EXAMPLES)
@pytest.mark.uses_network
def test_install_sdist(package, version, tmp_path, venv_python, setuptools_wheel):
    venv_pip = (venv_python, "-m", "pip")
    sdist = retrieve_sdist(package, version, tmp_path)
    deps = build_deps(package, sdist)
    if deps:
        print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
        print("Dependencies:", deps)
        run([*venv_pip, "install", *deps])

    # Use a virtualenv to simulate PEP 517 isolation
    # but install fresh setuptools wheel to ensure the version under development
    run([*venv_pip, "install", "-I", setuptools_wheel])
    run([*venv_pip, "install", *SDIST_OPTIONS, sdist])

    # Execute a simple script to make sure the package was installed correctly
    script = f"import {package}; print(getattr({package}, '__version__', 0))"
    run([venv_python, "-c", script])

# ---- Helper Functions ----

def retrieve_sdist(package, version, tmp_path):
    """Either use cached sdist file or download it from PyPI"""
    # `pip download` cannot be used due to
    # https://github.com/pypa/pip/issues/1884
    # https://discuss.python.org/t/pep-625-file-name-of-a-source-distribution/4686
    # We have to find the correct distribution file and download it
    download_path = os.getenv("DOWNLOAD_PATH", str(tmp_path))
    dist = retrieve_pypi_sdist_metadata(package, version)

    # Remove old files to prevent cache to grow indefinitely
    for file in glob(os.path.join(download_path, f"{package}*")):
        if dist["filename"] != file:
            os.unlink(file)

    dist_file = os.path.join(download_path, dist["filename"])
    if not os.path.exists(dist_file):
        download(dist["url"], dist_file, dist["md5_digest"])
    return dist_file

def retrieve_pypi_sdist_metadata(package, version):
    # https://warehouse.pypa.io/api-reference/json.html
    id_ = package if version is LATEST else f"{package}/{version}"
    with urlopen(f"https://pypi.org/pypi/{id_}/json") as f:
        metadata = json.load(f)

    if metadata["info"]["yanked"]:
        raise ValueError(f"Release for {package} {version} was yanked")

    version = metadata["info"]["version"]
    release = metadata["releases"][version]
    dists = [d for d in release if d["packagetype"] == "sdist"]
    if len(dists) == 0:
        raise ValueError(f"No sdist found for {package} {version}")

    for dist in dists:
        if dist["filename"].endswith(".tar.gz"):
            return dist

    # Not all packages are publishing tar.gz
    return dist

def download(url, dest, md5_digest):
    with urlopen(url) as f:
        data = f.read()

    assert md5(data).hexdigest() == md5_digest

    with open(dest, "wb") as f:
        f.write(data)

    assert os.path.exists(dest)

def build_deps(package, sdist_file):
    """Find out what are the build dependencies for a package.

    We need to "manually" install them, since pip will not install build
    deps with `--no-build-isolation`.
    """
    import tomli as toml

    # delay importing, since pytest discovery phase may hit this file from a
    # testenv without tomli

    archive = Archive(sdist_file)
    pyproject = _read_pyproject(archive)

    info = toml.loads(pyproject)
    deps = info.get("build-system", {}).get("requires", [])
    deps += EXTRA_BUILD_DEPS.get(package, [])
    # Remove setuptools from requirements (and deduplicate)
    requirements = {Requirement(d).name: d for d in deps}
    return [v for k, v in requirements.items() if k != "setuptools"]

def _read_pyproject(archive):
    for member in archive:
        if os.path.basename(archive.get_name(member)) == "pyproject.toml":
            return archive.get_content(member)
    return ""
<commit_msg>Add another conditional to resolve the release for version-specific packages. Fixes #3448.<commit_after>"""Integration tests for setuptools that focus on building packages via pip.

The idea behind these tests is not to exhaustively check all the possible
combinations of packages, operating systems, supporting libraries, etc, but
rather check a limited number of popular packages and how they interact with
the exposed public API. This way if any change in API is introduced, we hope to
identify backward compatibility problems before publishing a release.

The number of tested packages is purposefully kept small, to minimise duration
and the associated maintenance cost (changes in the way these packages define
their build process may require changes in the tests).
"""
import json
import os
import shutil
import sys
from enum import Enum
from glob import glob
from hashlib import md5
from urllib.request import urlopen

import pytest
from packaging.requirements import Requirement

from .helpers import Archive, run

pytestmark = pytest.mark.integration

LATEST, = list(Enum("v", "LATEST"))
"""Default version to be checked"""
# There are positive and negative aspects of checking the latest version of the
# packages.
# The main positive aspect is that the latest version might have already
# removed the use of APIs deprecated in previous releases of setuptools.

# Packages to be tested:
# (Please notice the test environment cannot support EVERY library required for
# compiling binary extensions. In Ubuntu/Debian nomenclature, we only assume
# that `build-essential`, `gfortran` and `libopenblas-dev` are installed,
# due to their relevance to the numerical/scientific programming ecosystem)
EXAMPLES = [
    ("pandas", LATEST),  # cython + custom build_ext
    ("sphinx", LATEST),  # custom setup.py
    ("pip", LATEST),  # just in case...
    ("pytest", LATEST),  # uses setuptools_scm
    ("mypy", LATEST),  # custom build_py + ext_modules

    # --- Popular packages: https://hugovk.github.io/top-pypi-packages/ ---
    ("botocore", LATEST),
    ("kiwisolver", "1.3.2"),  # build_ext, version pinned due to setup_requires
    ("brotli", LATEST),  # not in the list but used by urllib3

    # When adding packages to this list, make sure they expose a `__version__`
    # attribute, or modify the tests below
]

# Some packages have "optional" dependencies that modify their build behaviour
# and are not listed in pyproject.toml, others still use `setup_requires`
EXTRA_BUILD_DEPS = {
    "sphinx": ("babel>=1.3",),
    "kiwisolver": ("cppy>=1.1.0",)
}

VIRTUALENV = (sys.executable, "-m", "virtualenv")

# By default, pip will try to build packages in isolation (PEP 517), which
# means it will download the previous stable version of setuptools.
# `pip` flags can avoid that (the version of setuptools under test
# should be the one to be used)
SDIST_OPTIONS = (
    "--ignore-installed",
    "--no-build-isolation",
    # We don't need "--no-binary :all:" since we specify the path to the sdist.
    # It also helps with performance, since dependencies can come from wheels.
)
# The downside of `--no-build-isolation` is that pip will not download build
# dependencies. The test script will have to also handle that.

@pytest.fixture
def venv_python(tmp_path):
    run([*VIRTUALENV, str(tmp_path / ".venv")])
    possible_path = (str(p.parent) for p in tmp_path.glob(".venv/*/python*"))
    return shutil.which("python", path=os.pathsep.join(possible_path))

@pytest.fixture(autouse=True)
def _prepare(tmp_path, venv_python, monkeypatch, request):
    download_path = os.getenv("DOWNLOAD_PATH", str(tmp_path))
    os.makedirs(download_path, exist_ok=True)

    # Environment vars used for building some of the packages
    monkeypatch.setenv("USE_MYPYC", "1")

    def _debug_info():
        # Let's provide the maximum amount of information possible in the case
        # it is necessary to debug the tests directly from the CI logs.
        print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
        print("Temporary directory:")
        map(print, tmp_path.glob("*"))
        print("Virtual environment:")
        run([venv_python, "-m", "pip", "freeze"])
    request.addfinalizer(_debug_info)

ALREADY_LOADED = ("pytest", "mypy")  # loaded by pytest/pytest-enabler

@pytest.mark.parametrize('package, version', EXAMPLES)
@pytest.mark.uses_network
def test_install_sdist(package, version, tmp_path, venv_python, setuptools_wheel):
    venv_pip = (venv_python, "-m", "pip")
    sdist = retrieve_sdist(package, version, tmp_path)
    deps = build_deps(package, sdist)
    if deps:
        print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
        print("Dependencies:", deps)
        run([*venv_pip, "install", *deps])

    # Use a virtualenv to simulate PEP 517 isolation
    # but install fresh setuptools wheel to ensure the version under development
    run([*venv_pip, "install", "-I", setuptools_wheel])
    run([*venv_pip, "install", *SDIST_OPTIONS, sdist])

    # Execute a simple script to make sure the package was installed correctly
    script = f"import {package}; print(getattr({package}, '__version__', 0))"
    run([venv_python, "-c", script])

# ---- Helper Functions ----

def retrieve_sdist(package, version, tmp_path):
    """Either use cached sdist file or download it from PyPI"""
    # `pip download` cannot be used due to
    # https://github.com/pypa/pip/issues/1884
    # https://discuss.python.org/t/pep-625-file-name-of-a-source-distribution/4686
    # We have to find the correct distribution file and download it
    download_path = os.getenv("DOWNLOAD_PATH", str(tmp_path))
    dist = retrieve_pypi_sdist_metadata(package, version)

    # Remove old files to prevent cache to grow indefinitely
    for file in glob(os.path.join(download_path, f"{package}*")):
        if dist["filename"] != file:
            os.unlink(file)

    dist_file = os.path.join(download_path, dist["filename"])
    if not os.path.exists(dist_file):
        download(dist["url"], dist_file, dist["md5_digest"])
    return dist_file

def retrieve_pypi_sdist_metadata(package, version):
    # https://warehouse.pypa.io/api-reference/json.html
    id_ = package if version is LATEST else f"{package}/{version}"
    with urlopen(f"https://pypi.org/pypi/{id_}/json") as f:
        metadata = json.load(f)

    if metadata["info"]["yanked"]:
        raise ValueError(f"Release for {package} {version} was yanked")

    version = metadata["info"]["version"]
    release = metadata["releases"][version] if version is LATEST else metadata["urls"]
    dists = [d for d in release if d["packagetype"] == "sdist"]
    if len(dists) == 0:
        raise ValueError(f"No sdist found for {package} {version}")

    for dist in dists:
        if dist["filename"].endswith(".tar.gz"):
            return dist

    # Not all packages are publishing tar.gz
    return dist

def download(url, dest, md5_digest):
    with urlopen(url) as f:
        data = f.read()

    assert md5(data).hexdigest() == md5_digest

    with open(dest, "wb") as f:
        f.write(data)

    assert os.path.exists(dest)

def build_deps(package, sdist_file):
    """Find out what are the build dependencies for a package.

    We need to "manually" install them, since pip will not install build
    deps with `--no-build-isolation`.
    """
    import tomli as toml

    # delay importing, since pytest discovery phase may hit this file from a
    # testenv without tomli

    archive = Archive(sdist_file)
    pyproject = _read_pyproject(archive)

    info = toml.loads(pyproject)
    deps = info.get("build-system", {}).get("requires", [])
    deps += EXTRA_BUILD_DEPS.get(package, [])
    # Remove setuptools from requirements (and deduplicate)
    requirements = {Requirement(d).name: d for d in deps}
    return [v for k, v in requirements.items() if k != "setuptools"]

def _read_pyproject(archive):
    for member in archive:
        if os.path.basename(archive.get_name(member)) == "pyproject.toml":
            return archive.get_content(member)
    return ""
<|endoftext|>


<commit_before># Copyright (C) 2010 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Utility module to import a JSON module

Hides all the messy details of exactly where
we get a simplejson module from.
"""

__author__ = 'jcgregorio@google.com (Joe Gregorio)'

try: # pragma: no cover
  # Should work for Python2.6 and higher.
  import json as simplejson
except ImportError: # pragma: no cover
  try:
    import simplejson
  except ImportError:
    # Try to import from django, should work on App Engine
    from django.utils import simplejson
<commit_msg>remove oauth2client/anyjson.py<commit_after><|endoftext|>