<commit_before>from silopub import util
from silopub.ext import csrf, redis
from silopub.models import Site, Token
from flask import Blueprint, redirect, url_for, current_app, request, abort
from flask import session, make_response
import datetime
import json
import sys
import binascii
import os
import urllib

SERVICES = {}
micropub = Blueprint('micropub', __name__)

def register_service(name, service):
    """Register the services by name so we can delegate to their specific
    implementations
    """
    SERVICES[name] = service

def guess_service(me):
    me = deproxyify(me)
    service = None

    # if we've previously registered, then it's easy
    site = Site.lookup_by_url(me)
    if site:
        service = site.service
    else:
        # otherwise we have to take an educated guess
        domain = util.domain_for_url(me).lower()
        current_app.logger.debug('guessing service by domain %s', domain)
        if domain.endswith('tumblr.com'):
            service = 'tumblr'
        elif domain.endswith('wordpress.com'):
            service = 'wordpress'
        elif domain.endswith('blogger.com'):
            service = 'blogger'
        elif domain == 'twitter.com':
            service = 'twitter'
        elif domain == 'facebook.com':
            service = 'facebook'
        elif domain == 'flickr.com':
            service = 'flickr'
        elif domain == 'github.com':
            service = 'github'
        elif domain == 'goodreads.com':
            service = 'goodreads'

    return service and SERVICES[service]

@csrf.exempt
@micropub.route('/indieauth', methods=['GET', 'POST'])
def indieauth():
    # verify authorization
    if request.method == 'POST':
        code = request.form.get('code')
        client_id = request.form.get('client_id')
        redirect_uri = request.form.get('redirect_uri')
        state = request.form.get('state', '')

        datastr = redis.get('indieauth-code:{}'.format(code))
        if not datastr:
            current_app.logger.warn('unrecognized auth code %s', code)
            return util.urlenc_response(
                {'error': 'Unrecognized or expired authorization code'}, 400)

        data = json.loads(datastr.decode('utf-8'))
        for key, value in [('client_id', client_id),
                           ('redirect_uri', redirect_uri), ('state', state)]:
            if data.get(key) != value:
                current_app.logger.warn(
                    '%s mismatch. expected=%s, received=%s',
                    key, data.get(key), value)
                return util.urlenc_response({'error': key + ' mismatch'}, 400)

        me = data.get('me')
        return util.urlenc_response({'me': me})

    # indieauth via the silo's authenication mechanism
    try:
        me = request.args.get('me')
        redirect_uri = request.args.get('redirect_uri')

        current_app.logger.info('get indieauth with me=%s and redirect=%s', me,
                                redirect_uri)
        if not me or not redirect_uri:
            resp = make_response(
                "This is SiloPub's authorization endpoint. At least 'me' and "
                "'redirect_uri' are required.")
            resp.headers['IndieAuth'] = 'authorization_endpoint'
            return resp

        session['indieauth_params'] = {
            'me': me,
            'redirect_uri': redirect_uri,
            'client_id': request.args.get('client_id'),
            'state': request.args.get('state', ''),
            'scope': request.args.get('scope', ''),
        }

        service = guess_service(me)
        if not service:
            resp = make_response("Could not find a service for URL {}"
                                 .format(me))
            return resp

        return redirect(service.get_authorize_url(
            url_for('.indieauth_callback', _external=True), me=me))

    except:
        current_app.logger.exception('Starting IndieAuth')
        if not redirect_uri:
            resp = make_response('Exception starting indieauth: {}'.format(
                str(sys.exc_info()[0])), 400)
            resp.headers['Content-Type'] = 'text/plain'
            return resp

        return redirect(util.set_query_params(
            redirect_uri, error=str(sys.exc_info()[0])))

@micropub.route('/indieauth/callback')
def indieauth_callback():
    ia_params = session.get('indieauth_params', {})
    me = ia_params.get('me')
    client_id = ia_params.get('client_id')
    redirect_uri = ia_params.get('redirect_uri')
    state = ia_params.get('state', '')
    scope = ia_params.get('scope', '')

    if not me:
        current_app.logger.warn('No "me" value in session')
        return abort(400)

    service = guess_service(me)
    if not service:
        current_app.logger.warn('Could not guess service for URL %s', me)
        return abort(400)

    result = service.process_callback(
        url_for('.indieauth_callback', _external=True))

    if 'error' in result:
        current_app.logger.warn('error on callback %s', result['error'])
        return redirect(
            util.set_query_params(redirect_uri, error=result['error']))

    parsed = urllib.parse.urlparse(deproxyify(me))
    domain = parsed.netloc
    if parsed.path != '/':
        domain += parsed.path

    account = result['account']
    my_site = result.get('site') or next(
        (s for s in account.sites if s.domain == domain), None)

    if not my_site:
        error = 'Authed account does not have a site for domain {}'.format(
            domain)
        current_app.logger.warn('error on callback %s', error)
        return redirect(
            util.set_query_params(redirect_uri, error=error))

    current_app.logger.debug('auth callback got account %s and site %s',
                             account, my_site)

    # hand back a code to the micropub client
    code = binascii.hexlify(os.urandom(16)).decode()
    redis.setex('indieauth-code:{}'.format(code),
                datetime.timedelta(minutes=5),
                json.dumps({
                    'site': my_site.id,
                    'me': me,
                    'redirect_uri': redirect_uri,
                    'client_id': client_id,
                    'state': state,
                    'scope': scope,
                }))

    return redirect(util.set_query_params(
        redirect_uri, me=me, state=state, code=code))

@csrf.exempt
@micropub.route('/token', methods=['POST'])
def token_endpoint():
    code = request.form.get('code')
    me = request.form.get('me')
    redirect_uri = request.form.get('redirect_uri')
    client_id = request.form.get('client_id')
    state = request.form.get('state', '')

    datastr = redis.get('indieauth-code:{}'.format(code))
    if not datastr:
        current_app.logger.warn('unrecognized or expired code %s', code)
        return util.urlenc_response(
            {'error': 'Unrecognized or expired authorization code'}, 400)

    data = json.loads(datastr.decode('utf-8'))
    for key, value in [('me', me), ('client_id', client_id),
                       ('redirect_uri', redirect_uri), ('state', state)]:
        if data.get(key) != value:
            current_app.logger.warn('%s mismatch. expected=%s, received=%s',
                                    key, data.get(key), value)
            return util.urlenc_response({'error': key + ' mismatch'}, 400)

    # ok we're confirmed, create an access token
    scope = data.get('scope', '')
    site_id = data.get('site')
    site = Site.query.get(site_id)

    if not site_id or not site:
        return util.urlenc_response(
            {'error': 'No site for authorization code!'}, 400)

    token = Token.create_or_update(site, scope, client_id)
    return util.urlenc_response({
        'access_token': token.token,
        'me': me,
        'scope': scope,
    })

@csrf.exempt
@micropub.route('/micropub', methods=['GET', 'POST'])
def micropub_endpoint():
    current_app.logger.info(
        "received micropub request %s, args=%s, form=%s, headers=%s",
        request, request.args, request.form, request.headers)

    if request.method == 'GET':
        return 'This is the micropub endpoint.'

    bearer_prefix = 'Bearer '
    header_token = request.headers.get('authorization')
    if header_token and header_token.startswith(bearer_prefix):
        token = header_token[len(bearer_prefix):]
    else:
        token = request.form.get('access_token')

    if not token:
        err_msg = 'Micropub request is missing access_token'
        current_app.logger.warn(err_msg)
        return err_msg, 401

    # indieauth has confirmed that me is who they say they are, so if we have
    # an access token for their api, then we should be good to go

    # new short tokens
    token_data = Token.query.get(token)
    if token_data:
        site = token_data.site

    # old JWT tokens (TODO how do I get rid of these??)
    else:
        try:
            token_data = util.jwt_decode(token)
        except:
            current_app.logger.debug('could not find or decode token: %s', token)

        if not token_data:
            err_msg = 'Unrecognized token: {}' .format(token)
            current_app.logger.warn(err_msg)
            return err_msg, 401
        site_id = token_data.get('site')
        site = Site.query.get(site_id)

        if not site:
            err_msg = 'Could not find a site for site id {}.'.format(site_id)
            current_app.logger.warn(err_msg)
            return err_msg, 401

    current_app.logger.info('Success! Publishing to %s', site)
    resp = SERVICES[site.service].publish(site)
    if not resp:
        return util.make_publish_error_response(
            'Unspecified failure. Maybe an invalid or unsupported request?')
    return resp

def deproxyify(url):
    if url.startswith(request.url_root):
        return 'http://' + url[len(request.url_root):]
    return url
<commit_msg>state is not sent to the server (#55)<commit_after>from silopub import util
from silopub.ext import csrf, redis
from silopub.models import Site, Token
from flask import Blueprint, redirect, url_for, current_app, request, abort
from flask import session, make_response
import datetime
import json
import sys
import binascii
import os
import urllib

SERVICES = {}
micropub = Blueprint('micropub', __name__)

def register_service(name, service):
    """Register the services by name so we can delegate to their specific
    implementations
    """
    SERVICES[name] = service

def guess_service(me):
    me = deproxyify(me)
    service = None

    # if we've previously registered, then it's easy
    site = Site.lookup_by_url(me)
    if site:
        service = site.service
    else:
        # otherwise we have to take an educated guess
        domain = util.domain_for_url(me).lower()
        current_app.logger.debug('guessing service by domain %s', domain)
        if domain.endswith('tumblr.com'):
            service = 'tumblr'
        elif domain.endswith('wordpress.com'):
            service = 'wordpress'
        elif domain.endswith('blogger.com'):
            service = 'blogger'
        elif domain == 'twitter.com':
            service = 'twitter'
        elif domain == 'facebook.com':
            service = 'facebook'
        elif domain == 'flickr.com':
            service = 'flickr'
        elif domain == 'github.com':
            service = 'github'
        elif domain == 'goodreads.com':
            service = 'goodreads'

    return service and SERVICES[service]

@csrf.exempt
@micropub.route('/indieauth', methods=['GET', 'POST'])
def indieauth():
    # verify authorization
    if request.method == 'POST':
        code = request.form.get('code')
        client_id = request.form.get('client_id')
        redirect_uri = request.form.get('redirect_uri')

        datastr = redis.get('indieauth-code:{}'.format(code))
        if not datastr:
            current_app.logger.warn('unrecognized auth code %s', code)
            return util.urlenc_response(
                {'error': 'Unrecognized or expired authorization code'}, 400)

        data = json.loads(datastr.decode('utf-8'))
        for key, value in [('client_id', client_id),
                           ('redirect_uri', redirect_uri)]:
            if data.get(key) != value:
                current_app.logger.warn(
                    '%s mismatch. expected=%s, received=%s',
                    key, data.get(key), value)
                return util.urlenc_response({'error': key + ' mismatch'}, 400)

        me = data.get('me')
        return util.urlenc_response({'me': me})

    # indieauth via the silo's authenication mechanism
    try:
        me = request.args.get('me')
        redirect_uri = request.args.get('redirect_uri')

        current_app.logger.info('get indieauth with me=%s and redirect=%s', me,
                                redirect_uri)
        if not me or not redirect_uri:
            resp = make_response(
                "This is SiloPub's authorization endpoint. At least 'me' and "
                "'redirect_uri' are required.")
            resp.headers['IndieAuth'] = 'authorization_endpoint'
            return resp

        session['indieauth_params'] = {
            'me': me,
            'redirect_uri': redirect_uri,
            'client_id': request.args.get('client_id'),
            'state': request.args.get('state', ''),
            'scope': request.args.get('scope', ''),
        }

        service = guess_service(me)
        if not service:
            resp = make_response("Could not find a service for URL {}"
                                 .format(me))
            return resp

        return redirect(service.get_authorize_url(
            url_for('.indieauth_callback', _external=True), me=me))

    except:
        current_app.logger.exception('Starting IndieAuth')
        if not redirect_uri:
            resp = make_response('Exception starting indieauth: {}'.format(
                str(sys.exc_info()[0])), 400)
            resp.headers['Content-Type'] = 'text/plain'
            return resp

        return redirect(util.set_query_params(
            redirect_uri, error=str(sys.exc_info()[0])))

@micropub.route('/indieauth/callback')
def indieauth_callback():
    ia_params = session.get('indieauth_params', {})
    me = ia_params.get('me')
    client_id = ia_params.get('client_id')
    redirect_uri = ia_params.get('redirect_uri')
    state = ia_params.get('state', '')
    scope = ia_params.get('scope', '')

    if not me:
        current_app.logger.warn('No "me" value in session')
        return abort(400)

    service = guess_service(me)
    if not service:
        current_app.logger.warn('Could not guess service for URL %s', me)
        return abort(400)

    result = service.process_callback(
        url_for('.indieauth_callback', _external=True))

    if 'error' in result:
        current_app.logger.warn('error on callback %s', result['error'])
        return redirect(
            util.set_query_params(redirect_uri, error=result['error']))

    parsed = urllib.parse.urlparse(deproxyify(me))
    domain = parsed.netloc
    if parsed.path != '/':
        domain += parsed.path

    account = result['account']
    my_site = result.get('site') or next(
        (s for s in account.sites if s.domain == domain), None)

    if not my_site:
        error = 'Authed account does not have a site for domain {}'.format(
            domain)
        current_app.logger.warn('error on callback %s', error)
        return redirect(
            util.set_query_params(redirect_uri, error=error))

    current_app.logger.debug('auth callback got account %s and site %s',
                             account, my_site)

    # hand back a code to the micropub client
    code = binascii.hexlify(os.urandom(16)).decode()
    redis.setex('indieauth-code:{}'.format(code),
                datetime.timedelta(minutes=5),
                json.dumps({
                    'site': my_site.id,
                    'me': me,
                    'redirect_uri': redirect_uri,
                    'client_id': client_id,
                    'state': state,
                    'scope': scope,
                }))

    return redirect(util.set_query_params(
        redirect_uri, me=me, state=state, code=code))

@csrf.exempt
@micropub.route('/token', methods=['POST'])
def token_endpoint():
    code = request.form.get('code')
    me = request.form.get('me')
    redirect_uri = request.form.get('redirect_uri')
    client_id = request.form.get('client_id')

    datastr = redis.get('indieauth-code:{}'.format(code))
    if not datastr:
        current_app.logger.warn('unrecognized or expired code %s', code)
        return util.urlenc_response(
            {'error': 'Unrecognized or expired authorization code'}, 400)

    data = json.loads(datastr.decode('utf-8'))
    for key, value in [('me', me), ('client_id', client_id),
                       ('redirect_uri', redirect_uri)]:
        if data.get(key) != value:
            current_app.logger.warn('%s mismatch. expected=%s, received=%s',
                                    key, data.get(key), value)
            return util.urlenc_response({'error': key + ' mismatch'}, 400)

    # ok we're confirmed, create an access token
    scope = data.get('scope', '')
    site_id = data.get('site')
    site = Site.query.get(site_id)

    if not site_id or not site:
        return util.urlenc_response(
            {'error': 'No site for authorization code!'}, 400)

    token = Token.create_or_update(site, scope, client_id)
    return util.urlenc_response({
        'access_token': token.token,
        'me': me,
        'scope': scope,
    })

@csrf.exempt
@micropub.route('/micropub', methods=['GET', 'POST'])
def micropub_endpoint():
    current_app.logger.info(
        "received micropub request %s, args=%s, form=%s, headers=%s",
        request, request.args, request.form, request.headers)

    if request.method == 'GET':
        return 'This is the micropub endpoint.'

    bearer_prefix = 'Bearer '
    header_token = request.headers.get('authorization')
    if header_token and header_token.startswith(bearer_prefix):
        token = header_token[len(bearer_prefix):]
    else:
        token = request.form.get('access_token')

    if not token:
        err_msg = 'Micropub request is missing access_token'
        current_app.logger.warn(err_msg)
        return err_msg, 401

    # indieauth has confirmed that me is who they say they are, so if we have
    # an access token for their api, then we should be good to go

    # new short tokens
    token_data = Token.query.get(token)
    if token_data:
        site = token_data.site

    # old JWT tokens (TODO how do I get rid of these??)
    else:
        try:
            token_data = util.jwt_decode(token)
        except:
            current_app.logger.debug('could not find or decode token: %s', token)

        if not token_data:
            err_msg = 'Unrecognized token: {}' .format(token)
            current_app.logger.warn(err_msg)
            return err_msg, 401
        site_id = token_data.get('site')
        site = Site.query.get(site_id)

        if not site:
            err_msg = 'Could not find a site for site id {}.'.format(site_id)
            current_app.logger.warn(err_msg)
            return err_msg, 401

    current_app.logger.info('Success! Publishing to %s', site)
    resp = SERVICES[site.service].publish(site)
    if not resp:
        return util.make_publish_error_response(
            'Unspecified failure. Maybe an invalid or unsupported request?')
    return resp

def deproxyify(url):
    if url.startswith(request.url_root):
        return 'http://' + url[len(request.url_root):]
    return url
<|endoftext|>


<commit_before>import os
import os.path as op
from mne.utils import ProgressBar

class Client():
    def __init__(self, server, credentials=None, bucket=None):
        if server == 'S3':
            self.route = S3_client(credentials, bucket)
        elif server == 'Dropbox':
            self.route = Dropbox_client(credentials, bucket)
        else:
            raise ValueError('Unknown server')

    def download(self, f_server, f_client, overwrite='auto'):
        # connect
        self.route.connect()
        # check that file exists online
        meta_data = self.route.metadata(f_server)
        # don't download if files are identical
        client_path = '/'.join(f_client.split('/')[:-1])
        if op.exists(f_client):
            if overwrite is False:
                print('%s already exists & no overwrite: skip' % f_client)
                return False
            elif overwrite == 'auto':
                fsize_client = op.getsize(f_client)
                if meta_data['bytes'] == fsize_client:
                    print('Identical files, download skipped %s' % f_client)
                    return False
        # create directory if doesn't exist already
        if ((not op.exists(client_path)) and (client_path is not None) and
                (client_path != '')):
            os.makedirs(client_path)
        # download
        self.route.download(f_server, f_client)
        print('Downloaded: %s > %s' % (f_server, f_client))
        return True

    def upload(self, f_client, f_server, overwrite='auto'):
        if op.isfile(f_client):
            return self._upload_file(f_client, f_server, overwrite=overwrite)
        elif op.isdir(f_client):
            results = list()
            for root, dirs, files in os.walk(f_client):
                for filename in files:
                    # construct the full local path
                    local_path = op.join(root, filename)
                    # construct the relative server path
                    relative_path = op.relpath(local_path, f_client)
                    # upload the file
                    results.append(self._upload_file(local_path,
                                                     relative_path))
            return sum(results)
        else:
            raise ValueError('File not found %s' % f_client)

    def _upload_file(self, f_client, f_server, overwrite='auto'):
        # connect
        self.route.connect()
        # check whether file exists online
        if overwrite is not True:
            metadata = self.route.metadata(f_server)
            # don't upload if files are identical
            if metadata['exist']:
                if overwrite is False:
                    print('File already exists & overwrite is False: skipped')
                    return False
                elif overwrite == 'auto':
                    fsize_client = op.getsize(f_client)
                    if metadata['bytes'] == fsize_client:
                        print('Identical files, upload skipped %s' % f_server)
                        return False
                else:
                    raise ValueError('overwrite must be bool or `auto`')
        # upload
        print('Uploading %s > %s' % (f_client,
                                     op.join(self.route.bucket, f_server)))
        self.route.upload(f_client, f_server)
        return True

    def metadata(self, f_server):
        return self.route.metadata(f_server)

    def delete(self, f_server):
        self.route.connect()
        return self.route.delete(f_server)

class S3_client():

    def __init__(self, credentials=None, bucket=None):
        if credentials is None:
            credentials = op.expanduser('~/.credentials/boto.cfg')
        self.credentials = credentials
        self.bucket = bucket
        self.client = self.connect()

    def connect(self):
        import boto
        if isinstance(self.credentials, str):
            with open(self.credentials, 'rb') as f:
                auth = f.read()
                auth = auth.split('\n')
                AWSAccessKeyId = auth[0].split('AWSAccessKeyId=')[1]
                AWSSecretKey = auth[1].split('AWSSecretKey=')[1]
        elif isinstance(self.credentials, dict):
            AWSAccessKeyId = self.credentials['AWSAccessKeyId']
            AWSSecretKey = self.credentials['AWSSecretKey']
        calling_format = 'boto.s3.connection.OrdinaryCallingFormat'
        client_ = boto.connect_s3(AWSAccessKeyId, AWSSecretKey,
                                  calling_format=calling_format)
        return client_.get_bucket(self.bucket)

    def metadata(self, f_server):
        key = self.client.get_key(f_server)
        if key is None:
            metadata = dict(exist=False, bytes=0)
        else:
            metadata = dict(exist=True, bytes=key.size)
        return metadata

    def download(self, f_server, f_client):
        key = self.client.get_key(f_server)
        key.get_contents_to_filename(f_client)

    def upload(self, f_client, f_server):
        key = self.client.get_key(f_server)
        if key is None:
            key = self.client.new_key(f_server)
        key.set_contents_from_filename(f_client)

    def delete(self, f_server):
        key = self.client.get_key(f_server)
        key = key is not None
        self.client.delete_key(f_server)
        return key

class Dropbox_client():
    def __init__(self, credentials=None, bucket=None):
        if credentials is None:
            credentials = op.expanduser('~/.credentials/dropbox.pem')
        self.credentials = credentials
        self.bucket = bucket
        self.client = self.connect()

    def connect(self):
        import dropbox
        if self.credentials is None:
            self.credentials = '~/.credentials/dropbox.pem'
        if isinstance(self.credentials, dict):
            APP_KEY = self.credentials['APP_KEY']
            APP_SECRET = self.credentials['APP_SECRET']
            TOKEN_KEY = self.credentials['TOKEN_KEY']
            TOKEN_SECRET = self.credentials['TOKEN_SECRET']
        elif isinstance(self.credentials, str):
            with open(self.credentials, 'rb') as f:
                auth = f.read()
                auth = auth.split('\n')
                APP_KEY = auth[0].split('APP_KEY=')[1]
                APP_SECRET = auth[1].split('APP_SECRET=')[1]
                TOKEN_KEY = auth[2].split('TOKEN_KEY=')[1]
                TOKEN_SECRET = auth[3].split('TOKEN_SECRET=')[1]
        else:
            raise ValueError('Specify credentials with a file or a dict')
        session = dropbox.session.DropboxSession(
            APP_KEY, APP_SECRET, 'dropbox')
        # request_token = session.obtain_request_token()
        # url = session.build_authorize_url(request_token)
        # access_token = session.obtain_access_token(request_token)
        session.set_token(TOKEN_KEY, TOKEN_SECRET)
        self.client = dropbox.client.DropboxClient(session)

    def metadata(self, f_server):
        from dropbox.client import ErrorResponse
        try:
            metadata = self.client.metadata(
                op.join(self.bucket, f_server))
            metadata['exist'] = True
            if 'is_deleted' in metadata.keys():
                metadata['exist'] = not metadata['is_deleted']
        except ErrorResponse:
            metadata = dict(exist=False, bytes=0)
        return metadata

    def download(self, f_server, f_client):
        f = self.client.get_file(op.join(self.bucket, f_server))
        with open(f_client, 'wb') as out:
            out.write(f.read())

    def upload(self, f_client, f_server):
        f_server = op.join(self.bucket, f_server)
        if self.metadata(f_server)['exist']:
            self.delete(f_server)
        self._upload_chunk(self.client, f_client, f_server)

    def _upload_chunk(self, client, f_client, f_server):
        from StringIO import StringIO
        file_obj = open(f_client, 'rb')
        target_length = os.path.getsize(f_client)
        chunk_size = 10 * 1024 * 1024
        offset = 0
        uploader = client.get_chunked_uploader(file_obj, target_length)
        last_block = None
        params = dict()
        pbar = ProgressBar(target_length, spinner=True)
        error_count = 0
        while offset < target_length:
            if error_count > 3:
                raise RuntimeError
            pbar.update(offset)
            next_chunk_size = min(chunk_size, target_length - offset)
            # read data if last chunk passed
            if last_block is None:
                last_block = file_obj.read(next_chunk_size)
            # set parameters
            if offset > 0:
                params = dict(upload_id=uploader.upload_id, offset=offset)
            try:
                url, ignored_params, headers = client.request(
                    "/chunked_upload", params, method='PUT',
                    content_server=True)
                reply = client.rest_client.PUT(url, StringIO(last_block),
                                               headers)
                new_offset = reply['offset']
                uploader.upload_id = reply['upload_id']
                # avoid reading data if last chunk didn't pass
                if new_offset > offset:
                    offset = new_offset
                    last_block = None
                    error_count == 0
                else:
                    error_count += 1
            except Exception:
                error_count += 1
        if target_length > 0:
            pbar.update(target_length)
        print('')
        file_obj.close()
        uploader.finish(f_server, overwrite=True)

    def delete(self, f_server):
        metadata = self.metadata(f_server)
        if metadata['exist']:
            self.client.file_delete(op.join(self.bucket, f_server))
            return True
        else:
            return False

    def _create_token():
        import dropbox
        credentials = '.credentials/dropbox.pem'
        with open(credentials, 'rb') as f:
            auth = f.read()
            auth = auth.split('\n')
            APP_KEY = auth[0].split('APP_KEY=')[1]
            APP_SECRET = auth[1].split('APP_SECRET=')[1]
        session = dropbox.session.DropboxSession(
            APP_KEY, APP_SECRET, 'dropbox')
        request_token = session.obtain_request_token()
        url = session.build_authorize_url(request_token)
        raw_input(url)
        access_token = session.obtain_access_token(request_token)
        print access_token.key, access_token.secret
        session.set_token(access_token.key, access_token.secret)
        client = dropbox.client.DropboxClient(session)
        print client.account_info()
<commit_msg>fix relative path upload<commit_after>import os
import os.path as op
from mne.utils import ProgressBar

class Client():
    def __init__(self, server, credentials=None, bucket=None):
        if server == 'S3':
            self.route = S3_client(credentials, bucket)
        elif server == 'Dropbox':
            self.route = Dropbox_client(credentials, bucket)
        else:
            raise ValueError('Unknown server')

    def download(self, f_server, f_client, overwrite='auto'):
        # connect
        self.route.connect()
        # check that file exists online
        meta_data = self.route.metadata(f_server)
        # don't download if files are identical
        client_path = '/'.join(f_client.split('/')[:-1])
        if op.exists(f_client):
            if overwrite is False:
                print('%s already exists & no overwrite: skip' % f_client)
                return False
            elif overwrite == 'auto':
                fsize_client = op.getsize(f_client)
                if meta_data['bytes'] == fsize_client:
                    print('Identical files, download skipped %s' % f_client)
                    return False
        # create directory if doesn't exist already
        if ((not op.exists(client_path)) and (client_path is not None) and
                (client_path != '')):
            os.makedirs(client_path)
        # download
        self.route.download(f_server, f_client)
        print('Downloaded: %s > %s' % (f_server, f_client))
        return True

    def upload(self, f_client, f_server=None, overwrite='auto'):
        if f_server is None:
            f_server = f_client.split('/')[-1]
        if op.isfile(f_client):
            return self._upload_file(f_client, f_server, overwrite=overwrite)
        elif op.isdir(f_client):
            results = list()
            for root, dirs, files in os.walk(f_client):
                for filename in files:
                    # construct the full local path
                    local_path = op.join(root, filename)
                    # upload the file
                    results.append(self._upload_file(
                        local_path,
                        f_server + local_path.split(f_client)[-1]))
            return sum(results)
        else:
            raise ValueError('File not found %s' % f_client)

    def _upload_file(self, f_client, f_server, overwrite='auto'):
        # connect
        self.route.connect()
        # check whether file exists online
        if overwrite is not True:
            metadata = self.route.metadata(f_server)
            # don't upload if files are identical
            if metadata['exist']:
                if overwrite is False:
                    print('File already exists & overwrite is False: skipped')
                    return False
                elif overwrite == 'auto':
                    fsize_client = op.getsize(f_client)
                    if metadata['bytes'] == fsize_client:
                        print('Identical files, upload skipped %s' % f_server)
                        return False
                else:
                    raise ValueError('overwrite must be bool or `auto`')
        # upload
        print('Uploading %s > %s' % (f_client,
                                     op.join(self.route.bucket, f_server)))
        self.route.upload(f_client, f_server)
        return True

    def metadata(self, f_server):
        return self.route.metadata(f_server)

    def delete(self, f_server):
        self.route.connect()
        return self.route.delete(f_server)

class S3_client():

    def __init__(self, credentials=None, bucket=None):
        if credentials is None:
            credentials = op.expanduser('~/.credentials/boto.cfg')
        self.credentials = credentials
        self.bucket = bucket
        self.client = self.connect()

    def connect(self):
        import boto
        if isinstance(self.credentials, str):
            with open(self.credentials, 'rb') as f:
                auth = f.read()
                auth = auth.split('\n')
                AWSAccessKeyId = auth[0].split('AWSAccessKeyId=')[1]
                AWSSecretKey = auth[1].split('AWSSecretKey=')[1]
        elif isinstance(self.credentials, dict):
            AWSAccessKeyId = self.credentials['AWSAccessKeyId']
            AWSSecretKey = self.credentials['AWSSecretKey']
        calling_format = 'boto.s3.connection.OrdinaryCallingFormat'
        client_ = boto.connect_s3(AWSAccessKeyId, AWSSecretKey,
                                  calling_format=calling_format)
        return client_.get_bucket(self.bucket)

    def metadata(self, f_server):
        key = self.client.get_key(f_server)
        if key is None:
            metadata = dict(exist=False, bytes=0)
        else:
            metadata = dict(exist=True, bytes=key.size)
        return metadata

    def download(self, f_server, f_client):
        key = self.client.get_key(f_server)
        key.get_contents_to_filename(f_client)

    def upload(self, f_client, f_server):
        key = self.client.get_key(f_server)
        if key is None:
            key = self.client.new_key(f_server)
        key.set_contents_from_filename(f_client)

    def delete(self, f_server):
        key = self.client.get_key(f_server)
        key = key is not None
        self.client.delete_key(f_server)
        return key

class Dropbox_client():
    def __init__(self, credentials=None, bucket=None):
        if credentials is None:
            credentials = op.expanduser('~/.credentials/dropbox.pem')
        self.credentials = credentials
        self.bucket = bucket
        self.client = self.connect()

    def connect(self):
        import dropbox
        if self.credentials is None:
            self.credentials = '~/.credentials/dropbox.pem'
        if isinstance(self.credentials, dict):
            APP_KEY = self.credentials['APP_KEY']
            APP_SECRET = self.credentials['APP_SECRET']
            TOKEN_KEY = self.credentials['TOKEN_KEY']
            TOKEN_SECRET = self.credentials['TOKEN_SECRET']
        elif isinstance(self.credentials, str):
            with open(self.credentials, 'rb') as f:
                auth = f.read()
                auth = auth.split('\n')
                APP_KEY = auth[0].split('APP_KEY=')[1]
                APP_SECRET = auth[1].split('APP_SECRET=')[1]
                TOKEN_KEY = auth[2].split('TOKEN_KEY=')[1]
                TOKEN_SECRET = auth[3].split('TOKEN_SECRET=')[1]
        else:
            raise ValueError('Specify credentials with a file or a dict')
        session = dropbox.session.DropboxSession(
            APP_KEY, APP_SECRET, 'dropbox')
        # request_token = session.obtain_request_token()
        # url = session.build_authorize_url(request_token)
        # access_token = session.obtain_access_token(request_token)
        session.set_token(TOKEN_KEY, TOKEN_SECRET)
        self.client = dropbox.client.DropboxClient(session)

    def metadata(self, f_server):
        from dropbox.client import ErrorResponse
        try:
            metadata = self.client.metadata(
                op.join(self.bucket, f_server))
            metadata['exist'] = True
            if 'is_deleted' in metadata.keys():
                metadata['exist'] = not metadata['is_deleted']
        except ErrorResponse:
            metadata = dict(exist=False, bytes=0)
        return metadata

    def download(self, f_server, f_client):
        f = self.client.get_file(op.join(self.bucket, f_server))
        with open(f_client, 'wb') as out:
            out.write(f.read())

    def upload(self, f_client, f_server):
        f_server = op.join(self.bucket, f_server)
        if self.metadata(f_server)['exist']:
            self.delete(f_server)
        self._upload_chunk(self.client, f_client, f_server)

    def _upload_chunk(self, client, f_client, f_server):
        from StringIO import StringIO
        file_obj = open(f_client, 'rb')
        target_length = os.path.getsize(f_client)
        chunk_size = 10 * 1024 * 1024
        offset = 0
        uploader = client.get_chunked_uploader(file_obj, target_length)
        last_block = None
        params = dict()
        pbar = ProgressBar(target_length, spinner=True)
        error_count = 0
        while offset < target_length:
            if error_count > 3:
                raise RuntimeError
            pbar.update(offset)
            next_chunk_size = min(chunk_size, target_length - offset)
            # read data if last chunk passed
            if last_block is None:
                last_block = file_obj.read(next_chunk_size)
            # set parameters
            if offset > 0:
                params = dict(upload_id=uploader.upload_id, offset=offset)
            try:
                url, ignored_params, headers = client.request(
                    "/chunked_upload", params, method='PUT',
                    content_server=True)
                reply = client.rest_client.PUT(url, StringIO(last_block),
                                               headers)
                new_offset = reply['offset']
                uploader.upload_id = reply['upload_id']
                # avoid reading data if last chunk didn't pass
                if new_offset > offset:
                    offset = new_offset
                    last_block = None
                    error_count == 0
                else:
                    error_count += 1
            except Exception:
                error_count += 1
        if target_length > 0:
            pbar.update(target_length)
        print('')
        file_obj.close()
        uploader.finish(f_server, overwrite=True)

    def delete(self, f_server):
        metadata = self.metadata(f_server)
        if metadata['exist']:
            self.client.file_delete(op.join(self.bucket, f_server))
            return True
        else:
            return False

    def _create_token():
        import dropbox
        credentials = '.credentials/dropbox.pem'
        with open(credentials, 'rb') as f:
            auth = f.read()
            auth = auth.split('\n')
            APP_KEY = auth[0].split('APP_KEY=')[1]
            APP_SECRET = auth[1].split('APP_SECRET=')[1]
        session = dropbox.session.DropboxSession(
            APP_KEY, APP_SECRET, 'dropbox')
        request_token = session.obtain_request_token()
        url = session.build_authorize_url(request_token)
        raw_input(url)
        access_token = session.obtain_access_token(request_token)
        print access_token.key, access_token.secret
        session.set_token(access_token.key, access_token.secret)
        client = dropbox.client.DropboxClient(session)
        print client.account_info()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2015  ADHOC SA  (http://www.adhoc.com.ar)
#    All Rights Reserved.
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
{
    'author': 'ADHOC SA',
    'website': 'www.adhoc.com.ar',
    'license': 'AGPL-3',
    'category': 'Accounting & Finance',
    'data': [
    ],
    'demo': [],
    'depends': ['account'],
    'description': '''
Account General Ledger Fix
==========================
When getting odoo general ledger with initial balance WITH DATES or PERIODS
and WITHOUT fiscalyear dates, initial balance was not shown, this module
fix that.

IMPORTANI: It wont works with oca webkit financial reports. If you use oca
reports, you can have initial balance if you get report by period or no filter
and account user type close method is not "none".
''',
    'installable': True,
    'name': 'Account General Ledger Fix',
    'test': [],
    'version': '8.0.0.0.0'}
<commit_msg>IMP ggeneral ledger not installable<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2015  ADHOC SA  (http://www.adhoc.com.ar)
#    All Rights Reserved.
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
{
    'author': 'ADHOC SA',
    'website': 'www.adhoc.com.ar',
    'license': 'AGPL-3',
    'category': 'Accounting & Finance',
    'data': [
    ],
    'demo': [],
    'depends': ['account'],
    'description': '''
Account General Ledger Fix
==========================
When getting odoo general ledger with initial balance WITH DATES or PERIODS
and WITHOUT fiscalyear dates, initial balance was not shown, this module
fix that.

IMPORTANI: It wont works with oca webkit financial reports. If you use oca
reports, you can have initial balance if you get report by period or no filter
and account user type close method is not "none".
''',
    'installable': False,
    'name': 'Account General Ledger Fix',
    'test': [],
    'version': '8.0.0.0.0'}
<|endoftext|>


<commit_before>""":mod:`wand.api` --- Low-level interfaces
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

"""
import os
import os.path
import platform
import ctypes
import ctypes.util

__all__ = 'load_library', 'MagickPixelPacket', 'library', 'libc'

def load_library():
    """Loads the MagickWand library.

    :returns: the MagickWand library and the ImageMagick library

    """
    libpath = None
    system = platform.system()

    magick_home = os.environ.get('MAGICK_HOME')
    if magick_home:
        if system == 'Windows':
            libpath = 'CORE_RL_wand_.dll',
        elif system == 'Darwin':
            libpath = 'lib', 'libMagickWand.dylib',
        else:
            libpath = 'lib', 'libMagickWand.so',
        libpath = os.path.join(magick_home, *libpath)
    else:
        if system == 'Windows':
            libpath = ctypes.util.find_library('CORE_RL_wand_')
        else:
            libpath = ctypes.util.find_library('MagickWand')
    libwand = ctypes.CDLL(libpath)

    if system == 'Windows':
        # On Windows, the API is split between two libs. On other platforms,
        # it's all contained in one.
        libmagick_filename = 'CORE_RL_magick_'
        if magick_home:
            libmagick_path = os.path.join(magick_home,
                                          libmagick_filename + '.dll')
        else:
            libmagick_path = ctypes.util.find_library(libmagick_filename)
        libmagick = ctypes.CDLL(libmagick_path)
        return libwand, libmagick

    return libwand, libwand

if not hasattr(ctypes, 'c_ssize_t'):
    if ctypes.sizeof(ctypes.c_uint) == ctypes.sizeof(ctypes.c_void_p):
        ctypes.c_ssize_t = ctypes.c_int
    elif ctypes.sizeof(ctypes.c_ulong) == ctypes.sizeof(ctypes.c_void_p):
        ctypes.c_ssize_t = ctypes.c_long
    elif ctypes.sizeof(ctypes.c_ulonglong) == ctypes.sizeof(ctypes.c_void_p):
        ctypes.c_ssize_t = ctypes.c_longlong

class MagickPixelPacket(ctypes.Structure):

    _fields_ = [('storage_class', ctypes.c_int),
                ('colorspace', ctypes.c_int),
                ('matte', ctypes.c_int),
                ('fuzz', ctypes.c_double),
                ('depth', ctypes.c_size_t),
                ('red', ctypes.c_double),
                ('green', ctypes.c_double),
                ('blue', ctypes.c_double),
                ('opacity', ctypes.c_double),
                ('index', ctypes.c_double)]

libraries = load_library()

#: (:class:`ctypes.CDLL`) The MagickWand library.
library = libraries[0]

#: (:class:`ctypes.CDLL`) The ImageMagick library.  It is the same with
#: :data:`library` on platforms other than Windows.
libmagick = libraries[1]

try:
    library.NewMagickWand.restype = ctypes.c_void_p

    library.DestroyMagickWand.argtypes = [ctypes.c_void_p]
    library.DestroyMagickWand.restype = ctypes.c_void_p

    library.CloneMagickWand.argtypes = [ctypes.c_void_p]
    library.CloneMagickWand.restype = ctypes.c_void_p

    library.IsMagickWand.argtypes = [ctypes.c_void_p]

    library.MagickGetException.argtypes = [ctypes.c_void_p,
                                           ctypes.POINTER(ctypes.c_int)]
    library.MagickGetException.restype = ctypes.c_char_p

    library.MagickClearException.argtypes = [ctypes.c_void_p]

    library.MagickReadImageBlob.argtypes = [ctypes.c_void_p, ctypes.c_void_p,
                                            ctypes.c_size_t]

    library.MagickReadImage.argtypes = [ctypes.c_void_p, ctypes.c_char_p]

    library.MagickReadImageFile.argtypes = [ctypes.c_void_p, ctypes.c_void_p]

    library.MagickGetImageFormat.argtypes = [ctypes.c_void_p]
    library.MagickGetImageFormat.restype = ctypes.c_char_p

    library.MagickSetImageFormat.argtypes = [ctypes.c_void_p, ctypes.c_char_p]

    libmagick.MagickToMime.argtypes = [ctypes.c_char_p]
    libmagick.MagickToMime.restype = ctypes.POINTER(ctypes.c_char)

    library.MagickGetImageSignature.argtypes = [ctypes.c_void_p]
    library.MagickGetImageSignature.restype = ctypes.c_char_p

    library.MagickGetImageBackgroundColor.argtypes = [ctypes.c_void_p,
                                                      ctypes.c_void_p]

    library.MagickSetImageBackgroundColor.argtypes = [ctypes.c_void_p,
                                                      ctypes.c_void_p]

    library.MagickGetImageBlob.argtypes = [ctypes.c_void_p,
                                           ctypes.POINTER(ctypes.c_size_t)]
    library.MagickGetImageBlob.restype = ctypes.POINTER(ctypes.c_ubyte)

    library.MagickWriteImage.argtypes = [ctypes.c_void_p, ctypes.c_char_p]

    library.MagickWriteImageFile.argtypes = [ctypes.c_void_p, ctypes.c_void_p]

    library.MagickGetImageWidth.argtypes = [ctypes.c_void_p]
    library.MagickGetImageWidth.restype = ctypes.c_size_t

    library.MagickGetImageHeight.argtypes = [ctypes.c_void_p]
    library.MagickGetImageHeight.restype = ctypes.c_size_t

    library.MagickCropImage.argtypes = [ctypes.c_void_p, ctypes.c_size_t,
                                        ctypes.c_size_t, ctypes.c_ssize_t,
                                        ctypes.c_ssize_t]

    library.MagickResizeImage.argtypes = [ctypes.c_void_p, ctypes.c_size_t,
                                          ctypes.c_size_t, ctypes.c_int,
                                          ctypes.c_double]

    library.MagickRotateImage.argtypes = [ctypes.c_void_p, ctypes.c_void_p,
                                          ctypes.c_double]

    library.MagickResetIterator.argtypes = [ctypes.c_void_p]

    library.MagickIdentifyImage.argtypes = [ctypes.c_void_p]
    library.MagickIdentifyImage.restype = ctypes.c_char_p

    library.MagickRelinquishMemory.argtypes = [ctypes.c_void_p]
    library.MagickRelinquishMemory.restype = ctypes.c_void_p

    library.NewPixelIterator.argtypes = [ctypes.c_void_p]
    library.NewPixelIterator.restype = ctypes.c_void_p

    library.DestroyPixelIterator.argtypes = [ctypes.c_void_p]
    library.DestroyPixelIterator.restype = ctypes.c_void_p

    library.ClonePixelIterator.argtypes = [ctypes.c_void_p]
    library.ClonePixelIterator.restype = ctypes.c_void_p

    library.IsPixelIterator.argtypes = [ctypes.c_void_p]

    library.PixelGetIteratorException.argtypes = [ctypes.c_void_p,
                                                  ctypes.POINTER(ctypes.c_int)]
    library.PixelGetIteratorException.restype = ctypes.c_char_p

    library.PixelClearIteratorException.argtypes = [ctypes.c_void_p]

    library.PixelSetFirstIteratorRow.argtypes = [ctypes.c_void_p]

    library.PixelSetIteratorRow.argtypes = [ctypes.c_void_p, ctypes.c_ssize_t]

    library.PixelGetNextIteratorRow.argtypes = [ctypes.c_void_p,
                                                ctypes.POINTER(ctypes.c_size_t)]
    library.PixelGetNextIteratorRow.restype = ctypes.POINTER(ctypes.c_void_p)

    library.NewPixelWand.restype = ctypes.c_void_p

    library.DestroyPixelWand.argtypes = [ctypes.c_void_p]
    library.DestroyPixelWand.restype = ctypes.c_void_p

    library.IsPixelWand.argtypes = [ctypes.c_void_p]

    library.PixelGetException.argtypes = [ctypes.c_void_p,
                                          ctypes.POINTER(ctypes.c_int)]
    library.PixelGetException.restype = ctypes.c_char_p

    library.PixelClearException.argtypes = [ctypes.c_void_p]

    library.IsPixelWandSimilar.argtypes = [ctypes.c_void_p, ctypes.c_void_p,
                                           ctypes.c_double]

    library.PixelGetMagickColor.argtypes = [ctypes.c_void_p, ctypes.c_void_p]

    library.PixelSetMagickColor.argtypes = [ctypes.c_void_p, ctypes.c_void_p]

    library.PixelSetColor.argtypes = [ctypes.c_void_p, ctypes.c_char_p]

    library.PixelGetColorAsString.argtypes = [ctypes.c_void_p]
    library.PixelGetColorAsString.restype = ctypes.c_char_p

    library.PixelGetAlpha.argtypes = [ctypes.c_void_p]
    library.PixelGetAlpha.restype = ctypes.c_double
except AttributeError:
    raise ImportError('MagickWand shared library not found or incompatible')

#: (:class:`ctypes.CDLL`) The C standard library.
libc = None

if platform.system() == 'Windows':
    libc = ctypes.CDLL(ctypes.util.find_msvcrt())
else:
    if platform.system() == 'Darwin':
        libc = ctypes.cdll.LoadLibrary('libc.dylib')
    else:
        libc = ctypes.cdll.LoadLibrary('libc.so.6')
    libc.fdopen.argtypes = [ctypes.c_int, ctypes.c_char_p]
    libc.fdopen.restype = ctypes.c_void_p

libc.free.argtypes = [ctypes.c_void_p]

<commit_msg>added argtypes definitions for MagickWand methods needed for watermarking<commit_after>""":mod:`wand.api` --- Low-level interfaces
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

"""
import os
import os.path
import platform
import ctypes
import ctypes.util

__all__ = 'load_library', 'MagickPixelPacket', 'library', 'libc'

def load_library():
    """Loads the MagickWand library.

    :returns: the MagickWand library and the ImageMagick library

    """
    libpath = None
    system = platform.system()

    magick_home = os.environ.get('MAGICK_HOME')
    if magick_home:
        if system == 'Windows':
            libpath = 'CORE_RL_wand_.dll',
        elif system == 'Darwin':
            libpath = 'lib', 'libMagickWand.dylib',
        else:
            libpath = 'lib', 'libMagickWand.so',
        libpath = os.path.join(magick_home, *libpath)
    else:
        if system == 'Windows':
            libpath = ctypes.util.find_library('CORE_RL_wand_')
        else:
            libpath = ctypes.util.find_library('MagickWand')
    libwand = ctypes.CDLL(libpath)

    if system == 'Windows':
        # On Windows, the API is split between two libs. On other platforms,
        # it's all contained in one.
        libmagick_filename = 'CORE_RL_magick_'
        if magick_home:
            libmagick_path = os.path.join(magick_home,
                                          libmagick_filename + '.dll')
        else:
            libmagick_path = ctypes.util.find_library(libmagick_filename)
        libmagick = ctypes.CDLL(libmagick_path)
        return libwand, libmagick

    return libwand, libwand

if not hasattr(ctypes, 'c_ssize_t'):
    if ctypes.sizeof(ctypes.c_uint) == ctypes.sizeof(ctypes.c_void_p):
        ctypes.c_ssize_t = ctypes.c_int
    elif ctypes.sizeof(ctypes.c_ulong) == ctypes.sizeof(ctypes.c_void_p):
        ctypes.c_ssize_t = ctypes.c_long
    elif ctypes.sizeof(ctypes.c_ulonglong) == ctypes.sizeof(ctypes.c_void_p):
        ctypes.c_ssize_t = ctypes.c_longlong

class MagickPixelPacket(ctypes.Structure):

    _fields_ = [('storage_class', ctypes.c_int),
                ('colorspace', ctypes.c_int),
                ('matte', ctypes.c_int),
                ('fuzz', ctypes.c_double),
                ('depth', ctypes.c_size_t),
                ('red', ctypes.c_double),
                ('green', ctypes.c_double),
                ('blue', ctypes.c_double),
                ('opacity', ctypes.c_double),
                ('index', ctypes.c_double)]

libraries = load_library()

#: (:class:`ctypes.CDLL`) The MagickWand library.
library = libraries[0]

#: (:class:`ctypes.CDLL`) The ImageMagick library.  It is the same with
#: :data:`library` on platforms other than Windows.
libmagick = libraries[1]

try:
    library.NewMagickWand.restype = ctypes.c_void_p

    library.DestroyMagickWand.argtypes = [ctypes.c_void_p]
    library.DestroyMagickWand.restype = ctypes.c_void_p

    library.CloneMagickWand.argtypes = [ctypes.c_void_p]
    library.CloneMagickWand.restype = ctypes.c_void_p

    library.IsMagickWand.argtypes = [ctypes.c_void_p]

    library.MagickGetException.argtypes = [ctypes.c_void_p,
                                           ctypes.POINTER(ctypes.c_int)]
    library.MagickGetException.restype = ctypes.c_char_p

    library.MagickClearException.argtypes = [ctypes.c_void_p]

    library.MagickReadImageBlob.argtypes = [ctypes.c_void_p, ctypes.c_void_p,
                                            ctypes.c_size_t]

    library.MagickReadImage.argtypes = [ctypes.c_void_p, ctypes.c_char_p]

    library.MagickReadImageFile.argtypes = [ctypes.c_void_p, ctypes.c_void_p]

    library.MagickGetImageFormat.argtypes = [ctypes.c_void_p]
    library.MagickGetImageFormat.restype = ctypes.c_char_p

    library.MagickSetImageFormat.argtypes = [ctypes.c_void_p, ctypes.c_char_p]

    libmagick.MagickToMime.argtypes = [ctypes.c_char_p]
    libmagick.MagickToMime.restype = ctypes.POINTER(ctypes.c_char)

    library.MagickGetImageSignature.argtypes = [ctypes.c_void_p]
    library.MagickGetImageSignature.restype = ctypes.c_char_p

    library.MagickGetImageBackgroundColor.argtypes = [ctypes.c_void_p,
                                                      ctypes.c_void_p]

    library.MagickSetImageBackgroundColor.argtypes = [ctypes.c_void_p,
                                                      ctypes.c_void_p]

    library.MagickGetImageBlob.argtypes = [ctypes.c_void_p,
                                           ctypes.POINTER(ctypes.c_size_t)]
    library.MagickGetImageBlob.restype = ctypes.POINTER(ctypes.c_ubyte)

    library.MagickWriteImage.argtypes = [ctypes.c_void_p, ctypes.c_char_p]

    library.MagickWriteImageFile.argtypes = [ctypes.c_void_p, ctypes.c_void_p]

    library.MagickGetImageWidth.argtypes = [ctypes.c_void_p]
    library.MagickGetImageWidth.restype = ctypes.c_size_t

    library.MagickGetImageHeight.argtypes = [ctypes.c_void_p]
    library.MagickGetImageHeight.restype = ctypes.c_size_t

    library.MagickCropImage.argtypes = [ctypes.c_void_p, ctypes.c_size_t,
                                        ctypes.c_size_t, ctypes.c_ssize_t,
                                        ctypes.c_ssize_t]

    library.MagickResizeImage.argtypes = [ctypes.c_void_p, ctypes.c_size_t,
                                          ctypes.c_size_t, ctypes.c_int,
                                          ctypes.c_double]

    library.MagickRotateImage.argtypes = [ctypes.c_void_p, ctypes.c_void_p,
                                          ctypes.c_double]

    library.MagickResetIterator.argtypes = [ctypes.c_void_p]

    library.MagickIdentifyImage.argtypes = [ctypes.c_void_p]
    library.MagickIdentifyImage.restype = ctypes.c_char_p

    library.MagickRelinquishMemory.argtypes = [ctypes.c_void_p]
    library.MagickRelinquishMemory.restype = ctypes.c_void_p

    library.NewPixelIterator.argtypes = [ctypes.c_void_p]
    library.NewPixelIterator.restype = ctypes.c_void_p

    library.DestroyPixelIterator.argtypes = [ctypes.c_void_p]
    library.DestroyPixelIterator.restype = ctypes.c_void_p

    library.ClonePixelIterator.argtypes = [ctypes.c_void_p]
    library.ClonePixelIterator.restype = ctypes.c_void_p

    library.IsPixelIterator.argtypes = [ctypes.c_void_p]

    library.PixelGetIteratorException.argtypes = [ctypes.c_void_p,
                                                  ctypes.POINTER(ctypes.c_int)]
    library.PixelGetIteratorException.restype = ctypes.c_char_p

    library.PixelClearIteratorException.argtypes = [ctypes.c_void_p]

    library.PixelSetFirstIteratorRow.argtypes = [ctypes.c_void_p]

    library.PixelSetIteratorRow.argtypes = [ctypes.c_void_p, ctypes.c_ssize_t]

    library.PixelGetNextIteratorRow.argtypes = [ctypes.c_void_p,
                                                ctypes.POINTER(ctypes.c_size_t)]
    library.PixelGetNextIteratorRow.restype = ctypes.POINTER(ctypes.c_void_p)

    library.NewPixelWand.restype = ctypes.c_void_p

    library.DestroyPixelWand.argtypes = [ctypes.c_void_p]
    library.DestroyPixelWand.restype = ctypes.c_void_p

    library.IsPixelWand.argtypes = [ctypes.c_void_p]

    library.PixelGetException.argtypes = [ctypes.c_void_p,
                                          ctypes.POINTER(ctypes.c_int)]
    library.PixelGetException.restype = ctypes.c_char_p

    library.PixelClearException.argtypes = [ctypes.c_void_p]

    library.IsPixelWandSimilar.argtypes = [ctypes.c_void_p, ctypes.c_void_p,
                                           ctypes.c_double]

    library.PixelGetMagickColor.argtypes = [ctypes.c_void_p, ctypes.c_void_p]

    library.PixelSetMagickColor.argtypes = [ctypes.c_void_p, ctypes.c_void_p]

    library.PixelSetColor.argtypes = [ctypes.c_void_p, ctypes.c_char_p]

    library.PixelGetColorAsString.argtypes = [ctypes.c_void_p]
    library.PixelGetColorAsString.restype = ctypes.c_char_p

    library.PixelGetAlpha.argtypes = [ctypes.c_void_p]
    library.PixelGetAlpha.restype = ctypes.c_double

    library.MagickGetQuantumRange.argtypes = [ctypes.POINTER(ctypes.c_size_t)]

    library.MagickSetIteratorIndex.argtypes = [ctypes.c_void_p,
                                               ctypes.c_ssize_t]

    library.MagickSetImageType.argtypes = [ctypes.c_void_p, ctypes.c_int]

    library.MagickEvaluateImageChannel.argtypes = [ctypes.c_void_p,
                                                   ctypes.c_int,
                                                   ctypes.c_int,
                                                   ctypes.c_double]

    library.MagickCompositeImage.argtypes = [ctypes.c_void_p, ctypes.c_void_p,
                                             ctypes.c_int, ctypes.c_ssize_t,
                                             ctypes.c_ssize_t]
except AttributeError:
    raise ImportError('MagickWand shared library not found or incompatible')

#: (:class:`ctypes.CDLL`) The C standard library.
libc = None

if platform.system() == 'Windows':
    libc = ctypes.CDLL(ctypes.util.find_msvcrt())
else:
    if platform.system() == 'Darwin':
        libc = ctypes.cdll.LoadLibrary('libc.dylib')
    else:
        libc = ctypes.cdll.LoadLibrary('libc.so.6')
    libc.fdopen.argtypes = [ctypes.c_int, ctypes.c_char_p]
    libc.fdopen.restype = ctypes.c_void_p

libc.free.argtypes = [ctypes.c_void_p]

<|endoftext|>


<commit_before># Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Module to provide implicit behavior based on enviroment.

Acts as a mutable namespace to allow the datastore package to
imply the current dataset ID and connection from the enviroment.
"""

import httplib
import socket

try:
    from google.appengine.api import app_identity
except ImportError:
    app_identity = None

DATASET_ID = None
"""Module global to allow persistent implied dataset ID from enviroment."""

CONNECTION = None
"""Module global to allow persistent implied connection from enviroment."""

def app_engine_id():
    """Gets the App Engine application ID if it can be inferred.

    :rtype: string or ``NoneType``
    :returns: App Engine application ID if running in App Engine,
              else ``None``.
    """
    if app_identity is None:
        return None

    return app_identity.get_application_id()

def compute_engine_id():
    """Gets the Compute Engine project ID if it can be inferred.

    Uses 169.254.169.254 for the metadata server to avoid request
    latency from DNS lookup.

    See https://cloud.google.com/compute/docs/metadata#metadataserver
    for information about this IP address. (This IP is also used for
    Amazon EC2 instances, so the metadata flavor is crucial.)

    See https://github.com/google/oauth2client/issues/93 for context about
    DNS latency.

    :rtype: string or ``NoneType``
    :returns: Compute Engine project ID if the metadata service is available,
              else ``None``.
    """
    host = '169.254.169.254'
    uri_path = '/computeMetadata/v1/project/project-id'
    headers = {'Metadata-Flavor': 'Google'}
    connection = httplib.HTTPConnection(host, timeout=0.1)

    content = None
    try:
        connection.request('GET', uri_path, headers=headers)
        response = connection.getresponse()
        if response.status == 200:
            content = response.read()
    except socket.error:  # socket.timeout or socket.error(64, 'Host is down')
        pass
    finally:
        connection.close()

    return content
<commit_msg>Returning response value in compute_engine_id() instead of using var.<commit_after># Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Module to provide implicit behavior based on enviroment.

Acts as a mutable namespace to allow the datastore package to
imply the current dataset ID and connection from the enviroment.
"""

import httplib
import socket

try:
    from google.appengine.api import app_identity
except ImportError:
    app_identity = None

DATASET_ID = None
"""Module global to allow persistent implied dataset ID from enviroment."""

CONNECTION = None
"""Module global to allow persistent implied connection from enviroment."""

def app_engine_id():
    """Gets the App Engine application ID if it can be inferred.

    :rtype: string or ``NoneType``
    :returns: App Engine application ID if running in App Engine,
              else ``None``.
    """
    if app_identity is None:
        return None

    return app_identity.get_application_id()

def compute_engine_id():
    """Gets the Compute Engine project ID if it can be inferred.

    Uses 169.254.169.254 for the metadata server to avoid request
    latency from DNS lookup.

    See https://cloud.google.com/compute/docs/metadata#metadataserver
    for information about this IP address. (This IP is also used for
    Amazon EC2 instances, so the metadata flavor is crucial.)

    See https://github.com/google/oauth2client/issues/93 for context about
    DNS latency.

    :rtype: string or ``NoneType``
    :returns: Compute Engine project ID if the metadata service is available,
              else ``None``.
    """
    host = '169.254.169.254'
    uri_path = '/computeMetadata/v1/project/project-id'
    headers = {'Metadata-Flavor': 'Google'}
    connection = httplib.HTTPConnection(host, timeout=0.1)

    try:
        connection.request('GET', uri_path, headers=headers)
        response = connection.getresponse()
        if response.status == 200:
            return response.read()
    except socket.error:  # socket.timeout or socket.error(64, 'Host is down')
        pass
    finally:
        connection.close()
<|endoftext|>


<commit_before>import fnmatch
import glob
import io
import itertools
import logging
import os
import subprocess

from sqlalchemy.orm.exc import NoResultFound

from aeromancer.db.models import Project, File, Line
from aeromancer import filehandler
from aeromancer import utils

LOG = logging.getLogger(__name__)

def discover(repo_root, organizations):
    """Discover project-like directories under the repository root"""
    glob_patterns = ['%s/*' % o for o in organizations]
    with utils.working_dir(repo_root):
        return itertools.ifilter(
            lambda x: os.path.isdir(os.path.join(repo_root, x)),
            itertools.chain(*(glob.glob(g) for g in glob_patterns))
        )

def _find_files_in_project(path):
    """Return a list of the files managed in the project and their sha hash.

    Uses 'git ls-files -s'
    """
    with utils.working_dir(path):
        # Ask git to tell us the sha hash so we can tell if the file
        # has changed since we looked at it last.
        cmd = subprocess.Popen(['git', 'ls-files', '-z', '-s'],
                               stdout=subprocess.PIPE)
        output = cmd.communicate()[0]
        entries = output.split('\0')
        for e in entries:
            if not e:
                continue
            metadata, ignore, filename = e.partition('\t')
            sha = metadata.split(' ')[1]
            yield (filename, sha)

class ProjectManager(object):

    _DO_NOT_READ = [
        '*.doc', '*.docx', '*.graffle', '*.odp', '*.pptx', '*.vsd', '*.xsd',
        '*.gif', '*.ico', '*.jpeg', '*.jpg', '*.png', '*.tiff',
        '*.gpg',
        '*.jar',  # Why do we check in jar files?!
        '*.swf', '*.eot',
        '*.ttf', '*.woff',  # webfont; horizon
        '*.xml',
        '*.gz', '*.zip', '*.z',
        '*.mo', '*.db',
    ]

    def __init__(self, session):
        self.file_handlers = filehandler.load_handlers()
        self.session = session

    def get_project(self, name):
        """Return an existing project, if there is one"""
        query = self.session.query(Project).filter(Project.name == name)
        try:
            return query.one()
        except NoResultFound:
            return None

    def add_or_update(self, name, path, force=False):
        """Create a new project definition or update an existing one"""
        proj_obj = self.get_project(name)
        if proj_obj:
            proj_obj.path = path
            LOG.info('updating project %s from %s', name, path)
        else:
            proj_obj = Project(name=name, path=path)
            LOG.info('adding project %s from %s', name, path)
            self.session.add(proj_obj)
        self.update(proj_obj, force=force)
        return proj_obj

    def update(self, proj_obj, force=False):
        """Update the settings for an existing project"""
        self._update_project_files(proj_obj, force=force)

    def remove(self, name):
        """Delete stored data for the named project"""
        query = self.session.query(Project).filter(Project.name == name)
        try:
            proj_obj = query.one()
            LOG.info('removing project %s', name)
        except NoResultFound:
            return
        self._delete_filehandler_data_from_project(proj_obj)
        self.session.delete(proj_obj)

    def _remove_filehandler_data_for_file(self, file_obj):
        # We have to explicitly have the handlers delete their data
        # because the parent-child relationship of the tables is reversed
        # because the plugins define the relationships.
        LOG.debug('removing plugin data for %s', file_obj.name)
        for fh in self.file_handlers:
            if fh.obj.supports_file(file_obj):
                fh.obj.delete_data_for_file(self.session, file_obj)

    def _remove_file_data(self, file_obj, reason='file has changed'):
        """Delete the data associated with the file, including plugin data and
        file contents.

        """
        LOG.debug('removing cached contents of %s: %s', file_obj.name, reason)
        self._remove_filehandler_data_for_file(file_obj)
        self.session.delete(file_obj)

    def _update_project_files(self, proj_obj, force):
        """Update the files stored for each project"""
        LOG.debug('reading file contents in %s', proj_obj.name)

        # Collect the known files in a project so we can test their
        # SHAs quickly.
        known = {f.name: f for f in proj_obj.files}

        # Track the files we've seen so we can delete any files that
        # are no longer present.
        seen = set()

        # Now load the files currently being managed by git.
        for filename, sha in _find_files_in_project(proj_obj.path):
            # Remember that we have seen the file in the project.
            seen.add(filename)
            # Skip things that are not files (usually symlinks).
            fullname = os.path.join(proj_obj.path, filename)
            if not os.path.isfile(fullname):
                continue
            try:
                existing_file = known[filename]
                if existing_file.sha == sha and not force:
                    # File has not changed, we can use the content we
                    # already have.
                    LOG.debug('using cached version of %s', filename)
                    continue
                self._remove_file_data(existing_file)
            except KeyError:
                pass
            new_file = File(project=proj_obj, name=filename, path=fullname, sha=sha)
            self.session.add(new_file)
            if any(fnmatch.fnmatch(filename, dnr) for dnr in self._DO_NOT_READ):
                LOG.debug('ignoring contents of %s', fullname)
            else:
                with io.open(fullname, mode='r', encoding='utf-8') as f:
                    try:
                        body = f.read()
                    except UnicodeDecodeError:
                        # FIXME(dhellmann): Be smarter about trying other
                        # encodings?
                        LOG.warn('Could not read %s as a UTF-8 encoded file, ignoring',
                                 fullname)
                        continue
                    lines = body.splitlines()
                    LOG.debug('%s/%s has %s lines', proj_obj.name, filename, len(lines))
                    for num, content in enumerate(lines, 1):
                        self.session.add(Line(file=new_file, number=num, content=content))

            # Invoke plugins for processing files in special ways
            for fh in self.file_handlers:
                if fh.obj.supports_file(new_file):
                    fh.obj.process_file(self.session, new_file)

            self.session.flush()

        # Remove files that we have in the database but that were no
        # longer seen in the git repository.
        for name, obj in known.items():
            if name not in seen:
                self._remove_file_data(obj, reason='file no longer exists')
<commit_msg>a few more performance tweaks for loading<commit_after>import fnmatch
import glob
import io
import itertools
import logging
import os
import subprocess

from sqlalchemy.orm.exc import NoResultFound

from aeromancer.db.models import Project, File, Line
from aeromancer import filehandler
from aeromancer import utils

LOG = logging.getLogger(__name__)

def discover(repo_root, organizations):
    """Discover project-like directories under the repository root"""
    glob_patterns = ['%s/*' % o for o in organizations]
    with utils.working_dir(repo_root):
        return itertools.ifilter(
            lambda x: os.path.isdir(os.path.join(repo_root, x)),
            itertools.chain(*(glob.glob(g) for g in glob_patterns))
        )

def _find_files_in_project(path):
    """Return a list of the files managed in the project and their sha hash.

    Uses 'git ls-files -s'
    """
    with utils.working_dir(path):
        # Ask git to tell us the sha hash so we can tell if the file
        # has changed since we looked at it last.
        cmd = subprocess.Popen(['git', 'ls-files', '-z', '-s'],
                               stdout=subprocess.PIPE)
        output = cmd.communicate()[0]
        entries = output.split('\0')
        for e in entries:
            if not e:
                continue
            metadata, ignore, filename = e.partition('\t')
            sha = metadata.split(' ')[1]
            yield (filename, sha)

class ProjectManager(object):

    _DO_NOT_READ = [
        '*.doc', '*.docx', '*.graffle', '*.odp', '*.pptx', '*.vsd', '*.xsd',
        '*.gif', '*.ico', '*.jpeg', '*.jpg', '*.png', '*.tiff',
        '*.gpg',
        '*.jar',  # Why do we check in jar files?!
        '*.swf', '*.eot',
        '*.ttf', '*.woff',  # webfont; horizon
        '*.xml',
        '*.gz', '*.zip', '*.z',
        '*.mo', '*.db',
    ]

    def __init__(self, session):
        self.file_handlers = filehandler.load_handlers()
        self.session = session

    def get_project(self, name):
        """Return an existing project, if there is one"""
        query = self.session.query(Project).filter(Project.name == name)
        try:
            return query.one()
        except NoResultFound:
            return None

    def add_or_update(self, name, path, force=False):
        """Create a new project definition or update an existing one"""
        proj_obj = self.get_project(name)
        if proj_obj:
            proj_obj.path = path
            LOG.info('updating project %s from %s', name, path)
        else:
            proj_obj = Project(name=name, path=path)
            LOG.info('adding project %s from %s', name, path)
            self.session.add(proj_obj)
        self.update(proj_obj, force=force)
        return proj_obj

    def update(self, proj_obj, force=False):
        """Update the settings for an existing project"""
        self._update_project_files(proj_obj, force=force)

    def remove(self, name):
        """Delete stored data for the named project"""
        query = self.session.query(Project).filter(Project.name == name)
        try:
            proj_obj = query.one()
            LOG.info('removing project %s', name)
        except NoResultFound:
            return
        self._delete_filehandler_data_from_project(proj_obj)
        self.session.delete(proj_obj)

    def _remove_file_data(self, file_obj, reason='file has changed'):
        """Delete the data associated with the file, including plugin data and
        file contents.

        """
        LOG.debug('removing cached contents of %s: %s', file_obj.name, reason)
        # We have to explicitly have the handlers delete their data
        # because the parent-child relationship of the tables is reversed
        # because the plugins define the relationships.
        for fh in self.file_handlers:
            if fh.obj.supports_file(file_obj):
                LOG.debug('removing %s plugin data for %s', fh.name, file_obj.name)
                fh.obj.delete_data_for_file(self.session, file_obj)
        self.session.query(Line).filter(Line.file_id == file_obj.id).delete()
        self.session.delete(file_obj)

    def _update_project_files(self, proj_obj, force):
        """Update the files stored for each project"""
        LOG.debug('reading file contents in %s', proj_obj.name)

        # Collect the known files in a project so we can test their
        # SHAs quickly.
        known = {f.name: f for f in proj_obj.files}

        # Track the files we've seen so we can delete any files that
        # are no longer present.
        seen = set()

        # Now load the files currently being managed by git.
        for filename, sha in _find_files_in_project(proj_obj.path):
            # Remember that we have seen the file in the project.
            seen.add(filename)
            # Skip things that are not files (usually symlinks).
            fullname = os.path.join(proj_obj.path, filename)
            if not os.path.isfile(fullname):
                continue
            try:
                existing_file = known[filename]
                if existing_file.sha == sha and not force:
                    # File has not changed, we can use the content we
                    # already have.
                    LOG.debug('using cached version of %s', filename)
                    continue
                self._remove_file_data(existing_file)
            except KeyError:
                pass
            new_file = File(project=proj_obj, name=filename, path=fullname, sha=sha)
            self.session.add(new_file)
            if any(fnmatch.fnmatch(filename, dnr) for dnr in self._DO_NOT_READ):
                LOG.debug('ignoring contents of %s', fullname)
            else:
                LOG.debug('reading %s', fullname)
                with io.open(fullname, mode='r', encoding='utf-8') as f:
                    try:
                        body = f.read()
                    except UnicodeDecodeError:
                        # FIXME(dhellmann): Be smarter about trying other
                        # encodings?
                        LOG.warn('Could not read %s as a UTF-8 encoded file, ignoring',
                                 fullname)
                        continue
                    lines = body.splitlines()
                    # Use SQLalchemy's core mode to bulk insert the lines.
                    if lines:
                        self.session.execute(
                            Line.__table__.insert(),
                            [{'file_id': new_file.id,
                              'number': num,
                              'content': content}
                             for num, content in enumerate(lines, 1)]
                        )
                    LOG.debug('%s/%s has %s lines', proj_obj.name, filename, len(lines))

            # Invoke plugins for processing files in special ways
            for fh in self.file_handlers:
                if fh.obj.supports_file(new_file):
                    fh.obj.process_file(self.session, new_file)

            self.session.flush()

        # Remove files that we have in the database but that were no
        # longer seen in the git repository.
        for name, obj in known.items():
            if name not in seen:
                self._remove_file_data(obj, reason='file no longer exists')
                self.session.flush()
<|endoftext|>