<commit_before>'''
Created on June 14, 2012

@package: superdesk media archive image
@copyright: 2012 Sourcefabric o.p.s.
@license: http://www.gnu.org/licenses/gpl-3.0.txt
@author: Mugur Rus
'''

# --------------------------------------------------------------------

from setuptools import setup, find_packages

# --------------------------------------------------------------------

setup(
    name="media_archive_image",
    version="1.0",
    packages=find_packages(),
    install_requires=['ally_api >= 1.0', 'ally_core_sqlalchemy >= 1.0', 'gui_action >= 1.0',
                      'gui_core >= 1.0', 'internationalization >= 1.0'],
    platforms=['all'],
    zip_safe=True,
    package_data={
        '': ['*.gif', '*.png', '*.jpg', '*.jpeg', '*.js', '*.css', '*.less', '*.dust', '*.html',
             '*.jar', '*.exe', '*.so*', '*.a*', '*.txt', 'AUTHORS', 'COPYING', 'README'],
    },

    # metadata for upload to PyPI
    author="Gabriel Nistor",
    author_email="gabriel.nistor@sourcefabric.org",
    description="Media archive image plugin",
    long_description='Image files management functionality',
    license="GPL v3",
    keywords="Ally REST framework plugin Livedesk",
    url="http://www.sourcefabric.org/en/superdesk/", # project home page
)
<commit_msg>NR-436 : Image metadata reading tool not inserted in egg<commit_after>'''
Created on June 14, 2012

@package: superdesk media archive image
@copyright: 2012 Sourcefabric o.p.s.
@license: http://www.gnu.org/licenses/gpl-3.0.txt
@author: Mugur Rus
'''

# --------------------------------------------------------------------

from setuptools import setup, find_packages

# --------------------------------------------------------------------

setup(
    name="media_archive_image",
    version="1.0",
    packages=find_packages(),
    install_requires=['ally_api >= 1.0', 'ally_core_sqlalchemy >= 1.0', 'gui_action >= 1.0',
                      'gui_core >= 1.0', 'internationalization >= 1.0'],
    platforms=['all'],
    zip_safe=True,
    package_data={
        '': ['*.gif', '*.png', '*.jpg', '*.jpeg', '*.js', '*.css', '*.less', '*.dust', '*.html',
             '*.jar', '*.exe', '*.so*', '*.a*', '*.txt', 'AUTHORS', 'COPYING*', 'README*'],
    },

    # metadata for upload to PyPI
    author="Gabriel Nistor",
    author_email="gabriel.nistor@sourcefabric.org",
    description="Media archive image plugin",
    long_description='Image files management functionality',
    license="GPL v3",
    keywords="Ally REST framework plugin Livedesk",
    url="http://www.sourcefabric.org/en/superdesk/", # project home page
)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import argparse
import random
import subprocess
import os
import sys

PROJECT_NAME = "cla_backend"

# use python scripts/jenkins/build.py integration

# args
parser = argparse.ArgumentParser(description='Build project ready for testing by Jenkins.')
parser.add_argument('envname', metavar='envname', type=str, nargs=1, help='e.g. integration, production, etc.')
args = parser.parse_args()

env = args.envname[0]
env_name = "%s-%s" % (PROJECT_NAME, env)
env_path = "/tmp/jenkins/envs/%s" % env_name
bin_path = "%s/bin" % env_path

def run(command, **kwargs):
	defaults = {
		'shell': True
	}
	defaults.update(kwargs)

	return_code = subprocess.call(command, **defaults)
	if return_code:
		sys.exit(return_code)

# setting up virtualenv
if not os.path.isdir(env_path):
	run('virtualenv --no-site-packages %s' % env_path)

run('%s/pip install -r requirements/jenkins.txt' % bin_path)

# Remove .pyc files from the project
run("find . -name '*.pyc' -delete")

# run tests
print 'starting...'

backend_port = random.randint(8005, 8999)
frontend_port = backend_port + 1
os.environ['BACKEND_BASE_PORT'] = '%s' % backend_port
os.environ['FRONTEND_BASE_PORT'] = '%s' % frontend_port

run("%s/python manage.py jenkins --coverage-rcfile=.coveragerc --settings=cla_backend.settings.jenkins" % bin_path)
<commit_msg>minor jenkins build script changes<commit_after>#!/usr/bin/env python
import argparse
import random
import subprocess
import os
import sys

PROJECT_NAME = 'cla_backend'

# use python scripts/jenkins/build.py integration

# args
parser = argparse.ArgumentParser(
    description='Build project ready for testing by Jenkins.'
)
parser.add_argument('envname', type=str,
                    help='e.g. integration, production, etc.')
args = parser.parse_args()

env = args.envname
env_name = '%s-%s' % (PROJECT_NAME, env)
env_path = '/tmp/jenkins/envs/%s' % env_name
bin_path = '%s/bin' % env_path

def run(command, **kwargs):
    defaults = {
        'shell': True
    }
    defaults.update(kwargs)

    return_code = subprocess.call(command, **defaults)
    if return_code:
        sys.exit(return_code)

# setting up virtualenv
if not os.path.isdir(env_path):
    run('virtualenv %s' % env_path)

run('%s/pip install -U setuptools pip wheel' % bin_path)
run('%s/pip install -r requirements/jenkins.txt' % bin_path)

# Remove .pyc files from the project
run("find . -name '*.pyc' -delete")

# run tests
print 'starting...'

backend_port = random.randint(8005, 8999)
frontend_port = backend_port + 1
os.environ['BACKEND_BASE_PORT'] = str(backend_port)
os.environ['FRONTEND_BASE_PORT'] = str(frontend_port)

run('%s/python manage.py jenkins '
    '--coverage-rcfile=.coveragerc '
    '--settings=cla_backend.settings.jenkins' % bin_path)
<|endoftext|>


<commit_before>import json
import os
import time

import requests

from .errors import *

class BaseAPI(object):
    """
    Base class for the pyfcm API wrapper for FCM
    """

    CONTENT_TYPE = "application/json"
    FCM_END_POINT = "https://fcm.googleapis.com/fcm/send"
    # FCM only allows up to 1000 reg ids per bulk message.
    FCM_MAX_RECIPIENTS = 1000

    #: Indicates that the push message should be sent with low priority. Low
    #: priority optimizes the client app's battery consumption, and should be used
    #: unless immediate delivery is required. For messages with low priority, the
    #: app may receive the message with unspecified delay.
    FCM_LOW_PRIORITY = 'normal'

    #: Indicates that the push message should be sent with a high priority. When a
    #: message is sent with high priority, it is sent immediately, and the app can
    #: wake a sleeping device and open a network connection to your server.
    FCM_HIGH_PRIORITY = 'high'

    def __init__(self, api_key=None, proxy_dict=None, env=None, json_encoder=None):
        """

        :type proxy_dict: dict, api_key: string
        """
        if api_key:
            self._FCM_API_KEY = api_key
        elif os.getenv('FCM_API_KEY', None):
            self._FCM_API_KEY = os.getenv('FCM_API_KEY', None)
        else:
            raise AuthenticationError("Please provide the api_key in the google-services.json file")
        self.FCM_REQ_PROXIES = None
        if proxy_dict and isinstance(proxy_dict, dict) and (('http' in proxy_dict) or ('https' in proxy_dict)):
            self.FCM_REQ_PROXIES = proxy_dict
        self.send_request_responses = list()
        if env == 'app_engine':
            try:
                from requests_toolbelt.adapters import appengine
                appengine.monkeypatch()
            except:
                pass
        self.json_encoder = json_encoder

    def request_headers(self):
        return {
            "Content-Type": self.CONTENT_TYPE,
            "Authorization": "key=" + self._FCM_API_KEY,
        }

    def registration_id_chunks(self, registration_ids):
        try:
            xrange
        except NameError:
            xrange = range
        """Yield successive 1000-sized (max fcm recipients per request) chunks from registration_ids."""
        for i in xrange(0, len(registration_ids), self.FCM_MAX_RECIPIENTS):
            yield registration_ids[i:i + self.FCM_MAX_RECIPIENTS]

    def json_dumps(self, data):
        """Standardized json.dumps function with separators and sorted keys set."""
        return (json.dumps(data, separators=(',', ':'), sort_keys=True, cls=self.json_encoder)
                .encode('utf8'))

    def parse_payload(self,
                      registration_ids=None,
                      topic_name=None,
                      message_body=None,
                      message_title=None,
                      message_icon=None,
                      sound=None,
                      condition=None,
                      collapse_key=None,
                      delay_while_idle=False,
                      time_to_live=None,
                      restricted_package_name=None,
                      low_priority=False,
                      dry_run=False,
                      data_message=None,
                      click_action=None,
                      badge=None,
                      color=None,
                      tag=None,
                      body_loc_key=None,
                      body_loc_args=None,
                      title_loc_key=None,
                      title_loc_args=None,
                      content_available=None,
                      remove_notification=False,
                      **extra_kwargs):

        """

        :rtype: json
        """
        fcm_payload = dict()
        if registration_ids:
            if len(registration_ids) > 1:
                fcm_payload['registration_ids'] = registration_ids
            else:
                fcm_payload['to'] = registration_ids[0]
        if condition:
            fcm_payload['condition'] = condition
        else:
            # In the `to` reference at: https://firebase.google.com/docs/cloud-messaging/http-server-ref#send-downstream
            # We have `Do not set this field (to) when sending to multiple topics`
            # Which is why it's in the `else` block since `condition` is used when multiple topics are being targeted
            if topic_name:
                fcm_payload['to'] = '/topics/%s' % topic_name
        # Add a object within the payload to set priority of the messages with reference to
        # https://firebase.google.com/docs/cloud-messaging/concept-options#setting-the-priority-of-a-message
        # required for legacy http otherwise data messages with priority as high are sent with normal priority
        if low_priority:
            fcm_payload['android'] = dict(priority=self.FCM_LOW_PRIORITY)
        else:
            fcm_payload['android'] = dict(priority=self.FCM_HIGH_PRIORITY)

        if delay_while_idle:
            fcm_payload['delay_while_idle'] = delay_while_idle
        if collapse_key:
            fcm_payload['collapse_key'] = collapse_key
        if time_to_live:
            if isinstance(time_to_live, int):
                fcm_payload['time_to_live'] = time_to_live
            else:
                raise InvalidDataError("Provided time_to_live is not an integer")
        if restricted_package_name:
            fcm_payload['restricted_package_name'] = restricted_package_name
        if dry_run:
            fcm_payload['dry_run'] = dry_run

        if data_message:
            if isinstance(data_message, dict):
                fcm_payload['data'] = data_message
            else:
                raise InvalidDataError("Provided data_message is in the wrong format")

        fcm_payload['notification'] = {}
        if message_icon:
            fcm_payload['notification']['icon'] = message_icon
        # If body is present, use it
        if message_body:
            fcm_payload['notification']['body'] = message_body
        # Else use body_loc_key and body_loc_args for body
        else:
            if body_loc_key:
                fcm_payload['notification']['body_loc_key'] = body_loc_key
            if body_loc_args:
                if isinstance(body_loc_args, list):
                    fcm_payload['notification']['body_loc_args'] = body_loc_args
                else:
                    raise InvalidDataError('body_loc_args should be an array')
        # If title is present, use it
        if message_title:
            fcm_payload['notification']['title'] = message_title
        # Else use title_loc_key and title_loc_args for title
        else:
            if title_loc_key:
                fcm_payload['notification']['title_loc_key'] = title_loc_key
            if title_loc_args:
                if isinstance(title_loc_args, list):
                    fcm_payload['notification']['title_loc_args'] = title_loc_args
                else:
                    raise InvalidDataError('title_loc_args should be an array')

        # This is needed for iOS when we are sending only custom data messages
        if content_available and isinstance(content_available, bool):
            fcm_payload['content_available'] = content_available

        if click_action:
            fcm_payload['notification']['click_action'] = click_action
        if isinstance(badge, int) and badge >= 0:
            fcm_payload['notification']['badge'] = badge
        if color:
            fcm_payload['notification']['color'] = color
        if tag:
            fcm_payload['notification']['tag'] = tag
        # only add the 'sound' key if sound is not None
        # otherwise a default sound will play -- even with empty string args.
        if sound:
            fcm_payload['notification']['sound'] = sound

        if extra_kwargs:
            fcm_payload['notification'].update(extra_kwargs)

        # Do this if you only want to send a data message.
        if remove_notification:
            del fcm_payload['notification']

        return self.json_dumps(fcm_payload)

    def do_request(self, payload, timeout):
        if self.FCM_REQ_PROXIES:
            response = requests.post(self.FCM_END_POINT, headers=self.request_headers(), data=payload,
                                     proxies=self.FCM_REQ_PROXIES, timeout=timeout)
        else:
            response = requests.post(self.FCM_END_POINT, headers=self.request_headers(), data=payload, timeout=timeout)
        if 'Retry-After' in response.headers and int(response.headers['Retry-After']) > 0:
            sleep_time = int(response.headers['Retry-After'])
            time.sleep(sleep_time)
            return self.do_request(payload, timeout)
        return response

    def send_request(self, payloads=None, timeout=None):
        self.send_request_responses = []
        for payload in payloads:
            response = self.do_request(payload, timeout)
            self.send_request_responses.append(response)

    def registration_info_request(self, registration_id):
        """ Makes a request for registration info and returns the response
            object
        """
        response = requests.get('https://iid.googleapis.com/iid/info/' + registration_id,
                               headers=self.request_headers(),
                               params={'details': 'true'})
        return response

    def clean_registration_ids(self, registration_ids=[]):
        """ Return list of active IDS from the list of registration_ids
        """
        valid_registration_ids = []
        for registration_id in registration_ids:
            details = self.registration_info_request(registration_id)
            if details.status_code == 200:
                valid_registration_ids.append(registration_id)
        return valid_registration_ids

    def get_registration_id_info(self, registration_id):
        """ Returns details related to a registration id if it exists
            otherwise return None
        """
        details = self.registration_info_request(registration_id)
        if details.status_code == 200:
            return details.json()
        return None

    def parse_responses(self):
        """
        Returns a python dict of multicast_ids(list), success(int), failure(int), canonical_ids(int), results(list) and optional topic_message_id(str but None by default)
        """
        response_dict = {
            'multicast_ids': list(),
            'success': 0,
            'failure': 0,
            'canonical_ids': 0,
            'results': list(),
            'topic_message_id': None
        }

        for response in self.send_request_responses:
            if response.status_code == 200:
                """
                Parses the json response sent back by the
                server and tries to get out the important return variables
                """
                if 'content-length' in response.headers and int(response.headers['content-length']) <= 0:
                    raise FCMServerError("FCM server connection error, the response is empty")
                else:
                    parsed_response = response.json()

                    multicast_id = parsed_response.get('multicast_id', None)
                    success = parsed_response.get('success', 0)
                    failure = parsed_response.get('failure', 0)
                    canonical_ids = parsed_response.get('canonical_ids', 0)
                    results = parsed_response.get('results', [])
                    message_id = parsed_response.get('message_id', None)  # for topic messages
                    if message_id:
                        success = 1
                    if multicast_id:
                        response_dict['multicast_ids'].append(multicast_id)
                    response_dict['success'] += success
                    response_dict['failure'] += failure
                    response_dict['canonical_ids'] += canonical_ids
                    response_dict['results'].extend(results)
                    response_dict['topic_message_id'] = message_id
            elif response.status_code == 401:
                raise AuthenticationError("There was an error authenticating the sender account")
            elif response.status_code == 400:
                raise InternalPackageError(response.text)
            else:
                raise FCMServerError("FCM server is temporarily unavailable")
        return response_dict
<commit_msg>Add topic subscription management.<commit_after>import json
import os
import time

import requests

from .errors import *

class BaseAPI(object):
    """
    Base class for the pyfcm API wrapper for FCM
    """

    CONTENT_TYPE = "application/json"
    FCM_END_POINT = "https://fcm.googleapis.com/fcm/send"
    # FCM only allows up to 1000 reg ids per bulk message.
    FCM_MAX_RECIPIENTS = 1000

    #: Indicates that the push message should be sent with low priority. Low
    #: priority optimizes the client app's battery consumption, and should be used
    #: unless immediate delivery is required. For messages with low priority, the
    #: app may receive the message with unspecified delay.
    FCM_LOW_PRIORITY = 'normal'

    #: Indicates that the push message should be sent with a high priority. When a
    #: message is sent with high priority, it is sent immediately, and the app can
    #: wake a sleeping device and open a network connection to your server.
    FCM_HIGH_PRIORITY = 'high'

    def __init__(self, api_key=None, proxy_dict=None, env=None, json_encoder=None):
        """

        :type proxy_dict: dict, api_key: string
        """
        if api_key:
            self._FCM_API_KEY = api_key
        elif os.getenv('FCM_API_KEY', None):
            self._FCM_API_KEY = os.getenv('FCM_API_KEY', None)
        else:
            raise AuthenticationError("Please provide the api_key in the google-services.json file")
        self.FCM_REQ_PROXIES = None
        if proxy_dict and isinstance(proxy_dict, dict) and (('http' in proxy_dict) or ('https' in proxy_dict)):
            self.FCM_REQ_PROXIES = proxy_dict
        self.send_request_responses = list()
        if env == 'app_engine':
            try:
                from requests_toolbelt.adapters import appengine
                appengine.monkeypatch()
            except:
                pass
        self.json_encoder = json_encoder

    def request_headers(self):
        return {
            "Content-Type": self.CONTENT_TYPE,
            "Authorization": "key=" + self._FCM_API_KEY,
        }

    def registration_id_chunks(self, registration_ids):
        try:
            xrange
        except NameError:
            xrange = range
        """Yield successive 1000-sized (max fcm recipients per request) chunks from registration_ids."""
        for i in xrange(0, len(registration_ids), self.FCM_MAX_RECIPIENTS):
            yield registration_ids[i:i + self.FCM_MAX_RECIPIENTS]

    def json_dumps(self, data):
        """Standardized json.dumps function with separators and sorted keys set."""
        return (json.dumps(data, separators=(',', ':'), sort_keys=True, cls=self.json_encoder)
                .encode('utf8'))

    def parse_payload(self,
                      registration_ids=None,
                      topic_name=None,
                      message_body=None,
                      message_title=None,
                      message_icon=None,
                      sound=None,
                      condition=None,
                      collapse_key=None,
                      delay_while_idle=False,
                      time_to_live=None,
                      restricted_package_name=None,
                      low_priority=False,
                      dry_run=False,
                      data_message=None,
                      click_action=None,
                      badge=None,
                      color=None,
                      tag=None,
                      body_loc_key=None,
                      body_loc_args=None,
                      title_loc_key=None,
                      title_loc_args=None,
                      content_available=None,
                      remove_notification=False,
                      **extra_kwargs):

        """

        :rtype: json
        """
        fcm_payload = dict()
        if registration_ids:
            if len(registration_ids) > 1:
                fcm_payload['registration_ids'] = registration_ids
            else:
                fcm_payload['to'] = registration_ids[0]
        if condition:
            fcm_payload['condition'] = condition
        else:
            # In the `to` reference at: https://firebase.google.com/docs/cloud-messaging/http-server-ref#send-downstream
            # We have `Do not set this field (to) when sending to multiple topics`
            # Which is why it's in the `else` block since `condition` is used when multiple topics are being targeted
            if topic_name:
                fcm_payload['to'] = '/topics/%s' % topic_name
        # Add a object within the payload to set priority of the messages with reference to
        # https://firebase.google.com/docs/cloud-messaging/concept-options#setting-the-priority-of-a-message
        # required for legacy http otherwise data messages with priority as high are sent with normal priority
        if low_priority:
            fcm_payload['android'] = dict(priority=self.FCM_LOW_PRIORITY)
        else:
            fcm_payload['android'] = dict(priority=self.FCM_HIGH_PRIORITY)

        if delay_while_idle:
            fcm_payload['delay_while_idle'] = delay_while_idle
        if collapse_key:
            fcm_payload['collapse_key'] = collapse_key
        if time_to_live:
            if isinstance(time_to_live, int):
                fcm_payload['time_to_live'] = time_to_live
            else:
                raise InvalidDataError("Provided time_to_live is not an integer")
        if restricted_package_name:
            fcm_payload['restricted_package_name'] = restricted_package_name
        if dry_run:
            fcm_payload['dry_run'] = dry_run

        if data_message:
            if isinstance(data_message, dict):
                fcm_payload['data'] = data_message
            else:
                raise InvalidDataError("Provided data_message is in the wrong format")

        fcm_payload['notification'] = {}
        if message_icon:
            fcm_payload['notification']['icon'] = message_icon
        # If body is present, use it
        if message_body:
            fcm_payload['notification']['body'] = message_body
        # Else use body_loc_key and body_loc_args for body
        else:
            if body_loc_key:
                fcm_payload['notification']['body_loc_key'] = body_loc_key
            if body_loc_args:
                if isinstance(body_loc_args, list):
                    fcm_payload['notification']['body_loc_args'] = body_loc_args
                else:
                    raise InvalidDataError('body_loc_args should be an array')
        # If title is present, use it
        if message_title:
            fcm_payload['notification']['title'] = message_title
        # Else use title_loc_key and title_loc_args for title
        else:
            if title_loc_key:
                fcm_payload['notification']['title_loc_key'] = title_loc_key
            if title_loc_args:
                if isinstance(title_loc_args, list):
                    fcm_payload['notification']['title_loc_args'] = title_loc_args
                else:
                    raise InvalidDataError('title_loc_args should be an array')

        # This is needed for iOS when we are sending only custom data messages
        if content_available and isinstance(content_available, bool):
            fcm_payload['content_available'] = content_available

        if click_action:
            fcm_payload['notification']['click_action'] = click_action
        if isinstance(badge, int) and badge >= 0:
            fcm_payload['notification']['badge'] = badge
        if color:
            fcm_payload['notification']['color'] = color
        if tag:
            fcm_payload['notification']['tag'] = tag
        # only add the 'sound' key if sound is not None
        # otherwise a default sound will play -- even with empty string args.
        if sound:
            fcm_payload['notification']['sound'] = sound

        if extra_kwargs:
            fcm_payload['notification'].update(extra_kwargs)

        # Do this if you only want to send a data message.
        if remove_notification:
            del fcm_payload['notification']

        return self.json_dumps(fcm_payload)

    def do_request(self, payload, timeout):
        if self.FCM_REQ_PROXIES:
            response = requests.post(self.FCM_END_POINT, headers=self.request_headers(), data=payload,
                                     proxies=self.FCM_REQ_PROXIES, timeout=timeout)
        else:
            response = requests.post(self.FCM_END_POINT, headers=self.request_headers(), data=payload, timeout=timeout)
        if 'Retry-After' in response.headers and int(response.headers['Retry-After']) > 0:
            sleep_time = int(response.headers['Retry-After'])
            time.sleep(sleep_time)
            return self.do_request(payload, timeout)
        return response

    def send_request(self, payloads=None, timeout=None):
        self.send_request_responses = []
        for payload in payloads:
            response = self.do_request(payload, timeout)
            self.send_request_responses.append(response)

    def registration_info_request(self, registration_id):
        """ Makes a request for registration info and returns the response
            object
        """
        response = requests.get('https://iid.googleapis.com/iid/info/' + registration_id,
                               headers=self.request_headers(),
                               params={'details': 'true'})
        return response

    def clean_registration_ids(self, registration_ids=[]):
        """ Return list of active IDS from the list of registration_ids
        """
        valid_registration_ids = []
        for registration_id in registration_ids:
            details = self.registration_info_request(registration_id)
            if details.status_code == 200:
                valid_registration_ids.append(registration_id)
        return valid_registration_ids

    def get_registration_id_info(self, registration_id):
        """ Returns details related to a registration id if it exists
            otherwise return None
        """
        details = self.registration_info_request(registration_id)
        if details.status_code == 200:
            return details.json()
        return None

    def subscribe_registration_ids_to_topic(self, registration_ids, topic_name):
        """ Subscribes a list of registration ids to a topic
        """
        url = '''https://iid.googleapis.com/iid/v1:batchAdd'''
        payload = json.dumps({
            'to': '/topics/'+topic_name,
            'registration_tokens': registration_ids,
        })
        response = requests.post(
           url,
           headers=self.request_headers(),
           data=payload,
        )
        if response.status_code == 200:
            return True
        elif response.status_code == 400:
            error = json.loads( response.content )
            raise InvalidDataError(error['error'])
        else:
            raise FCMError()

    def unsubscribe_registration_ids_from_topic(self, registration_ids, topic_name):
        """ Unsubscribes a list of registration ids from a topic
        """
        url = '''https://iid.googleapis.com/iid/v1:batchRemove'''
        payload = json.dumps({
            'to': '/topics/'+topic_name,
            'registration_tokens': registration_ids,
        })
        response = requests.post(
           url,
           headers=self.request_headers(),
           data=payload,
        )
        if response.status_code == 200:
            return True
        elif response.status_code == 400:
            error = json.loads( response.content )
            raise InvalidDataError(error['error'])
        else:
            raise FCMError()

    def parse_responses(self):
        """
        Returns a python dict of multicast_ids(list), success(int), failure(int), canonical_ids(int), results(list) and optional topic_message_id(str but None by default)
        """
        response_dict = {
            'multicast_ids': list(),
            'success': 0,
            'failure': 0,
            'canonical_ids': 0,
            'results': list(),
            'topic_message_id': None
        }

        for response in self.send_request_responses:
            if response.status_code == 200:
                """
                Parses the json response sent back by the
                server and tries to get out the important return variables
                """
                if 'content-length' in response.headers and int(response.headers['content-length']) <= 0:
                    raise FCMServerError("FCM server connection error, the response is empty")
                else:
                    parsed_response = response.json()

                    multicast_id = parsed_response.get('multicast_id', None)
                    success = parsed_response.get('success', 0)
                    failure = parsed_response.get('failure', 0)
                    canonical_ids = parsed_response.get('canonical_ids', 0)
                    results = parsed_response.get('results', [])
                    message_id = parsed_response.get('message_id', None)  # for topic messages
                    if message_id:
                        success = 1
                    if multicast_id:
                        response_dict['multicast_ids'].append(multicast_id)
                    response_dict['success'] += success
                    response_dict['failure'] += failure
                    response_dict['canonical_ids'] += canonical_ids
                    response_dict['results'].extend(results)
                    response_dict['topic_message_id'] = message_id
            elif response.status_code == 401:
                raise AuthenticationError("There was an error authenticating the sender account")
            elif response.status_code == 400:
                raise InternalPackageError(response.text)
            else:
                raise FCMServerError("FCM server is temporarily unavailable")
        return response_dict
<|endoftext|>


<commit_before>#!/usr/bin/env python
import argparse
import subprocess
import os
import sys

PROJECT_NAME = "cla_frontend"

# use python scripts/jenkins/build.py integration

# args
parser = argparse.ArgumentParser(description='Build project ready for testing by Jenkins.')
parser.add_argument('envname', metavar='envname', type=str, nargs=1, help='e.g. integration, production, etc.')
args = parser.parse_args()

env = args.envname[0]
env_name = "%s-%s" % (PROJECT_NAME, env)
env_path = "/var/lib/jenkins/envs/%s" % env_name
bin_path = "%s/bin" % env_path

def run(command, **kwargs):
	defaults = {
		'shell': True
	}
	defaults.update(kwargs)

	return_code = subprocess.call(command, **defaults)
	if return_code:
		sys.exit(return_code)

# setting up virtualenv
if not os.path.isdir(env_path):
	run('virtualenv --no-site-packages %s' % env_path)

run('%s/pip install -r requirements/jenkins.txt' % bin_path)

# Remove .pyc files from the project
run("find . -name '*.pyc' -delete")

# run tests
run("%s/python manage.py jenkins --coverage-rcfile=.coveragerc --settings=cla_frontend.settings.jenkins" % bin_path)
<commit_msg>fix jenkins<commit_after>#!/usr/bin/env python
import argparse
import subprocess
import os
import sys

PROJECT_NAME = "cla_backend"

# use python scripts/jenkins/build.py integration

# args
parser = argparse.ArgumentParser(description='Build project ready for testing by Jenkins.')
parser.add_argument('envname', metavar='envname', type=str, nargs=1, help='e.g. integration, production, etc.')
args = parser.parse_args()

env = args.envname[0]
env_name = "%s-%s" % (PROJECT_NAME, env)
env_path = "/tmp/jenkins/envs/%s" % env_name
bin_path = "%s/bin" % env_path

def run(command, **kwargs):
	defaults = {
		'shell': True
	}
	defaults.update(kwargs)

	return_code = subprocess.call(command, **defaults)
	if return_code:
		sys.exit(return_code)

# setting up virtualenv
if not os.path.isdir(env_path):
	run('virtualenv --no-site-packages %s' % env_path)

run('%s/pip install -r requirements/jenkins.txt' % bin_path)

# Remove .pyc files from the project
run("find . -name '*.pyc' -delete")

# run tests
run("%s/python manage.py jenkins --coverage-rcfile=.coveragerc --settings=cla_backend.settings.jenkins --all" % bin_path)
<|endoftext|>


<commit_before>from os.path import dirname, join
from PyQt4.QtGui import QIcon, QPixmap, QCursor, QFileDialog, QApplication, \
        qRgb, QDialog
from PyQt4.QtCore import Qt, QSettings, QVariant, PYQT_VERSION

def icon_from_res(path):
    """Create an icon from a resource path"""
    icon = QIcon()
    icon.addPixmap(QPixmap(path), QIcon.Normal, QIcon.Off)
    return icon

def long_operation(func):
    """
    Decorator that changes the application cursor during the execution of a
    function.
    """
    def f(*args, **kwargs):
        QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
        try:
            res = func(*args, **kwargs)
        finally:
            QApplication.restoreOverrideCursor()
        return res
    return f

def get_save_path(parent, settings_path, default_filename=None,
        default_suffix=None):
    """
    Shows a 'save' dialog and returns the selected path.

    Also remembers the selected folder, so that subsequent calls to this
    function with the same ``settings_path`` will open in the last selected
    directory.

    Returns None if the user canceled.
    """
    # Get the last used save directory
    settings = QSettings()
    last_dir = unicode(settings.value(settings_path, QVariant(u"")).toString())
    # Get save location
    dlg = QFileDialog(parent)
    dlg.setWindowTitle(parent.trUtf8("Save as"))
    dlg.setFileMode(QFileDialog.AnyFile)
    dlg.setAcceptMode(QFileDialog.AcceptSave)
    dlg.setDirectory(last_dir)
    if default_suffix is not None:
        dlg.setDefaultSuffix(default_suffix)
    if default_filename is not None:
        dlg.selectFile(default_filename)
    if dlg.exec_() != QDialog.Accepted:
        return None
    save_path = unicode(dlg.selectedFiles()[0])
    # Remember save location directory
    settings.setValue(settings_path, QVariant(dirname(save_path)))
    return save_path

def get_open_path(parent, settings_path):
    """
    Shows an 'open file' dialog and returns the selected path.

    Also remembers the selected folder, so that subsequent calls to this
    function with the same ``settings_path`` will open in the last selected
    directory.

    Returns None if the user canceled.
    """
    # Get the last used save directory
    settings = QSettings()
    last_dir = unicode(settings.value(settings_path, QVariant(u"")).toString())
    # Get save location
    open_path = QFileDialog.getOpenFileName(parent, parent.trUtf8("Open"),
            last_dir)
    if open_path.isNull():
        return None
    open_path = unicode(open_path)
    # Remember save location directory
    settings.setValue(settings_path, QVariant(dirname(open_path)))
    return open_path

def rgb(r, g, b):
    if PYQT_VERSION <= 263172:
        return (qRgb(r, g, b) & 0xffffff) - 0x1000000
    return qRgb(r, g, b)

def get_or_create_app(args=None):
    if args is None:
        args = []
    app = QApplication.instance()
    if app is None:
        app = QApplication(args)
    return app

<commit_msg>pyflu: added qt.util.rgba<commit_after>from os.path import dirname, join
from PyQt4.QtGui import QIcon, QPixmap, QCursor, QFileDialog, QApplication, \
        qRgb, qRgba, QDialog
from PyQt4.QtCore import Qt, QSettings, QVariant, PYQT_VERSION
import warnings

def icon_from_res(path):
    """Create an icon from a resource path"""
    icon = QIcon()
    icon.addPixmap(QPixmap(path), QIcon.Normal, QIcon.Off)
    return icon

def long_operation(func):
    """
    Decorator that changes the application cursor during the execution of a
    function.
    """
    def f(*args, **kwargs):
        QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
        try:
            res = func(*args, **kwargs)
        finally:
            QApplication.restoreOverrideCursor()
        return res
    return f

def get_save_path(parent, settings_path, default_filename=None,
        default_suffix=None):
    """
    Shows a 'save' dialog and returns the selected path.

    Also remembers the selected folder, so that subsequent calls to this
    function with the same ``settings_path`` will open in the last selected
    directory.

    Returns None if the user canceled.
    """
    # Get the last used save directory
    settings = QSettings()
    last_dir = unicode(settings.value(settings_path, QVariant(u"")).toString())
    # Get save location
    dlg = QFileDialog(parent)
    dlg.setWindowTitle(parent.trUtf8("Save as"))
    dlg.setFileMode(QFileDialog.AnyFile)
    dlg.setAcceptMode(QFileDialog.AcceptSave)
    dlg.setDirectory(last_dir)
    if default_suffix is not None:
        dlg.setDefaultSuffix(default_suffix)
    if default_filename is not None:
        dlg.selectFile(default_filename)
    if dlg.exec_() != QDialog.Accepted:
        return None
    save_path = unicode(dlg.selectedFiles()[0])
    # Remember save location directory
    settings.setValue(settings_path, QVariant(dirname(save_path)))
    return save_path

def get_open_path(parent, settings_path):
    """
    Shows an 'open file' dialog and returns the selected path.

    Also remembers the selected folder, so that subsequent calls to this
    function with the same ``settings_path`` will open in the last selected
    directory.

    Returns None if the user canceled.
    """
    # Get the last used save directory
    settings = QSettings()
    last_dir = unicode(settings.value(settings_path, QVariant(u"")).toString())
    # Get save location
    open_path = QFileDialog.getOpenFileName(parent, parent.trUtf8("Open"),
            last_dir)
    if open_path.isNull():
        return None
    open_path = unicode(open_path)
    # Remember save location directory
    settings.setValue(settings_path, QVariant(dirname(open_path)))
    return open_path

def rgba(r, g, b, a=255):
    if PYQT_VERSION <= 263172:
        warnings.warn("using untested qRgba fix")
        return (qRgba(r, g, b, a) & 0xffffff) - 0x1000000
    return qRgba(r, g, b, a)

def rgb(r, g, b):
    if PYQT_VERSION <= 263172:
        return (qRgb(r, g, b) & 0xffffff) - 0x1000000
    return qRgb(r, g, b)

def get_or_create_app(args=None):
    if args is None:
        args = []
    app = QApplication.instance()
    if app is None:
        app = QApplication(args)
    return app

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os
import tempfile
from copy import copy
from reportlab.platypus.flowables import Image, Flowable
from log import log, nodeid
from reportlab.lib.units import *

try:
    from PythonMagick import Image as PMImage
    HAS_MAGICK = True
except ImportError:
    HAS_MAGICK = False

HAS_PIL = True
try:
    from PIL import Image as PILImage
except ImportError:
    try:
        import Image as PILImage
    except ImportError:
        if not HAS_MAGICK:
            log.warning("Support for images other than JPG,"
                " is now limited. Please install PIL.")
        HAS_PIL = False

from svgimage import SVGImage, VectorImage

missing = os.path.join(os.path.join(os.path.abspath(os.path.dirname(__file__))),
          'images', 'image-missing.jpg')

class MyImage (Flowable):
    """A Image subclass that can:

    1. Take a 'percentage_of_container' kind,
       which resizes it on wrap() to use... well, a percentage of the
       container's width.

    2. Take vector formats and instantiates the right "backend" flowable

    """

    def __init__(self, filename, width=None, height=None,
                 kind='direct', mask="auto", lazy=1, client=None):
        self.__kind=kind

        self.filename, self._backend=self.get_backend(filename, client)
        if kind == 'percentage_of_container':
            self.image=self._backend(self.filename, width, height,
                'direct', mask, lazy)
            self.image.drawWidth=width
            self.image.drawHeight=height
            self.__width=width
            self.__height=height
        else:
            self.image=self._backend(self.filename, width, height,
                kind, mask, lazy)
        self.__ratio=float(self.image.imageWidth)/self.image.imageHeight
        self.__wrappedonce=False

    @classmethod
    def raster(self, filename, client):
	"""Takes a filename and converts it to a raster image
        reportlab can process"""
        if not os.path.exists(filename):
            log.error("Missing image file: %s",filename)
            return missing
	if HAS_PIL:
	    ext='.png'
	else:
	    ext='.jpg'
	if HAS_MAGICK:
	    img = PMImage()
	    img.density("%s"%client.styles.def_dpi)
	    img.read(str(filename))
	    _, tmpname = tempfile.mkstemp(suffix=ext)
	    img.write(tmpname)
	    client.to_unlink.append(tmpname)
	    return tmpname
	else:
	    #FIXME: if there is no MAGICK, convert via PIL
	    pass

    @classmethod
    def get_backend(self,filename, client):
        '''Given the filename of an image, returns (fname, backend)
        where fname is the filename to be used (could be the same as
        filename, or something different if the image had to be converted
        or is missing), and backend is an Image class that can handle
        fname.'''

        # If the image doesn't exist, we use a 'missing' image
        if not os.path.exists(filename):
            log.error("Missing image file: %s",filename)
            filename = missing

        # Decide what class of backend image we will use
        extension = os.path.splitext(filename)[-1][1:].lower()

        if extension in ['svg','svgz'] and SVGImage.available():
            backend=SVGImage

        elif extension in ['ai', 'ccx', 'cdr', 'cgm', 'cmx', 'fig',
                'sk1', 'sk', 'xml', 'wmf']:
            backend=VectorImage

        elif extension in ['pdf']:
            # PDF images are implemented by converting via PythonMagick
            # w,h are in pixels. I need to set the density
            # of the image to  the right dpi so this
            # looks decent
            if HAS_MAGICK:
                filename=self.raster(filename, client)
                backend=Image
            else:
                log.warning("Minimal PDF image support "\
                    "requires PythonMagick [%s]", filename)
                filename = missing
        elif not HAS_PIL and HAS_MAGICK and extension != 'jpg':
            # Need to convert to JPG via PythonMagick
            filename=self.raster(filename)
            backend=Image

        elif HAS_PIL or extension == 'jpg':
            backend=Image
        else:
            # No way to make this work
            log.error('To use a %s image you need PIL installed [%s]',extension,filename)
            backend=Image
            filename=missing
        return filename, backend

    @classmethod
    def size_for_node(self, node, client):
        '''Given a docutils image node, returns the size the image should have
        in the PDF document, and what 'kind' of size that is.
        That involves lots of guesswork'''

        imgname = os.path.join(client.basedir,str(node.get("uri")))

        if not os.path.isfile(imgname):
            imgname = missing

        scale = float(node.get('scale', 100))/100

        # Figuring out the size to display of an image is ... annoying.
        # If the user provides a size with a unit, it's simple, adjustUnits
        # will return it in points and we're done.

        # However, often the unit wil be "%" (specially if it's meant for
        # HTML originally. In which case, we will use a percentage of
        # the containing frame.

        # Find the image size in pixels:
        kind = 'direct'
        xdpi, ydpi = client.styles.def_dpi, client.styles.def_dpi
        extension = imgname.split('.')[-1].lower()
        if extension in ['svg','svgz'] and SVGImage.available():
            iw, ih = SVGImage(imgname).wrap(0, 0)
            # These are in pt, so convert to px
            iw = iw * xdpi / 72
            ih = ih * ydpi / 72

        elif extension in [
                "ai", "ccx", "cdr", "cgm", "cmx",
                "sk1", "sk", "xml", "wmf", "fig"] and VectorImage.available():
            iw, ih = VectorImage(imgname).wrap(0, 0)
            # These are in pt, so convert to px
            iw = iw * xdpi / 72
            ih = ih * ydpi / 72

        elif extension == 'pdf':
            try:
                from pyPdf import pdf
            except:
                log.warning('PDF images are not supported without pypdf [%s]', nodeid(node))
                return 0, 0, 'direct'
            reader = pdf.PdfFileReader(open(imgname))
            x1, y1, x2, y2 = reader.getPage(0)['/MediaBox']
            # These are in pt, so convert to px
            iw = float((x2-x1) * xdpi / 72)
            ih = float((y2-y1) * ydpi / 72)

        else:
            if HAS_PIL:
                img = PILImage.open(imgname)
                iw, ih = img.size
                xdpi, ydpi = img.info.get('dpi', (xdpi, ydpi))
            elif HAS_MAGICK:
                img = PMImage(imgname)
                iw = img.size().width()
                ih = img.size().height()
		density=img.density()
		# The density is in pixelspercentimeter (!?)
		xdpi=density.width()*2.54
		ydpi=density.height()*2.54
            else:
                log.warning("Sizing images without PIL "
                            "or PythonMagick, using 100x100 [%s]"
                            , nodeid(node))
                iw, ih = 100., 100.

        # Try to get the print resolution from the image itself via PIL.
        # If it fails, assume a DPI of 300, which is pretty much made up,
        # and then a 100% size would be iw*inch/300, so we pass
        # that as the second parameter to adjustUnits
        #
        # Some say the default DPI should be 72. That would mean
        # the largest printable image in A4 paper would be something
        # like 480x640. That would be awful.
        #

        w = node.get('width')
        if w is not None:
            # In this particular case, we want the default unit
            # to be pixels so we work like rst2html
            if w[-1] == '%':
                kind = 'percentage_of_container'
                w=int(w[:-1])
            else:
                # This uses default DPI setting because we
                # are not using the image's "natural size"
                # this is what LaTeX does, according to the
                # docutils mailing list discussion
                w = client.styles.adjustUnits(w, client.styles.tw,
                                            default_unit='px')
        else:
            log.warning("Using image %s without specifying size."
                "Calculating based on image size at %ddpi [%s]",
                imgname, xdpi, nodeid(node))
            # No width specified at all, use w in px
            w = iw*inch/xdpi

        h = node.get('height')
        if h is not None and h[-1] != '%':
            h = client.styles.adjustUnits(h, ih*inch/ydpi)
        else:
            # Now, often, only the width is specified!
            # if we don't have a height, we need to keep the
            # aspect ratio, or else it will look ugly
            if h and h[-1]=='%':
                log.error('Setting height as a percentage does **not** work. '\
                          'ignoring height parameter [%s]', nodeid(node))
            h = w*ih/iw

        # Apply scale factor
        w = w*scale
        h = h*scale

        # And now we have this probably completely bogus size!
        log.info("Image %s size calculated:  %fcm by %fcm [%s]",
            imgname, w/cm, h/cm, nodeid(node))

        return w, h, kind

    def __deepcopy__(self, *whatever):
        # ImageCore class is not deep copyable.  Stop the copy at this
        # class.  If you remove this, re-test for issue #126.
        return copy(self)

    def wrap(self, availWidth, availHeight):
        if self.__kind=='percentage_of_container':
            w, h= self.__width, self.__height
            if not w:
                log.warning('Scaling image as % of container with w unset.'
                'This should not happen, setting to 100')
                w = 100
            scale=w/100.
            w = availWidth*scale
            h = w/self.__ratio
            self.image.drawWidth, self.image.drawHeight = w, h
            return w, h
        else:
            if self.image.drawHeight > availHeight:
                if not self._atTop:
                    return self.image.wrap(availWidth, availHeight)
                else:
                    # It's the first thing in the frame, probably
                    # Wrapping it will not make it work, so we
                    # adjust by height
                    # FIXME get rst file info (line number)
                    # here for better error message
                    log.warning('image %s is too tall for the '\
                                'frame, rescaling'%\
                                self.filename)
                    self.image.drawHeight = availHeight
                    self.image.drawWidth = availHeight*self.__ratio
            elif self.image.drawWidth > availWidth:
                log.warning('image %s is too wide for the frame, rescaling'%\
                            self.filename)
                self.image.drawWidth = availWidth
                self.image.drawHeight = availWidth / self.__ratio
            return self.image.wrap(availWidth, availHeight)

    def drawOn(self, canv, x, y, _sW=0):
        return self.image.drawOn(canv, x, y, _sW)

<commit_msg>added fixme<commit_after># -*- coding: utf-8 -*-

import os
import tempfile
from copy import copy
from reportlab.platypus.flowables import Image, Flowable
from log import log, nodeid
from reportlab.lib.units import *

try:
    from PythonMagick import Image as PMImage
    HAS_MAGICK = True
except ImportError:
    HAS_MAGICK = False

HAS_PIL = True
try:
    from PIL import Image as PILImage
except ImportError:
    try:
        import Image as PILImage
    except ImportError:
        if not HAS_MAGICK:
            log.warning("Support for images other than JPG,"
                " is now limited. Please install PIL.")
        HAS_PIL = False

from svgimage import SVGImage, VectorImage

missing = os.path.join(os.path.join(os.path.abspath(os.path.dirname(__file__))),
          'images', 'image-missing.jpg')

class MyImage (Flowable):
    """A Image subclass that can:

    1. Take a 'percentage_of_container' kind,
       which resizes it on wrap() to use... well, a percentage of the
       container's width.

    2. Take vector formats and instantiates the right "backend" flowable

    """

    def __init__(self, filename, width=None, height=None,
                 kind='direct', mask="auto", lazy=1, client=None):
        self.__kind=kind

        self.filename, self._backend=self.get_backend(filename, client)
        if kind == 'percentage_of_container':
            self.image=self._backend(self.filename, width, height,
                'direct', mask, lazy)
            self.image.drawWidth=width
            self.image.drawHeight=height
            self.__width=width
            self.__height=height
        else:
            self.image=self._backend(self.filename, width, height,
                kind, mask, lazy)
        self.__ratio=float(self.image.imageWidth)/self.image.imageHeight
        self.__wrappedonce=False

    @classmethod
    def raster(self, filename, client):
	"""Takes a filename and converts it to a raster image
        reportlab can process"""
        if not os.path.exists(filename):
            log.error("Missing image file: %s",filename)
            return missing

        #FIXME: don't convert things that already are raster

	if HAS_PIL:
	    ext='.png'
	else:
	    ext='.jpg'
	if HAS_MAGICK:
	    img = PMImage()
	    img.density("%s"%client.styles.def_dpi)
	    img.read(str(filename))
	    _, tmpname = tempfile.mkstemp(suffix=ext)
	    img.write(tmpname)
	    client.to_unlink.append(tmpname)
	    return tmpname
	else:
	    #FIXME: if there is no MAGICK, convert via PIL
	    pass

    @classmethod
    def get_backend(self,filename, client):
        '''Given the filename of an image, returns (fname, backend)
        where fname is the filename to be used (could be the same as
        filename, or something different if the image had to be converted
        or is missing), and backend is an Image class that can handle
        fname.'''

        # If the image doesn't exist, we use a 'missing' image
        if not os.path.exists(filename):
            log.error("Missing image file: %s",filename)
            filename = missing

        # Decide what class of backend image we will use
        extension = os.path.splitext(filename)[-1][1:].lower()

        if extension in ['svg','svgz'] and SVGImage.available():
            backend=SVGImage

        elif extension in ['ai', 'ccx', 'cdr', 'cgm', 'cmx', 'fig',
                'sk1', 'sk', 'xml', 'wmf']:
            backend=VectorImage

        elif extension in ['pdf']:
            # PDF images are implemented by converting via PythonMagick
            # w,h are in pixels. I need to set the density
            # of the image to  the right dpi so this
            # looks decent
            if HAS_MAGICK:
                filename=self.raster(filename, client)
                backend=Image
            else:
                log.warning("Minimal PDF image support "\
                    "requires PythonMagick [%s]", filename)
                filename = missing
        elif not HAS_PIL and HAS_MAGICK and extension != 'jpg':
            # Need to convert to JPG via PythonMagick
            filename=self.raster(filename)
            backend=Image

        elif HAS_PIL or extension == 'jpg':
            backend=Image
        else:
            # No way to make this work
            log.error('To use a %s image you need PIL installed [%s]',extension,filename)
            backend=Image
            filename=missing
        return filename, backend

    @classmethod
    def size_for_node(self, node, client):
        '''Given a docutils image node, returns the size the image should have
        in the PDF document, and what 'kind' of size that is.
        That involves lots of guesswork'''

        imgname = os.path.join(client.basedir,str(node.get("uri")))

        if not os.path.isfile(imgname):
            imgname = missing

        scale = float(node.get('scale', 100))/100

        # Figuring out the size to display of an image is ... annoying.
        # If the user provides a size with a unit, it's simple, adjustUnits
        # will return it in points and we're done.

        # However, often the unit wil be "%" (specially if it's meant for
        # HTML originally. In which case, we will use a percentage of
        # the containing frame.

        # Find the image size in pixels:
        kind = 'direct'
        xdpi, ydpi = client.styles.def_dpi, client.styles.def_dpi
        extension = imgname.split('.')[-1].lower()
        if extension in ['svg','svgz'] and SVGImage.available():
            iw, ih = SVGImage(imgname).wrap(0, 0)
            # These are in pt, so convert to px
            iw = iw * xdpi / 72
            ih = ih * ydpi / 72

        elif extension in [
                "ai", "ccx", "cdr", "cgm", "cmx",
                "sk1", "sk", "xml", "wmf", "fig"] and VectorImage.available():
            iw, ih = VectorImage(imgname).wrap(0, 0)
            # These are in pt, so convert to px
            iw = iw * xdpi / 72
            ih = ih * ydpi / 72

        elif extension == 'pdf':
            try:
                from pyPdf import pdf
            except:
                log.warning('PDF images are not supported without pypdf [%s]', nodeid(node))
                return 0, 0, 'direct'
            reader = pdf.PdfFileReader(open(imgname))
            x1, y1, x2, y2 = reader.getPage(0)['/MediaBox']
            # These are in pt, so convert to px
            iw = float((x2-x1) * xdpi / 72)
            ih = float((y2-y1) * ydpi / 72)

        else:
            if HAS_PIL:
                img = PILImage.open(imgname)
                iw, ih = img.size
                xdpi, ydpi = img.info.get('dpi', (xdpi, ydpi))
            elif HAS_MAGICK:
                img = PMImage(imgname)
                iw = img.size().width()
                ih = img.size().height()
		density=img.density()
		# The density is in pixelspercentimeter (!?)
		xdpi=density.width()*2.54
		ydpi=density.height()*2.54
            else:
                log.warning("Sizing images without PIL "
                            "or PythonMagick, using 100x100 [%s]"
                            , nodeid(node))
                iw, ih = 100., 100.

        # Try to get the print resolution from the image itself via PIL.
        # If it fails, assume a DPI of 300, which is pretty much made up,
        # and then a 100% size would be iw*inch/300, so we pass
        # that as the second parameter to adjustUnits
        #
        # Some say the default DPI should be 72. That would mean
        # the largest printable image in A4 paper would be something
        # like 480x640. That would be awful.
        #

        w = node.get('width')
        if w is not None:
            # In this particular case, we want the default unit
            # to be pixels so we work like rst2html
            if w[-1] == '%':
                kind = 'percentage_of_container'
                w=int(w[:-1])
            else:
                # This uses default DPI setting because we
                # are not using the image's "natural size"
                # this is what LaTeX does, according to the
                # docutils mailing list discussion
                w = client.styles.adjustUnits(w, client.styles.tw,
                                            default_unit='px')
        else:
            log.warning("Using image %s without specifying size."
                "Calculating based on image size at %ddpi [%s]",
                imgname, xdpi, nodeid(node))
            # No width specified at all, use w in px
            w = iw*inch/xdpi

        h = node.get('height')
        if h is not None and h[-1] != '%':
            h = client.styles.adjustUnits(h, ih*inch/ydpi)
        else:
            # Now, often, only the width is specified!
            # if we don't have a height, we need to keep the
            # aspect ratio, or else it will look ugly
            if h and h[-1]=='%':
                log.error('Setting height as a percentage does **not** work. '\
                          'ignoring height parameter [%s]', nodeid(node))
            h = w*ih/iw

        # Apply scale factor
        w = w*scale
        h = h*scale

        # And now we have this probably completely bogus size!
        log.info("Image %s size calculated:  %fcm by %fcm [%s]",
            imgname, w/cm, h/cm, nodeid(node))

        return w, h, kind

    def __deepcopy__(self, *whatever):
        # ImageCore class is not deep copyable.  Stop the copy at this
        # class.  If you remove this, re-test for issue #126.
        return copy(self)

    def wrap(self, availWidth, availHeight):
        if self.__kind=='percentage_of_container':
            w, h= self.__width, self.__height
            if not w:
                log.warning('Scaling image as % of container with w unset.'
                'This should not happen, setting to 100')
                w = 100
            scale=w/100.
            w = availWidth*scale
            h = w/self.__ratio
            self.image.drawWidth, self.image.drawHeight = w, h
            return w, h
        else:
            if self.image.drawHeight > availHeight:
                if not self._atTop:
                    return self.image.wrap(availWidth, availHeight)
                else:
                    # It's the first thing in the frame, probably
                    # Wrapping it will not make it work, so we
                    # adjust by height
                    # FIXME get rst file info (line number)
                    # here for better error message
                    log.warning('image %s is too tall for the '\
                                'frame, rescaling'%\
                                self.filename)
                    self.image.drawHeight = availHeight
                    self.image.drawWidth = availHeight*self.__ratio
            elif self.image.drawWidth > availWidth:
                log.warning('image %s is too wide for the frame, rescaling'%\
                            self.filename)
                self.image.drawWidth = availWidth
                self.image.drawHeight = availWidth / self.__ratio
            return self.image.wrap(availWidth, availHeight)

    def drawOn(self, canv, x, y, _sW=0):
        return self.image.drawOn(canv, x, y, _sW)

<|endoftext|>


<commit_before>#!/usr/bin/python
#
# FSF Client for sending information and generating a report
#
# Jason Batchelor
# Emerson Corporation
# 02/09/2016
'''
   Copyright 2016 Emerson Electric Co.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
'''

import os
import sys
import socket
import argparse
import struct
import json
import time
import hashlib
import random
from conf import config
from datetime import datetime as dt

class FSFClient:
   def __init__(self, fullpath, filename, delete, source, archive, suppress_report, full, file):

         self.fullpath = fullpath
         self.filename = filename
         self.delete = delete
         self.source = source
         self.archive = archive
         self.suppress_report = suppress_report
         self.full = full
         self.file = file
         # will hold host after verifying connection to server
         self.host = ''
         self.port = config.SERVER_CONFIG['PORT']
         self.logfile = config.CLIENT_CONFIG['LOG_FILE']
         self.server_list = config.SERVER_CONFIG['IP_ADDRESS']

         archive_options = ['none', 'file-on-alert', 'all-on-alert', 'all-the-files', 'all-the-things']
         if args.archive not in archive_options:
            error = '\033[31m'+'%s Please specify a valid archive option: \'none\', \'file-on-alert\', \'all-on-alert\', \'all-the-files\' or \'all-the-things\'.\n' % dt.now()
            self.issue_error(error)
            sys.exit(1)

# Test connection to randomized server and rudimentary fail over
   def test_server_connection(self):
      sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

      random.shuffle(self.server_list)
      attempts = 0
      for server in self.server_list:
          success = 1
          try:
              sock.connect((server, self.port))
          except:
              warning ='\033[93m'+ '%s There was a problem connecting to %s on port %s. Trying another server. <WARN>' % (dt.now(), server, self.port)
              self.issue_error(warning)
              success = 0
              attempts += 1
          if success:
              self.host = server
              self.process_files()
              break
          elif attempts == len(self.server_list):
              e = sys.exc_info()[0]
              error = '\033[31m'+'%s There are not servers available to send files too. Error: %s\n' % (dt.now(), e)
              self.issue_error(error)

   # Send files to server for processing and await results
   def process_files(self):

      sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
      msg = '%sFSF_RPC%sFSF_RPC%sFSF_RPC%sFSF_RPC%sFSF_RPC%s' % (self.filename, self.source, self.archive, self.suppress_report, self.full, self.file)
      buffer = struct.pack('>I', len(msg)) + 'FSF_RPC' + msg

      try:
         sock.connect((self.host, self.port))
         sock.sendall(buffer)
      except:
         e = sys.exc_info()[0]
         error = '\033[31m'+'%s There was a problem sending file %s to %s on port %s. Error: %s\n' % (dt.now(), self.filename, self.host, self.port, e)
         self.issue_error(error)

      finally:

         if self.delete:
            os.remove(self.fullpath)

         if not self.suppress_report:
            self.process_results(sock)

         sock.close()

   # Process the results sent back from the FSF server
   def process_results(self, sock):

      try:
         raw_msg_len = sock.recv(4)
         msg_len = struct.unpack('>I', raw_msg_len)[0]
         data = ''

         while len(data) < msg_len:
            recv_buff = sock.recv(msg_len - len(data))
            data += recv_buff

         print data

         # Does the user want all sub objects?
         if self.full:
            # Generate dirname by calculating epoch time and hash of results
            dirname = 'fsf_dump_%s_%s' % (int(time.time()), hashlib.md5(data).hexdigest())
            self.dump_subobjects(sock, dirname)

      except:
         e = sys.exc_info()[0]
         error = '\033[31m'+'%s There was a problem getting data for %s from %s on port %s. Error: %s' % (dt.now(), self.filename, self.host, self.port, e)
         self.issue_error(error)

   # Dump all subobjects returned by the scanner server
   def dump_subobjects(self, sock, dirname):

      sub_status = sock.recv(4)
      if sub_status == 'Null':
         print 'No subobjects were returned from scanner for %s.' % self.filename
         return

      os.mkdir(dirname)

      while self.full:
         raw_sub_count = sock.recv(4)
         sub_count = struct.unpack('>I', raw_sub_count)[0]
         raw_msg_len = sock.recv(4)
         msg_len = struct.unpack('>I', raw_msg_len)[0]
         data = ''

         while len(data) < msg_len:
            recv_buff = sock.recv(msg_len - len(data))
            data += recv_buff

         fname = hashlib.md5(data).hexdigest()
         with open('%s/%s' % (dirname, fname), 'w') as f:
            f.write(data)
            f.close

         if sub_count == 0:
            self.full = False

      print 'Sub objects of %s successfully written to: %s' % (self.filename, dirname)

   # Either log to log file or print to stdout depending on flags used
   def issue_error(self, error):

      if self.suppress_report:
         with open(self.logfile, 'a') as f:
            f.write(error)
            f.close()
      else:
         print error+'\033[0m'

if __name__ == '__main__':

   parser = argparse.ArgumentParser(prog='fsf_client', description='Uploads files to scanner server and returns the results to the user if desired. Results will always be written to a server side log file. Default options for each flag are designed to accommodate easy analyst interaction. Adjustments can be made to accommodate larger operations. Read the documentation for more details!')
   parser.add_argument('file', nargs='*', type=argparse.FileType('r'), help='Full path to file(s) to be processed.')
   parser.add_argument('--delete', default=False, action='store_true', help='Remove file from client after sending to the FSF server. Data can be archived later on server depending on selected options.')
   parser.add_argument('--source', nargs='?', type=str, default='Analyst', help='Specify the source of the input. Useful when scaling up to larger operations or supporting multiple input sources, such as; integrating with a sensor grid or other network defense solutions. Defaults to \'Analyst\' as submission source.')
   parser.add_argument('--archive', nargs='?', type=str, default='none', help='Specify the archive option to use. The most common option is \'none\' which will tell the server not to archive for this submission (default). \'file-on-alert\' will archive the file only if the alert flag is set. \'all-on-alert\' will archive the file and all sub objects if the alert flag is set. \'all-the-files\' will archive all the files sent to the scanner regardless of the alert flag. \'all-the-things\' will archive the file and all sub objects regardless of the alert flag.')
   parser.add_argument('--suppress-report', default=False, action='store_true', help='Don\'t return a JSON report back to the client and log client-side errors to the locally configured log directory. Choosing this will log scan results server-side only. Needed for automated scanning use cases when sending large amount of files for bulk collection. Set to false by default.')
   parser.add_argument('--full', default=False, action='store_true', help='Dump all sub objects of submitted file to current directory of the client. Format or directory name is \'fsf_dump_[epoch time]_[md5 hash of scan results]\'. Only supported when suppress-report option is false (default).')

   if len(sys.argv) == 1:
      parser.print_help()
      sys.exit(1)

   try:
      args = parser.parse_args()
   except IOError:
      e = sys.exc_info()[1]
      print 'The file provided could not be found. Error: %s' % e
      sys.exit(1)

   if len(args.file) == 0:
      print 'A file to scan needs to be provided!'

   for f in args.file:
      filename = os.path.basename(f.name)
      file = f.read()
      fsf = FSFClient(f.name, filename, args.delete, args.source, args.archive, args.suppress_report, args.full, file)
      fsf.test_server_connection()
<commit_msg>Update fsf_client.py<commit_after>#!/usr/bin/env python
#
# FSF Client for sending information and generating a report
#
# Jason Batchelor
# Emerson Corporation
# 02/09/2016
'''
   Copyright 2016 Emerson Electric Co.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
'''

import os
import sys
import socket
import argparse
import struct
import json
import time
import hashlib
import random
from conf import config
from datetime import datetime as dt

class FSFClient:
   def __init__(self, fullpath, filename, delete, source, archive, suppress_report, full, file):

         self.fullpath = fullpath
         self.filename = filename
         self.delete = delete
         self.source = source
         self.archive = archive
         self.suppress_report = suppress_report
         self.full = full
         self.file = file
         # will hold host after verifying connection to server
         self.host = ''
         self.port = config.SERVER_CONFIG['PORT']
         self.logfile = config.CLIENT_CONFIG['LOG_FILE']
         self.server_list = config.SERVER_CONFIG['IP_ADDRESS']

         archive_options = ['none', 'file-on-alert', 'all-on-alert', 'all-the-files', 'all-the-things']
         if args.archive not in archive_options:
            error = '%s Please specify a valid archive option: \'none\', \'file-on-alert\', \'all-on-alert\', \'all-the-files\' or \'all-the-things\'.\n' % dt.now()
            self.issue_error(error)
            sys.exit(1)

# Test connection to randomized server and rudimentary fail over
   def initiate_submission(self):
      sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

      random.shuffle(self.server_list)
      attempts = 0
      for server in self.server_list:
          success = 1
          try:
              sock.connect((server, self.port))
          except:
              warning ='%s There was a problem connecting to %s on port %s. Trying another server. <WARN>' % (dt.now(), server, self.port)
              self.issue_error(warning)
              success = 0
              attempts += 1
          if success:
              self.host = server
              self.process_files()
              break
          elif attempts == len(self.server_list):
              e = sys.exc_info()[0]
              error = '%s There are not servers available to send files too. Error: %s\n' % (dt.now(), e)
              self.issue_error(error)

   # Send files to server for processing and await results
   def process_files(self):

      sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
      msg = '%sFSF_RPC%sFSF_RPC%sFSF_RPC%sFSF_RPC%sFSF_RPC%s' % (self.filename, self.source, self.archive, self.suppress_report, self.full, self.file)
      buffer = struct.pack('>I', len(msg)) + 'FSF_RPC' + msg

      try:
         sock.connect((self.host, self.port))
         sock.sendall(buffer)
      except:
         e = sys.exc_info()[0]
         error = '%s There was a problem sending file %s to %s on port %s. Error: %s\n' % (dt.now(), self.filename, self.host, self.port, e)
         self.issue_error(error)

      finally:

         if self.delete:
            os.remove(self.fullpath)

         if not self.suppress_report:
            self.process_results(sock)

         sock.close()

   # Process the results sent back from the FSF server
   def process_results(self, sock):

      try:
         raw_msg_len = sock.recv(4)
         msg_len = struct.unpack('>I', raw_msg_len)[0]
         data = ''

         while len(data) < msg_len:
            recv_buff = sock.recv(msg_len - len(data))
            data += recv_buff

         print data

         # Does the user want all sub objects?
         if self.full:
            # Generate dirname by calculating epoch time and hash of results
            dirname = 'fsf_dump_%s_%s' % (int(time.time()), hashlib.md5(data).hexdigest())
            self.dump_subobjects(sock, dirname)

      except:
         e = sys.exc_info()[0]
         error = '%s There was a problem getting data for %s from %s on port %s. Error: %s' % (dt.now(), self.filename, self.host, self.port, e)
         self.issue_error(error)

   # Dump all subobjects returned by the scanner server
   def dump_subobjects(self, sock, dirname):

      sub_status = sock.recv(4)
      if sub_status == 'Null':
         print 'No subobjects were returned from scanner for %s.' % self.filename
         return

      os.mkdir(dirname)

      while self.full:
         raw_sub_count = sock.recv(4)
         sub_count = struct.unpack('>I', raw_sub_count)[0]
         raw_msg_len = sock.recv(4)
         msg_len = struct.unpack('>I', raw_msg_len)[0]
         data = ''

         while len(data) < msg_len:
            recv_buff = sock.recv(msg_len - len(data))
            data += recv_buff

         fname = hashlib.md5(data).hexdigest()
         with open('%s/%s' % (dirname, fname), 'w') as f:
            f.write(data)
            f.close

         if sub_count == 0:
            self.full = False

      print 'Sub objects of %s successfully written to: %s' % (self.filename, dirname)

   # Either log to log file or print to stdout depending on flags used
   def issue_error(self, error):

      if self.suppress_report:
         with open(self.logfile, 'a') as f:
            f.write(error)
            f.close()
      else:
         print error

if __name__ == '__main__':

   parser = argparse.ArgumentParser(prog='fsf_client', description='Uploads files to scanner server and returns the results to the user if desired. Results will always be written to a server side log file. Default options for each flag are designed to accommodate easy analyst interaction. Adjustments can be made to accommodate larger operations. Read the documentation for more details!')
   parser.add_argument('file', nargs='*', type=argparse.FileType('r'), help='Full path to file(s) to be processed.')
   parser.add_argument('--delete', default=False, action='store_true', help='Remove file from client after sending to the FSF server. Data can be archived later on server depending on selected options.')
   parser.add_argument('--source', nargs='?', type=str, default='Analyst', help='Specify the source of the input. Useful when scaling up to larger operations or supporting multiple input sources, such as; integrating with a sensor grid or other network defense solutions. Defaults to \'Analyst\' as submission source.')
   parser.add_argument('--archive', nargs='?', type=str, default='none', help='Specify the archive option to use. The most common option is \'none\' which will tell the server not to archive for this submission (default). \'file-on-alert\' will archive the file only if the alert flag is set. \'all-on-alert\' will archive the file and all sub objects if the alert flag is set. \'all-the-files\' will archive all the files sent to the scanner regardless of the alert flag. \'all-the-things\' will archive the file and all sub objects regardless of the alert flag.')
   parser.add_argument('--suppress-report', default=False, action='store_true', help='Don\'t return a JSON report back to the client and log client-side errors to the locally configured log directory. Choosing this will log scan results server-side only. Needed for automated scanning use cases when sending large amount of files for bulk collection. Set to false by default.')
   parser.add_argument('--full', default=False, action='store_true', help='Dump all sub objects of submitted file to current directory of the client. Format or directory name is \'fsf_dump_[epoch time]_[md5 hash of scan results]\'. Only supported when suppress-report option is false (default).')

   if len(sys.argv) == 1:
      parser.print_help()
      sys.exit(1)

   try:
      args = parser.parse_args()
   except IOError:
      e = sys.exc_info()[1]
      print 'The file provided could not be found. Error: %s' % e
      sys.exit(1)

   if len(args.file) == 0:
      print 'A file to scan needs to be provided!'

   for f in args.file:
      filename = os.path.basename(f.name)
      file = f.read()
      fsf = FSFClient(f.name, filename, args.delete, args.source, args.archive, args.suppress_report, args.full, file)
      fsf.test_server_connection()
<|endoftext|>


<commit_before>import argparse
import logging
import os
import time

import zmq
from managers import (ServiceManager, WorkflowManager)
from routes import (route, dispatch, RouteNotFound)
from workqueue import WorkQueueService

import utils
import core

logger = logging.getLogger('yerba')

def listen_forever(port, options=None):
    WorkQueueService.set_project(options['queue_prefix'])
    ServiceManager.register(WorkQueueService())
    ServiceManager.start()

    connection_string = "tcp://*:{}".format(port)
    context = zmq.Context()
    socket = context.socket(zmq.REP)
    socket.set(zmq.LINGER, 1000)
    socket.bind(connection_string)
    poller = zmq.Poller()
    poller.register(socket, zmq.POLLIN)

    while True:
        ServiceManager.update()
        if socket in dict(poller.poll(timeout=10)):
            msg = socket.recv_json(flags=zmq.NOBLOCK)
            response = None

            with utils.ignored(RouteNotFound):
                response = dispatch(msg)

            if not response:
                logger.info("Invalid request: %s", msg)
                response = {"status" : "error"}

            try:
                socket.send_json(response, flags=zmq.NOBLOCK)
            except zmq.Again:
                if logger.isEnabledFor(logging.DEBUG):
                    logger.exception("Failed to respond with response %s",
                        response)

@route("schedule")
def schedule_workflow(data):
    '''Returns the job id'''
    status = WorkflowManager.submit(data)
    return {"status" : core.status_name(status)}

@route("cancel")
def cancel_workflow(data):
    '''Cancels the job if it is running.'''
    try:
        identity = data['id']
        status = WorkflowManager.cancel(identity)
        logger.info(core.status_message(identity, status))
        return {"status" : core.status_name(status)}
    except KeyError:
        return {"status" : 'NotFound'}

@route("get_status")
def get_workflow_status(data):
    '''Gets the status of the workflow.'''
    try:
        identity = data['id']
        (status, jobs) = WorkflowManager.status(identity)
        logger.info(core.status_message(identity, status))
        return {"status" : core.status_name(status), "jobs" : jobs}
    except KeyError:
        return {"status" : 'NotFound', "jobs" : {}}
<commit_msg>Added additional logging.<commit_after>import argparse
import logging
import os
import time

import zmq
from managers import (ServiceManager, WorkflowManager)
from routes import (route, dispatch, RouteNotFound)
from workqueue import WorkQueueService

import utils
import core

logger = logging.getLogger('yerba')

def listen_forever(port, options=None):
    WorkQueueService.set_project(options['queue_prefix'])
    ServiceManager.register(WorkQueueService())
    ServiceManager.start()

    connection_string = "tcp://*:{}".format(port)
    context = zmq.Context()
    socket = context.socket(zmq.REP)
    socket.set(zmq.LINGER, 1000)
    socket.bind(connection_string)
    poller = zmq.Poller()
    poller.register(socket, zmq.POLLIN)

    while True:
        ServiceManager.update()
        if socket in dict(poller.poll(timeout=10)):
            msg = socket.recv_json(flags=zmq.NOBLOCK)
            response = None

            if not msg:
                logger.info("The message was not recieved.")
                continue

            with utils.ignored(RouteNotFound):
                response = dispatch(msg)

            if not response:
                logger.info("Invalid request: %s", msg)
                response = {"status" : "error"}

            try:
                socket.send_json(response, flags=zmq.NOBLOCK)
            except zmq.Again:
                logger.exception("Failed to respond with response %s",
                    response)

@route("schedule")
def schedule_workflow(data):
    '''Returns the job id'''
    status = WorkflowManager.submit(data)
    return {"status" : core.status_name(status)}

@route("cancel")
def cancel_workflow(data):
    '''Cancels the job if it is running.'''
    try:
        identity = data['id']
        status = WorkflowManager.cancel(identity)
        logger.info(core.status_message(identity, status))
        return {"status" : core.status_name(status)}
    except KeyError:
        return {"status" : 'NotFound'}

@route("get_status")
def get_workflow_status(data):
    '''Gets the status of the workflow.'''
    try:
        identity = data['id']
        (status, jobs) = WorkflowManager.status(identity)
        logger.info(core.status_message(identity, status))
        return {"status" : core.status_name(status), "jobs" : jobs}
    except KeyError:
        return {"status" : 'NotFound', "jobs" : {}}
<|endoftext|>