<commit_before># -*- coding: utf-8; -*-
#
# This file is part of Superdesk.
#
# Copyright 2013, 2014, 2015 Sourcefabric z.u. and contributors.
#
# For the full copyright and license information, please see the
# AUTHORS and LICENSE files distributed with this source code, or
# at https://www.sourcefabric.org/superdesk/license

from collections import namedtuple
import json
import logging
import flask
from superdesk import get_resource_service
import superdesk
from superdesk.errors import SuperdeskApiError
from superdesk.metadata.item import not_analyzed, ITEM_STATE, PUBLISH_STATES
from superdesk.metadata.utils import aggregations, get_elastic_highlight_query
from superdesk.resource import Resource
from superdesk.services import BaseService
from superdesk.utc import utcnow

from bson.objectid import ObjectId
from eve.utils import ParsedRequest, config
from flask import current_app as app

from apps.archive.archive import SOURCE as ARCHIVE
from apps.archive.common import handle_existing_data, item_schema
from superdesk.publish.publish_queue import PUBLISHED_IN_PACKAGE
from flask_babel import _

logger = logging.getLogger(__name__)

PUBLISHED = 'published'
LAST_PUBLISHED_VERSION = 'last_published_version'
QUEUE_STATE = 'queue_state'
ERROR_MESSAGE = 'error_message'
queue_states = ['pending', 'in_progress', 'queued', 'queued_not_transmitted', 'error']
PUBLISH_STATE = namedtuple('PUBLISH_STATE', ['PENDING',
                                             'IN_PROGRESS',
                                             'QUEUED',
                                             'QUEUED_NOT_TRANSMITTED',
                                             'ERROR'])(*queue_states)

published_item_fields = {
    'item_id': {
        'type': 'string',
        'mapping': not_analyzed
    },
    'publish_state': {
        'type': 'string'
    },
    # last_published_version field is set to true for last published version of the item in the published collection
    # and for the older version is set to false. This field is used to display the last version of the digital copy
    # in the published view.
    LAST_PUBLISHED_VERSION: {
        'type': 'boolean',
        'default': True
    },
    QUEUE_STATE: {
        'type': 'string',
        'default': 'pending',
        'allowed': queue_states,
    },
    ERROR_MESSAGE: {
        'type': 'string'
    },
    'is_take_item': {  # deprecated
        'type': 'boolean',
        'default': False,
    },
    'digital_item_id': {  # deprecated
        'type': 'string'
    },
    'publish_sequence_no': {
        'type': 'integer',
        'readonly': True
    },
    'last_queue_event': {
        'type': 'datetime'
    },
    # item is moved to legal. True if moved else false.
    'moved_to_legal': {
        'type': 'boolean',
        'default': False
    },
    PUBLISHED_IN_PACKAGE: {
        'type': 'string'
    }
}

def get_content_filter():
    """Filter out content of stages not visible to current user (if any).

    :return:
    """
    user = getattr(flask.g, 'user', None)
    if user:
        if 'invisible_stages' in user:
            stages = user.get('invisible_stages')
        else:
            stages = get_resource_service('users').get_invisible_stages_ids(user.get('_id'))

        if stages:
            return {'bool': {'must_not': {'terms': {'task.stage': stages}}}}

class PublishedItemResource(Resource):
    datasource = {
        'search_backend': 'elastic',
        'aggregations': aggregations,
        'es_highlight': get_elastic_highlight_query,
        'default_sort': [('_updated', -1)],
        'elastic_filter_callback': get_content_filter,
        'projection': {
            'old_version': 0,
            'last_version': 0
        }
    }

    schema = item_schema(published_item_fields)
    etag_ignore_fields = [config.ID_FIELD, 'highlights', 'item_id', LAST_PUBLISHED_VERSION, 'moved_to_legal']

    privileges = {'POST': 'publish_queue', 'PATCH': 'publish_queue'}
    item_methods = ['GET', 'PATCH']
    additional_lookup = {
        'url': 'regex("[\w,.:-]+")',
        'field': 'item_id'
    }

class PublishedItemService(BaseService):
    """
    PublishedItemService class is the base class for ArchivedService.
    """

    SEQ_KEY_NAME = 'published_item_sequence_no'

    def on_fetched(self, docs):
        """
        Overriding this to enhance the published article with the one in archive collection
        """

        self.enhance_with_archive_items(docs[config.ITEMS])

    def on_fetched_item(self, doc):
        """
        Overriding this to enhance the published article with the one in archive collection
        """

        self.enhance_with_archive_items([doc])

    def on_create(self, docs):
        """Runs on create.

        An article can be published multiple times in its lifetime. So, it's necessary to preserve the _id which comes
        from archive collection. Also, sets the expiry on the published item and removes the lock information.
        """

        for doc in docs:
            self.raise_if_not_marked_for_publication(doc)
            doc[config.LAST_UPDATED] = doc[config.DATE_CREATED] = utcnow()
            self.set_defaults(doc)

    def on_update(self, updates, original):
        if ITEM_STATE in updates:
            self.raise_if_not_marked_for_publication(updates)

    def raise_if_not_marked_for_publication(self, doc):
        """
        Item should be one of the PUBLISH_STATES. If not raise error.
        """
        if doc.get(ITEM_STATE) not in PUBLISH_STATES:
            raise SuperdeskApiError.badRequestError(
                _('Invalid state ({state}) for the Published item.').format(state=doc.get(ITEM_STATE)))

    def set_defaults(self, doc):
        doc['item_id'] = doc[config.ID_FIELD]
        doc['versioncreated'] = utcnow()
        doc['publish_sequence_no'] = get_resource_service('sequences').get_next_sequence_number(self.SEQ_KEY_NAME)
        doc.pop(config.ID_FIELD, None)
        doc.pop('lock_user', None)
        doc.pop('lock_time', None)
        doc.pop('lock_action', None)
        doc.pop('lock_session', None)

    def enhance_with_archive_items(self, items):
        if items:
            ids = list(set([item.get('item_id') for item in items if item.get('item_id')]))
            archive_items = []
            archive_lookup = {}
            if ids:
                query = {'$and': [{config.ID_FIELD: {'$in': ids}}]}
                archive_req = ParsedRequest()
                archive_req.max_results = len(ids)
                # can't access published from elastic due filter on the archive resource hence going to mongo
                archive_items = list(superdesk.get_resource_service(ARCHIVE)
                                     .get_from_mongo(req=archive_req, lookup=query))

                for item in archive_items:
                    handle_existing_data(item)
                    archive_lookup[item[config.ID_FIELD]] = item

            for item in items:
                archive_item = archive_lookup.get(item.get('item_id'), {config.VERSION: item.get(config.VERSION, 1)})

                updates = {
                    config.ID_FIELD: item.get('item_id'),
                    'item_id': item.get(config.ID_FIELD),
                    'lock_user': archive_item.get('lock_user', None),
                    'lock_time': archive_item.get('lock_time', None),
                    'lock_action': archive_item.get('lock_action', None),
                    'lock_session': archive_item.get('lock_session', None),
                    'archive_item': archive_item if archive_item else None
                }

                item.update(updates)
                handle_existing_data(item)

    def on_delete(self, doc):
        """Deleting a published item has a workflow which is implemented in remove_expired().

        Overriding to avoid other services from invoking this method accidentally.
        """

        if app.testing:
            super().on_delete(doc)
        else:
            raise NotImplementedError(
                _("Deleting a published item has a workflow which is implemented in remove_expired()."))

    def delete_action(self, lookup=None):
        """Deleting a published item has a workflow which is implemented in remove_expired().

        Overriding to avoid other services from invoking this method accidentally.
        """

        if app.testing:
            super().delete_action(lookup)
        else:
            raise NotImplementedError(
                _("Deleting a published item has a workflow which is implemented in remove_expired()."))

    def on_deleted(self, doc):
        """Deleting a published item has a workflow which is implemented in remove_expired().

        Overriding to avoid other services from invoking this method accidentally.
        """

        if app.testing:
            super().on_deleted(doc)
        else:
            raise NotImplementedError(
                _("Deleting a published item has a workflow which is implemented in remove_expired()."))

    def get_other_published_items(self, _id):
        try:
            query = {'query': {'filtered': {'filter': {'term': {'item_id': _id}}}}}
            request = ParsedRequest()
            request.args = {'source': json.dumps(query)}
            return super().get(req=request, lookup=None)
        except Exception:
            return []

    def get_last_published_version(self, _id):
        """Returns the last published entry for the passed item id

        :param _id:
        :return:
        """
        try:
            query = {'query':
                     {'filtered':
                      {'filter':
                       {'bool':
                        {'must': [
                            {'term': {'item_id': _id}},
                            {'term': {LAST_PUBLISHED_VERSION: True}}
                        ]}}}}}

            request = ParsedRequest()
            request.args = {'source': json.dumps(query), 'repo': 'published'}
            return get_resource_service('search').get(req=request, lookup=None)
        except Exception:
            return None

    def get_rewritten_items_by_event_story(self, event_id, rewrite_id, rewrite_field):
        """Returns all the rewritten stories from published and archive for a given event and rewrite_id.

        :param str event_id: event id of the document
        :param str rewrite_id: rewrite id of the document
        :param str rewrite_field: field name 'rewrite_of' or 'rewritten_by'
        """
        try:
            query = {'query':
                     {'filtered':
                      {'filter':
                       {'bool':
                        {'must': [
                            {'term': {'event_id': event_id}},
                            {'term': {rewrite_field: rewrite_id}}
                        ]}}}}}

            request = ParsedRequest()
            request.args = {'source': json.dumps(query), 'repo': 'archive,published'}
            return list(get_resource_service('search').get(req=request, lookup=None))
        except Exception:
            return []

    def is_rewritten_before(self, item_id):
        """Checks if the published item is rewritten before.

        :param _id: item_id of the published item
        :return: True is it is rewritten before
        """
        doc = self.find_one(req=None, item_id=item_id)
        return doc and 'rewritten_by' in doc and doc['rewritten_by']

    def update_published_items(self, _id, field, state):
        items = self.get_other_published_items(_id)
        for item in items:
            try:
                super().system_update(ObjectId(item[config.ID_FIELD]), {field: state}, item)
            except Exception:
                # This part is used in unit testing
                super().system_update(item[config.ID_FIELD], {field: state}, item)

    def delete_by_article_id(self, _id):
        """Removes the article from the published collection.

        Removes published queue entries.

        :param str _id: id of the document to be deleted. In mongo, it is the item_id
        """
        lookup = {'item_id': _id}
        self.delete(lookup=lookup)
        get_resource_service('publish_queue').delete_by_article_id(_id)

    def find_one(self, req, **lookup):
        item = super().find_one(req, **lookup)
        handle_existing_data(item)

        return item

    def move_to_archived(self, _id):
        published_items = list(self.get_from_mongo(req=None, lookup={'item_id': _id}))
        if not published_items:
            return
        get_resource_service('archived').post(published_items)
        self.delete_by_article_id(_id)

    def set_moved_to_legal(self, item_id, version, status):
        """Update the legal flag.

        :param str item_id: id of the document
        :param int version: version of the document
        :param boolean status: True if the item is moved to legal else false
        """
        items = self.get_other_published_items(item_id)

        for item in items:
            try:
                if item.get(config.VERSION) <= version and not item.get('moved_to_legal', False):
                    super().system_update(ObjectId(item.get(config.ID_FIELD)), {'moved_to_legal': status}, item)
            except Exception:
                logger.exception('Failed to set the moved_to_legal flag '
                                 'for item {} and version {}'.format(item_id, version))

    def get_published_items_by_moved_to_legal(self, item_ids, move_to_legal):
        """Get the pulished items where flag is moved.

        :param list item_ids: List of item
        :param bool move_to_legal: move_to_legal boolean flag
        :return: list of published items
        """
        if item_ids:
            try:
                query = {
                    'query': {
                        'filtered': {
                            'filter': {
                                'and': [
                                    {'terms': {'item_id': item_ids}},
                                    {'term': {'moved_to_legal': move_to_legal}}
                                ]
                            }
                        }
                    }
                }

                request = ParsedRequest()
                request.args = {'source': json.dumps(query)}
                return list(super().get(req=request, lookup=None))
            except Exception:
                logger.exception('Failed to get published items '
                                 'by moved to legal: {} -- ids: {}.'.format(move_to_legal, item_ids))

        return []
<commit_msg>send mark-for-user notification for publsihed items. (SDBELGA-124) (#1670)<commit_after># -*- coding: utf-8; -*-
#
# This file is part of Superdesk.
#
# Copyright 2013, 2014, 2015 Sourcefabric z.u. and contributors.
#
# For the full copyright and license information, please see the
# AUTHORS and LICENSE files distributed with this source code, or
# at https://www.sourcefabric.org/superdesk/license

from collections import namedtuple
import json
import logging
import flask
from superdesk import get_resource_service
import superdesk
from superdesk.errors import SuperdeskApiError
from superdesk.metadata.item import not_analyzed, ITEM_STATE, PUBLISH_STATES
from superdesk.metadata.utils import aggregations, get_elastic_highlight_query
from superdesk.resource import Resource
from superdesk.services import BaseService
from superdesk.utc import utcnow

from bson.objectid import ObjectId
from eve.utils import ParsedRequest, config
from flask import current_app as app

from apps.archive.archive import SOURCE as ARCHIVE
from apps.archive.common import handle_existing_data, item_schema
from superdesk.publish.publish_queue import PUBLISHED_IN_PACKAGE
from flask_babel import _

logger = logging.getLogger(__name__)

PUBLISHED = 'published'
LAST_PUBLISHED_VERSION = 'last_published_version'
QUEUE_STATE = 'queue_state'
ERROR_MESSAGE = 'error_message'
queue_states = ['pending', 'in_progress', 'queued', 'queued_not_transmitted', 'error']
PUBLISH_STATE = namedtuple('PUBLISH_STATE', ['PENDING',
                                             'IN_PROGRESS',
                                             'QUEUED',
                                             'QUEUED_NOT_TRANSMITTED',
                                             'ERROR'])(*queue_states)

published_item_fields = {
    'item_id': {
        'type': 'string',
        'mapping': not_analyzed
    },
    'publish_state': {
        'type': 'string'
    },
    # last_published_version field is set to true for last published version of the item in the published collection
    # and for the older version is set to false. This field is used to display the last version of the digital copy
    # in the published view.
    LAST_PUBLISHED_VERSION: {
        'type': 'boolean',
        'default': True
    },
    QUEUE_STATE: {
        'type': 'string',
        'default': 'pending',
        'allowed': queue_states,
    },
    ERROR_MESSAGE: {
        'type': 'string'
    },
    'is_take_item': {  # deprecated
        'type': 'boolean',
        'default': False,
    },
    'digital_item_id': {  # deprecated
        'type': 'string'
    },
    'publish_sequence_no': {
        'type': 'integer',
        'readonly': True
    },
    'last_queue_event': {
        'type': 'datetime'
    },
    # item is moved to legal. True if moved else false.
    'moved_to_legal': {
        'type': 'boolean',
        'default': False
    },
    PUBLISHED_IN_PACKAGE: {
        'type': 'string'
    }
}

def get_content_filter():
    """Filter out content of stages not visible to current user (if any).

    :return:
    """
    user = getattr(flask.g, 'user', None)
    if user:
        if 'invisible_stages' in user:
            stages = user.get('invisible_stages')
        else:
            stages = get_resource_service('users').get_invisible_stages_ids(user.get('_id'))

        if stages:
            return {'bool': {'must_not': {'terms': {'task.stage': stages}}}}

class PublishedItemResource(Resource):
    datasource = {
        'search_backend': 'elastic',
        'aggregations': aggregations,
        'es_highlight': get_elastic_highlight_query,
        'default_sort': [('_updated', -1)],
        'elastic_filter_callback': get_content_filter,
        'projection': {
            'old_version': 0,
            'last_version': 0
        }
    }

    schema = item_schema(published_item_fields)
    etag_ignore_fields = [config.ID_FIELD, 'highlights', 'item_id', LAST_PUBLISHED_VERSION, 'moved_to_legal']

    privileges = {'POST': 'publish_queue', 'PATCH': 'publish_queue'}
    item_methods = ['GET', 'PATCH']
    additional_lookup = {
        'url': 'regex("[\w,.:-]+")',
        'field': 'item_id'
    }

class PublishedItemService(BaseService):
    """
    PublishedItemService class is the base class for ArchivedService.
    """

    SEQ_KEY_NAME = 'published_item_sequence_no'

    def on_fetched(self, docs):
        """
        Overriding this to enhance the published article with the one in archive collection
        """

        self.enhance_with_archive_items(docs[config.ITEMS])

    def on_fetched_item(self, doc):
        """
        Overriding this to enhance the published article with the one in archive collection
        """

        self.enhance_with_archive_items([doc])

    def on_create(self, docs):
        """Runs on create.

        An article can be published multiple times in its lifetime. So, it's necessary to preserve the _id which comes
        from archive collection. Also, sets the expiry on the published item and removes the lock information.
        """

        for doc in docs:
            self.raise_if_not_marked_for_publication(doc)
            doc[config.LAST_UPDATED] = doc[config.DATE_CREATED] = utcnow()
            self.set_defaults(doc)

    def on_update(self, updates, original):
        if ITEM_STATE in updates:
            self.raise_if_not_marked_for_publication(updates)

    def on_updated(self, updates, original):
        if 'marked_for_user' in updates:
            # Send notification on mark-for-user operation
            get_resource_service('archive').handle_mark_user_notifications(updates, original)

    def raise_if_not_marked_for_publication(self, doc):
        """
        Item should be one of the PUBLISH_STATES. If not raise error.
        """
        if doc.get(ITEM_STATE) not in PUBLISH_STATES:
            raise SuperdeskApiError.badRequestError(
                _('Invalid state ({state}) for the Published item.').format(state=doc.get(ITEM_STATE)))

    def set_defaults(self, doc):
        doc['item_id'] = doc[config.ID_FIELD]
        doc['versioncreated'] = utcnow()
        doc['publish_sequence_no'] = get_resource_service('sequences').get_next_sequence_number(self.SEQ_KEY_NAME)
        doc.pop(config.ID_FIELD, None)
        doc.pop('lock_user', None)
        doc.pop('lock_time', None)
        doc.pop('lock_action', None)
        doc.pop('lock_session', None)

    def enhance_with_archive_items(self, items):
        if items:
            ids = list(set([item.get('item_id') for item in items if item.get('item_id')]))
            archive_items = []
            archive_lookup = {}
            if ids:
                query = {'$and': [{config.ID_FIELD: {'$in': ids}}]}
                archive_req = ParsedRequest()
                archive_req.max_results = len(ids)
                # can't access published from elastic due filter on the archive resource hence going to mongo
                archive_items = list(superdesk.get_resource_service(ARCHIVE)
                                     .get_from_mongo(req=archive_req, lookup=query))

                for item in archive_items:
                    handle_existing_data(item)
                    archive_lookup[item[config.ID_FIELD]] = item

            for item in items:
                archive_item = archive_lookup.get(item.get('item_id'), {config.VERSION: item.get(config.VERSION, 1)})

                updates = {
                    config.ID_FIELD: item.get('item_id'),
                    'item_id': item.get(config.ID_FIELD),
                    'lock_user': archive_item.get('lock_user', None),
                    'lock_time': archive_item.get('lock_time', None),
                    'lock_action': archive_item.get('lock_action', None),
                    'lock_session': archive_item.get('lock_session', None),
                    'archive_item': archive_item if archive_item else None
                }

                item.update(updates)
                handle_existing_data(item)

    def on_delete(self, doc):
        """Deleting a published item has a workflow which is implemented in remove_expired().

        Overriding to avoid other services from invoking this method accidentally.
        """

        if app.testing:
            super().on_delete(doc)
        else:
            raise NotImplementedError(
                _("Deleting a published item has a workflow which is implemented in remove_expired()."))

    def delete_action(self, lookup=None):
        """Deleting a published item has a workflow which is implemented in remove_expired().

        Overriding to avoid other services from invoking this method accidentally.
        """

        if app.testing:
            super().delete_action(lookup)
        else:
            raise NotImplementedError(
                _("Deleting a published item has a workflow which is implemented in remove_expired()."))

    def on_deleted(self, doc):
        """Deleting a published item has a workflow which is implemented in remove_expired().

        Overriding to avoid other services from invoking this method accidentally.
        """

        if app.testing:
            super().on_deleted(doc)
        else:
            raise NotImplementedError(
                _("Deleting a published item has a workflow which is implemented in remove_expired()."))

    def get_other_published_items(self, _id):
        try:
            query = {'query': {'filtered': {'filter': {'term': {'item_id': _id}}}}}
            request = ParsedRequest()
            request.args = {'source': json.dumps(query)}
            return super().get(req=request, lookup=None)
        except Exception:
            return []

    def get_last_published_version(self, _id):
        """Returns the last published entry for the passed item id

        :param _id:
        :return:
        """
        try:
            query = {'query':
                     {'filtered':
                      {'filter':
                       {'bool':
                        {'must': [
                            {'term': {'item_id': _id}},
                            {'term': {LAST_PUBLISHED_VERSION: True}}
                        ]}}}}}

            request = ParsedRequest()
            request.args = {'source': json.dumps(query), 'repo': 'published'}
            return get_resource_service('search').get(req=request, lookup=None)
        except Exception:
            return None

    def get_rewritten_items_by_event_story(self, event_id, rewrite_id, rewrite_field):
        """Returns all the rewritten stories from published and archive for a given event and rewrite_id.

        :param str event_id: event id of the document
        :param str rewrite_id: rewrite id of the document
        :param str rewrite_field: field name 'rewrite_of' or 'rewritten_by'
        """
        try:
            query = {'query':
                     {'filtered':
                      {'filter':
                       {'bool':
                        {'must': [
                            {'term': {'event_id': event_id}},
                            {'term': {rewrite_field: rewrite_id}}
                        ]}}}}}

            request = ParsedRequest()
            request.args = {'source': json.dumps(query), 'repo': 'archive,published'}
            return list(get_resource_service('search').get(req=request, lookup=None))
        except Exception:
            return []

    def is_rewritten_before(self, item_id):
        """Checks if the published item is rewritten before.

        :param _id: item_id of the published item
        :return: True is it is rewritten before
        """
        doc = self.find_one(req=None, item_id=item_id)
        return doc and 'rewritten_by' in doc and doc['rewritten_by']

    def update_published_items(self, _id, field, state):
        items = self.get_other_published_items(_id)
        for item in items:
            try:
                super().system_update(ObjectId(item[config.ID_FIELD]), {field: state}, item)
            except Exception:
                # This part is used in unit testing
                super().system_update(item[config.ID_FIELD], {field: state}, item)

    def delete_by_article_id(self, _id):
        """Removes the article from the published collection.

        Removes published queue entries.

        :param str _id: id of the document to be deleted. In mongo, it is the item_id
        """
        lookup = {'item_id': _id}
        self.delete(lookup=lookup)
        get_resource_service('publish_queue').delete_by_article_id(_id)

    def find_one(self, req, **lookup):
        item = super().find_one(req, **lookup)
        handle_existing_data(item)

        return item

    def move_to_archived(self, _id):
        published_items = list(self.get_from_mongo(req=None, lookup={'item_id': _id}))
        if not published_items:
            return
        get_resource_service('archived').post(published_items)
        self.delete_by_article_id(_id)

    def set_moved_to_legal(self, item_id, version, status):
        """Update the legal flag.

        :param str item_id: id of the document
        :param int version: version of the document
        :param boolean status: True if the item is moved to legal else false
        """
        items = self.get_other_published_items(item_id)

        for item in items:
            try:
                if item.get(config.VERSION) <= version and not item.get('moved_to_legal', False):
                    super().system_update(ObjectId(item.get(config.ID_FIELD)), {'moved_to_legal': status}, item)
            except Exception:
                logger.exception('Failed to set the moved_to_legal flag '
                                 'for item {} and version {}'.format(item_id, version))

    def get_published_items_by_moved_to_legal(self, item_ids, move_to_legal):
        """Get the pulished items where flag is moved.

        :param list item_ids: List of item
        :param bool move_to_legal: move_to_legal boolean flag
        :return: list of published items
        """
        if item_ids:
            try:
                query = {
                    'query': {
                        'filtered': {
                            'filter': {
                                'and': [
                                    {'terms': {'item_id': item_ids}},
                                    {'term': {'moved_to_legal': move_to_legal}}
                                ]
                            }
                        }
                    }
                }

                request = ParsedRequest()
                request.args = {'source': json.dumps(query)}
                return list(super().get(req=request, lookup=None))
            except Exception:
                logger.exception('Failed to get published items '
                                 'by moved to legal: {} -- ids: {}.'.format(move_to_legal, item_ids))

        return []
<|endoftext|>


<commit_before>import settings

import json
import os.path
import sqlite3
import sys
import time
import urllib2
import urlparse

sys.path.insert(1,'lib')
import flickrapi

def get_flickr():
    """Connect to flickr, prompting the user to authorize if necessary"""
    sys.stdout.write("Connecting to Flickr...\n");
    flickr = flickrapi.FlickrAPI(settings.api_key, settings.secret,
                                 format='json')

    # Check for permissions
    (token, frob) = flickr.get_token_part_one(perms='read')

    if not token:
        raw_input('Press ENTER once you have authorized the program')

    # Complete
    flickr.get_token_part_two((token, frob))

    return flickr

def get_database():
    """Fetch and/or create the database for holding photo info"""
    existed = os.path.exists(settings.database_filename)
    conn = sqlite3.connect(settings.database_filename)

    if existed:
        sys.stdout.write("Using existing database: %s\n" %
                         settings.database_filename)
        return conn

    else:
        sys.stdout.write("Creating new database: %s\n" %
                         settings.database_filename)
        # Create tables
        conn.execute('CREATE TABLE info(key text, value text)')
        conn.execute('''CREATE TABLE photos(id text,
                                            img_url text,
                                            media text,
                                            json_data text,
                                            fetched integer,
                                            downloaded integer,
                                            saved_path text
                    )''')
        conn.commit()

        return conn

def populate_photos(flickr, db):
    """Return the total number of photos that the user has"""
    # Do one quick request just to get the list of total photos
    sys.stdout.write("Fetching list of photos from Flickr\n")

    # Flickr begins at 1
    page_number = 1
    total_pages = None
    photos = []

    # Flags for the metadata we need
    extras = 'date_taken,original_format,geo,tags,machine_tags,views,media,description,url_o'

    # Query flickr a bunch of times to get the info we need
    while not total_pages or page_number <= total_pages:
        sys.stdout.write("\rFetching page %s of %s ...      " % (page_number, total_pages or '?'))
        sys.stdout.flush()

        search_res = json.loads(
            flickr.photos_search(user_id=settings.user_id, per_page='100',
                                 page=page_number, nojsoncallback=True,
                                 extras=extras)
        )

        if not total_pages:
            total_pages = search_res['photos']['pages']

        pics_arr = search_res['photos']['photo']
        pic_tuples = [(photo['id'], photo['url_o'], photo['media'], json.dumps(photo)) for photo in pics_arr]

        # Write records into database
        db.executemany('INSERT INTO photos(id, img_url, media, json_data) VALUES(?,?,?,?)', pic_tuples)

        # Increment
        page_number += 1

    sys.stdout.write("\n\rFetched all %s pages of info      \n" % page_number)
    sys.stdout.flush()

    # Save out to disk
    db.commit()

    return photos

def get_photo_list(flickr, db):
    """Returns an array of photos that need to be downloaded"""
    photos = db.execute('SELECT * FROM photos WHERE downloaded IS NULL').fetchall()

    if len(photos):
        return photos

    # If this is the first time, then there might not have been any photos
    all_photos = db.execute('SELECT * FROM photos').fetchall()

    if len(all_photos):
        # Great, photos have been fetched before, so we're all good
        return []

    return populate_photos(flickr, db)

def download_photos(photos):
    """Downloads a list of photos to disk"""

    sys.stdout.write("%s photos to download\n" % len(photos))
    sys.stdout.flush()

    photo_count = len(photos)

    for index, photo in enumerate(photos):
        json_info = json.loads(photo[3])
        url = photo[1] # URL
        filename = os.path.basename(urlparse.urlparse(url).path)
        folderpath = os.path.join(settings.photo_root, json_info['datetaken'][:10])
        # Create folder, if not there
        if not os.path.exists(folderpath):
            os.makedirs(folderpath)
        filepath = os.path.join(folderpath, filename)
        sys.stdout.write("\rDownloading file %s of %s [%s]" % (index + 1, photo_count, filepath))
        sys.stdout.flush()

        try:
            # Download File
            remote_image = urllib2.urlopen(url)
            image_data = remote_image.read()
            # Save File
            image_file = open(filepath, 'wb')
            image_file.write(image_data)
            image_file.close()
            # Save path into db
            db.execute('UPDATE photos SET saved_path=?,downloaded=1 WHERE id=?', (filepath, photo[0]))
            db.commit()

            # Add geodata
            # Add tag data
            # Other flickr-specific data? (id, url)
            # Write file out in picasa-like directory structure?
            # Mark as downloaded
            pass

        except:
            sys.stdout.write("\r")
            sys.stdout.flush()
            sys.stderr.write("Download failed for: %s\n" % url)
            sys.stderr.flush()

    sys.stdout.write("\n%s photos downloaded\n" % len(photos))

if __name__ == '__main__':
    # Connect to Flickr
    flickr = get_flickr()

    # Get a connection to the database, creating one if it does not exist
    db = get_database()

    # Retrieve list of photos that still need to be downloaded
    photos = get_photo_list(flickr, db)

    # Do the actual downloading
    if len(photos):
        download_photos(photos)
    else:
        print "No photos left to download"

    db.close()

    print "\nCompleted"
<commit_msg>Write metadata to files using exiftool<commit_after>import settings

import json
import os.path
import sqlite3
import sys
import time
import urllib2
import urlparse
import subprocess
import shlex
import pipes

sys.path.insert(1,'lib')
import flickrapi

def get_flickr():
    """Connect to flickr, prompting the user to authorize if necessary"""
    sys.stdout.write("Connecting to Flickr...\n");
    flickr = flickrapi.FlickrAPI(settings.api_key, settings.secret,
                                 format='json')

    # Check for permissions
    (token, frob) = flickr.get_token_part_one(perms='read')

    if not token:
        raw_input('Press ENTER once you have authorized the program')

    # Complete
    flickr.get_token_part_two((token, frob))

    return flickr

def get_database():
    """Fetch and/or create the database for holding photo info"""
    existed = os.path.exists(settings.database_filename)
    conn = sqlite3.connect(settings.database_filename)

    if existed:
        sys.stdout.write("Using existing database: %s\n" %
                         settings.database_filename)
        return conn

    else:
        sys.stdout.write("Creating new database: %s\n" %
                         settings.database_filename)
        # Create tables
        conn.execute('CREATE TABLE info(key text, value text)')
        conn.execute('''CREATE TABLE photos(id text,
                                            img_url text,
                                            media text,
                                            json_data text,
                                            downloaded integer,
                                            metadata integer,
                                            saved_path text
                    )''')
        conn.commit()

        return conn

def populate_photos(flickr, db):
    """Return the total number of photos that the user has"""
    # Do one quick request just to get the list of total photos
    sys.stdout.write("Fetching list of photos from Flickr\n")

    # Flickr begins at 1
    page_number = 1
    total_pages = None
    photos = []

    # Flags for the metadata we need
    extras = 'date_taken,original_format,geo,tags,machine_tags,views,media,description,url_o'

    # Query flickr a bunch of times to get the info we need
    while not total_pages or page_number <= total_pages:
        sys.stdout.write("\rFetching page %s of %s ...      " % (page_number, total_pages or '?'))
        sys.stdout.flush()

        search_res = json.loads(
            flickr.photos_search(user_id=settings.user_id, per_page='100',
                                 page=page_number, nojsoncallback=True,
                                 extras=extras)
        )

        if not total_pages:
            total_pages = search_res['photos']['pages']

        pics_arr = search_res['photos']['photo']
        pic_tuples = [(photo['id'], photo['url_o'], photo['media'], json.dumps(photo)) for photo in pics_arr]

        # Write records into database
        db.executemany('INSERT INTO photos(id, img_url, media, json_data) VALUES(?,?,?,?)', pic_tuples)

        # Increment
        page_number += 1

    sys.stdout.write("\n\rFetched all %s pages of info      \n" % page_number)
    sys.stdout.flush()

    # Save out to disk
    db.commit()

    return photos

def get_photo_list(flickr, db):
    """Returns an array of photos that need to be downloaded"""
    photos = db.execute('SELECT * FROM photos WHERE downloaded IS NULL').fetchall()

    if len(photos):
        return photos

    # If this is the first time, then there might not have been any photos
    all_photos = db.execute('SELECT * FROM photos').fetchall()

    if len(all_photos):
        # Great, photos have been fetched before, so we're all good
        return []

    return populate_photos(flickr, db)

def download_photos(photos, db):
    """Downloads a list of photos to disk"""

    sys.stdout.write("%s photos to download\n" % len(photos))
    sys.stdout.flush()

    photo_count = len(photos)

    for index, photo in enumerate(photos):
        json_info = json.loads(photo[3])
        url = photo[1] # URL
        filename = os.path.basename(urlparse.urlparse(url).path)
        folderpath = os.path.join(settings.photo_root, json_info['datetaken'][:10])
        # Create folder, if not there
        if not os.path.exists(folderpath):
            os.makedirs(folderpath)
        filepath = os.path.join(folderpath, filename)
        sys.stdout.write("\rDownloading file %s of %s [%s]" % (index + 1, photo_count, filepath))
        sys.stdout.flush()

        try:
            # Download File
            remote_image = urllib2.urlopen(url)
            image_data = remote_image.read()
            # Save File
            image_file = open(filepath, 'wb')
            image_file.write(image_data)
            image_file.close()
            # Save path into db and mark as downloaded
            db.execute('UPDATE photos SET saved_path=?,downloaded=1 WHERE id=?', (filepath, photo[0]))
            db.commit()

        except:
            sys.stdout.write("\r")
            sys.stdout.flush()
            sys.stderr.write("Download failed for: %s\n" % url)
            sys.stderr.flush()

    sys.stdout.write("\n%s photos downloaded\n" % len(photos))

base58letters = '123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ'
base58len = len(base58letters)

def base58encode(id):
    """
    Encoding for Flickr short urls described here:
    http://www.flickr.com/groups/api/discuss/72157616713786392/
    """
    str = ''

    while id >= base58len:
        div, mod = divmod(id, base58len)
        str = base58letters[mod] + str
        id = int(div)

    if (id):
        str = base58letters[id] + str

    return str

def add_metadata(db):
    # Only get photos w/o metadata written
    photos = db.execute('SELECT * FROM photos WHERE metadata IS NULL').fetchall()

    photo_count = len(photos)
    if not photo_count:
        print 'No photos need metadata'

    print "%s photos need metadata" % photo_count
    for index, photo in enumerate(photos):
        metadata = json.loads(photo[3])
        # Becomes Caption-Abstract in EXIF
        title = metadata['title']
        if metadata['description'] and metadata['description']['_content']:
            title += ": " + metadata['description']['_content']
        # Keywords (comma-sep)
        tags = metadata['tags'].split(' ')
        # ID in comment
        id = photo[0]
        # URL in Source
        short_url = "http://flic.kr/p/%s" % base58encode(int(id))
        # DateTimeOriginal
        date_taken = metadata['datetaken'].replace('-', ':')
        # Latitude & Longitude
        latitude = metadata['latitude']
        longitude = metadata['longitude']

        sys.stdout.write("\rWriting metadata %s of %s [%s]                    " % (index + 1, photo_count, photo[6]))
        sys.stdout.flush()

        # Run exiftool command
        cmd = "exiftool -overwrite_original_in_place"
        if title:
            cmd += " -description=%s " % pipes.quote(title)
        if tags and len(tags):
            for tag in tags:
                cmd += """ -keywords=%s """ % pipes.quote(tag)
        cmd += """ -Source=%s """ % pipes.quote(short_url)
        cmd += """ -Comment="Flickr ID: %s" """ % id
        cmd += """ -DateTimeOriginal=%s """ % pipes.quote(date_taken)
        if latitude and longitude:
            if latitude > 0:
                cmd += """ -gps:GPSLatitudeRef="N" """
            else:
                cmd += """ -gps:GPSLatitudeRef="S" """
            if longitude > 0:
                cmd += """ -gps:GPSLongitudeRef="E" """
            else:
                cmd += """ -gps:GPSLongitudeRef="W" """

            cmd += """ -gpslatitude="%s" """ % abs(latitude)
            cmd += """ -gpslongitude="%s" """ % abs(longitude)

        # Privacy
        if metadata['ispublic']:
            cmd += ' -keywords="privacy:public"'
        elif not (metadata['isfriend'] or metadata['isfamily']):
            cmd += ' -keywords="privacy:private"'
        else:
            if metadata['isfriend']:
                cmd += ' -keywords="privacy:friend"'
            if metadata['isfamily']:
                cmd += ' -keywords="privacy:family"'

        # Add filename
        cmd += """ %s """ % photo[6]

        # Run command
        try:
            fnull = open(os.devnull, 'w')
            subprocess.call(cmd, shell=True, stdout=fnull, stderr=fnull)
            fnull.close()

            # Save path into db and mark as metadata'd
            db.execute('UPDATE photos SET metadata=1 WHERE id=?', [photo[0]])
            db.commit()
        except sqlite3.ProgrammingError as e:
            print e
            print photo[0]
        except:
            sys.stderr.write("\rMetadata failed for: %s                \n" % photo[6])
            sys.stderr.write("%s\n\r" % sys.exc_info()[0])
            sys.stderr.flush()

if __name__ == '__main__':
    # Connect to Flickr
    flickr = get_flickr()

    # Get a connection to the database, creating one if it does not exist
    db = get_database()

    # Retrieve list of photos that still need to be downloaded
    photos = get_photo_list(flickr, db)

    # Do the actual downloading
    if len(photos):
        download_photos(photos, db)
    else:
        print "No photos left to download"

    # Now add metadata
    add_metadata(db)

    db.close()

    print "\nCompleted"
<|endoftext|>


<commit_before>#
#  Copyright 2019 The FATE Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

import unittest

from fate_arch.session import computing_session as session
from federatedml.ensemble import FeatureHistogram
from federatedml.feature.instance import Instance
from federatedml.feature.sparse_vector import SparseVector
from federatedml.util import consts
import copy
import numpy as np
import random

class TestFeatureHistogram(unittest.TestCase):

    def setUp(self):
        self.feature_histogram = FeatureHistogram()
        session.init("test_feature_histogram")
        data_insts = []
        for i in range(1000):
            indices = []
            data = []
            for j in range(10):
                x = random.randint(0, 5)
                if x != 0:
                    data.append(x)
                    indices.append(j)
            sparse_vec = SparseVector(indices, data, shape=10)
            data_insts.append((Instance(features=sparse_vec), (1, random.randint(0, 3))))
        self.node_map = {0: 0, 1: 1, 2: 2, 3: 3}
        self.data_insts = data_insts
        self.data_bin = session.parallelize(data_insts, include_key=False, partition=16)

        self.grad_and_hess_list = [(random.random(), random.random()) for i in range(1000)]
        self.grad_and_hess = session.parallelize(self.grad_and_hess_list, include_key=False, partition=16)

        bin_split_points = []
        for i in range(10):
            bin_split_points.append(np.array([i for i in range(6)]))
        self.bin_split_points = np.array(bin_split_points)
        self.bin_sparse = [0 for i in range(10)]

    def test_accumulate_histogram(self):
        data = [[[[random.randint(0, 10) for i in range(2)]
                  for j in range(3)]
                 for k in range(4)]
                for r in range(5)]
        histograms = copy.deepcopy(data)
        for i in range(len(data)):
            for j in range(len(data[i])):
                histograms[i][j] = self.feature_histogram._tensor_histogram_cumsum(histograms[i][j])
                for k in range(1, len(data[i][j])):
                    for r in range(len(data[i][j][k])):
                        data[i][j][k][r] += data[i][j][k - 1][r]
                        self.assertTrue(data[i][j][k][r] == histograms[i][j][k][r])

    def test_calculate_histogram(self):
        histograms = self.feature_histogram.calculate_histogram(
            self.data_bin, self.grad_and_hess,
            self.bin_split_points, self.bin_sparse,
            node_map=self.node_map)

        his2 = [[[[0 for i in range(3)]
                  for j in range(6)]
                 for k in range(10)]
                for r in range(4)]
        for i in range(1000):
            grad, hess = self.grad_and_hess_list[i]
            id = self.node_map[self.data_insts[i][1][1]]
            for fid, bid in self.data_insts[i][0].features.get_all_data():
                his2[id][fid][bid][0] += grad
                his2[id][fid][bid][1] += hess
                his2[id][fid][bid][2] += 1

        for i in range(len(his2)):
            for j in range(len(his2[i])):
                his2[i][j] = self.feature_histogram._tensor_histogram_cumsum(his2[i][j])
                for k in range(len(his2[i][j])):
                    for r in range(len(his2[i][j][k])):
                        self.assertTrue(np.fabs(his2[i][j][k][r] - histograms[i][j][k][r]) < consts.FLOAT_ZERO)

    def test_aggregate_histogram(self):

        fake_fid = 114
        data1 = [[random.randint(0, 10) for i in range(2)] for j in range(3)]

        data2 = [[random.randint(0, 10) for i in range(2)] for j in range(3)]

        fid, agg_histograms = self.feature_histogram._aggregate_histogram((fake_fid, data1),
                                                                          (fake_fid, data2),
                                                                          node_map=None)
        for i in range(len(data1)):
            for j in range(len(data1[i])):
                data1[i][j] += data2[i][j]
                self.assertTrue(data1[i][j] == agg_histograms[i][j])

    def tearDown(self):
        session.stop()

if __name__ == '__main__':
    unittest.main()
<commit_msg>fix pipeline example<commit_after>#
#  Copyright 2019 The FATE Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

import unittest

from fate_arch.session import computing_session as session
from federatedml.ensemble import FeatureHistogram
from federatedml.feature.instance import Instance
from federatedml.feature.sparse_vector import SparseVector
from federatedml.util import consts
import copy
import numpy as np
import random

class TestFeatureHistogram(unittest.TestCase):

    def setUp(self):
        self.feature_histogram = FeatureHistogram()
        session.init("test_feature_histogram")
        data_insts = []
        for i in range(1000):
            indices = []
            data = []
            for j in range(10):
                x = random.randint(0, 5)
                if x != 0:
                    data.append(x)
                    indices.append(j)
            sparse_vec = SparseVector(indices, data, shape=10)
            data_insts.append((Instance(features=sparse_vec), (1, random.randint(0, 3))))
        self.node_map = {0: 0, 1: 1, 2: 2, 3: 3}
        self.data_insts = data_insts
        self.data_bin = session.parallelize(data_insts, include_key=False, partition=16)

        self.grad_and_hess_list = [(random.random(), random.random()) for i in range(1000)]
        self.grad_and_hess = session.parallelize(self.grad_and_hess_list, include_key=False, partition=16)

        bin_split_points = []
        for i in range(10):
            bin_split_points.append(np.array([i for i in range(6)]))
        self.bin_split_points = np.array(bin_split_points)
        self.bin_sparse = [0 for i in range(10)]

    def test_accumulate_histogram(self):
        data = [[[[random.randint(0, 10) for i in range(2)]
                  for j in range(3)]
                 for k in range(4)]
                for r in range(5)]
        histograms = copy.deepcopy(data)
        for i in range(len(data)):
            for j in range(len(data[i])):
                histograms[i][j] = self.feature_histogram._tensor_histogram_cumsum(histograms[i][j])
                for k in range(1, len(data[i][j])):
                    for r in range(len(data[i][j][k])):
                        data[i][j][k][r] += data[i][j][k - 1][r]
                        self.assertTrue(data[i][j][k][r] == histograms[i][j][k][r])

    def test_calculate_histogram(self):
        histograms = self.feature_histogram.calculate_histogram(
            self.data_bin, self.grad_and_hess,
            self.bin_split_points, self.bin_sparse,
            node_map=self.node_map)

        his2 = [[[[0 for i in range(3)]
                  for j in range(6)]
                 for k in range(10)]
                for r in range(4)]
        for i in range(1000):
            grad, hess = self.grad_and_hess_list[i]
            id = self.node_map[self.data_insts[i][1][1]]
            for fid, bid in self.data_insts[i][0].features.get_all_data():
                his2[id][fid][bid][0] += grad
                his2[id][fid][bid][1] += hess
                his2[id][fid][bid][2] += 1

        for i in range(len(his2)):
            for j in range(len(his2[i])):
                his2[i][j] = self.feature_histogram._tensor_histogram_cumsum(his2[i][j])
                for k in range(len(his2[i][j])):
                    for r in range(len(his2[i][j][k])):
                        self.assertTrue(np.fabs(his2[i][j][k][r] - histograms[i][j][k][r]) < consts.FLOAT_ZERO)

    def test_aggregate_histogram(self):

        fake_fid = 114
        data1 = [[random.randint(0, 10) for i in range(2)] for j in range(3)]

        data2 = [[random.randint(0, 10) for i in range(2)] for j in range(3)]

        fid, agg_histograms = self.feature_histogram._hist_aggregate((fake_fid, data1),
                                                                          (fake_fid, data2),
                                                                          node_map=None)
        for i in range(len(data1)):
            for j in range(len(data1[i])):
                data1[i][j] += data2[i][j]
                self.assertTrue(data1[i][j] == agg_histograms[i][j])

    def tearDown(self):
        session.stop()

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from synapse.http.agent_name import AGENT_NAME
from synapse.api.errors import (
    cs_exception, SynapseError, CodeMessageException
)
from synapse.util.logcontext import LoggingContext

from syutil.jsonutil import (
    encode_canonical_json, encode_pretty_printed_json
)

from twisted.internet import defer, reactor
from twisted.web import server, resource
from twisted.web.server import NOT_DONE_YET
from twisted.web.util import redirectTo

import collections
import logging
import urllib

logger = logging.getLogger(__name__)

class HttpServer(object):
    """ Interface for registering callbacks on a HTTP server
    """

    def register_path(self, method, path_pattern, callback):
        """ Register a callback that get's fired if we receive a http request
        with the given method for a path that matches the given regex.

        If the regex contains groups these get's passed to the calback via
        an unpacked tuple.

        Args:
            method (str): The method to listen to.
            path_pattern (str): The regex used to match requests.
            callback (function): The function to fire if we receive a matched
                request. The first argument will be the request object and
                subsequent arguments will be any matched groups from the regex.
                This should return a tuple of (code, response).
        """
        pass

class JsonResource(HttpServer, resource.Resource):
    """ This implements the HttpServer interface and provides JSON support for
    Resources.

    Register callbacks via register_path()
    """

    isLeaf = True

    _PathEntry = collections.namedtuple("_PathEntry", ["pattern", "callback"])

    def __init__(self):
        resource.Resource.__init__(self)

        self.path_regexs = {}

    def register_path(self, method, path_pattern, callback):
        self.path_regexs.setdefault(method, []).append(
            self._PathEntry(path_pattern, callback)
        )

    def start_listening(self, port):
        """ Registers the http server with the twisted reactor.

        Args:
            port (int): The port to listen on.

        """
        reactor.listenTCP(port, server.Site(self))

    # Gets called by twisted
    def render(self, request):
        """ This get's called by twisted every time someone sends us a request.
        """
        self._async_render_with_logging_context(request)
        return server.NOT_DONE_YET

    _request_id = 0

    @defer.inlineCallbacks
    def _async_render_with_logging_context(self, request):
        request_id = "%s-%s" % (request.method, JsonResource._request_id)
        JsonResource._request_id += 1
        with LoggingContext(request_id) as request_context:
            request_context.request = request_id
            yield self._async_render(request)

    @defer.inlineCallbacks
    def _async_render(self, request):
        """ This get's called by twisted every time someone sends us a request.
            This checks if anyone has registered a callback for that method and
            path.
        """
        try:
            # Just say yes to OPTIONS.
            if request.method == "OPTIONS":
                self._send_response(request, 200, {})
                return

            # Loop through all the registered callbacks to check if the method
            # and path regex match
            for path_entry in self.path_regexs.get(request.method, []):
                m = path_entry.pattern.match(request.path)
                if m:
                    # We found a match! Trigger callback and then return the
                    # returned response. We pass both the request and any
                    # matched groups from the regex to the callback.

                    args = [
                        urllib.unquote(u).decode("UTF-8") for u in m.groups()
                    ]

                    code, response = yield path_entry.callback(
                        request,
                        *args
                    )

                    self._send_response(request, code, response)
                    return

            # Huh. No one wanted to handle that? Fiiiiiine. Send 400.
            self._send_response(
                request,
                400,
                {"error": "Unrecognized request"}
            )
        except CodeMessageException as e:
            if isinstance(e, SynapseError):
                logger.info("%s SynapseError: %s - %s", request, e.code, e.msg)
            else:
                logger.exception(e)
            self._send_response(
                request,
                e.code,
                cs_exception(e),
                response_code_message=e.response_code_message
            )
        except Exception as e:
            logger.exception(e)
            self._send_response(
                request,
                500,
                {"error": "Internal server error"}
            )

    def _send_response(self, request, code, response_json_object,
                       response_code_message=None):
        # could alternatively use request.notifyFinish() and flip a flag when
        # the Deferred fires, but since the flag is RIGHT THERE it seems like
        # a waste.
        if request._disconnected:
            logger.warn(
                "Not sending response to request %s, already disconnected.",
                request)
            return

        # TODO: Only enable CORS for the requests that need it.
        respond_with_json(request, code, response_json_object, send_cors=True,
                          response_code_message=response_code_message,
                          pretty_print=self._request_user_agent_is_curl)

    @staticmethod
    def _request_user_agent_is_curl(request):
        user_agents = request.requestHeaders.getRawHeaders(
            "User-Agent", default=[]
        )
        for user_agent in user_agents:
            if "curl" in user_agent:
                return True
        return False

class RootRedirect(resource.Resource):
    """Redirects the root '/' path to another path."""

    def __init__(self, path):
        resource.Resource.__init__(self)
        self.url = path

    def render_GET(self, request):
        return redirectTo(self.url, request)

    def getChild(self, name, request):
        if len(name) == 0:
            return self  # select ourselves as the child to render
        return resource.Resource.getChild(self, name, request)

def respond_with_json(request, code, json_object, send_cors=False,
                      response_code_message=None, pretty_print=False):
    if not pretty_print:
        json_bytes = encode_pretty_printed_json(json_object)
    else:
        json_bytes = encode_canonical_json(json_object)

    return respond_with_json_bytes(request, code, json_bytes, send_cors,
                                   response_code_message=response_code_message)

def respond_with_json_bytes(request, code, json_bytes, send_cors=False,
                            response_code_message=None):
    """Sends encoded JSON in response to the given request.

    Args:
        request (twisted.web.http.Request): The http request to respond to.
        code (int): The HTTP response code.
        json_bytes (bytes): The json bytes to use as the response body.
        send_cors (bool): Whether to send Cross-Origin Resource Sharing headers
            http://www.w3.org/TR/cors/
    Returns:
        twisted.web.server.NOT_DONE_YET"""

    request.setResponseCode(code, message=response_code_message)
    request.setHeader(b"Content-Type", b"application/json")
    request.setHeader(b"Server", AGENT_NAME)

    if send_cors:
        request.setHeader("Access-Control-Allow-Origin", "*")
        request.setHeader("Access-Control-Allow-Methods",
                          "GET, POST, PUT, DELETE, OPTIONS")
        request.setHeader("Access-Control-Allow-Headers",
                          "Origin, X-Requested-With, Content-Type, Accept")

    request.write(json_bytes)
    request.finish()
    return NOT_DONE_YET
<commit_msg>Set a content-length for JSON responses<commit_after># -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from synapse.http.agent_name import AGENT_NAME
from synapse.api.errors import (
    cs_exception, SynapseError, CodeMessageException
)
from synapse.util.logcontext import LoggingContext

from syutil.jsonutil import (
    encode_canonical_json, encode_pretty_printed_json
)

from twisted.internet import defer, reactor
from twisted.web import server, resource
from twisted.web.server import NOT_DONE_YET
from twisted.web.util import redirectTo

import collections
import logging
import urllib

logger = logging.getLogger(__name__)

class HttpServer(object):
    """ Interface for registering callbacks on a HTTP server
    """

    def register_path(self, method, path_pattern, callback):
        """ Register a callback that get's fired if we receive a http request
        with the given method for a path that matches the given regex.

        If the regex contains groups these get's passed to the calback via
        an unpacked tuple.

        Args:
            method (str): The method to listen to.
            path_pattern (str): The regex used to match requests.
            callback (function): The function to fire if we receive a matched
                request. The first argument will be the request object and
                subsequent arguments will be any matched groups from the regex.
                This should return a tuple of (code, response).
        """
        pass

class JsonResource(HttpServer, resource.Resource):
    """ This implements the HttpServer interface and provides JSON support for
    Resources.

    Register callbacks via register_path()
    """

    isLeaf = True

    _PathEntry = collections.namedtuple("_PathEntry", ["pattern", "callback"])

    def __init__(self):
        resource.Resource.__init__(self)

        self.path_regexs = {}

    def register_path(self, method, path_pattern, callback):
        self.path_regexs.setdefault(method, []).append(
            self._PathEntry(path_pattern, callback)
        )

    def start_listening(self, port):
        """ Registers the http server with the twisted reactor.

        Args:
            port (int): The port to listen on.

        """
        reactor.listenTCP(port, server.Site(self))

    # Gets called by twisted
    def render(self, request):
        """ This get's called by twisted every time someone sends us a request.
        """
        self._async_render_with_logging_context(request)
        return server.NOT_DONE_YET

    _request_id = 0

    @defer.inlineCallbacks
    def _async_render_with_logging_context(self, request):
        request_id = "%s-%s" % (request.method, JsonResource._request_id)
        JsonResource._request_id += 1
        with LoggingContext(request_id) as request_context:
            request_context.request = request_id
            yield self._async_render(request)

    @defer.inlineCallbacks
    def _async_render(self, request):
        """ This get's called by twisted every time someone sends us a request.
            This checks if anyone has registered a callback for that method and
            path.
        """
        try:
            # Just say yes to OPTIONS.
            if request.method == "OPTIONS":
                self._send_response(request, 200, {})
                return

            # Loop through all the registered callbacks to check if the method
            # and path regex match
            for path_entry in self.path_regexs.get(request.method, []):
                m = path_entry.pattern.match(request.path)
                if m:
                    # We found a match! Trigger callback and then return the
                    # returned response. We pass both the request and any
                    # matched groups from the regex to the callback.

                    args = [
                        urllib.unquote(u).decode("UTF-8") for u in m.groups()
                    ]

                    code, response = yield path_entry.callback(
                        request,
                        *args
                    )

                    self._send_response(request, code, response)
                    return

            # Huh. No one wanted to handle that? Fiiiiiine. Send 400.
            self._send_response(
                request,
                400,
                {"error": "Unrecognized request"}
            )
        except CodeMessageException as e:
            if isinstance(e, SynapseError):
                logger.info("%s SynapseError: %s - %s", request, e.code, e.msg)
            else:
                logger.exception(e)
            self._send_response(
                request,
                e.code,
                cs_exception(e),
                response_code_message=e.response_code_message
            )
        except Exception as e:
            logger.exception(e)
            self._send_response(
                request,
                500,
                {"error": "Internal server error"}
            )

    def _send_response(self, request, code, response_json_object,
                       response_code_message=None):
        # could alternatively use request.notifyFinish() and flip a flag when
        # the Deferred fires, but since the flag is RIGHT THERE it seems like
        # a waste.
        if request._disconnected:
            logger.warn(
                "Not sending response to request %s, already disconnected.",
                request)
            return

        # TODO: Only enable CORS for the requests that need it.
        respond_with_json(request, code, response_json_object, send_cors=True,
                          response_code_message=response_code_message,
                          pretty_print=self._request_user_agent_is_curl)

    @staticmethod
    def _request_user_agent_is_curl(request):
        user_agents = request.requestHeaders.getRawHeaders(
            "User-Agent", default=[]
        )
        for user_agent in user_agents:
            if "curl" in user_agent:
                return True
        return False

class RootRedirect(resource.Resource):
    """Redirects the root '/' path to another path."""

    def __init__(self, path):
        resource.Resource.__init__(self)
        self.url = path

    def render_GET(self, request):
        return redirectTo(self.url, request)

    def getChild(self, name, request):
        if len(name) == 0:
            return self  # select ourselves as the child to render
        return resource.Resource.getChild(self, name, request)

def respond_with_json(request, code, json_object, send_cors=False,
                      response_code_message=None, pretty_print=False):
    if not pretty_print:
        json_bytes = encode_pretty_printed_json(json_object)
    else:
        json_bytes = encode_canonical_json(json_object)

    return respond_with_json_bytes(request, code, json_bytes, send_cors,
                                   response_code_message=response_code_message)

def respond_with_json_bytes(request, code, json_bytes, send_cors=False,
                            response_code_message=None):
    """Sends encoded JSON in response to the given request.

    Args:
        request (twisted.web.http.Request): The http request to respond to.
        code (int): The HTTP response code.
        json_bytes (bytes): The json bytes to use as the response body.
        send_cors (bool): Whether to send Cross-Origin Resource Sharing headers
            http://www.w3.org/TR/cors/
    Returns:
        twisted.web.server.NOT_DONE_YET"""

    request.setResponseCode(code, message=response_code_message)
    request.setHeader(b"Content-Type", b"application/json")
    request.setHeader(b"Server", AGENT_NAME)
    request.setHeader(b"Content-Length", b"%d" % len(json_bytes))

    if send_cors:
        request.setHeader("Access-Control-Allow-Origin", "*")
        request.setHeader("Access-Control-Allow-Methods",
                          "GET, POST, PUT, DELETE, OPTIONS")
        request.setHeader("Access-Control-Allow-Headers",
                          "Origin, X-Requested-With, Content-Type, Accept")

    request.write(json_bytes)
    request.finish()
    return NOT_DONE_YET
<|endoftext|>


<commit_before># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Service functions for managing pack configuration inside the datastore.
"""

import json

from st2common import log as logging
from st2common.services import keyvalues as keyvalue_service
from st2common.constants.keyvalue import USER_SCOPE
from st2common.constants.keyvalue import SYSTEM_SCOPE
from st2common.models.db.keyvalue import KeyValuePairDB
from st2common.persistence.keyvalue import KeyValuePair
from st2common.constants.keyvalue import DATASTORE_KEY_SEPARATOR

__all__ = [
    'get_datastore_key_prefix_for_pack',
    'get_datastore_key_name',

    'get_datastore_value_for_config_key',
    'set_datastore_value_for_config_key',

    'get_datastore_value'
]

LOG = logging.getLogger(__name__)

# Prefix for datastore items which store config values
# Full keys follow this format: pack_config.<pack name>.<config key name>
# For example: pack_config.aws.setup.region
DATASTORE_CONFIG_KEY_PREFIX = 'pack_config'

def get_datastore_key_prefix_for_pack(pack_name):
    prefix = [DATASTORE_CONFIG_KEY_PREFIX, pack_name]
    prefix = DATASTORE_KEY_SEPARATOR.join(prefix)
    return prefix

def get_datastore_key_name(pack_name, key_name, user=None):
    """
    Retrieve datastore key name based on the config key name.

    :param user: Optional username if working on a user-scoped config item.
    :type user: ``str``

    :rtype: ``str``
    """
    values = []

    prefix = get_datastore_key_prefix_for_pack(pack_name=pack_name)
    values.append(prefix)

    if user:
        values.append(user)

    values.append(key_name)

    return DATASTORE_KEY_SEPARATOR.join(values)

def get_datastore_value_for_config_key(pack_name, key_name):
    """
    Retrieve config value for the provided config key from the datastore.

    :param pack_name: Pack name.
    :type pack_name: ``str``

    :param key_name: Config key name.
    :type key_name: ``str``
    """
    name = get_datastore_key_name(pack_name=pack_name, key_name=key_name)
    kvp_db = get_datastore_value(key_name=name)
    return kvp_db

def get_datastore_value(key_name):
    """
    Retrieve config value with the matching datastore key name.

    :param key_name: Full key name in the datastore. Note: This must already be the fully qualified
                     name of the key in the datastore.
    :tupe key_name: ``str``
    """
    kvp_db = keyvalue_service.get_kvp_for_name(name=key_name)

    if not kvp_db:
        # Item doesn't exist
        return None

    if not kvp_db.value:
        # Item doesn't contain a value
        return None

    value = deserialize_key_value(value=kvp_db.value)
    return value

def set_datastore_value_for_config_key(pack_name, key_name, value, user=None):
    """
    Set config value in the datastore.

    This function takes care of correctly encoding the key name, serializing the
    value, etc.

    :param pack_name: Pack name.
    :type pack_name: ``str``

    :param key_name: Config key name.
    :type key_name: ``str``

    :param user: Optional username if working on a user-scoped config item.
    :type user: ``str``

    :rtype: :class:`KeyValuePairDB`
    """
    # TODO: Once and if config schema is available, validate the provided type against the one
    # specified in the schema.
    name = get_datastore_key_name(pack_name=pack_name, key_name=key_name, user=user)

    if user:
        scope = USER_SCOPE
    else:
        scope = SYSTEM_SCOPE

    value = json.dumps({'value': value})
    kvp_db = KeyValuePairDB(name=name, value=value, scope=scope)
    kvp_db = KeyValuePair.add_or_update(kvp_db)

    return kvp_db

def deserialize_kvp_db(kvp_db):
    """
    Deserialize the datastore item value.

    :param kvp_db: KeyValuePairDB object.
    :type kvp_db: :class:`KeyValuePairDB`
    """
    value = kvp_db.value
    value = deserialize_key_value(value=value)
    return value

def deserialize_key_value(value):
    """
    Deserialize the datastore item value.

    Values are serialized as a JSON object where the actual value is stored under top-level key
    value.

    This introduces some space-related overhead, but it's transparent and preferred over custom
    serialization format.

    :param value: Value to deserialize.
    :type value: ``str``
    """
    try:
        value = json.loads(value)
    except Exception as e:
        # Value is not serialized correctly
        LOG.debug('Failed to de-serialize datastore: %s' % (str(e)),
                  exc_info=True)
        return None

    try:
        value = value['value']
    except KeyError:
        # Value is not serialized correctly
        LOG.debug('Datastore item is missing "value" attribute :%s' % (str(e)),
                  exc_info=True)
        return None

    return value
<commit_msg>Add support for config items being a secret.<commit_after># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Service functions for managing pack configuration inside the datastore.
"""

import json

from st2common import log as logging
from st2common.services import keyvalues as keyvalue_service
from st2common.constants.keyvalue import USER_SCOPE
from st2common.constants.keyvalue import SYSTEM_SCOPE
from st2common.models.api.keyvalue import KeyValuePairAPI
from st2common.persistence.keyvalue import KeyValuePair
from st2common.constants.keyvalue import DATASTORE_KEY_SEPARATOR

__all__ = [
    'get_datastore_key_prefix_for_pack',
    'get_datastore_key_name',

    'get_datastore_value_for_config_key',
    'set_datastore_value_for_config_key',

    'get_datastore_value'
]

LOG = logging.getLogger(__name__)

# Prefix for datastore items which store config values
# Full keys follow this format: pack_config.<pack name>.<config key name>
# For example: pack_config.aws.setup.region
DATASTORE_CONFIG_KEY_PREFIX = 'pack_config'

def get_datastore_key_prefix_for_pack(pack_name):
    prefix = [DATASTORE_CONFIG_KEY_PREFIX, pack_name]
    prefix = DATASTORE_KEY_SEPARATOR.join(prefix)
    return prefix

def get_datastore_key_name(pack_name, key_name, user=None):
    """
    Retrieve datastore key name based on the config key name.

    :param user: Optional username if working on a user-scoped config item.
    :type user: ``str``

    :rtype: ``str``
    """
    values = []

    prefix = get_datastore_key_prefix_for_pack(pack_name=pack_name)
    values.append(prefix)

    if user:
        values.append(user)

    values.append(key_name)

    return DATASTORE_KEY_SEPARATOR.join(values)

def get_datastore_value_for_config_key(pack_name, key_name):
    """
    Retrieve config value for the provided config key from the datastore.

    :param pack_name: Pack name.
    :type pack_name: ``str``

    :param key_name: Config key name.
    :type key_name: ``str``
    """
    name = get_datastore_key_name(pack_name=pack_name, key_name=key_name)
    kvp_db = get_datastore_value(key_name=name)
    return kvp_db

def get_datastore_value(key_name):
    """
    Retrieve config value with the matching datastore key name.

    :param key_name: Full key name in the datastore. Note: This must already be the fully qualified
                     name of the key in the datastore.
    :tupe key_name: ``str``
    """
    kvp_db = keyvalue_service.get_kvp_for_name(name=key_name)

    if not kvp_db:
        # Item doesn't exist
        return None

    if not kvp_db.value:
        # Item doesn't contain a value
        return None

    value = deserialize_key_value(value=kvp_db.value)
    return value

def set_datastore_value_for_config_key(pack_name, key_name, value, secret=False, user=None):
    """
    Set config value in the datastore.

    This function takes care of correctly encoding the key name, serializing the
    value, etc.

    :param pack_name: Pack name.
    :type pack_name: ``str``

    :param key_name: Config key name.
    :type key_name: ``str``

    :param secret: True if this value is a secret.
    :type secret: ``bool``

    :param user: Optional username if working on a user-scoped config item.
    :type user: ``str``

    :rtype: :class:`KeyValuePairDB`
    """
    # TODO: Once and if config schema is available, validate the provided type against the one
    # specified in the schema.
    name = get_datastore_key_name(pack_name=pack_name, key_name=key_name, user=user)

    if user:
        scope = USER_SCOPE
    else:
        scope = SYSTEM_SCOPE

    value = json.dumps({'value': value})
    kvp_api = KeyValuePairAPI(name=name, value=value, scope=scope, secret=secret)
    kvp_db = KeyValuePairAPI.to_model(kvp_api)
    kvp_db = KeyValuePair.add_or_update(kvp_db)

    return kvp_db

def deserialize_kvp_db(kvp_db):
    """
    Deserialize the datastore item value.

    :param kvp_db: KeyValuePairDB object.
    :type kvp_db: :class:`KeyValuePairDB`
    """
    value = kvp_db.value
    value = deserialize_key_value(value=value)
    return value

def deserialize_key_value(value):
    """
    Deserialize the datastore item value.

    Values are serialized as a JSON object where the actual value is stored under top-level key
    value.

    This introduces some space-related overhead, but it's transparent and preferred over custom
    serialization format.

    :param value: Value to deserialize.
    :type value: ``str``
    """
    try:
        value = json.loads(value)
    except Exception as e:
        # Value is not serialized correctly
        LOG.debug('Failed to de-serialize datastore: %s' % (str(e)),
                  exc_info=True)
        return None

    try:
        value = value['value']
    except KeyError:
        # Value is not serialized correctly
        LOG.debug('Datastore item is missing "value" attribute :%s' % (str(e)),
                  exc_info=True)
        return None

    return value
<|endoftext|>