<commit_before>import os
import tempfile

import gevent
import pytest
import requests

from volttron.platform.keystore import KeyStore
from volttron.platform.web import DiscoveryInfo
from volttrontesting.fixtures.vc_fixtures import PLATFORM_AGENT_CONFIG

def validate_instances(wrapper1, wrapper2):
    assert wrapper1.bind_web_address
    assert wrapper2.bind_web_address
    assert wrapper1.bind_web_address != wrapper2.bind_web_address
    assert wrapper1.volttron_home
    assert wrapper2.volttron_home
    assert wrapper1.volttron_home != wrapper2.volttron_home

@pytest.mark.vc
def test_platform_responds_to_discover_endpoint(pa_instance):
    """
    Successful when the response is json and has the same server key as the
    one in pa_instance.
    """
    wrapper = pa_instance['wrapper']
    info = DiscoveryInfo.request_discovery_info(wrapper.bind_web_address)

    assert wrapper.bind_web_address == info.discovery_address
    assert wrapper.publickey == info.serverkey

@pytest.mark.vc
def test_publickey_retrieval(vc_instance, pa_instance):
    """ This method tests that the /discovery addresses.

    The discovery now should return the server key for the bus as well as
    for the volttron.central and platform.agent public keys if they are
    available.

    Parameters
    ----------
    :vc_instance:
        A dictionary featuring a wrapper and ...
    :vc_instance:
        A dictionary featuring a wrapper and ...
    """
    vc_discovery = vc_instance['wrapper'].bind_web_address+"/discovery/"
    pa_discovery = pa_instance['wrapper'].bind_web_address+"/discovery/"
    response = requests.get(vc_discovery)
    assert response.json()['serverkey']
    assert response.json()['vcpublickey']
    assert response.json()['vcpublickey'] != response.json()['serverkey']

    response2 = requests.get(pa_discovery)
    assert response2.json()['serverkey']
    assert response2.json()['papublickey']
    assert response2.json()['papublickey'] != response.json()['serverkey']

    assert response2.json()['serverkey'] != response.json()['serverkey']
    assert response2.json()['papublickey'] != response.json()['vcpublickey']

    vc_wrapper = vc_instance['wrapper']
    vc_wrapper.install_agent(
        agent_dir='services/core/Platform',
        config_file=PLATFORM_AGENT_CONFIG)
    response3 = requests.get(vc_discovery)
    assert response3.json()['serverkey'] == response.json()['serverkey']
    assert response3.json()['vcpublickey'] == response.json()['vcpublickey']
    assert response3.json()['papublickey']
    assert response3.json()['vcpublickey'] != response2.json()['papublickey']

@pytest.mark.vc
def test_autoregistered_peer_platform(vc_instance):
    vc_wrapper = vc_instance['wrapper']

    caller_agent = vc_wrapper.build_agent(
        address=vc_wrapper.local_vip_address)
    assert caller_agent

    platforms = caller_agent.vip.rpc.call(
        'volttron.central', 'get_platforms').get(timeout=2)

    assert not platforms # no platforms should be registered.

    platform_uuid = vc_wrapper.install_agent(
        agent_dir="services/core/Platform", config_file=PLATFORM_AGENT_CONFIG)

    # must wait for the registration to be added.
    gevent.sleep(6)
    platforms = caller_agent.vip.rpc.call(
        'volttron.central', 'list_platforms').get(timeout=2)

    assert platforms
    p = platforms[0]
    print('A PLATFORM IS: {}'.format(p))
    assert p['is_local']
    assert p['tags']['available']
    assert p['tags']['created']
    assert p['vip_address'] == vc_wrapper.local_vip_address

@pytest.mark.vc
def test_vc_started(vc_instance):
    vc_wrapper = vc_instance['wrapper']
    vc_uuid = vc_instance['vc_uuid']

    assert vc_wrapper.is_agent_running(vc_uuid)
    tf = tempfile.NamedTemporaryFile()
    ks = KeyStore(tf.name)
    ks.generate()  #needed because using a temp file!!!!
    print('Checking peers on vc using:\nserverkey: {}\npublickey: {}\n'
          'secretkey: {}'.format(
        vc_wrapper.publickey,
        ks.public(),
        ks.secret()
    ))
    paagent = vc_wrapper.build_agent(serverkey=vc_wrapper.publickey,
                                     publickey=ks.public(),
                                     secretkey=ks.secret())
    peers = paagent.vip.peerlist().get(timeout=3)
    print(peers)
    assert "volttron.central" in peers
    paagent.core.stop()
    del paagent

def onmessage(self, peer, sender, bus, topic, headers, message):
    print('topic: {} message: {}'.format(topic, message))

@pytest.mark.vc
@pytest.mark.parametrize("display_name", [None, "happydays"])
def test_register_instance(vc_instance, pa_instance, display_name):
    vc_wrapper = vc_instance['wrapper']
    pa_wrapper = pa_instance['wrapper']

    validate_instances(vc_wrapper, pa_wrapper)

    print("connecting to vc instance with vip_adddress: {}".format(
        pa_wrapper.vip_address)
    )

    controlagent = vc_wrapper.build_agent()
    controlagent.vip.pubsub.subscribe('', onmessage)
    controlagent.vip.rpc.call(
        'volttron.central', pa_wrapper.bind_web_address).get(timeout=5)

    gevent.sleep(20)

    # res = requests.get(pa_wrapper.bind_web_address+"/discovery/")
    #
    # assert res.ok
    #
    # viptcp = "{vip_address}?serverkey={serverkey}&secretkey
    #
    # authfile = os.path.join(vc_wrapper.volttron_home, "auth.json")
    # with open(authfile) as f:
    #     print("vc authfile: {}".format(f.read()))
    #
    # tf = tempfile.NamedTemporaryFile()
    # paks = KeyStore(tf.name)
    # paks.generate()  #needed because using a temp file!!!!
    # print('Checking peers on pa using:\nserverkey: {}\npublickey: {}\n'
    #       'secretkey: {}'.format(
    #     pa_wrapper.publickey,
    #     paks.public(),
    #     paks.secret()
    # ))
    # paagent = pa_wrapper.build_agent(serverkey=pa_wrapper.publickey,
    #                                  publickey=paks.public(),
    #                                  secretkey=paks.secret())
    # peers = paagent.vip.peerlist().get(timeout=3)
    # assert "platform.agent" in peers
    # paagent.core.stop()
    # del paagent
    #
    # tf = tempfile.NamedTemporaryFile()
    # ks = KeyStore(tf.name)
    # ks.generate()  #needed because using a temp file!!!!!
    # print('Checking peers on vc using:\nserverkey: {}\npublickey: {}\n'
    #       'secretkey: {}'.format(
    #     vc_wrapper.publickey,
    #     ks.public(),
    #     ks.secret()
    # ))
    #
    # # Create an agent to use for calling rpc methods on volttron.central.
    # controlagent = vc_wrapper.build_agent(serverkey=vc_wrapper.publickey,
    #                                       publickey=ks.public(),
    #                                       secretkey=ks.secret())
    # plist = controlagent.vip.peerlist().get(timeout=2)
    # assert "volttron.central" in plist
    #
    # print('Attempting to manage platform now.')
    # print('display_name is now: ', display_name)
    # dct = dict(peer="volttron.central", method="register_instance",
    #                                     discovery_address=pa_wrapper.bind_web_address,
    #                                     display_name=display_name)
    # print(jsonapi.dumps(dct))
    # if display_name:
    #     retval = controlagent.vip.rpc.call("volttron.central", "register_instance",
    #                                     discovery_address=pa_wrapper.bind_web_address,
    #                                     display_name=display_name).get(timeout=10)
    # else:
    #
    #     retval = controlagent.vip.rpc.call("volttron.central", "register_instance",
    #                                     discovery_address=pa_wrapper.bind_web_address).get(timeout=10)
    #
    # assert retval
    # if display_name:
    #     assert display_name == retval['display_name']
    # else:
    #     assert pa_wrapper.bind_web_address == retval['display_name']
    # assert retval['success']
    #
    # print('Testing that we now have a single entry in the platform_details')
    # retval = controlagent.vip.rpc.call("volttron.central",
    #                                    "list_platform_details").get(timeout=10)
    # print("From vc list_platform_details: {}".format(retval))
    #
    # assert len(retval) == 1
    # assert 'hushpuppy' == retval[0]['display_name']
    # assert retval[0]['vip_address']
    # assert not retval[0]['tags']
    # assert retval[0]['serverkey']
    #
    # controlagent.core.stop()

    # # build agent to interact with the vc agent on the vc_wrapper instance.
    # #agent = vc_wrapper.build_agent(**params)
    # # serverkey=vc_wrapper.publickey,
    # #                                publickey=ks.public(),
    # #                                secretkey=ks.secret())
    # with open(authfile) as f:
    #     print("vc authfile: {}".format(f.read()))
    # peers = agent.vip.peerlist().get(timeout=2)
    # assert "volttron.central" in peers

<commit_msg>Fixed with the new tuple return.<commit_after>import os
import tempfile

import gevent
import pytest
import requests

from volttron.platform.keystore import KeyStore
from volttron.platform.web import DiscoveryInfo
from volttrontesting.fixtures.vc_fixtures import PLATFORM_AGENT_CONFIG

def validate_instances(wrapper1, wrapper2):
    assert wrapper1.bind_web_address
    assert wrapper2.bind_web_address
    assert wrapper1.bind_web_address != wrapper2.bind_web_address
    assert wrapper1.volttron_home
    assert wrapper2.volttron_home
    assert wrapper1.volttron_home != wrapper2.volttron_home

@pytest.mark.vc
def test_platform_responds_to_discover_endpoint(pa_instance):
    """
    Successful when the response is json and has the same server key as the
    one in pa_instance.
    """
    wrapper, pa_uuid = pa_instance
    info = DiscoveryInfo.request_discovery_info(wrapper.bind_web_address)

    assert wrapper.bind_web_address == info.discovery_address
    assert wrapper.publickey == info.serverkey

@pytest.mark.vc
def test_publickey_retrieval(vc_instance, pa_instance):
    """ This method tests that the /discovery addresses.

    The discovery now should return the server key for the bus as well as
    for the volttron.central and platform.agent public keys if they are
    available.

    Parameters
    ----------
    :vc_instance:
        A dictionary featuring a wrapper and ...
    :vc_instance:
        A dictionary featuring a wrapper and ...
    """
    vc_discovery = vc_instance['wrapper'].bind_web_address+"/discovery/"
    pa_discovery = pa_instance['wrapper'].bind_web_address+"/discovery/"
    response = requests.get(vc_discovery)
    assert response.json()['serverkey']
    assert response.json()['vcpublickey']
    assert response.json()['vcpublickey'] != response.json()['serverkey']

    response2 = requests.get(pa_discovery)
    assert response2.json()['serverkey']
    assert response2.json()['papublickey']
    assert response2.json()['papublickey'] != response.json()['serverkey']

    assert response2.json()['serverkey'] != response.json()['serverkey']
    assert response2.json()['papublickey'] != response.json()['vcpublickey']

    vc_wrapper = vc_instance['wrapper']
    vc_wrapper.install_agent(
        agent_dir='services/core/Platform',
        config_file=PLATFORM_AGENT_CONFIG)
    response3 = requests.get(vc_discovery)
    assert response3.json()['serverkey'] == response.json()['serverkey']
    assert response3.json()['vcpublickey'] == response.json()['vcpublickey']
    assert response3.json()['papublickey']
    assert response3.json()['vcpublickey'] != response2.json()['papublickey']

@pytest.mark.vc
def test_autoregistered_peer_platform(vc_instance):
    vc_wrapper, vc_uuid = vc_instance

    caller_agent = vc_wrapper.build_agent(
        address=vc_wrapper.local_vip_address)
    assert caller_agent

    platforms = caller_agent.vip.rpc.call(
        'volttron.central', 'get_platforms').get(timeout=2)

    assert not platforms # no platforms should be registered.

    platform_uuid = vc_wrapper.install_agent(
        agent_dir="services/core/Platform", config_file=PLATFORM_AGENT_CONFIG)

    # must wait for the registration to be added.
    gevent.sleep(6)
    platforms = caller_agent.vip.rpc.call(
        'volttron.central', 'list_platforms').get(timeout=2)

    assert platforms
    p = platforms[0]
    print('A PLATFORM IS: {}'.format(p))
    assert p['is_local']
    assert p['tags']['available']
    assert p['tags']['created']
    assert p['vip_address'] == vc_wrapper.local_vip_address

@pytest.mark.vc
def test_vc_started(vc_instance):
    vc_wrapper, vc_uuid = vc_instance

    assert vc_wrapper.is_agent_running(vc_uuid)
    tf = tempfile.NamedTemporaryFile()
    ks = KeyStore(tf.name)
    ks.generate()  #needed because using a temp file!!!!
    print('Checking peers on vc using:\nserverkey: {}\npublickey: {}\n'
          'secretkey: {}'.format(
        vc_wrapper.publickey,
        ks.public(),
        ks.secret()
    ))
    paagent = vc_wrapper.build_agent(serverkey=vc_wrapper.publickey,
                                     publickey=ks.public(),
                                     secretkey=ks.secret())
    peers = paagent.vip.peerlist().get(timeout=3)
    print(peers)
    assert "volttron.central" in peers
    paagent.core.stop()
    del paagent

def onmessage(self, peer, sender, bus, topic, headers, message):
    print('topic: {} message: {}'.format(topic, message))

# @pytest.mark.vc
# @pytest.mark.parametrize("display_name", [None, "happydays"])
# def test_register_instance(vc_instance, pa_instance, display_name):
#     vc_wrapper = vc_instance['wrapper']
#     pa_wrapper = pa_instance['wrapper']
#
#     validate_instances(vc_wrapper, pa_wrapper)
#
#     print("connecting to vc instance with vip_adddress: {}".format(
#         pa_wrapper.vip_address)
#     )
#
#     controlagent = vc_wrapper.build_agent()
#     controlagent.vip.pubsub.subscribe('', onmessage)
#     controlagent.vip.rpc.call(
#         'volttron.central', pa_wrapper.bind_web_address).get(timeout=5)
#
#     gevent.sleep(20)

    # res = requests.get(pa_wrapper.bind_web_address+"/discovery/")
    #
    # assert res.ok
    #
    # viptcp = "{vip_address}?serverkey={serverkey}&secretkey
    #
    # authfile = os.path.join(vc_wrapper.volttron_home, "auth.json")
    # with open(authfile) as f:
    #     print("vc authfile: {}".format(f.read()))
    #
    # tf = tempfile.NamedTemporaryFile()
    # paks = KeyStore(tf.name)
    # paks.generate()  #needed because using a temp file!!!!
    # print('Checking peers on pa using:\nserverkey: {}\npublickey: {}\n'
    #       'secretkey: {}'.format(
    #     pa_wrapper.publickey,
    #     paks.public(),
    #     paks.secret()
    # ))
    # paagent = pa_wrapper.build_agent(serverkey=pa_wrapper.publickey,
    #                                  publickey=paks.public(),
    #                                  secretkey=paks.secret())
    # peers = paagent.vip.peerlist().get(timeout=3)
    # assert "platform.agent" in peers
    # paagent.core.stop()
    # del paagent
    #
    # tf = tempfile.NamedTemporaryFile()
    # ks = KeyStore(tf.name)
    # ks.generate()  #needed because using a temp file!!!!!
    # print('Checking peers on vc using:\nserverkey: {}\npublickey: {}\n'
    #       'secretkey: {}'.format(
    #     vc_wrapper.publickey,
    #     ks.public(),
    #     ks.secret()
    # ))
    #
    # # Create an agent to use for calling rpc methods on volttron.central.
    # controlagent = vc_wrapper.build_agent(serverkey=vc_wrapper.publickey,
    #                                       publickey=ks.public(),
    #                                       secretkey=ks.secret())
    # plist = controlagent.vip.peerlist().get(timeout=2)
    # assert "volttron.central" in plist
    #
    # print('Attempting to manage platform now.')
    # print('display_name is now: ', display_name)
    # dct = dict(peer="volttron.central", method="register_instance",
    #                                     discovery_address=pa_wrapper.bind_web_address,
    #                                     display_name=display_name)
    # print(jsonapi.dumps(dct))
    # if display_name:
    #     retval = controlagent.vip.rpc.call("volttron.central", "register_instance",
    #                                     discovery_address=pa_wrapper.bind_web_address,
    #                                     display_name=display_name).get(timeout=10)
    # else:
    #
    #     retval = controlagent.vip.rpc.call("volttron.central", "register_instance",
    #                                     discovery_address=pa_wrapper.bind_web_address).get(timeout=10)
    #
    # assert retval
    # if display_name:
    #     assert display_name == retval['display_name']
    # else:
    #     assert pa_wrapper.bind_web_address == retval['display_name']
    # assert retval['success']
    #
    # print('Testing that we now have a single entry in the platform_details')
    # retval = controlagent.vip.rpc.call("volttron.central",
    #                                    "list_platform_details").get(timeout=10)
    # print("From vc list_platform_details: {}".format(retval))
    #
    # assert len(retval) == 1
    # assert 'hushpuppy' == retval[0]['display_name']
    # assert retval[0]['vip_address']
    # assert not retval[0]['tags']
    # assert retval[0]['serverkey']
    #
    # controlagent.core.stop()

    # # build agent to interact with the vc agent on the vc_wrapper instance.
    # #agent = vc_wrapper.build_agent(**params)
    # # serverkey=vc_wrapper.publickey,
    # #                                publickey=ks.public(),
    # #                                secretkey=ks.secret())
    # with open(authfile) as f:
    #     print("vc authfile: {}".format(f.read()))
    # peers = agent.vip.peerlist().get(timeout=2)
    # assert "volttron.central" in peers

<|endoftext|>


<commit_before>__version__ = "0.20.0"
<commit_msg>Bump version for 0.20.1 release<commit_after>__version__ = "0.20.1"
<|endoftext|>


<commit_before>import requests
from bs4 import BeautifulSoup as bs
import xlrd
from format import search_form

class Crawler:
    search_url = 'http://sugang.snu.ac.kr/sugang/cc/cc100.action'
    excel_url = 'http://sugang.snu.ac.kr/sugang/cc/cc100excel.action'
    semester_name = {
        'U000200001U000300001': '1',
        'U000200001U000300002': 'S',
        'U000200002U000300001': '2',
        'U000200002U000300002': 'W'
    }

    def parse_option_tag(self, op):
            return {'code': op['value'], 'name': op.text.strip()}

    def get_areas(self, year, semester):
        form = search_form(year, semester)
        search_page = requests.post(self.search_url, data=form)
        soup = bs(search_page.text, 'html5lib')
        select = soup.find('select', {'name': 'srchOpenUpSbjtFldCd'})
        options = select.find_all('option')
        areas = list(map(self.parse_option_tag, options))
        return list(filter(lambda s: s['code'] != '', areas))

    def get_subareas(self, year, semester, area):
        form = search_form(year, semester, area['code'])
        search_page = requests.post(self.search_url, data=form)
        soup = bs(search_page.text, 'html5lib')
        select = soup.find('select', {'name': 'srchOpenSbjtFldCd'})
        options = select.find_all('option')
        subareas = list(map(self.parse_option_tag, options))
        return list(filter(lambda s: s['code'] != '', subareas))

    def get_courses(self, year, semester, area, subarea):
        form = search_form(year, semester, area['code'], subarea['code'])
        excel = requests.post(self.excel_url, form)
        try:
            workbook = xlrd.open_workbook(file_contents=excel.content)
        except Exception as e:
            print(e)
            return []
        sheet = workbook.sheet_by_index(0)
        res = []
        for row in range(3, sheet.nrows):
            course = {
                'year': year,
                'semester': self.semester_name[semester],
                'code': sheet.cell_value(row, 5),
                'number': sheet.cell_value(row, 6),
                'title': sheet.cell_value(row, 7),
                'credit': sheet.cell_value(row, 9),
                'category': sheet.cell_value(row, 0),
                'language': sheet.cell_value(row, 19),
                'area': area['name'],
                'subarea': subarea['name'],
                'collage': sheet.cell_value(row, 1),
                'dept': sheet.cell_value(row, 2)
            }
            res.append(course)
        return res

    def crawl_courses(self, start, end=None):
        if end is None:
            end = start + 1
        courses = []
        for year in range(start, end):
            for semester in list(self.semester_name.keys()):
                areas = self.get_areas(year, semester)
                for area in areas:
                    subareas = self.get_subareas(year, semester, area)
                    for subarea in subareas:
                        cs = self.get_courses(year, semester, area, subarea)
                        courses.extend(cs)
        return courses
<commit_msg>Handle error with empty search result<commit_after>import requests
from bs4 import BeautifulSoup as bs
import xlrd
from format import search_form

class Crawler:
    search_url = 'http://sugang.snu.ac.kr/sugang/cc/cc100.action'
    excel_url = 'http://sugang.snu.ac.kr/sugang/cc/cc100excel.action'
    semester_name = {
        'U000200001U000300001': '1',
        'U000200001U000300002': 'S',
        'U000200002U000300001': '2',
        'U000200002U000300002': 'W'
    }

    def parse_option_tag(self, op):
            return {'code': op['value'], 'name': op.text.strip()}

    def get_areas(self, year, semester):
        form = search_form(year, semester)
        search_page = requests.post(self.search_url, data=form)
        soup = bs(search_page.text, 'html5lib')
        select = soup.find('select', {'name': 'srchOpenUpSbjtFldCd'})
        options = select.find_all('option')
        areas = list(map(self.parse_option_tag, options))
        return list(filter(lambda s: s['code'] != '', areas))

    def get_subareas(self, year, semester, area):
        form = search_form(year, semester, area['code'])
        search_page = requests.post(self.search_url, data=form)
        soup = bs(search_page.text, 'html5lib')
        select = soup.find('select', {'name': 'srchOpenSbjtFldCd'})
        options = select.find_all('option')
        subareas = list(map(self.parse_option_tag, options))
        return list(filter(lambda s: s['code'] != '', subareas))

    def get_courses(self, year, semester, area, subarea):
        form = search_form(year, semester, area['code'], subarea['code'])
        excel = requests.post(self.excel_url, form)
        if excel.content == b'':
            return []
        workbook = xlrd.open_workbook(file_contents=excel.content)
        sheet = workbook.sheet_by_index(0)
        res = []
        for row in range(3, sheet.nrows):
            course = {
                'year': year,
                'semester': self.semester_name[semester],
                'code': sheet.cell_value(row, 5),
                'number': sheet.cell_value(row, 6),
                'title': sheet.cell_value(row, 7),
                'credit': sheet.cell_value(row, 9),
                'category': sheet.cell_value(row, 0),
                'language': sheet.cell_value(row, 19),
                'area': area['name'],
                'subarea': subarea['name'],
                'collage': sheet.cell_value(row, 1),
                'dept': sheet.cell_value(row, 2)
            }
            res.append(course)
        return res

    def crawl_courses(self, start, end=None):
        if end is None:
            end = start + 1
        courses = []
        for year in range(start, end):
            for semester in list(self.semester_name.keys()):
                areas = self.get_areas(year, semester)
                for area in areas:
                    subareas = self.get_subareas(year, semester, area)
                    for subarea in subareas:
                        cs = self.get_courses(year, semester, area, subarea)
                        courses.extend(cs)
        return courses
<|endoftext|>


<commit_before>import logging
from followthemoney import model
from followthemoney.util import merge_data

from aleph.core import es, db, celery
from aleph.model import Collection, Entity
from aleph.index import index_entity
from aleph.index.core import entities_index
from aleph.index.entities import index_bulk
from aleph.index.collections import index_collection
from aleph.index.util import authz_query, field_filter_query
from aleph.util import dict_list

log = logging.getLogger(__name__)
BULK_PAGE = 500

def update_entity(entity):
    return index_entity(entity)

def delete_entity(entity, deleted_at=None):
    entity.delete(deleted_at=deleted_at)

@celery.task(priority=1)
def process_entities(collection_id=None):
    # re-process entities
    q = db.session.query(Entity)
    if collection_id is not None:
        q = q.filter(Entity.collection_id == collection_id)
    for entity in q:
        index_entity(entity)

def bulk_load(config):
    """Bulk load entities from a CSV file or SQL database.

    This is done by mapping the rows in the source data to entities and links
    which can be understood by the entity index.
    """
    from aleph.logic.collections import create_collection
    for foreign_id, data in config.items():
        data['foreign_id'] = foreign_id
        data['label'] = data.get('label', foreign_id)
        collection = create_collection(data)
        for query in dict_list(data, 'queries', 'query'):
            bulk_load_query.apply_async([collection.id, query], priority=6)

@celery.task()
def bulk_load_query(collection_id, query):
    collection = Collection.by_id(collection_id)
    if collection is None:
        log.warning("Collection does not exist: %s", collection_id)
        return

    mapping = model.make_mapping(query, key_prefix=collection.foreign_id)
    records_total = len(mapping.source) or 'streaming'
    entities = {}
    entities_count = 0
    for records_index, record in enumerate(mapping.source.records, 1):
        for entity in mapping.map(record).values():
            entity_id = entity.get('id')
            if entity_id is None:
                continue
            # When loading from a tabular data source, we will often
            # encounter mappings where the same entity is emitted
            # multiple times in short sequence, e.g. when the data
            # describes all the directors of a single company.
            base = entities.get(entity_id, {})
            entities[entity_id] = merge_data(entity, base)
            entities_count += 1

        if records_index > 0 and records_index % 1000 == 0:
            log.info("[%s] Loaded %s records (%s), %s entities...",
                     collection.foreign_id,
                     records_index,
                     records_total,
                     entities_count)

        if len(entities) >= BULK_PAGE:
            index_bulk(collection, entities, chunk_size=BULK_PAGE)
            entities = {}

    if len(entities):
        index_bulk(collection, entities, chunk_size=BULK_PAGE)

    # Update collection stats
    index_collection(collection)

def entity_references(entity, authz):
    """Given a particular entity, find all the references to it from other
    entities, grouped by the property where they are used."""
    schema = model[entity.get('schema')]

    # Generate all the possible mention locations.
    properties = []
    queries = []
    for prop in model.properties:
        if not prop.is_entity:
            continue
        if not schema.is_a(prop.range):
            continue

        field = 'properties.%s' % prop.name
        queries.append({})
        queries.append({
            'size': 0,
            'query': {
                'bool': {
                    'filter': [
                        authz_query(authz),
                        {'term': {'schemata': prop.schema.name}},
                        {'term': {field: entity.get('id')}},
                    ]
                }
            }
        })
        properties.append(prop)

    if not len(queries):
        return

    # Run a count search (with schema facet?)
    res = es.msearch(index=entities_index(), body=queries)
    for prop, resp in zip(properties, res.get('responses', [])):
        total = resp.get('hits', {}).get('total')
        if total is not None and total > 0:
            yield (prop, total)

def entity_tags(entity, authz):
    """Do a search on tags of an entity."""
    # NOTE: This must also work for documents.
    FIELDS = [
        'names',
        'emails',
        'phones',
        'addresses',
        'identifiers'
    ]
    pivots = []
    queries = []
    # Go through all the tags which apply to this entity, and find how
    # often they've been mentioned in other entities.
    for field in FIELDS:
        for value in entity.get(field, []):
            if value is None or not len(value):
                continue
            queries.append({})
            queries.append({
                'size': 0,
                'query': {
                    'bool': {
                        'filter': [
                            authz_query(authz),
                            field_filter_query(field, value)
                        ],
                        'must_not': [
                            {'ids': {'values': [entity.get('id')]}},
                        ]
                    }
                }
            })
            pivots.append((field, value))

    if not len(queries):
        return

    res = es.msearch(index=entities_index(), body=queries)
    for (field, value), resp in zip(pivots, res.get('responses', [])):
        total = resp.get('hits', {}).get('total')
        if total is not None and total > 0:
            yield (field, value, total)
<commit_msg>wontfix for now.<commit_after>import logging
from followthemoney import model
from followthemoney.util import merge_data

from aleph.core import es, db, celery
from aleph.model import Collection, Entity
from aleph.index import index_entity
from aleph.index.core import entities_index
from aleph.index.entities import index_bulk
from aleph.index.collections import index_collection
from aleph.index.util import authz_query, field_filter_query
from aleph.util import dict_list

log = logging.getLogger(__name__)
BULK_PAGE = 500

def update_entity(entity):
    return index_entity(entity)

def delete_entity(entity, deleted_at=None):
    entity.delete(deleted_at=deleted_at)

@celery.task(priority=1)
def process_entities(collection_id=None):
    # re-process entities
    q = db.session.query(Entity)
    if collection_id is not None:
        q = q.filter(Entity.collection_id == collection_id)
    for entity in q:
        index_entity(entity)

def bulk_load(config):
    """Bulk load entities from a CSV file or SQL database.

    This is done by mapping the rows in the source data to entities and links
    which can be understood by the entity index.
    """
    from aleph.logic.collections import create_collection
    for foreign_id, data in config.items():
        data['foreign_id'] = foreign_id
        data['label'] = data.get('label', foreign_id)
        collection = create_collection(data)
        # FIXME: this does not perform collection metadata validation.
        for query in dict_list(data, 'queries', 'query'):
            bulk_load_query.apply_async([collection.id, query], priority=6)

@celery.task()
def bulk_load_query(collection_id, query):
    collection = Collection.by_id(collection_id)
    if collection is None:
        log.warning("Collection does not exist: %s", collection_id)
        return

    mapping = model.make_mapping(query, key_prefix=collection.foreign_id)
    records_total = len(mapping.source) or 'streaming'
    entities = {}
    entities_count = 0
    for records_index, record in enumerate(mapping.source.records, 1):
        for entity in mapping.map(record).values():
            entity_id = entity.get('id')
            if entity_id is None:
                continue
            # When loading from a tabular data source, we will often
            # encounter mappings where the same entity is emitted
            # multiple times in short sequence, e.g. when the data
            # describes all the directors of a single company.
            base = entities.get(entity_id, {})
            entities[entity_id] = merge_data(entity, base)
            entities_count += 1

        if records_index > 0 and records_index % 1000 == 0:
            log.info("[%s] Loaded %s records (%s), %s entities...",
                     collection.foreign_id,
                     records_index,
                     records_total,
                     entities_count)

        if len(entities) >= BULK_PAGE:
            index_bulk(collection, entities, chunk_size=BULK_PAGE)
            entities = {}

    if len(entities):
        index_bulk(collection, entities, chunk_size=BULK_PAGE)

    # Update collection stats
    index_collection(collection)

def entity_references(entity, authz):
    """Given a particular entity, find all the references to it from other
    entities, grouped by the property where they are used."""
    schema = model[entity.get('schema')]

    # Generate all the possible mention locations.
    properties = []
    queries = []
    for prop in model.properties:
        if not prop.is_entity:
            continue
        if not schema.is_a(prop.range):
            continue

        field = 'properties.%s' % prop.name
        queries.append({})
        queries.append({
            'size': 0,
            'query': {
                'bool': {
                    'filter': [
                        authz_query(authz),
                        {'term': {'schemata': prop.schema.name}},
                        {'term': {field: entity.get('id')}},
                    ]
                }
            }
        })
        properties.append(prop)

    if not len(queries):
        return

    # Run a count search (with schema facet?)
    res = es.msearch(index=entities_index(), body=queries)
    for prop, resp in zip(properties, res.get('responses', [])):
        total = resp.get('hits', {}).get('total')
        if total is not None and total > 0:
            yield (prop, total)

def entity_tags(entity, authz):
    """Do a search on tags of an entity."""
    # NOTE: This must also work for documents.
    FIELDS = [
        'names',
        'emails',
        'phones',
        'addresses',
        'identifiers'
    ]
    pivots = []
    queries = []
    # Go through all the tags which apply to this entity, and find how
    # often they've been mentioned in other entities.
    for field in FIELDS:
        for value in entity.get(field, []):
            if value is None or not len(value):
                continue
            queries.append({})
            queries.append({
                'size': 0,
                'query': {
                    'bool': {
                        'filter': [
                            authz_query(authz),
                            field_filter_query(field, value)
                        ],
                        'must_not': [
                            {'ids': {'values': [entity.get('id')]}},
                        ]
                    }
                }
            })
            pivots.append((field, value))

    if not len(queries):
        return

    res = es.msearch(index=entities_index(), body=queries)
    for (field, value), resp in zip(pivots, res.get('responses', [])):
        total = resp.get('hits', {}).get('total')
        if total is not None and total > 0:
            yield (field, value, total)
<|endoftext|>


<commit_before>import logging

logger = logging.getLogger('LanguageClient')
fileHandler = logging.FileHandler(filename='/tmp/LanguageClient.log')
fileHandler.setFormatter(
        logging.Formatter(
            '%(asctime)s %(levelname)-8s (%(name)s) %(message)s'))
logger.addHandler(fileHandler)
logger.setLevel(logging.ERROR)
<commit_msg>Shorten logging entry.<commit_after>import logging

logger = logging.getLogger("LanguageClient")
fileHandler = logging.FileHandler(filename="/tmp/LanguageClient.log")
fileHandler.setFormatter(
        logging.Formatter(
            "%(asctime)s %(levelname)-8s %(message)s",
            "%H:%M:%S"))
logger.addHandler(fileHandler)
logger.setLevel(logging.ERROR)
<|endoftext|>


<commit_before>"""jobs.py

Jobs for the Harvest System.
"""

import sys
import os
import requests
import urllib
import datetime
from dateutil.parser import parse
import RDF

from d1lod.sesame import Store, Repository
from glharvest import registry, void, util

# RQ
from redis import StrictRedis
from rq import Queue

conn = StrictRedis(host='redis', port='6379')
q = Queue(connection=conn)

SESAME_HOST = os.getenv('WEB_1_PORT_8080_TCP_ADDR', 'localhost')
SESAME_PORT = os.getenv('WEB_1_PORT_8080_TCP_PORT', '8080')
SESAME_REPOSITORY = 'glharvest'

def update():
    """Update the contents of the Harvester.

    Reads the registry file and does work depending on what it finds.
    """

    print "job.update()"
    print "Parsing registry file."

    # Establish the location of the registry file
    # The else case will run when not running in a container
    if os.path.isfile('/glharvest/registry.yml'):
        registry_filepath = '/glharvest/registry.yml'
    else:
        registry_filepath = 'registry.yml'

    if not os.path.isfile:
        print "Couldn't locate the registry file at the provided path: %s. Exiting." % registry_filepath
        return

    registry_file = registry.parse_registry_file(registry_filepath)

    if registry_file is None:
        print "Failed to parse registry file. Exiting."
        return

    print "Registry file parsed. %d provider(s) found." % len(registry_file)

    for provider in registry_file:
        # Validate and extract information for each provider
        print "Validating provider: %s." % provider

        # Check for VoID
        if 'void' not in registry_file[provider]:
            print "Location of VoID dataset not found for this provider in the registry. Skipping."
            continue

        voidfile = registry_file[provider]['void']

        # Check for modified value
        if 'modified' not in registry_file[provider]:
            print "Location of VoID dataset not found for this provider in the registry."
            registry_modified = None
        else:
            registry_modified = registry_file[provider]['modified']
            print registry_modified
            registry_modified = parse(str(registry_modified), ignoretz=True)

            print registry_modified
        print "Registry has a last modified value of %s." % registry_modified

        # Get and parse the VoID file
        print "Attemting to retrieve VoID file from location %s." % voidfile

        try:
            r = requests.get(voidfile)
        except:
            print "Failed to get voidfile located at `%s`." % voidfile
            continue

        void_string_format = "rdfxml"

        # Use another format is we detect a different one
        if voidfile.endswith('ttl'):
            void_string_format = 'turtle'

        print "Loading VoID file"
        model = util.load_string_into_model(r.text, fmt=void_string_format)
        void_model = void.parse_void_model(model)

        print void_model

        for provider_dataset in void_model:
            print "Processing provider VoID:Dataset %s." % provider_dataset

            if 'modified' not in void_model[provider_dataset]:
                print "VoID:Dataset found in VoID dump did not have dcterms:modified value. Skipping."
                continue

            if 'features' not in void_model[provider_dataset]:
                print "VoID:Dataset has no feature declaration. Skipping."
                continue

            modified = void_model[provider_dataset]['modified']

            try:
                modified = parse(modified, ignoretz=True)
            except:
                print "Failed to parse modified time string of %s. Skipping." % modified
                continue

            print "After parsing, `modified` value is %s." % modified

            # TODO process features
            features = void_model[provider_dataset]['features']
            print "Found features: %s" % features

            print "Updating dataset if %s is greater than %s." % (modified, registry_modified)

            if registry_modified is None or modified > registry_modified:
                print "Updating dataset %s." % provider_dataset

                # Set up repository connection
                s = Store(SESAME_HOST, SESAME_PORT)
                r = Repository(s, SESAME_REPOSITORY)

                data_dumps = void_model[provider_dataset]['dumps']

                for dump in data_dumps:
                    print "Processing %s %s %s." % (provider, provider_dataset, dump)

                    # Create a temporary name for the file
                    outfilename = datetime.datetime.now().strftime("%s-%f")

                    try:
                        # resp = requests.get(dump)
                        urllib.urlretrieve(dump, outfilename)
                    except:
                        print "Failed to fetch the void file at `%s`. Skipping." % dump
                        continue

                    # Decide the format (from looking at the URL string)
                    dump_file_format = "rdfxml"

                    # Use another format is we detect a different one
                    if dump.endswith('ttl'):
                        dump_file_format = 'turtle'

                    # parser = RDF.Parser(name=dump_file_format)

                    # Delete triples about each subject (streaming)
                    # for statement in parser.parse_as_stream('file:' + outfilename):
                    #     # Don't delete statements about non_URI subjects because
                    #     # we can't
                    #     if not statement.subject.is_resource():
                    #         continue
                    #
                    #     print "Deleting triples in context %s about %s." % (provider, str(statement.subject))
                    #     r.delete_triples_about(statement.subject, context=provider)

                    # Just delete all triples in the context
                    print "Deleting triples in context %s." % provider
                    r.delete_triples_about('?s', context=provider)

                    # Import the file
                    print "Importing temp file '%s' into named graph '%s'." % (outfilename, provider)
                    r.import_from_file(outfilename, context=provider, fmt=dump_file_format)

                    # Delete the temp file
                    os.remove(outfilename)

                # Update registry file on disk
                print "Updating registry with new modified datetime of %s." % modified
                registry_file[provider]['modified'] = modified
                registry.save_registry("registry.yml", registry_file)
            else:
                print "datadump has not been updated since the last visit. Doing nothing."

def export():
    """Export the repository as Turtle."""

    s = Store(SESAME_HOST, SESAME_PORT)
    r = Repository(s, SESAME_REPOSITORY)

    with open('/www/export.ttl', 'wb') as f:
        exported_text = r.export()
        f.write(exported_text.encode('utf-8'))

def status():
    """Gets the status of the system and prints to stdout."""
    print "Getting the status of the system."

    s = Store(SESAME_HOST, SESAME_PORT)
    r = Repository(s, SESAME_REPOSITORY)

    print "Repository size is %d" % r.size()
<commit_msg>Fix filepath for saving registry file so it actually saves<commit_after>"""jobs.py

Jobs for the Harvest System.
"""

import sys
import os
import requests
import urllib
import datetime
from dateutil.parser import parse
import RDF

from d1lod.sesame import Store, Repository
from glharvest import registry, void, util

# RQ
from redis import StrictRedis
from rq import Queue

conn = StrictRedis(host='redis', port='6379')
q = Queue(connection=conn)

SESAME_HOST = os.getenv('WEB_1_PORT_8080_TCP_ADDR', 'localhost')
SESAME_PORT = os.getenv('WEB_1_PORT_8080_TCP_PORT', '8080')
SESAME_REPOSITORY = 'glharvest'

def update():
    """Update the contents of the Harvester.

    Reads the registry file and does work depending on what it finds.
    """

    print "job.update()"
    print "Parsing registry file."

    # Establish the location of the registry file
    # The else case will run when not running in a container
    if os.path.isfile('/glharvest/registry.yml'):
        registry_filepath = '/glharvest/registry.yml'
    else:
        registry_filepath = 'registry.yml'

    if not os.path.isfile:
        print "Couldn't locate the registry file at the provided path: %s. Exiting." % registry_filepath
        return

    registry_file = registry.parse_registry_file(registry_filepath)

    if registry_file is None:
        print "Failed to parse registry file. Exiting."
        return

    print "Registry file parsed. %d provider(s) found." % len(registry_file)

    for provider in registry_file:
        # Validate and extract information for each provider
        print "Validating provider: %s." % provider

        # Check for VoID
        if 'void' not in registry_file[provider]:
            print "Location of VoID dataset not found for this provider in the registry. Skipping."
            continue

        voidfile = registry_file[provider]['void']

        # Check for modified value
        if 'modified' not in registry_file[provider]:
            print "Location of VoID dataset not found for this provider in the registry."
            registry_modified = None
        else:
            registry_modified = registry_file[provider]['modified']
            print registry_modified
            registry_modified = parse(str(registry_modified), ignoretz=True)

            print registry_modified
        print "Registry has a last modified value of %s." % registry_modified

        # Get and parse the VoID file
        print "Attemting to retrieve VoID file from location %s." % voidfile

        try:
            r = requests.get(voidfile)
        except:
            print "Failed to get voidfile located at `%s`." % voidfile
            continue

        void_string_format = "rdfxml"

        # Use another format is we detect a different one
        if voidfile.endswith('ttl'):
            void_string_format = 'turtle'

        print "Loading VoID file"
        model = util.load_string_into_model(r.text, fmt=void_string_format)
        void_model = void.parse_void_model(model)

        print void_model

        for provider_dataset in void_model:
            print "Processing provider VoID:Dataset %s." % provider_dataset

            if 'modified' not in void_model[provider_dataset]:
                print "VoID:Dataset found in VoID dump did not have dcterms:modified value. Skipping."
                continue

            if 'features' not in void_model[provider_dataset]:
                print "VoID:Dataset has no feature declaration. Skipping."
                continue

            modified = void_model[provider_dataset]['modified']

            try:
                modified = parse(modified, ignoretz=True)
            except:
                print "Failed to parse modified time string of %s. Skipping." % modified
                continue

            print "After parsing, `modified` value is %s." % modified

            # TODO process features
            features = void_model[provider_dataset]['features']
            print "Found features: %s" % features

            print "Updating dataset if %s is greater than %s." % (modified, registry_modified)

            if registry_modified is None or modified > registry_modified:
                print "Updating dataset %s." % provider_dataset

                # Set up repository connection
                s = Store(SESAME_HOST, SESAME_PORT)
                r = Repository(s, SESAME_REPOSITORY)

                data_dumps = void_model[provider_dataset]['dumps']

                for dump in data_dumps:
                    print "Processing %s %s %s." % (provider, provider_dataset, dump)

                    # Create a temporary name for the file
                    outfilename = datetime.datetime.now().strftime("%s-%f")

                    try:
                        # resp = requests.get(dump)
                        urllib.urlretrieve(dump, outfilename)
                    except:
                        print "Failed to fetch the void file at `%s`. Skipping." % dump
                        continue

                    # Decide the format (from looking at the URL string)
                    dump_file_format = "rdfxml"

                    # Use another format is we detect a different one
                    if dump.endswith('ttl'):
                        dump_file_format = 'turtle'

                    # parser = RDF.Parser(name=dump_file_format)

                    # Delete triples about each subject (streaming)
                    # for statement in parser.parse_as_stream('file:' + outfilename):
                    #     # Don't delete statements about non_URI subjects because
                    #     # we can't
                    #     if not statement.subject.is_resource():
                    #         continue
                    #
                    #     print "Deleting triples in context %s about %s." % (provider, str(statement.subject))
                    #     r.delete_triples_about(statement.subject, context=provider)

                    # Just delete all triples in the context
                    print "Deleting triples in context %s." % provider
                    r.delete_triples_about('?s', context=provider)

                    # Import the file
                    print "Importing temp file '%s' into named graph '%s'." % (outfilename, provider)
                    r.import_from_file(outfilename, context=provider, fmt=dump_file_format)

                    # Delete the temp file
                    os.remove(outfilename)

                # Update registry file on disk
                print "Updating registry with new modified datetime of %s." % modified
                registry_file[provider]['modified'] = modified
                registry.save_registry("/glharvest/registry.yml", registry_file)
            else:
                print "datadump has not been updated since the last visit. Doing nothing."

def export():
    """Export the repository as Turtle."""

    s = Store(SESAME_HOST, SESAME_PORT)
    r = Repository(s, SESAME_REPOSITORY)

    with open('/www/export.ttl', 'wb') as f:
        exported_text = r.export()
        f.write(exported_text.encode('utf-8'))

def status():
    """Gets the status of the system and prints to stdout."""
    print "Getting the status of the system."

    s = Store(SESAME_HOST, SESAME_PORT)
    r = Repository(s, SESAME_REPOSITORY)

    print "Repository size is %d" % r.size()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from baseframe import _, __
from baseframe.forms.sqlalchemy import QuerySelectField
import baseframe.forms as forms

class NewProfileForm(forms.Form):
    """Create a new profile."""

    profile = forms.RadioField(
        __("Organization"),
        validators=[forms.validators.DataRequired("Select an organization")],
        description=__(
            u"Select the organization you’d like to create a Talkfunnel for"
        ),
    )

class EditProfileForm(forms.Form):
    """Edit a profile."""

    description = forms.MarkdownField(
        __("Welcome message"),
        validators=[
            forms.validators.DataRequired(
                _(u"Please write a message for the landing page")
            )
        ],
        description=__("This welcome message will be shown on the landing page."),
    )
    logo_url = forms.URLField(
        __("Logo URL"),
        description=__("Profile logo"),
        validators=[
            forms.validators.Optional(),
            forms.validators.ValidUrl(),
            forms.validators.Length(max=2000),
        ],
    )
    admin_team = QuerySelectField(
        u"Admin Team",
        validators=[forms.validators.DataRequired(_(u"Please select a team"))],
        get_label='title',
        allow_blank=True,
        blank_text=__("Choose a team…"),
        description=__(
            "The team of users with administrative rights to this Profile (owners always have admin access)"
        ),
    )

    def set_queries(self):
        self.admin_team.query = self.edit_obj.teams
<commit_msg>fixed unicode error<commit_after># -*- coding: utf-8 -*-

from baseframe import _, __
from baseframe.forms.sqlalchemy import QuerySelectField
import baseframe.forms as forms

class NewProfileForm(forms.Form):
    """Create a new profile."""

    profile = forms.RadioField(
        __("Organization"),
        validators=[forms.validators.DataRequired("Select an organization")],
        description=__(
            u"Select the organization you’d like to create a Talkfunnel for"
        ),
    )

class EditProfileForm(forms.Form):
    """Edit a profile."""

    description = forms.MarkdownField(
        __("Welcome message"),
        validators=[
            forms.validators.DataRequired(
                _(u"Please write a message for the landing page")
            )
        ],
        description=__("This welcome message will be shown on the landing page."),
    )
    logo_url = forms.URLField(
        __("Logo URL"),
        description=__("Profile logo"),
        validators=[
            forms.validators.Optional(),
            forms.validators.ValidUrl(),
            forms.validators.Length(max=2000),
        ],
    )
    admin_team = QuerySelectField(
        u"Admin Team",
        validators=[forms.validators.DataRequired(_(u"Please select a team"))],
        get_label='title',
        allow_blank=True,
        blank_text=__(u"Choose a team…"),
        description=__(
            u"The team of users with administrative rights to this Profile (owners always have admin access)"
        ),
    )

    def set_queries(self):
        self.admin_team.query = self.edit_obj.teams
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations
from django import VERSION as DJANGO_VERSION

def load_initial_data(apps, schema_editor):
    from django.apps import apps
    Page = apps.get_model("wagtailcore", "Page")
    ContentType = apps.get_model('contenttypes.ContentType')

    # Get blogpage content type
    blogpage_content_type, created = ContentType.objects.get_or_create(
        model='blogpage',
        app_label='puput',
        defaults={'name': 'page'} if DJANGO_VERSION < (1, 8) else {}
    )

    # Edit home page
    homepage = Page.objects.get(slug='home')
    homepage.title = "Puput blog manager"
    homepage.save()

    # Create example blog page
    blogpage = Page(
        title="Blog",
        content_type=blogpage_content_type,
        slug='blog',
    )

    # Add blog page as a child for homepage
    homepage.add_child(instance=blogpage)
    revision = blogpage.save_revision()
    revision.publish()

class Migration(migrations.Migration):

    dependencies = [
        ('puput', '0001_initial'),
    ]

    operations = [
        migrations.RunPython(load_initial_data),
    ]
<commit_msg>Modified initial_data<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations
from django import VERSION as DJANGO_VERSION

def load_initial_data(apps, schema_editor):
    from django.apps import apps
    Page = apps.get_model("wagtailcore", "Page")
    Site = apps.get_model("wagtailcore", "Site")
    ContentType = apps.get_model('contenttypes.ContentType')

    # Get blogpage content type
    blogpage_content_type, created = ContentType.objects.get_or_create(
        model='blogpage',
        app_label='puput',
        defaults={'name': 'page'} if DJANGO_VERSION < (1, 8) else {}
    )

    # Get root page
    rootpage = Page.objects.first()

    # Set site root page as root site page
    site = Site.objects.first()
    site.root_page = rootpage
    site.save()

    # Create example blog page
    blogpage = Page(
        title="Blog",
        content_type=blogpage_content_type,
        slug='blog',
    )

    # Add blog page as a child for homepage
    rootpage.add_child(instance=blogpage)
    revision = blogpage.save_revision()
    revision.publish()

class Migration(migrations.Migration):

    dependencies = [
        ('puput', '0001_initial'),
    ]

    operations = [
        migrations.RunPython(load_initial_data),
    ]
<|endoftext|>


<commit_before>import sys
import json
import six.moves.urllib as urllib
import requests
import datetime

# Override this directly to set the token
access_token = None

# Override this for other environments
flatfox_server = 'https://flatfox.ch'

class ApiError(Exception):

    def __init__(self, message, json_response=None, status_code=None):
        super(ApiError, self).__init__(message)

        self.message = message
        self.json_response = json_response
        self.status_code = status_code

    def __str__(self):
        return "{0}: {1} {2}".format(
            type(self).__name__,
            self.message,
            "({0})".format(self.status_code) if self.status_code else "")

class InvalidRequestError(ApiError):
    pass

class PermissionError(ApiError):
    pass

class ApiRequestor(object):
    def __init__(self, access_token=None):
        self._token = access_token
        self._api_base = flatfox_server + '/api/v1/'

    def request(self, method, url, data=None):

        # Construct full url
        full_url = self._api_base + url
        if not full_url[-1] == '/':
            full_url += '/'

        # Construct authentication
        if self._token:
            sk = self._token
        else:
            sk = access_token
        auth = (sk, '')

        # Serialize and prepare data and files.
        if data:
            serialized_data, files = serialize_object(data)
        else:
            serialized_data, files = None, None

        # sys.stdout.write("Flatfox Api: {0} {1}\n".format(
        #     method.upper(), full_url))

        if not files:
            res = self._raw_json_request(method=method, full_url=full_url,
                                         auth=auth, data=serialized_data)
        else:
            res = self._raw_multipart_request(method=method, full_url=full_url,
                                              auth=auth, data=serialized_data,
                                              files=files)

        # Handle response
        return self.interpret_response(response=res)

    def _raw_json_request(self, method, full_url, auth, data):
        request_func = getattr(requests, method.lower())
        return request_func(url=full_url, auth=auth, data=json.dumps(data),
                            headers={'Content-Type': 'application/json',
                                     'Accept': 'application/json'})

    def _raw_multipart_request(self, method, full_url, auth, data, files):
        request_func = getattr(requests, method.lower())
        return request_func(url=full_url, auth=auth, data=data, files=files)

    def interpret_response(self, response):
        try:
            json_resp = response.json()
        except Exception:
            raise ApiError(
                "Invalid response: %{0} (HTTP response code was {1})".format(
                    response.content, response.status_code))

        if not (200 <= response.status_code < 300):
            self.handle_api_error(response.status_code, json_resp)

        return json_resp

    def handle_api_error(self, status_code, json_resp):
        if status_code == 400:
            raise InvalidRequestError(str(json_resp),
                                      json_resp,
                                      status_code)
        if status_code == 404:
            raise InvalidRequestError(json_resp.get('detail'),
                                      json_resp,
                                      status_code)
        elif status_code == 403:
            raise PermissionError(json_resp.get('detail'),
                                  json_resp,
                                  status_code)
        else:
            raise ApiError(str(json_resp), json_resp, status_code)

class FlatfoxObject(dict):
    def __init__(self, id=None, key=None, **kwargs):
        super(FlatfoxObject, self).__init__(**kwargs)

        object.__setattr__(self, 'access_token', key if key else access_token)

        if id:
            self['id'] = id

    def __setattr__(self, key, value):
        """ Override for dot notation """
        # Special handling of special functions and non-value items
        if key[0] == '_' or key in self.__dict__:
            return super(FlatfoxObject, self).__setattr__(key, value)
        else:
            self[key] = value

    def __getattr__(self, key):
        """ Override for dot notation """
        # Special handling of special attrs
        if key[0] == '_':
            raise AttributeError(key)

        try:
            return self[key]
        except KeyError as err:
            raise AttributeError(*err.args)

    def refresh_from_data(self, values, access_token=None):
        self.access_token = access_token or getattr(values, 'access_token', None)

        self.clear()
        for key, value in values.items():
            super(FlatfoxObject, self).__setitem__(key, value)

    def request(self, method, url, data=None):
        requestor = ApiRequestor(access_token=self.access_token)
        response = requestor.request(method=method, url=url, data=data)
        return response

    @classmethod
    def init_from_response(cls, values, key):
        instance = cls(id=values.get('id'), key=key)
        instance.refresh_from_data(values, access_token=key)
        return instance

    @property
    def flatfox_id(self):
        return self.id

def serialize_object(obj):
    params = {}
    files = {}

    for k, v in obj.items():
        if k == 'id' or k.startswith('_'):
            continue
        elif hasattr(v, 'serialize'):
            params[k] = v.serialize()
        elif hasattr(v, 'read'):  # Check for files
            files[k] = v
        elif isinstance(v, (datetime.datetime, datetime.date,)):
            params[k] = v.isoformat()
        else:
            params[k] = v

    return params, files

def deserialize_object(klass, response, access_token):
    if isinstance(response, list):
        return [
            deserialize_object(klass, item, access_token)
            for item in response]

    elif isinstance(response, dict) and not isinstance(response, FlatfoxObject):
        return klass.init_from_response(response, key=access_token)
    else:
        return response

class ApiResource(FlatfoxObject):

    @classmethod
    def retrieve(cls, access_token=None, **params):
        instance = cls(access_token=access_token, **params)
        instance.refresh()
        return instance

    @classmethod
    def exists(cls, **params):
        try:
            instance = cls(**params)
            instance.refresh()
            return True
        except InvalidRequestError:
            return False

    def refresh(self):
        self.refresh_from_data(self.request('get', self.instance_url()))
        return self

    @classmethod
    def class_name(cls):
        if cls == ApiResource:
            raise NotImplementedError

        return str(urllib.parse.quote_plus(cls.__name__.lower()))

    @classmethod
    def class_url(cls):
        return "{0}".format(cls.class_name().lower())

    def instance_url(self):
        id = self.get('id')
        external_id = self.get('external_id')
        if not id and not external_id:
            raise InvalidRequestError(
                'Cannot get instance url of object {0}: no id is set.'.format(
                    type(self).__name__))
        base = self.class_url()
        id_part = ApiResource.format_id(id=id, external_id=external_id)
        return "{base}/{id}".format(base=base, id=id_part)

    @staticmethod
    def format_id(id=None, external_id=None):
        if id:
            res = str(id)
        else:
            res = u'ext-{0}'.format(external_id)
        return urllib.parse.quote_plus(res)

class ListableApiResource(ApiResource):
    @classmethod
    def list(cls, access_token=None, **params):
        requestor = ApiRequestor(access_token=access_token)
        url = cls.class_url()
        response = requestor.request('get', url, params)
        return deserialize_object(
            klass=cls,
            response=response,
            access_token=access_token)

class CreateableApiResource(ApiResource):
    @classmethod
    def create(cls, access_token=None, **params):
        requestor = ApiRequestor(access_token=access_token)
        url = cls.class_url()
        response = requestor.request(method='post', url=url, data=params)
        return cls(access_token=access_token, **response)

class UpdateableApiResource(ApiResource):

    def save(self):
        # params, files = serialize_object(self)
        res = self.request(method='put', url=self.instance_url(), data=self)
        self.refresh_from_data(res)
        return self

class Flat(ListableApiResource, UpdateableApiResource, CreateableApiResource):

    @classmethod
    def class_url(cls):
        return "my-flat"

    @classmethod
    def init_from_response(cls, values, key):
        images = values.pop('images', [])
        flat = super(Flat, cls).init_from_response(values, key)
        flat.images = deserialize_object(FlatImage, images, key)
        return flat

class FlatImage(CreateableApiResource):

    @classmethod
    def class_url(cls):
        return "flat-image"
<commit_msg>api change, related to flatfox-ag/flatfox-website@c65be78888665f0d2439016d0672d328841fa5b5<commit_after>import sys
import json
import six.moves.urllib as urllib
import requests
import datetime

# Override this directly to set the token
access_token = None

# Override this for other environments
flatfox_server = 'https://flatfox.ch'

class ApiError(Exception):

    def __init__(self, message, json_response=None, status_code=None):
        super(ApiError, self).__init__(message)

        self.message = message
        self.json_response = json_response
        self.status_code = status_code

    def __str__(self):
        return "{0}: {1} {2}".format(
            type(self).__name__,
            self.message,
            "({0})".format(self.status_code) if self.status_code else "")

class InvalidRequestError(ApiError):
    pass

class PermissionError(ApiError):
    pass

class ApiRequestor(object):
    def __init__(self, access_token=None):
        self._token = access_token
        self._api_base = flatfox_server + '/api/v1/'

    def request(self, method, url, data=None):

        # Construct full url
        full_url = self._api_base + url
        if not full_url[-1] == '/':
            full_url += '/'

        # Construct authentication
        if self._token:
            sk = self._token
        else:
            sk = access_token
        auth = (sk, '')

        # Serialize and prepare data and files.
        if data:
            serialized_data, files = serialize_object(data)
        else:
            serialized_data, files = None, None

        # sys.stdout.write("Flatfox Api: {0} {1}\n".format(
        #     method.upper(), full_url))

        if not files:
            res = self._raw_json_request(method=method, full_url=full_url,
                                         auth=auth, data=serialized_data)
        else:
            res = self._raw_multipart_request(method=method, full_url=full_url,
                                              auth=auth, data=serialized_data,
                                              files=files)

        # Handle response
        return self.interpret_response(response=res)

    def _raw_json_request(self, method, full_url, auth, data):
        request_func = getattr(requests, method.lower())
        return request_func(url=full_url, auth=auth, data=json.dumps(data),
                            headers={'Content-Type': 'application/json',
                                     'Accept': 'application/json'})

    def _raw_multipart_request(self, method, full_url, auth, data, files):
        request_func = getattr(requests, method.lower())
        return request_func(url=full_url, auth=auth, data=data, files=files)

    def interpret_response(self, response):
        try:
            json_resp = response.json()
        except Exception:
            raise ApiError(
                "Invalid response: %{0} (HTTP response code was {1})".format(
                    response.content, response.status_code))

        if not (200 <= response.status_code < 300):
            self.handle_api_error(response.status_code, json_resp)

        return json_resp

    def handle_api_error(self, status_code, json_resp):
        if status_code == 400:
            raise InvalidRequestError(str(json_resp),
                                      json_resp,
                                      status_code)
        if status_code == 404:
            raise InvalidRequestError(json_resp.get('detail'),
                                      json_resp,
                                      status_code)
        elif status_code == 403:
            raise PermissionError(json_resp.get('detail'),
                                  json_resp,
                                  status_code)
        else:
            raise ApiError(str(json_resp), json_resp, status_code)

class FlatfoxObject(dict):
    def __init__(self, id=None, key=None, **kwargs):
        super(FlatfoxObject, self).__init__(**kwargs)

        object.__setattr__(self, 'access_token', key if key else access_token)

        if id:
            self['id'] = id

    def __setattr__(self, key, value):
        """ Override for dot notation """
        # Special handling of special functions and non-value items
        if key[0] == '_' or key in self.__dict__:
            return super(FlatfoxObject, self).__setattr__(key, value)
        else:
            self[key] = value

    def __getattr__(self, key):
        """ Override for dot notation """
        # Special handling of special attrs
        if key[0] == '_':
            raise AttributeError(key)

        try:
            return self[key]
        except KeyError as err:
            raise AttributeError(*err.args)

    def refresh_from_data(self, values, access_token=None):
        self.access_token = access_token or getattr(values, 'access_token', None)

        self.clear()
        for key, value in values.items():
            super(FlatfoxObject, self).__setitem__(key, value)

    def request(self, method, url, data=None):
        requestor = ApiRequestor(access_token=self.access_token)
        response = requestor.request(method=method, url=url, data=data)
        return response

    @classmethod
    def init_from_response(cls, values, key):
        instance = cls(id=values.get('id'), key=key)
        instance.refresh_from_data(values, access_token=key)
        return instance

    @property
    def flatfox_id(self):
        return self.id

def serialize_object(obj):
    params = {}
    files = {}

    for k, v in obj.items():
        if k == 'id' or k.startswith('_'):
            continue
        elif hasattr(v, 'serialize'):
            params[k] = v.serialize()
        elif hasattr(v, 'read'):  # Check for files
            files[k] = v
        elif isinstance(v, (datetime.datetime, datetime.date,)):
            params[k] = v.isoformat()
        else:
            params[k] = v

    return params, files

def deserialize_object(klass, response, access_token):
    if isinstance(response, list):
        return [
            deserialize_object(klass, item, access_token)
            for item in response]

    elif isinstance(response, dict) and not isinstance(response, FlatfoxObject):
        return klass.init_from_response(response, key=access_token)
    else:
        return response

class ApiResource(FlatfoxObject):

    @classmethod
    def retrieve(cls, access_token=None, **params):
        instance = cls(access_token=access_token, **params)
        instance.refresh()
        return instance

    @classmethod
    def exists(cls, **params):
        try:
            instance = cls(**params)
            instance.refresh()
            return True
        except InvalidRequestError:
            return False

    def refresh(self):
        self.refresh_from_data(self.request('get', self.instance_url()))
        return self

    @classmethod
    def class_name(cls):
        if cls == ApiResource:
            raise NotImplementedError

        return str(urllib.parse.quote_plus(cls.__name__.lower()))

    @classmethod
    def class_url(cls):
        return "{0}".format(cls.class_name().lower())

    def instance_url(self):
        id = self.get('id')
        external_id = self.get('external_id')
        if not id and not external_id:
            raise InvalidRequestError(
                'Cannot get instance url of object {0}: no id is set.'.format(
                    type(self).__name__))
        base = self.class_url()
        id_part = ApiResource.format_id(id=id, external_id=external_id)
        return "{base}/{id}".format(base=base, id=id_part)

    @staticmethod
    def format_id(id=None, external_id=None):
        if id:
            res = str(id)
        else:
            res = u'ext-{0}'.format(external_id)
        return urllib.parse.quote_plus(res)

class ListableApiResource(ApiResource):
    @classmethod
    def list(cls, access_token=None, **params):
        requestor = ApiRequestor(access_token=access_token)
        url = cls.class_url()
        response = requestor.request('get', url, params)
        return deserialize_object(
            klass=cls,
            response=response,
            access_token=access_token)

class CreateableApiResource(ApiResource):
    @classmethod
    def create(cls, access_token=None, **params):
        requestor = ApiRequestor(access_token=access_token)
        url = cls.class_url()
        response = requestor.request(method='post', url=url, data=params)
        return cls(access_token=access_token, **response)

class UpdateableApiResource(ApiResource):

    def save(self):
        # params, files = serialize_object(self)
        res = self.request(method='put', url=self.instance_url(), data=self)
        self.refresh_from_data(res)
        return self

class Flat(ListableApiResource, UpdateableApiResource, CreateableApiResource):

    @classmethod
    def class_url(cls):
        return "my-flat"

    @classmethod
    def init_from_response(cls, values, key):
        images = values.pop('images', [])
        flat = super(Flat, cls).init_from_response(values, key)
        flat.images = deserialize_object(FlatImage, images, key)
        return flat

class FlatImage(CreateableApiResource):

    @classmethod
    def class_url(cls):
        return "upload/flat_image"
<|endoftext|>


<commit_before>"""
Django-AlphaFilter provides an admin widget for alphabetical filtering that
works like date_hierarchy and an template tag for use elsewhere.
"""

__version_info__ = {
    'major': 0,
    'minor': 5,
    'micro': 3,
    'releaselevel': 'final',
    'serial': 0
}

def get_version():
    """
    Return the formatted version information
    """
    vers = ["%(major)i.%(minor)i" % __version_info__, ]

    if __version_info__['micro']:
        vers.append(".%(micro)i" % __version_info__)
    if __version_info__['releaselevel'] != 'final':
        vers.append('%(releaselevel)s%(serial)i' % __version_info__)
    return ''.join(vers)

__version__ = get_version()
<commit_msg>Version bump to 0.5.4<commit_after>"""
Django-AlphaFilter provides an admin widget for alphabetical filtering that
works like date_hierarchy and an template tag for use elsewhere.
"""

__version_info__ = {
    'major': 0,
    'minor': 5,
    'micro': 4,
    'releaselevel': 'final',
    'serial': 0
}

def get_version():
    """
    Return the formatted version information
    """
    vers = ["%(major)i.%(minor)i" % __version_info__, ]

    if __version_info__['micro']:
        vers.append(".%(micro)i" % __version_info__)
    if __version_info__['releaselevel'] != 'final':
        vers.append('%(releaselevel)s%(serial)i' % __version_info__)
    return ''.join(vers)

__version__ = get_version()
<|endoftext|>


<commit_before>import json
import logging
import os
import pkgutil
from collections import deque
from functools import wraps

from flask import Flask, request
from flask.ext.cors import CORS
from flask_compress import Compress
from flask_restplus import Api as RestPlusAPI
from flask_restplus.model import Model
from flask_restplus.resource import Resource
from jsonschema.exceptions import RefResolutionError

from flexget import manager
from flexget.config_schema import process_config, register_config_key
from flexget.event import event
from flexget.utils.database import with_session
from flexget.webserver import User
from flexget.webserver import register_app, get_secret

__version__ = '0.4.0-beta'

log = logging.getLogger('api')

api_config = {}
api_config_schema = {
    'type': 'boolean',
    'additionalProperties': False
}

@with_session
def api_key(session=None):
    log.debug('fetching token for internal lookup')
    return session.query(User).first().token

@event('config.register')
def register_config():
    register_config_key('api', api_config_schema)

class ApiSchemaModel(Model, object):
    """A flask restplus :class:`flask_restplus.models.ApiModel` which can take a json schema directly."""

    def __init__(self, name, schema, *args, **kwargs):
        super(ApiSchemaModel, self).__init__(name, *args, **kwargs)
        self._schema = schema

    @property
    def __schema__(self):
        if self.__parent__:
            return {
                'allOf': [
                    {'$ref': '#/definitions/{0}'.format(self.__parent__.name)},
                    self._schema
                ]
            }
        else:
            return self._schema

    def __nonzero__(self):
        return bool(self._schema)

    def __repr__(self):
        return '<ApiSchemaModel(%r)>' % self._schema

class Api(RestPlusAPI, object):
    """
    Extends a flask restplus :class:`flask_restplus.Api` with:
      - methods to make using json schemas easier
      - methods to auto document and handle :class:`ApiError` responses
    """

    def _rewrite_refs(self, schema):
        if isinstance(schema, list):
            for value in schema:
                self._rewrite_refs(value)

        if isinstance(schema, dict):
            for key, value in schema.iteritems():
                if isinstance(value, (list, dict)):
                    self._rewrite_refs(value)

                if key == '$ref' and value.startswith('/'):
                    schema[key] = '#definitions%s' % value

    def schema(self, name, schema, **kwargs):
        """
        Register a json schema.

        Usable like :meth:`flask_restplus.Api.model`, except takes a json schema as its argument.

        :returns: An :class:`ApiSchemaModel` instance registered to this api.
        """
        model = ApiSchemaModel(name, schema, **kwargs)
        model.__apidoc__.update(kwargs)
        self.models[name] = model
        return model

    def inherit(self, name, parent, fields):
        """
        Extends :meth:`flask_restplus.Api.inherit` to allow `fields` to be a json schema, if `parent` is a
        :class:`ApiSchemaModel`.
        """
        if isinstance(parent, ApiSchemaModel):
            model = ApiSchemaModel(name, fields)
            model.__apidoc__['name'] = name
            model.__parent__ = parent
            self.models[name] = model
            return model
        return super(Api, self).inherit(name, parent, fields)

    def validate(self, model, schema_override=None, description=None):
        """
        When a method is decorated with this, json data submitted to the endpoint will be validated with the given
        `model`. This also auto-documents the expected model, as well as the possible :class:`ValidationError` response.
        """

        def decorator(func):
            @api.expect((model, description))
            @api.response(ValidationError)
            @wraps(func)
            def wrapper(*args, **kwargs):
                payload = request.json
                try:
                    schema = schema_override if schema_override else model.__schema__
                    errors = process_config(config=payload, schema=schema, set_defaults=False)

                    if errors:
                        raise ValidationError(errors)
                except RefResolutionError as e:
                    raise ApiError(str(e))
                return func(*args, **kwargs)

            return wrapper

        return decorator

    def response(self, code_or_apierror, description=None, model=None, **kwargs):
        """
        Extends :meth:`flask_restplus.Api.response` to allow passing an :class:`ApiError` class instead of
        response code. If an `ApiError` is used, the response code, and expected response model, is automatically
        documented.
        """
        try:
            if issubclass(code_or_apierror, ApiError):
                description = description or code_or_apierror.description
                return self.doc(responses={code_or_apierror.code: (description, code_or_apierror.response_model)})
        except TypeError:
            # If first argument isn't a class this happens
            pass
        return super(Api, self).response(code_or_apierror, description, model=model)

class APIResource(Resource, object):
    """All api resources should subclass this class."""
    method_decorators = [with_session]

    def __init__(self, api, *args, **kwargs):
        self.manager = manager.manager
        super(APIResource, self).__init__(api, *args, **kwargs)

app = Flask(__name__, template_folder=os.path.join(os.path.dirname(__path__[0]), 'templates'))
app.config['REMEMBER_COOKIE_NAME'] = 'flexgetToken'
app.config['DEBUG'] = True

CORS(app)
Compress(app)

api = Api(
    app,
    catch_all_404s=True,
    title='API',
    version=__version__,
    description='Warning: under development, subject to change without notice.'
)

class ApiError(Exception):
    code = 500
    description = 'server error'

    response_model = api.schema('error', {
        'type': 'object',
        'properties': {
            'code': {'type': 'integer'},
            'message': {'type': 'string'}
        },
        'required': ['code', 'error']
    })

    def __init__(self, message, payload=None):
        self.message = message
        self.payload = payload

    def to_dict(self):
        rv = self.payload or {}
        rv.update(code=self.code, message=self.message)
        return rv

    @classmethod
    def schema(cls):
        return cls.response_model.__schema__

class NotFoundError(ApiError):
    code = 404
    description = 'not found'

class ValidationError(ApiError):
    code = 400
    description = 'validation error'

    response_model = api.inherit('validation_error', ApiError.response_model, {
        'type': 'object',
        'properties': {
            'validation_errors': {
                'type': 'array',
                'items': {
                    'type': 'object',
                    'properties': {
                        'message': {'type': 'string', 'description': 'A human readable message explaining the error.'},
                        'validator': {'type': 'string', 'description': 'The name of the failed validator.'},
                        'validator_value': {
                            'type': 'string', 'description': 'The value for the failed validator in the schema.'
                        },
                        'path': {'type': 'string'},
                        'schema_path': {'type': 'string'},
                    }
                }
            }
        },
        'required': ['validation_errors']
    })

    verror_attrs = (
        'message', 'cause', 'validator', 'validator_value',
        'path', 'schema_path', 'parent'
    )

    def __init__(self, validation_errors, message='validation error'):
        payload = {'validation_errors': [self._verror_to_dict(error) for error in validation_errors]}
        super(ValidationError, self).__init__(message, payload=payload)

    def _verror_to_dict(self, error):
        error_dict = {}
        for attr in self.verror_attrs:
            if isinstance(getattr(error, attr), deque):
                error_dict[attr] = list(getattr(error, attr))
            else:
                error_dict[attr] = getattr(error, attr)
        return error_dict

@api.errorhandler(ApiError)
@api.errorhandler(NotFoundError)
@api.errorhandler(ValidationError)
def api_errors(error):
    return error.to_dict(), error.code

@event('manager.daemon.started')
def register_api(mgr):
    global api_config
    api_config = mgr.config.get('api')

    app.secret_key = get_secret()

    if api_config:
        register_app('/api', app)

class ApiClient(object):
    """
    This is an client which can be used as a more pythonic interface to the rest api.

    It skips http, and is only usable from within the running flexget process.
    """

    def __init__(self):
        self.app = app.test_client()

    def __getattr__(self, item):
        return ApiEndopint('/api/' + item, self.get_endpoint)

    def get_endpoint(self, url, data=None, method=None):
        if method is None:
            method = 'POST' if data is not None else 'GET'
        auth_header = dict(Authorization='Token %s' % api_key())
        response = self.app.open(url, data=data, follow_redirects=True, method=method, headers=auth_header)
        result = json.loads(response.data)
        # TODO: Proper exceptions
        if 200 > response.status_code >= 300:
            raise Exception(result['error'])
        return result

class ApiEndopint(object):
    def __init__(self, endpoint, caller):
        self.endpoint = endpoint
        self.caller = caller

    def __getattr__(self, item):
        return self.__class__(self.endpoint + '/' + item, self.caller)

    __getitem__ = __getattr__

    def __call__(self, data=None, method=None):
        return self.caller(self.endpoint, data=data, method=method)

# Import API Sub Modules
for loader, module_name, is_pkg in pkgutil.walk_packages(__path__):
    loader.find_module(module_name).load_module(module_name)
<commit_msg>Changed api ver to 0.5.0<commit_after>import json
import logging
import os
import pkgutil
from collections import deque
from functools import wraps

from flask import Flask, request
from flask.ext.cors import CORS
from flask_compress import Compress
from flask_restplus import Api as RestPlusAPI
from flask_restplus.model import Model
from flask_restplus.resource import Resource
from jsonschema.exceptions import RefResolutionError

from flexget import manager
from flexget.config_schema import process_config, register_config_key
from flexget.event import event
from flexget.utils.database import with_session
from flexget.webserver import User
from flexget.webserver import register_app, get_secret

__version__ = '0.5.0-beta'

log = logging.getLogger('api')

api_config = {}
api_config_schema = {
    'type': 'boolean',
    'additionalProperties': False
}

@with_session
def api_key(session=None):
    log.debug('fetching token for internal lookup')
    return session.query(User).first().token

@event('config.register')
def register_config():
    register_config_key('api', api_config_schema)

class ApiSchemaModel(Model, object):
    """A flask restplus :class:`flask_restplus.models.ApiModel` which can take a json schema directly."""

    def __init__(self, name, schema, *args, **kwargs):
        super(ApiSchemaModel, self).__init__(name, *args, **kwargs)
        self._schema = schema

    @property
    def __schema__(self):
        if self.__parent__:
            return {
                'allOf': [
                    {'$ref': '#/definitions/{0}'.format(self.__parent__.name)},
                    self._schema
                ]
            }
        else:
            return self._schema

    def __nonzero__(self):
        return bool(self._schema)

    def __repr__(self):
        return '<ApiSchemaModel(%r)>' % self._schema

class Api(RestPlusAPI, object):
    """
    Extends a flask restplus :class:`flask_restplus.Api` with:
      - methods to make using json schemas easier
      - methods to auto document and handle :class:`ApiError` responses
    """

    def _rewrite_refs(self, schema):
        if isinstance(schema, list):
            for value in schema:
                self._rewrite_refs(value)

        if isinstance(schema, dict):
            for key, value in schema.iteritems():
                if isinstance(value, (list, dict)):
                    self._rewrite_refs(value)

                if key == '$ref' and value.startswith('/'):
                    schema[key] = '#definitions%s' % value

    def schema(self, name, schema, **kwargs):
        """
        Register a json schema.

        Usable like :meth:`flask_restplus.Api.model`, except takes a json schema as its argument.

        :returns: An :class:`ApiSchemaModel` instance registered to this api.
        """
        model = ApiSchemaModel(name, schema, **kwargs)
        model.__apidoc__.update(kwargs)
        self.models[name] = model
        return model

    def inherit(self, name, parent, fields):
        """
        Extends :meth:`flask_restplus.Api.inherit` to allow `fields` to be a json schema, if `parent` is a
        :class:`ApiSchemaModel`.
        """
        if isinstance(parent, ApiSchemaModel):
            model = ApiSchemaModel(name, fields)
            model.__apidoc__['name'] = name
            model.__parent__ = parent
            self.models[name] = model
            return model
        return super(Api, self).inherit(name, parent, fields)

    def validate(self, model, schema_override=None, description=None):
        """
        When a method is decorated with this, json data submitted to the endpoint will be validated with the given
        `model`. This also auto-documents the expected model, as well as the possible :class:`ValidationError` response.
        """

        def decorator(func):
            @api.expect((model, description))
            @api.response(ValidationError)
            @wraps(func)
            def wrapper(*args, **kwargs):
                payload = request.json
                try:
                    schema = schema_override if schema_override else model.__schema__
                    errors = process_config(config=payload, schema=schema, set_defaults=False)

                    if errors:
                        raise ValidationError(errors)
                except RefResolutionError as e:
                    raise ApiError(str(e))
                return func(*args, **kwargs)

            return wrapper

        return decorator

    def response(self, code_or_apierror, description=None, model=None, **kwargs):
        """
        Extends :meth:`flask_restplus.Api.response` to allow passing an :class:`ApiError` class instead of
        response code. If an `ApiError` is used, the response code, and expected response model, is automatically
        documented.
        """
        try:
            if issubclass(code_or_apierror, ApiError):
                description = description or code_or_apierror.description
                return self.doc(responses={code_or_apierror.code: (description, code_or_apierror.response_model)})
        except TypeError:
            # If first argument isn't a class this happens
            pass
        return super(Api, self).response(code_or_apierror, description, model=model)

class APIResource(Resource, object):
    """All api resources should subclass this class."""
    method_decorators = [with_session]

    def __init__(self, api, *args, **kwargs):
        self.manager = manager.manager
        super(APIResource, self).__init__(api, *args, **kwargs)

app = Flask(__name__, template_folder=os.path.join(os.path.dirname(__path__[0]), 'templates'))
app.config['REMEMBER_COOKIE_NAME'] = 'flexgetToken'
app.config['DEBUG'] = True

CORS(app)
Compress(app)

api = Api(
    app,
    catch_all_404s=True,
    title='API',
    version=__version__,
    description='Warning: under development, subject to change without notice.'
)

class ApiError(Exception):
    code = 500
    description = 'server error'

    response_model = api.schema('error', {
        'type': 'object',
        'properties': {
            'code': {'type': 'integer'},
            'message': {'type': 'string'}
        },
        'required': ['code', 'error']
    })

    def __init__(self, message, payload=None):
        self.message = message
        self.payload = payload

    def to_dict(self):
        rv = self.payload or {}
        rv.update(code=self.code, message=self.message)
        return rv

    @classmethod
    def schema(cls):
        return cls.response_model.__schema__

class NotFoundError(ApiError):
    code = 404
    description = 'not found'

class ValidationError(ApiError):
    code = 400
    description = 'validation error'

    response_model = api.inherit('validation_error', ApiError.response_model, {
        'type': 'object',
        'properties': {
            'validation_errors': {
                'type': 'array',
                'items': {
                    'type': 'object',
                    'properties': {
                        'message': {'type': 'string', 'description': 'A human readable message explaining the error.'},
                        'validator': {'type': 'string', 'description': 'The name of the failed validator.'},
                        'validator_value': {
                            'type': 'string', 'description': 'The value for the failed validator in the schema.'
                        },
                        'path': {'type': 'string'},
                        'schema_path': {'type': 'string'},
                    }
                }
            }
        },
        'required': ['validation_errors']
    })

    verror_attrs = (
        'message', 'cause', 'validator', 'validator_value',
        'path', 'schema_path', 'parent'
    )

    def __init__(self, validation_errors, message='validation error'):
        payload = {'validation_errors': [self._verror_to_dict(error) for error in validation_errors]}
        super(ValidationError, self).__init__(message, payload=payload)

    def _verror_to_dict(self, error):
        error_dict = {}
        for attr in self.verror_attrs:
            if isinstance(getattr(error, attr), deque):
                error_dict[attr] = list(getattr(error, attr))
            else:
                error_dict[attr] = getattr(error, attr)
        return error_dict

@api.errorhandler(ApiError)
@api.errorhandler(NotFoundError)
@api.errorhandler(ValidationError)
def api_errors(error):
    return error.to_dict(), error.code

@event('manager.daemon.started')
def register_api(mgr):
    global api_config
    api_config = mgr.config.get('api')

    app.secret_key = get_secret()

    if api_config:
        register_app('/api', app)

class ApiClient(object):
    """
    This is an client which can be used as a more pythonic interface to the rest api.

    It skips http, and is only usable from within the running flexget process.
    """

    def __init__(self):
        self.app = app.test_client()

    def __getattr__(self, item):
        return ApiEndopint('/api/' + item, self.get_endpoint)

    def get_endpoint(self, url, data=None, method=None):
        if method is None:
            method = 'POST' if data is not None else 'GET'
        auth_header = dict(Authorization='Token %s' % api_key())
        response = self.app.open(url, data=data, follow_redirects=True, method=method, headers=auth_header)
        result = json.loads(response.data)
        # TODO: Proper exceptions
        if 200 > response.status_code >= 300:
            raise Exception(result['error'])
        return result

class ApiEndopint(object):
    def __init__(self, endpoint, caller):
        self.endpoint = endpoint
        self.caller = caller

    def __getattr__(self, item):
        return self.__class__(self.endpoint + '/' + item, self.caller)

    __getitem__ = __getattr__

    def __call__(self, data=None, method=None):
        return self.caller(self.endpoint, data=data, method=method)

# Import API Sub Modules
for loader, module_name, is_pkg in pkgutil.walk_packages(__path__):
    loader.find_module(module_name).load_module(module_name)
<|endoftext|>