<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2014-2018 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

import os
import re
import shutil
import logging
import tempfile
import unittest
import sys

import numpy

from openquake.calculators import base
from openquake.baselib import datastore, general, parallel
from openquake.commonlib import readinput, oqvalidation

NOT_DARWIN = sys.platform != 'darwin'

class DifferentFiles(Exception):
    pass

def strip_calc_id(fname):
    name = os.path.basename(fname)
    return re.sub('_\d+\.', '.', name)

def columns(line):
    data = []
    if ',' in line:  # csv file
        for column in line.split(','):
            try:
                floats = list(map(float, column.split(' ')))
            except ValueError:  # skip header
                pass
            else:
                data.append(numpy.array(floats))
    else:  # txt file
        for column in line.split(' '):
            try:
                data.append(float(column))
            except ValueError:
                pass  # ignore nonfloats
    return data

class CalculatorTestCase(unittest.TestCase):
    OVERWRITE_EXPECTED = False
    edir = None  # will be set to a temporary directory

    @classmethod
    def setUpClass(cls):
        cls.duration = general.AccumDict()
        parallel.Starmap.shutdown = lambda: None  # avoid restarting the pool

    def get_calc(self, testfile, job_ini, **kw):
        """
        Return the outputs of the calculation as a dictionary
        """
        self.testdir = os.path.dirname(testfile) if os.path.isfile(testfile) \
            else testfile
        inis = [os.path.join(self.testdir, ini) for ini in job_ini.split(',')]
        params = readinput.get_params(inis, **kw)

        oqvalidation.OqParam.calculation_mode.validator.choices = tuple(
            base.calculators)
        oq = oqvalidation.OqParam(**params)
        oq.validate()
        # change this when debugging the test
        return base.calculators(oq)

    def run_calc(self, testfile, job_ini, **kw):
        """
        Return the outputs of the calculation as a dictionary
        """
        inis = job_ini.split(',')
        assert len(inis) in (1, 2), inis
        self.calc = self.get_calc(testfile, inis[0], **kw)
        self.edir = tempfile.mkdtemp()
        with self.calc._monitor:
            result = self.calc.run(export_dir=self.edir)
        duration = {inis[0]: self.calc._monitor.duration}
        if len(inis) == 2:
            hc_id = self.calc.datastore.calc_id
            calc = self.get_calc(
                testfile, inis[1], hazard_calculation_id=str(hc_id), **kw)
            with calc._monitor:
                exported = calc.run(export_dir=self.edir)
                result.update(exported)
            duration[inis[1]] = calc._monitor.duration
            self.calc = calc

        # reopen datastore, since some tests need to export from it
        dstore = datastore.read(self.calc.datastore.calc_id)
        self.calc.datastore = dstore
        self.__class__.duration += duration
        return result

    def execute(self, testfile, job_ini):
        """
        Return the result of the calculation without exporting it
        """
        self.calc = self.get_calc(testfile, job_ini)
        self.calc.pre_execute()
        return self.calc.execute()

    def practicallyEqual(self, line1, line2, delta):
        """
        Compare lines containing numbers up to the given delta
        """
        columns1 = columns(line1)
        columns2 = columns(line2)
        for c1, c2 in zip(columns1, columns2):
            numpy.testing.assert_allclose(c1, c2, atol=delta, rtol=delta)

    def assertEqualFiles(
            self, fname1, fname2, make_comparable=lambda lines: lines,
            delta=None, lastline=None):
        """
        Make sure the expected and actual files have the same content.
        `make_comparable` is a function processing the lines of the
        files to make them comparable. By default it does nothing,
        but in some tests a sorting function is passed, because some
        files can be equal only up to the ordering.
        """
        expected = os.path.abspath(os.path.join(self.testdir, fname1))
        if not os.path.exists(expected) and self.OVERWRITE_EXPECTED:
            expected_dir = os.path.dirname(expected)
            if not os.path.exists(expected_dir):
                os.makedirs(expected_dir)
            open(expected, 'w').write('')
        actual = os.path.abspath(
            os.path.join(self.calc.oqparam.export_dir, fname2))
        expected_lines = make_comparable(open(expected).readlines())
        actual_lines = make_comparable(open(actual).readlines()[:lastline])
        try:
            self.assertEqual(len(expected_lines), len(actual_lines))
            for exp, got in zip(expected_lines, actual_lines):
                if delta:
                    self.practicallyEqual(exp, got, delta)
                else:
                    self.assertEqual(exp, got)
        except AssertionError:
            if self.OVERWRITE_EXPECTED:
                # use this path when the expected outputs have changed
                # for a good reason
                logging.info('overriding %s', expected)
                open(expected, 'w').write(''.join(actual_lines))
            else:
                # normally raise an exception
                raise DifferentFiles('%s %s' % (expected, actual))

    def assertGot(self, expected_content, fname):
        """
        Make sure the content of the exported file is the expected one
        """
        with open(os.path.join(self.calc.oqparam.export_dir, fname)) as actual:
            self.assertEqual(expected_content, actual.read())

    def run(self, result=None):
        res = super().run(result)
        if res is not None:  # for Python 3
            issues = len(res.errors) + len(res.failures)
        else:
            issues = 0  # this is bad, but Python 2 will die soon or later
        # remove temporary dir only for success
        if self.edir and not issues:
            shutil.rmtree(self.edir)
        return res

    @classmethod
    def tearDownClass(cls):
        print('durations =', cls.duration)
<commit_msg>Cleanup [skip CI]<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2014-2018 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

import os
import re
import shutil
import logging
import tempfile
import unittest
import sys

import numpy

from openquake.calculators import base
from openquake.baselib import datastore, general, parallel
from openquake.commonlib import readinput, oqvalidation

NOT_DARWIN = sys.platform != 'darwin'

class DifferentFiles(Exception):
    pass

def strip_calc_id(fname):
    name = os.path.basename(fname)
    return re.sub('_\d+\.', '.', name)

def columns(line):
    data = []
    if ',' in line:  # csv file
        for column in line.split(','):
            try:
                floats = list(map(float, column.split(' ')))
            except ValueError:  # skip header
                pass
            else:
                data.append(numpy.array(floats))
    else:  # txt file
        for column in line.split(' '):
            try:
                data.append(float(column))
            except ValueError:
                pass  # ignore nonfloats
    return data

class CalculatorTestCase(unittest.TestCase):
    OVERWRITE_EXPECTED = False
    edir = None  # will be set to a temporary directory

    @classmethod
    def setUpClass(cls):
        cls.duration = general.AccumDict()
        # avoid restarting the pool in the tests
        parallel.Starmap.shutdown = classmethod(lambda cls: None)

    def get_calc(self, testfile, job_ini, **kw):
        """
        Return the outputs of the calculation as a dictionary
        """
        self.testdir = os.path.dirname(testfile) if os.path.isfile(testfile) \
            else testfile
        inis = [os.path.join(self.testdir, ini) for ini in job_ini.split(',')]
        params = readinput.get_params(inis, **kw)

        oqvalidation.OqParam.calculation_mode.validator.choices = tuple(
            base.calculators)
        oq = oqvalidation.OqParam(**params)
        oq.validate()
        # change this when debugging the test
        return base.calculators(oq)

    def run_calc(self, testfile, job_ini, **kw):
        """
        Return the outputs of the calculation as a dictionary
        """
        inis = job_ini.split(',')
        assert len(inis) in (1, 2), inis
        self.calc = self.get_calc(testfile, inis[0], **kw)
        self.edir = tempfile.mkdtemp()
        with self.calc._monitor:
            result = self.calc.run(export_dir=self.edir)
        duration = {inis[0]: self.calc._monitor.duration}
        if len(inis) == 2:
            hc_id = self.calc.datastore.calc_id
            calc = self.get_calc(
                testfile, inis[1], hazard_calculation_id=str(hc_id), **kw)
            with calc._monitor:
                exported = calc.run(export_dir=self.edir)
                result.update(exported)
            duration[inis[1]] = calc._monitor.duration
            self.calc = calc

        # reopen datastore, since some tests need to export from it
        dstore = datastore.read(self.calc.datastore.calc_id)
        self.calc.datastore = dstore
        self.__class__.duration += duration
        return result

    def execute(self, testfile, job_ini):
        """
        Return the result of the calculation without exporting it
        """
        self.calc = self.get_calc(testfile, job_ini)
        self.calc.pre_execute()
        return self.calc.execute()

    def practicallyEqual(self, line1, line2, delta):
        """
        Compare lines containing numbers up to the given delta
        """
        columns1 = columns(line1)
        columns2 = columns(line2)
        for c1, c2 in zip(columns1, columns2):
            numpy.testing.assert_allclose(c1, c2, atol=delta, rtol=delta)

    def assertEqualFiles(
            self, fname1, fname2, make_comparable=lambda lines: lines,
            delta=None, lastline=None):
        """
        Make sure the expected and actual files have the same content.
        `make_comparable` is a function processing the lines of the
        files to make them comparable. By default it does nothing,
        but in some tests a sorting function is passed, because some
        files can be equal only up to the ordering.
        """
        expected = os.path.abspath(os.path.join(self.testdir, fname1))
        if not os.path.exists(expected) and self.OVERWRITE_EXPECTED:
            expected_dir = os.path.dirname(expected)
            if not os.path.exists(expected_dir):
                os.makedirs(expected_dir)
            open(expected, 'w').write('')
        actual = os.path.abspath(
            os.path.join(self.calc.oqparam.export_dir, fname2))
        expected_lines = make_comparable(open(expected).readlines())
        actual_lines = make_comparable(open(actual).readlines()[:lastline])
        try:
            self.assertEqual(len(expected_lines), len(actual_lines))
            for exp, got in zip(expected_lines, actual_lines):
                if delta:
                    self.practicallyEqual(exp, got, delta)
                else:
                    self.assertEqual(exp, got)
        except AssertionError:
            if self.OVERWRITE_EXPECTED:
                # use this path when the expected outputs have changed
                # for a good reason
                logging.info('overriding %s', expected)
                open(expected, 'w').write(''.join(actual_lines))
            else:
                # normally raise an exception
                raise DifferentFiles('%s %s' % (expected, actual))

    def assertGot(self, expected_content, fname):
        """
        Make sure the content of the exported file is the expected one
        """
        with open(os.path.join(self.calc.oqparam.export_dir, fname)) as actual:
            self.assertEqual(expected_content, actual.read())

    def run(self, result=None):
        res = super().run(result)
        if res is not None:  # for Python 3
            issues = len(res.errors) + len(res.failures)
        else:
            issues = 0  # this is bad, but Python 2 will die soon or later
        # remove temporary dir only for success
        if self.edir and not issues:
            shutil.rmtree(self.edir)
        return res

    @classmethod
    def tearDownClass(cls):
        print('durations =', cls.duration)
<|endoftext|>


<commit_before>from os.path import abspath, dirname, join

from eve import Eve
from eve.io.sql import SQL, ValidatorSQL
from eve_docs import eve_docs
from flask.config import Config
from flask.ext.bootstrap import Bootstrap
from flask import g

from amivapi import models, rest_logic, schemas

def get_config(environment):
    config_dir = abspath(join(dirname(__file__), "../config"))
    config = Config(config_dir)
    config.from_object("amivapi.settings")
    try:
        config.from_pyfile("%s.cfg" % environment)
    except IOError as e:
        raise IOError(str(e) + "\nYou can create it by running "
                             + "`python manage.py create_config`.")

    schemas.load_domain(config)

    return config

def create_app(environment, create_db=False):
    config = get_config(environment)
    app = Eve(settings=config, data=SQL, validator=ValidatorSQL)

    # Bind SQLAlchemy
    db = app.data.driver
    models.Base.metadata.bind = db.engine
    db.Model = models.Base
    if create_db:
        db.create_all()

    # Generate and expose docs via eve-docs extension
    Bootstrap(app)
    with app.app_context():
        g.db = db.session
    app.register_blueprint(eve_docs, url_prefix="/docs")
    app.register_blueprint(rest_logic.logics)

    # Add event hooks
    app.on_pre_GET_users += event_hooks.pre_users_get_callback
    app.on_post_GET_users += event_hooks.post_users_get_callback
    app.on_pre_POST_eventsignups += event_hooks.pre_signups_post_callback

    return app
<commit_msg>forgot to import event_hooks in the bootstrap, fix for previous commit<commit_after>from os.path import abspath, dirname, join

from eve import Eve
from eve.io.sql import SQL, ValidatorSQL
from eve_docs import eve_docs
from flask.config import Config
from flask.ext.bootstrap import Bootstrap
from flask import g

from amivapi import models, rest_logic, schemas, event_hooks

def get_config(environment):
    config_dir = abspath(join(dirname(__file__), "../config"))
    config = Config(config_dir)
    config.from_object("amivapi.settings")
    try:
        config.from_pyfile("%s.cfg" % environment)
    except IOError as e:
        raise IOError(str(e) + "\nYou can create it by running "
                             + "`python manage.py create_config`.")

    schemas.load_domain(config)

    return config

def create_app(environment, create_db=False):
    config = get_config(environment)
    app = Eve(settings=config, data=SQL, validator=ValidatorSQL)

    # Bind SQLAlchemy
    db = app.data.driver
    models.Base.metadata.bind = db.engine
    db.Model = models.Base
    if create_db:
        db.create_all()

    # Generate and expose docs via eve-docs extension
    Bootstrap(app)
    with app.app_context():
        g.db = db.session
    app.register_blueprint(eve_docs, url_prefix="/docs")
    app.register_blueprint(rest_logic.logics)

    # Add event hooks
    app.on_pre_GET_users += event_hooks.pre_users_get_callback
    app.on_post_GET_users += event_hooks.post_users_get_callback
    app.on_pre_POST_eventsignups += event_hooks.pre_signups_post_callback

    return app
<|endoftext|>


<commit_before>import os
from setuptools import setup

package_data = {}
start_dir = os.getcwd()
requirements=[
     "ccg-django-utils==0.4.2",
     "Django==1.10.3",
     "django-ajax-selects==1.5.2",
     "django-anymail==0.6.1",
     "django-countries",
     "django-extensions>=0.7.1",
     "django-iprestrict==1.1.0",
     "django-messages-ui==0.2.7",
     "django-nose==1.4.4",
     "django-positions==0.5.4",
     "django-registration-redux==1.4",
     "djangorestframework==3.5.3",
     "django-storages==1.4.1",
     "django-templatetag-handlebars==1.3.1",
     "django-templatetag-sugar==1.0",
     "django-useraudit==1.3.3",
     "geoip2==2.4.0",
     "openpyxl==2.3.5",
     "polib==1.0.8",
     "psycopg2==2.6.2",
     "pycountry==1.20",
     "pyinotify==0.9.6",
     "pymongo==2.9.3",
     "pyodbc==3.0.7",
     "pyparsing==2.1.10",
     "python-dateutil==2.5.3",
     "python-memcached==1.58",
     "pyyaml==3.12",
     "setuptools_scm==1.10.1",
     "six==1.10.0",
     "SQLAlchemy==1.0.16",
     "uwsgi==2.0.13.1",
]

def add_file_for_package(package, subdir, f):
    full_path = os.path.join(subdir, f)
    # print "%s: %s" % (package, full_path)
    return full_path

packages = ['rdrf',
            'registry',
            'registry.common',
            'registry.patients',
            'registry.groups',
            'registry.genetic',
            'explorer',
            ]

for package in ['rdrf', 'registry.common', 'registry.genetic',
                'registry.groups', 'registry.patients', 'registry.humangenome', 'explorer']:
    package_data[package] = []
    if "." in package:
        base_dir, package_dir = package.split(".")
        os.chdir(os.path.join(start_dir, base_dir, package_dir))
    else:
        base_dir = package
        os.chdir(os.path.join(start_dir, base_dir))

    for data_dir in (
            'templates',
            'static',
            'migrations',
            'fixtures',
            'features',
            'hooks',
            'templatetags',
            'management'):
        package_data[package].extend([add_file_for_package(package, subdir, f) for (
            subdir, dirs, files) in os.walk(data_dir) for f in files])

    os.chdir(start_dir)

setup(name='django-rdrf',
      version="1.7.15",
      packages=packages,
      description='RDRF',
      long_description='Rare Disease Registry Framework',
      author='Centre for Comparative Genomics',
      author_email='rdrf@ccg.murdoch.edu.au',
      package_data=package_data,
      zip_safe=False,
      install_requires=requirements
      )
<commit_msg>django 1.10.5 #350<commit_after>import os
from setuptools import setup

package_data = {}
start_dir = os.getcwd()
requirements=[
     "ccg-django-utils==0.4.2",
     "Django==1.10.5",
     "django-ajax-selects==1.5.2",
     "django-anymail==0.6.1",
     "django-countries",
     "django-extensions>=0.7.1",
     "django-iprestrict==1.1.0",
     "django-messages-ui==0.2.7",
     "django-nose==1.4.4",
     "django-positions==0.5.4",
     "django-registration-redux==1.4",
     "djangorestframework==3.5.3",
     "django-storages==1.4.1",
     "django-templatetag-handlebars==1.3.1",
     "django-templatetag-sugar==1.0",
     "django-useraudit==1.3.3",
     "geoip2==2.4.0",
     "openpyxl==2.3.5",
     "polib==1.0.8",
     "psycopg2==2.6.2",
     "pycountry==1.20",
     "pyinotify==0.9.6",
     "pymongo==2.9.3",
     "pyodbc==3.0.7",
     "pyparsing==2.1.10",
     "python-dateutil==2.5.3",
     "python-memcached==1.58",
     "pyyaml==3.12",
     "setuptools_scm==1.10.1",
     "six==1.10.0",
     "SQLAlchemy==1.0.16",
     "uwsgi==2.0.13.1",
]

def add_file_for_package(package, subdir, f):
    full_path = os.path.join(subdir, f)
    # print "%s: %s" % (package, full_path)
    return full_path

packages = ['rdrf',
            'registry',
            'registry.common',
            'registry.patients',
            'registry.groups',
            'registry.genetic',
            'explorer',
            ]

for package in ['rdrf', 'registry.common', 'registry.genetic',
                'registry.groups', 'registry.patients', 'registry.humangenome', 'explorer']:
    package_data[package] = []
    if "." in package:
        base_dir, package_dir = package.split(".")
        os.chdir(os.path.join(start_dir, base_dir, package_dir))
    else:
        base_dir = package
        os.chdir(os.path.join(start_dir, base_dir))

    for data_dir in (
            'templates',
            'static',
            'migrations',
            'fixtures',
            'features',
            'hooks',
            'templatetags',
            'management'):
        package_data[package].extend([add_file_for_package(package, subdir, f) for (
            subdir, dirs, files) in os.walk(data_dir) for f in files])

    os.chdir(start_dir)

setup(name='django-rdrf',
      version="1.7.15",
      packages=packages,
      description='RDRF',
      long_description='Rare Disease Registry Framework',
      author='Centre for Comparative Genomics',
      author_email='rdrf@ccg.murdoch.edu.au',
      package_data=package_data,
      zip_safe=False,
      install_requires=requirements
      )
<|endoftext|>


<commit_before>"""AMQPStorm Connection.Channel."""

import logging
import multiprocessing
from time import sleep

from pamqp import specification as pamqp_spec
from pamqp.header import ContentHeader

from amqpstorm import compatibility
from amqpstorm.base import BaseChannel
from amqpstorm.base import IDLE_WAIT
from amqpstorm.basic import Basic
from amqpstorm.compatibility import try_utf8_decode
from amqpstorm.exception import AMQPChannelError
from amqpstorm.exception import AMQPConnectionError
from amqpstorm.exception import AMQPInvalidArgument
from amqpstorm.exception import AMQPMessageError
from amqpstorm.exchange import Exchange
from amqpstorm.message import Message
from amqpstorm.queue import Queue
from amqpstorm.rpc import Rpc
from amqpstorm.tx import Tx

LOGGER = logging.getLogger(__name__)
CONTENT_FRAME = ['Basic.Deliver', 'ContentHeader', 'ContentBody']

class Channel(BaseChannel):
    """Connection.channel"""
    __slots__ = [
        'confirming_deliveries', 'consumer_callback', 'rpc', '_basic',
        '_connection', '_exchange', '_inbound', '_queue', '_tx', '_die'
    ]

    def __init__(self, channel_id, connection, rpc_timeout):
        super(Channel, self).__init__(channel_id)
        self.rpc = Rpc(self, timeout=rpc_timeout)
        self.confirming_deliveries = False
        self.consumer_callback = None
        self._inbound = []
        self._connection = connection
        self._basic = Basic(self)
        self._exchange = Exchange(self)
        self._tx = Tx(self)
        self._queue = Queue(self)

        self._die = multiprocessing.Value("b", 0)

    def __enter__(self):
        return self

    def __exit__(self, exception_type, exception_value, _):
        if exception_type:
            LOGGER.warning(
                'Closing channel due to an unhandled exception: %s',
                exception_value
            )
        if not self.is_open:
            return
        self.close()

    def __int__(self):
        return self._channel_id

    @property
    def basic(self):
        """RabbitMQ Basic Operations.

        :rtype: amqpstorm.basic.Basic
        """
        # print("Access to basic?")
        return self._basic

    @property
    def exchange(self):
        """RabbitMQ Exchange Operations.

        :rtype: amqpstorm.exchange.Exchange
        """
        return self._exchange

    @property
    def tx(self):
        """RabbitMQ Tx Operations.

        :rtype: amqpstorm.tx.Tx
        """
        return self._tx

    @property
    def queue(self):
        """RabbitMQ Queue Operations.

        :rtype: amqpstorm.queue.Queue
        """
        return self._queue

    def build_inbound_messages(self, break_on_empty=False, to_tuple=False):
        """Build messages in the inbound queue.

        :param bool break_on_empty: Should we break the loop when there are
                                    no more messages to consume.
        :param bool to_tuple: Should incoming messages be converted to a
                              tuple before delivery.

        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :rtype: :py:class:`generator`
        """
        self.check_for_errors()
        while not self.is_closed:
            # print("build_inbound_messages looping!", self._die.value, self.is_closed)
            if self._die.value != 0:
                return
            if self.is_closed:
                return

            message = self._build_message()
            if not message:
                if break_on_empty:
                    break
                self.check_for_errors()
                sleep(IDLE_WAIT)
                continue
            if to_tuple:
                yield message.to_tuple()
                continue
            yield message

    def kill(self):
        self._die.value = 1
        self.set_state(self.CLOSED)

    def close(self, reply_code=200, reply_text=''):
        """Close Channel.

        :param int reply_code: Close reply code (e.g. 200)
        :param str reply_text: Close reply text

        :raises AMQPInvalidArgument: Invalid Parameters
        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :return:
        """
        LOGGER.debug('Channel #%d Closing', self.channel_id)
        if not compatibility.is_integer(reply_code):
            raise AMQPInvalidArgument('reply_code should be an integer')
        elif not compatibility.is_string(reply_text):
            raise AMQPInvalidArgument('reply_text should be a string')

        if not self._connection.is_open or not self.is_open:
            self.remove_consumer_tag()
            self.set_state(self.CLOSED)
            return
        self.set_state(self.CLOSING)
        self.stop_consuming()
        self.rpc_request(pamqp_spec.Channel.Close(
            reply_code=reply_code,
            reply_text=reply_text))
        del self._inbound[:]
        self.set_state(self.CLOSED)
        LOGGER.debug('Channel #%d Closed', self.channel_id)

    def check_for_errors(self):
        """Check connection and channel for errors.

        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.
        :return:
        """
        try:
            self._connection.check_for_errors()
        except AMQPConnectionError:
            self.set_state(self.CLOSED)
            raise

        if self.exceptions:
            exception = self.exceptions[0]
            if self.is_open:
                self.exceptions.pop(0)
            raise exception

        if self.is_closed:
            raise AMQPChannelError('channel was closed')

    def confirm_deliveries(self):
        """Set the channel to confirm that each message has been
        successfully delivered.

        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :return:
        """
        self.confirming_deliveries = True
        confirm_frame = pamqp_spec.Confirm.Select()
        return self.rpc_request(confirm_frame)

    def on_frame(self, frame_in):
        """Handle frame sent to this specific channel.

        :param pamqp.Frame frame_in: Amqp frame.
        :return:
        """
        # print("on_frame: ", frame_in.name)
        if self.rpc.on_frame(frame_in):
            return

        if frame_in.name in CONTENT_FRAME:
            self._inbound.append(frame_in)
        elif frame_in.name == 'Basic.Cancel':
            self._basic_cancel(frame_in)
        elif frame_in.name == 'Basic.CancelOk':
            self.remove_consumer_tag(frame_in.consumer_tag)
        elif frame_in.name == 'Basic.ConsumeOk':
            self.add_consumer_tag(frame_in['consumer_tag'])
        elif frame_in.name == 'Basic.Return':
            self._basic_return(frame_in)
        elif frame_in.name == 'Channel.Close':
            self._close_channel(frame_in)
        elif frame_in.name == 'Channel.Flow':
            self.write_frame(pamqp_spec.Channel.FlowOk(frame_in.active))
        else:
            LOGGER.error(
                '[Channel%d] Unhandled Frame: %s -- %s',
                self.channel_id, frame_in.name, dict(frame_in)
            )

    def open(self):
        """Open Channel.

        :return:
        """
        self._inbound = []
        self._exceptions = []
        self.set_state(self.OPENING)
        self.rpc_request(pamqp_spec.Channel.Open())
        self.set_state(self.OPEN)

    def process_data_events(self, to_tuple=False):
        """Consume inbound messages.

            This is only required when consuming messages. All other
            events are automatically handled in the background.

        :param bool to_tuple: Should incoming messages be converted to a
                              tuple before delivery.

        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :return:
        """
        # print("process_data_events")
        if not self.consumer_callback:
            raise AMQPChannelError('no consumer_callback defined')
        for message in self.build_inbound_messages(break_on_empty=True):
            if self._die.value != 0:
                return

            if not to_tuple:
                # noinspection PyCallingNonCallable
                self.consumer_callback(message)
                continue
            # noinspection PyCallingNonCallable
            self.consumer_callback(*message.to_tuple())
        sleep(IDLE_WAIT)

    def rpc_request(self, frame_out):
        """Perform a RPC Request.

        :param pamqp_spec.Frame frame_out: Amqp frame.
        :rtype: dict
        """
        # print('rpc_request')
        with self.rpc.lock:
            uuid = self.rpc.register_request(frame_out.valid_responses)
            self.write_frame(frame_out)
            return self.rpc.get_request(uuid)

    def start_consuming(self, to_tuple=False):
        """Start consuming messages.

        :param bool to_tuple: Should incoming messages be converted to a
                              tuple before delivery.

        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :return:
        """
        while self.consumer_tags:
            if self.is_closed:
                print("start_consuming looping (is_closed: %s, state: %s, should-die: %s)" % (self.is_closed, self._state, self._die.value))
                break
            elif self._die.value != 0:
                print("start_consuming looping (is_closed: %s, state: %s, should-die: %s)" % (self.is_closed, self._state, self._die.value))
                break
            else:
                print("Looping?", self._die.value, self.is_closed)
            self.process_data_events(to_tuple=to_tuple)
            # print("start_consuming looping (is_closed: %s, state: %s, should-die: %s)" % (self.is_closed, self._state, self._die.value))
    def stop_consuming(self):
        """Stop consuming messages.

        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :return:
        """
        if not self.consumer_tags:
            return
        for tag in self.consumer_tags:
            self.basic.cancel(tag)
        self.remove_consumer_tag()

    def write_frame(self, frame_out):
        """Write a pamqp frame from the current channel.

        :param pamqp_spec.Frame frame_out: A single pamqp frame.
        :return:
        """
        self.check_for_errors()
        self._connection.write_frame(self.channel_id, frame_out)

    def write_frames(self, frames_out):
        """Write multiple pamqp frames from the current channel.

        :param list frames_out: A list of pamqp frames.
        :return:
        """
        self.check_for_errors()
        self._connection.write_frames(self.channel_id, frames_out)

    def _basic_cancel(self, frame_in):
        """Handle a Basic Cancel frame.

        :param pamqp_spec.Basic.Cancel frame_in: Amqp frame.
        :return:
        """
        LOGGER.warning(
            'Received Basic.Cancel on consumer_tag: %s',
            try_utf8_decode(frame_in.consumer_tag)
        )
        self.remove_consumer_tag(frame_in.consumer_tag)

    def _basic_return(self, frame_in):
        """Handle a Basic Return Frame and treat it as an error.

        :param pamqp_spec.Basic.Return frame_in: Amqp frame.
        :return:
        """
        reply_text = try_utf8_decode(frame_in.reply_text)
        message = (
            "Message not delivered: %s (%s) to queue '%s' from exchange '%s'" %
            (
                reply_text,
                                           frame_in.reply_code,
                                           frame_in.routing_key,
                frame_in.exchange
            )
        )
        exception = AMQPMessageError(message,
                                     reply_code=frame_in.reply_code)
        self.exceptions.append(exception)

    def _build_message(self):
        """Fetch and build a complete Message from the inbound queue.

        :rtype: Message
        """
        # print("_build_message call")
        with self.lock:
            if len(self._inbound) < 2:
                return None
            headers = self._build_message_headers()
            if not headers:
                return None
            basic_deliver, content_header = headers
            body = self._build_message_body(content_header.body_size)

        message = Message(channel=self,
                          body=body,
                          method=dict(basic_deliver),
                          properties=dict(content_header.properties))
        return message

    def _build_message_headers(self):
        """Fetch Message Headers (Deliver & Header Frames).

        :rtype: tuple|None
        """
        basic_deliver = self._inbound.pop(0)
        if not isinstance(basic_deliver, pamqp_spec.Basic.Deliver):
            LOGGER.warning(
                'Received an out-of-order frame: %s was '
                           'expecting a Basic.Deliver frame',
                type(basic_deliver)
            )
            return None
        content_header = self._inbound.pop(0)
        if not isinstance(content_header, ContentHeader):
            LOGGER.warning(
                'Received an out-of-order frame: %s was '
                           'expecting a ContentHeader frame',
                type(content_header)
            )
            return None

        return basic_deliver, content_header

    def _build_message_body(self, body_size):
        """Build the Message body from the inbound queue.

        :rtype: str
        """
        body = bytes()
        while len(body) < body_size:
            if self._die.value == 1:
                raise RuntimeError("Exiting due to exit flag!")
            if not self._inbound:
                sleep(IDLE_WAIT)
                continue
            body_piece = self._inbound.pop(0)
            if not body_piece.value:
                break
            body += body_piece.value
        return body

    def _close_channel(self, frame_in):
        """Close Channel.

        :param pamqp_spec.Channel.Close frame_in: Amqp frame.
        :return:
        """
        self.remove_consumer_tag()
        if frame_in.reply_code != 200:
            reply_text = try_utf8_decode(frame_in.reply_text)
            message = (
                'Channel %d was closed by remote server: %s' %
                (
                    self._channel_id,
                    reply_text
                )
            )
            exception = AMQPChannelError(message,
                                         reply_code=frame_in.reply_code)
            self.exceptions.append(exception)
        del self._inbound[:]
        self.set_state(self.CLOSED)
<commit_msg>Roll in upstream change from https://github.com/eandersson/amqpstorm/commit/d8462daa7c791d01671f6b4be4b7dd4a601a8c49<commit_after>"""AMQPStorm Connection.Channel."""

import logging
import multiprocessing
from time import sleep

from pamqp import specification as pamqp_spec
from pamqp.header import ContentHeader

from amqpstorm import compatibility
from amqpstorm.base import BaseChannel
from amqpstorm.base import IDLE_WAIT
from amqpstorm.basic import Basic
from amqpstorm.compatibility import try_utf8_decode
from amqpstorm.exception import AMQPChannelError
from amqpstorm.exception import AMQPConnectionError
from amqpstorm.exception import AMQPInvalidArgument
from amqpstorm.exception import AMQPMessageError
from amqpstorm.exchange import Exchange
from amqpstorm.message import Message
from amqpstorm.queue import Queue
from amqpstorm.rpc import Rpc
from amqpstorm.tx import Tx

LOGGER = logging.getLogger(__name__)
CONTENT_FRAME = ['Basic.Deliver', 'ContentHeader', 'ContentBody']

class Channel(BaseChannel):
    """Connection.channel"""
    __slots__ = [
        'confirming_deliveries', 'consumer_callback', 'rpc', '_basic',
        '_connection', '_exchange', '_inbound', '_queue', '_tx', '_die'
    ]

    def __init__(self, channel_id, connection, rpc_timeout):
        super(Channel, self).__init__(channel_id)
        self.rpc = Rpc(self, timeout=rpc_timeout)
        self.confirming_deliveries = False
        self.consumer_callback = None
        self._inbound = []
        self._connection = connection
        self._basic = Basic(self)
        self._exchange = Exchange(self)
        self._tx = Tx(self)
        self._queue = Queue(self)

        self._die = multiprocessing.Value("b", 0)

    def __enter__(self):
        return self

    def __exit__(self, exception_type, exception_value, _):
        if exception_type:
            LOGGER.warning(
                'Closing channel due to an unhandled exception: %s',
                exception_value
            )
        if not self.is_open:
            return
        self.close()

    def __int__(self):
        return self._channel_id

    @property
    def basic(self):
        """RabbitMQ Basic Operations.

        :rtype: amqpstorm.basic.Basic
        """
        # print("Access to basic?")
        return self._basic

    @property
    def exchange(self):
        """RabbitMQ Exchange Operations.

        :rtype: amqpstorm.exchange.Exchange
        """
        return self._exchange

    @property
    def tx(self):
        """RabbitMQ Tx Operations.

        :rtype: amqpstorm.tx.Tx
        """
        return self._tx

    @property
    def queue(self):
        """RabbitMQ Queue Operations.

        :rtype: amqpstorm.queue.Queue
        """
        return self._queue

    def build_inbound_messages(self, break_on_empty=False, to_tuple=False):
        """Build messages in the inbound queue.

        :param bool break_on_empty: Should we break the loop when there are
                                    no more messages to consume.
        :param bool to_tuple: Should incoming messages be converted to a
                              tuple before delivery.

        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :rtype: :py:class:`generator`
        """
        self.check_for_errors()
        while not self.is_closed:
            # print("build_inbound_messages looping!", self._die.value, self.is_closed)
            if self._die.value != 0:
                return
            if self.is_closed:
                return

            message = self._build_message()
            if not message:
                if break_on_empty:
                    break
                self.check_for_errors()
                sleep(IDLE_WAIT)
                continue
            if to_tuple:
                yield message.to_tuple()
                continue
            yield message

    def kill(self):
        self._die.value = 1
        self.set_state(self.CLOSED)

    def close(self, reply_code=200, reply_text=''):
        """Close Channel.

        :param int reply_code: Close reply code (e.g. 200)
        :param str reply_text: Close reply text

        :raises AMQPInvalidArgument: Invalid Parameters
        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :return:
        """
        LOGGER.debug('Channel #%d Closing', self.channel_id)
        if not compatibility.is_integer(reply_code):
            raise AMQPInvalidArgument('reply_code should be an integer')
        elif not compatibility.is_string(reply_text):
            raise AMQPInvalidArgument('reply_text should be a string')

        if not self._connection.is_open or not self.is_open:
            self.remove_consumer_tag()
            self.set_state(self.CLOSED)
            return
        self.set_state(self.CLOSING)
        self.stop_consuming()
        self.rpc_request(pamqp_spec.Channel.Close(
            reply_code=reply_code,
            reply_text=reply_text))
        del self._inbound[:]
        self.set_state(self.CLOSED)
        LOGGER.debug('Channel #%d Closed', self.channel_id)

    def check_for_errors(self):
        """Check connection and channel for errors.

        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.
        :return:
        """
        try:
            self._connection.check_for_errors()
        except AMQPConnectionError:
            self.set_state(self.CLOSED)
            raise

        if self.exceptions:
            exception = self.exceptions[0]
            if self.is_open:
                self.exceptions.pop(0)
            raise exception

        if self.is_closed:
            raise AMQPChannelError('channel was closed')

    def confirm_deliveries(self):
        """Set the channel to confirm that each message has been
        successfully delivered.

        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :return:
        """
        self.confirming_deliveries = True
        confirm_frame = pamqp_spec.Confirm.Select()
        return self.rpc_request(confirm_frame)

    def on_frame(self, frame_in):
        """Handle frame sent to this specific channel.

        :param pamqp.Frame frame_in: Amqp frame.
        :return:
        """
        # print("on_frame: ", frame_in.name)
        if self.rpc.on_frame(frame_in):
            return

        if frame_in.name in CONTENT_FRAME:
            self._inbound.append(frame_in)
        elif frame_in.name == 'Basic.Cancel':
            self._basic_cancel(frame_in)
        elif frame_in.name == 'Basic.CancelOk':
            self.remove_consumer_tag(frame_in.consumer_tag)
        elif frame_in.name == 'Basic.ConsumeOk':
            self.add_consumer_tag(frame_in['consumer_tag'])
        elif frame_in.name == 'Basic.Return':
            self._basic_return(frame_in)
        elif frame_in.name == 'Channel.Close':
            self._close_channel(frame_in)
        elif frame_in.name == 'Channel.Flow':
            self.write_frame(pamqp_spec.Channel.FlowOk(frame_in.active))
        else:
            LOGGER.error(
                '[Channel%d] Unhandled Frame: %s -- %s',
                self.channel_id, frame_in.name, dict(frame_in)
            )

    def open(self):
        """Open Channel.

        :return:
        """
        self._inbound = []
        self._exceptions = []
        self.set_state(self.OPENING)
        self.rpc_request(pamqp_spec.Channel.Open())
        self.set_state(self.OPEN)

    def process_data_events(self, to_tuple=False):
        """Consume inbound messages.

            This is only required when consuming messages. All other
            events are automatically handled in the background.

        :param bool to_tuple: Should incoming messages be converted to a
                              tuple before delivery.

        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :return:
        """
        # print("process_data_events")
        if not self.consumer_callback:
            raise AMQPChannelError('no consumer_callback defined')
        for message in self.build_inbound_messages(break_on_empty=True):
            if self._die.value != 0:
                return

            if not to_tuple:
                # noinspection PyCallingNonCallable
                self.consumer_callback(message)
                continue
            # noinspection PyCallingNonCallable
            self.consumer_callback(*message.to_tuple())
        sleep(IDLE_WAIT)

    def rpc_request(self, frame_out):
        """Perform a RPC Request.

        :param pamqp_spec.Frame frame_out: Amqp frame.
        :rtype: dict
        """
        # print('rpc_request')
        with self.rpc.lock:
            uuid = self.rpc.register_request(frame_out.valid_responses)
            self.write_frame(frame_out)
            return self.rpc.get_request(uuid)

    def start_consuming(self, to_tuple=False):
        """Start consuming messages.

        :param bool to_tuple: Should incoming messages be converted to a
                              tuple before delivery.

        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :return:
        """
        while self.consumer_tags:
            if self.is_closed:
                print("start_consuming looping (is_closed: %s, state: %s, should-die: %s)" % (self.is_closed, self._state, self._die.value))
                break
            elif self._die.value != 0:
                print("start_consuming looping (is_closed: %s, state: %s, should-die: %s)" % (self.is_closed, self._state, self._die.value))
                break
            else:
                print("Looping?", self._die.value, self.is_closed)
            self.process_data_events(to_tuple=to_tuple)
            # print("start_consuming looping (is_closed: %s, state: %s, should-die: %s)" % (self.is_closed, self._state, self._die.value))
    def stop_consuming(self):
        """Stop consuming messages.

        :raises AMQPChannelError: Raises if the channel encountered an error.
        :raises AMQPConnectionError: Raises if the connection
                                     encountered an error.

        :return:
        """
        if not self.consumer_tags:
            return
        for tag in self.consumer_tags:
            self.basic.cancel(tag)
        self.remove_consumer_tag()

    def write_frame(self, frame_out):
        """Write a pamqp frame from the current channel.

        :param pamqp_spec.Frame frame_out: A single pamqp frame.
        :return:
        """
        self.check_for_errors()
        self._connection.write_frame(self.channel_id, frame_out)

    def write_frames(self, frames_out):
        """Write multiple pamqp frames from the current channel.

        :param list frames_out: A list of pamqp frames.
        :return:
        """
        self.check_for_errors()
        self._connection.write_frames(self.channel_id, frames_out)

    def _basic_cancel(self, frame_in):
        """Handle a Basic Cancel frame.

        :param pamqp_spec.Basic.Cancel frame_in: Amqp frame.
        :return:
        """
        LOGGER.warning(
            'Received Basic.Cancel on consumer_tag: %s',
            try_utf8_decode(frame_in.consumer_tag)
        )
        self.remove_consumer_tag(frame_in.consumer_tag)

    def _basic_return(self, frame_in):
        """Handle a Basic Return Frame and treat it as an error.

        :param pamqp_spec.Basic.Return frame_in: Amqp frame.
        :return:
        """
        reply_text = try_utf8_decode(frame_in.reply_text)
        message = (
            "Message not delivered: %s (%s) to queue '%s' from exchange '%s'" %
            (
                reply_text,
                                           frame_in.reply_code,
                                           frame_in.routing_key,
                frame_in.exchange
            )
        )
        exception = AMQPMessageError(message,
                                     reply_code=frame_in.reply_code)
        self.exceptions.append(exception)

    def _build_message(self):
        """Fetch and build a complete Message from the inbound queue.

        :rtype: Message
        """
        # print("_build_message call")
        with self.lock:
            if len(self._inbound) < 2:
                return None
            headers = self._build_message_headers()
            if not headers:
                return None
            basic_deliver, content_header = headers
            body = self._build_message_body(content_header.body_size)

        message = Message(channel=self,
                          body=body,
                          method=dict(basic_deliver),
                          properties=dict(content_header.properties))
        return message

    def _build_message_headers(self):
        """Fetch Message Headers (Deliver & Header Frames).

        :rtype: tuple|None
        """
        basic_deliver = self._inbound.pop(0)
        if not isinstance(basic_deliver, pamqp_spec.Basic.Deliver):
            LOGGER.warning(
                'Received an out-of-order frame: %s was '
                           'expecting a Basic.Deliver frame',
                type(basic_deliver)
            )
            return None
        content_header = self._inbound.pop(0)
        if not isinstance(content_header, ContentHeader):
            LOGGER.warning(
                'Received an out-of-order frame: %s was '
                           'expecting a ContentHeader frame',
                type(content_header)
            )
            return None

        return basic_deliver, content_header

    def _build_message_body(self, body_size):
        """Build the Message body from the inbound queue.

        :rtype: str
        """
        body = bytes()
        while len(body) < body_size:
            if self._die.value == 1:
                raise RuntimeError("Exiting due to exit flag!")
            if not self._inbound:
                if self.is_closed:
                    self.check_for_errors()
                    break
                sleep(IDLE_WAIT)
                continue
            body_piece = self._inbound.pop(0)
            if not body_piece.value:
                break
            body += body_piece.value
        return body

    def _close_channel(self, frame_in):
        """Close Channel.

        :param pamqp_spec.Channel.Close frame_in: Amqp frame.
        :return:
        """
        self.remove_consumer_tag()
        if frame_in.reply_code != 200:
            reply_text = try_utf8_decode(frame_in.reply_text)
            message = (
                'Channel %d was closed by remote server: %s' %
                (
                    self._channel_id,
                    reply_text
                )
            )
            exception = AMQPChannelError(message,
                                         reply_code=frame_in.reply_code)
            self.exceptions.append(exception)
        del self._inbound[:]
        self.set_state(self.CLOSED)
<|endoftext|>


<commit_before>import os
from setuptools import setup

package_data = {}
start_dir = os.getcwd()
requirements=[
     "ccg-django-utils==0.4.2",
     "Django==1.10.3",
     "django-ajax-selects==1.5.2",
     "django-anymail==0.6.1",
     "django-countries",
     "django-extensions>=0.7.1",
     "django-iprestrict==1.1.0",
     "django-messages-ui==0.2.7",
     "django-nose==1.4.4",
     "django-positions==0.5.4",
     "django-registration-redux==1.3",
     "djangorestframework==3.4.7",
     "django-storages==1.4.1",
     "django-templatetag-handlebars==1.3.1",
     "django-templatetag-sugar==1.0",
     "django-useraudit==1.3.3",
     "geoip2==2.4.0",
     "openpyxl==2.3.5",
     "polib==1.0.7",
     "psycopg2==2.6.2",
     "pycountry==1.20",
     "pyinotify==0.9.6",
     "pymongo==2.9.3",
     "pyodbc==3.0.7",
     "pyparsing==2.1.9",
     "python-dateutil==2.5.3",
     "python-memcached==1.58",
     "pyyaml==3.12",
     "setuptools_scm==1.10.1",
     "six==1.10.0",
     "SQLAlchemy==1.0.15",
     "uwsgi==2.0.13.1",
]

def add_file_for_package(package, subdir, f):
    full_path = os.path.join(subdir, f)
    # print "%s: %s" % (package, full_path)
    return full_path

packages = ['rdrf',
            'registry',
            'registry.common',
            'registry.patients',
            'registry.groups',
            'registry.genetic',
            'explorer',
            ]

for package in ['rdrf', 'registry.common', 'registry.genetic',
                'registry.groups', 'registry.patients', 'registry.humangenome', 'explorer']:
    package_data[package] = []
    if "." in package:
        base_dir, package_dir = package.split(".")
        os.chdir(os.path.join(start_dir, base_dir, package_dir))
    else:
        base_dir = package
        os.chdir(os.path.join(start_dir, base_dir))

    for data_dir in (
            'templates',
            'static',
            'migrations',
            'fixtures',
            'features',
            'hooks',
            'templatetags',
            'management'):
        package_data[package].extend([add_file_for_package(package, subdir, f) for (
            subdir, dirs, files) in os.walk(data_dir) for f in files])

    os.chdir(start_dir)

setup(name='django-rdrf',
      version="1.7.11",
      packages=packages,
      description='RDRF',
      long_description='Rare Disease Registry Framework',
      author='Centre for Comparative Genomics',
      author_email='rdrf@ccg.murdoch.edu.au',
      package_data=package_data,
      zip_safe=False,
      install_requires=requirements
      )
<commit_msg>django-registration-redux==1.4 #327<commit_after>import os
from setuptools import setup

package_data = {}
start_dir = os.getcwd()
requirements=[
     "ccg-django-utils==0.4.2",
     "Django==1.10.3",
     "django-ajax-selects==1.5.2",
     "django-anymail==0.6.1",
     "django-countries",
     "django-extensions>=0.7.1",
     "django-iprestrict==1.1.0",
     "django-messages-ui==0.2.7",
     "django-nose==1.4.4",
     "django-positions==0.5.4",
     "django-registration-redux==1.4",
     "djangorestframework==3.4.7",
     "django-storages==1.4.1",
     "django-templatetag-handlebars==1.3.1",
     "django-templatetag-sugar==1.0",
     "django-useraudit==1.3.3",
     "geoip2==2.4.0",
     "openpyxl==2.3.5",
     "polib==1.0.7",
     "psycopg2==2.6.2",
     "pycountry==1.20",
     "pyinotify==0.9.6",
     "pymongo==2.9.3",
     "pyodbc==3.0.7",
     "pyparsing==2.1.9",
     "python-dateutil==2.5.3",
     "python-memcached==1.58",
     "pyyaml==3.12",
     "setuptools_scm==1.10.1",
     "six==1.10.0",
     "SQLAlchemy==1.0.15",
     "uwsgi==2.0.13.1",
]

def add_file_for_package(package, subdir, f):
    full_path = os.path.join(subdir, f)
    # print "%s: %s" % (package, full_path)
    return full_path

packages = ['rdrf',
            'registry',
            'registry.common',
            'registry.patients',
            'registry.groups',
            'registry.genetic',
            'explorer',
            ]

for package in ['rdrf', 'registry.common', 'registry.genetic',
                'registry.groups', 'registry.patients', 'registry.humangenome', 'explorer']:
    package_data[package] = []
    if "." in package:
        base_dir, package_dir = package.split(".")
        os.chdir(os.path.join(start_dir, base_dir, package_dir))
    else:
        base_dir = package
        os.chdir(os.path.join(start_dir, base_dir))

    for data_dir in (
            'templates',
            'static',
            'migrations',
            'fixtures',
            'features',
            'hooks',
            'templatetags',
            'management'):
        package_data[package].extend([add_file_for_package(package, subdir, f) for (
            subdir, dirs, files) in os.walk(data_dir) for f in files])

    os.chdir(start_dir)

setup(name='django-rdrf',
      version="1.7.11",
      packages=packages,
      description='RDRF',
      long_description='Rare Disease Registry Framework',
      author='Centre for Comparative Genomics',
      author_email='rdrf@ccg.murdoch.edu.au',
      package_data=package_data,
      zip_safe=False,
      install_requires=requirements
      )
<|endoftext|>


<commit_before>from featureforge import generate
from featureforge.feature import make_feature
import schema

EQ = 'EQ'
APPROX = 'APPROX'
IN = 'IN'
RAISES = 'RAISES'

EPSILON = 0.01

def _raise_predicate(spec, data, exception):
    try:
        spec(data)
    except exception:
        return True
    return False

_PREDICATES = {
    EQ: lambda f, d, v: f(d) == v,
    APPROX: lambda f, d, v: abs(f(d) - v) < EPSILON,
    IN: lambda f, d, v: f(d) in v,
    RAISES: _raise_predicate
}

_EXPLAIN_PREDICATE_FAIL = {
    EQ: 'is not equal',
    APPROX: 'is not approx',
    IN: 'is not in',
    RAISES: 'not raised'
}

class FeatureFixtureCheckMixin(object):

    def assert_feature_passes_fixture(self, feature_spec, fixture):
        failures = []
        feature_spec = make_feature(feature_spec)
        for label, (data_point, predicate, value) in fixture.items():
            if not _PREDICATES[predicate](feature_spec, data_point, value):
                msg = '%s failed, %s %s %s' % (
                    label, feature_spec(data_point),
                    _EXPLAIN_PREDICATE_FAIL[predicate],
                    value)
                failures.append(msg)
        self.assertFalse(failures, msg='; '.join(failures))

    def assert_passes_fuzz(self, feature_spec, tries=1000):
        feature_spec = make_feature(feature_spec)
        for i in xrange(tries):
            data_point = generate.generate(feature_spec.input_schema)
            try:
                feature = feature_spec(data_point)
            except Exception as e:
                self.fail("Error evaluating; input=%r error=%r" % (data_point, e))
            try:
                feature_spec.output_schema.validate(feature)
            except schema.SchemaError:
                self.fail("Invalid output schema; input=%r output=%r" %
                          (data_point, feature))

class BaseFeatureFixture(FeatureFixtureCheckMixin):
    feature = None  # Needs to be defined on subclasses

    def test_fixtures(self):
        self.assert_feature_passes_fixture(self.feature, self.fixtures)

    def test_fuzz(self):
        self.assert_passes_fuzz(self.feature)

### EXAMPLE ###

if __name__ == "__main__":
    from featureforge.feature import input_schema, output_schema
    from schema import Schema, And
    import unittest

    @input_schema(str)
    @output_schema(int, lambda n: n >= 0)
    def length(data_point):
        return len(data_point)

    # This is an example on how to use assertions directly
    class TestLength(unittest.TestCase, FeatureFixtureCheckMixin):

        def test_f(self):
            fixture = dict(
                test_eq=('hello', EQ, 5),
                test_approx=('world!', APPROX, 6.00001),
                test_in=('hello', IN, (5, 6, 1)),
                test_raise=(None, RAISES, ValueError),
            )
            self.assert_feature_passes_fixture(length, fixture)

        def test_fuzz(self):
            self.assert_passes_fuzz(length)

    class TestLength2(unittest.TestCase, BaseFeatureFixture):
        feature = length
        fixtures = dict(
                test_eq=('hello', EQ, 5),
                test_approx=('world!', APPROX, 6.00001),
                test_in=('hello', IN, (5, 6, 1)),
                test_raise=(None, RAISES, ValueError),
            )

    unittest.main()
<commit_msg>Additional docstrings<commit_after>from featureforge import generate
from featureforge.feature import make_feature
import schema

EQ = 'EQ'
APPROX = 'APPROX'
IN = 'IN'
RAISES = 'RAISES'

EPSILON = 0.01

def _raise_predicate(spec, data, exception):
    try:
        spec(data)
    except exception:
        return True
    return False

_PREDICATES = {
    EQ: lambda f, d, v: f(d) == v,
    APPROX: lambda f, d, v: abs(f(d) - v) < EPSILON,
    IN: lambda f, d, v: f(d) in v,
    RAISES: _raise_predicate
}

_EXPLAIN_PREDICATE_FAIL = {
    EQ: 'is not equal',
    APPROX: 'is not approx',
    IN: 'is not in',
    RAISES: 'not raised'
}

class FeatureFixtureCheckMixin(object):
    """
    This class is a TestCase mixin that provides some assertions to test
    features.

    In most cases, you shouldn't use this directly but BaseFeatureFixture
    instead
    """

    def assert_feature_passes_fixture(self, feature_spec, fixture):
        """
        Check that the given feature (function or Feature instance) passes
        all the conditions given in the fixture

        `fixture` is a dictionary where each key/value pair describes a simple
        example for the feature. The key should be a string (which will be
        reported in case of failure, so you know which case failed), and the
        value is a tuple (input, predicate, value). The `input` is the value
        that will be passed as argument as a feature. The predicate and the
        value give the condition, and should be one of the following:

         * (input, EQ, value) checks that feature(input) == value
         * (input, APPROX, value) checks that feature(input) == value approximately
           the error allowed is given by the constant EPSILON in this module
         * (input, IN, values) checks that feature(input) in values
         * (input, RAISES, eclass) checks that feature(input) raises an exception of
           eclass type. Note that input/output validation always raise an exception
           that subclasses ValueError

        """
        failures = []
        feature_spec = make_feature(feature_spec)
        for label, (data_point, predicate, value) in fixture.items():
            if not _PREDICATES[predicate](feature_spec, data_point, value):
                msg = '%s failed, %s %s %s' % (
                    label, feature_spec(data_point),
                    _EXPLAIN_PREDICATE_FAIL[predicate],
                    value)
                failures.append(msg)
        self.assertFalse(failures, msg='; '.join(failures))

    def assert_passes_fuzz(self, feature_spec, tries=1000):
        """
        Generates tries data points for the feature (which should have an
        input schema which allows generation) randomly, and applies those
        to the feature. It checks that the evaluation proceeds without raising
        exceptions and that it produces valid outputs according to the
        output schema.
        """
        feature_spec = make_feature(feature_spec)
        for i in xrange(tries):
            data_point = generate.generate(feature_spec.input_schema)
            try:
                feature = feature_spec(data_point)
            except Exception as e:
                self.fail("Error evaluating; input=%r error=%r" % (data_point, e))
            try:
                feature_spec.output_schema.validate(feature)
            except schema.SchemaError:
                self.fail("Invalid output schema; input=%r output=%r" %
                          (data_point, feature))

class BaseFeatureFixture(FeatureFixtureCheckMixin):
    """
    Inheriting this class together with unittest.TestCase allows you to
    quickly build test cases for features. Your subclass should define two
    class attributes:

    `feature` should be a function or a Feature() instance

    `fixture` has a list of cases to test; check the documentation
    of `assert_feature_passes_fixture` for more details.

    The class defined by this will validate all features in the fixture. It will
    also subject the feature to fuzzy testing if the input schema allows it.
    It's also possible to add additional tests to the testcase.

    If you want to have more control about how the fixture is applied or skip
    fuzzy testing, take a look at the FeatureFixtureCheckMixin.
    """

    feature = None  # Needs to be defined on subclasses

    def test_fixtures(self):
        self.assert_feature_passes_fixture(self.feature, self.fixtures)

    def test_fuzz(self):
        self.assert_passes_fuzz(self.feature)

### EXAMPLE ###

if __name__ == "__main__":
    from featureforge.feature import input_schema, output_schema
    from schema import Schema, And
    import unittest

    @input_schema(str)
    @output_schema(int, lambda n: n >= 0)
    def length(data_point):
        return len(data_point)

    # This is an example on how to use assertions directly
    class TestLength(unittest.TestCase, FeatureFixtureCheckMixin):

        def test_f(self):
            fixture = dict(
                test_eq=('hello', EQ, 5),
                test_approx=('world!', APPROX, 6.00001),
                test_in=('hello', IN, (5, 6, 1)),
                test_raise=(None, RAISES, ValueError),
            )
            self.assert_feature_passes_fixture(length, fixture)

        def test_fuzz(self):
            self.assert_passes_fuzz(length)

    class TestLength2(unittest.TestCase, BaseFeatureFixture):
        feature = length
        fixtures = dict(
                test_eq=('hello', EQ, 5),
                test_approx=('world!', APPROX, 6.00001),
                test_in=('hello', IN, (5, 6, 1)),
                test_raise=(None, RAISES, ValueError),
            )

    unittest.main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2018 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import logging
import operator

from openquake.baselib import parallel
from openquake.hazardlib.calc.hazard_curve import classical
from openquake.calculators import base
from openquake.calculators.classical import ClassicalCalculator
from openquake.calculators.ucerf_base import UcerfFilter
# FIXME: the counting of effective ruptures has to be revised

@base.calculators.add('ucerf_classical')
class UcerfClassicalCalculator(ClassicalCalculator):
    """
    UCERF classical calculator.
    """
    def pre_execute(self):
        super().pre_execute()
        self.csm_info = self.csm.info
        self.src_filter = UcerfFilter(
            self.sitecol, self.oqparam.maximum_distance)
        for sm in self.csm.source_models:  # one branch at the time
            [grp] = sm.src_groups
            for src in grp:
                grp.tot_ruptures += src.num_ruptures

    def execute(self):
        """
        Run in parallel `core_task(sources, sitecol, monitor)`, by
        parallelizing on the sources according to their weight and
        tectonic region type.
        """
        monitor = self.monitor(self.core_task.__name__)
        oq = self.oqparam
        acc = self.zerodict()
        self.nsites = []  # used in agg_dicts
        param = dict(imtls=oq.imtls, truncation_level=oq.truncation_level,
                     filter_distance=oq.filter_distance)
        for sm in self.csm.source_models:  # one branch at the time
            [grp] = sm.src_groups
            gsims = self.csm.info.get_gsims(sm.ordinal)
            acc = parallel.Starmap.apply(
                classical, (grp, self.src_filter, gsims, param, monitor),
                weight=operator.attrgetter('weight'),
                concurrent_tasks=oq.concurrent_tasks,
            ).reduce(self.agg_dicts, acc)
            ucerf = grp.sources[0].orig
            logging.info('Getting background sources from %s', ucerf.source_id)
            srcs = ucerf.get_background_sources(self.src_filter)
            acc = parallel.Starmap.apply(
                classical, (srcs, self.src_filter, gsims, param, monitor),
                weight=operator.attrgetter('weight'),
                concurrent_tasks=oq.concurrent_tasks,
            ).reduce(self.agg_dicts, acc)
        self.store_csm_info(acc)
        return acc  # {grp_id: pmap}
<commit_msg>[skip hazardlib]<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2018 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import logging
import operator

from openquake.baselib import parallel
from openquake.hazardlib.calc.hazard_curve import classical
from openquake.calculators import base
from openquake.calculators.classical import ClassicalCalculator
from openquake.calculators.ucerf_base import UcerfFilter
# FIXME: the counting of effective ruptures has to be revised

@base.calculators.add('ucerf_classical')
class UcerfClassicalCalculator(ClassicalCalculator):
    """
    UCERF classical calculator.
    """
    def pre_execute(self):
        super().pre_execute()
        self.csm_info = self.csm.info
        self.src_filter = UcerfFilter(
            self.sitecol, self.oqparam.maximum_distance)
        for sm in self.csm.source_models:  # one branch at the time
            [grp] = sm.src_groups
            for src in grp:
                grp.tot_ruptures += src.num_ruptures

    def execute(self):
        """
        Run in parallel `core_task(sources, sitecol, monitor)`, by
        parallelizing on the sources according to their weight and
        tectonic region type.
        """
        monitor = self.monitor(self.core_task.__name__)
        oq = self.oqparam
        acc = self.zerodict()
        self.nsites = []  # used in agg_dicts
        param = dict(imtls=oq.imtls, truncation_level=oq.truncation_level,
                     filter_distance=oq.filter_distance)
        for sm in self.csm.source_models:  # one branch at the time
            [grp] = sm.src_groups
            gsims = self.csm.info.get_gsims(sm.ordinal)
            acc = parallel.Starmap.apply(
                classical, (grp, self.src_filter, gsims, param, monitor),
                weight=operator.attrgetter('weight'),
                concurrent_tasks=oq.concurrent_tasks,
            ).reduce(self.agg_dicts, acc)
            ucerf = grp.sources[0].orig
            logging.info('Getting background sources from %s', ucerf.source_id)
            srcs = ucerf.get_background_sources(self.src_filter)
            acc = parallel.Starmap.apply(
                classical, (srcs, self.src_filter, gsims, param, monitor),
                weight=operator.attrgetter('weight'),
                concurrent_tasks=oq.concurrent_tasks,
            ).reduce(self.agg_dicts, acc)
        self.store_csm_info(acc.eff_ruptures)
        return acc  # {grp_id: pmap}
<|endoftext|>


<commit_before>#coding: utf-8
import random
import feedparser
import logging
import pytz
import json
# import uuid

from datetime import datetime
from time import mktime

from django.conf import settings
from django.utils import html
from django.utils import timezone
from django.utils.text import slugify

from .base import BaseProcessor
from .category_brasil import CATEGORY_BRASIL

from opps.articles.models import Post
from opps.channels.models import Channel
from opps.contrib.db_backend.postgres.base import DatabaseError
logger = logging.getLogger()

class RSSProcessor(BaseProcessor):

    def fetch(self):
        self.parsed = feedparser.parse(self.feed.source_url)
        if hasattr(self.parsed.feed, 'bozo_exception'):
            msg = "Malformed feed %s" % self.feed.source_url
            logger.warning(msg)
            self.verbose_print(msg)
            return
        self.verbose_print("Feed succesfully parsed")
        return self.parsed

    def update_feed(self):
        self.verbose_print("updating feed")
        if hasattr(self.parsed, 'published_parsed'):
            published_time = datetime.fromtimestamp(
                mktime(self.parsed.feed.published_parsed)
            )
            published_time = pytz.timezone(
                settings.TIME_ZONE
            ).localize(
                published_time,
                is_dst=None
            )

            if (self.feed.published_time and
                    self.feed.published_time >= published_time):
                return

            self.feed.published_time = published_time

        for attr in ['title', 'title_detail', 'link']:
            if not hasattr(self.parsed.feed, attr):
                msg = 'refresh_feeds. Feed "%s" has no %s'
                logger.error(msg % (self.feed.source_url, attr))
                self.verbose_print(msg % (self.feed.source_url, attr))
                return

        if self.parsed.feed.title_detail.type == 'text/plain':
            self.feed.title = html.escape(self.parsed.feed.title)[:150]
        else:
            self.feed.title = self.parsed.feed.title[:150]

        self.feed.link = self.feed.link or self.parsed.feed.link

        try:
            if self.parsed.feed.description_detail.type == 'text/plain':
                self.feed.description = html.escape(self.parsed.feed.description)
            else:
                self.feed.description = self.parsed.feed.description
        except:
            pass

        self.feed.last_polled_time = datetime.now()

        self.feed.save()

        self.verbose_print("Feed obj updated %s" % self.feed.last_polled_time)

        return len(self.parsed.entries)

    def create_entries(self):
        self.verbose_print("creating entry")
        count = 0
        for i, entry in enumerate(self.parsed.entries):

            e_id = getattr(entry, 'id', getattr(entry, 'guid', None))
            if self.log_created(e_id):
                self.verbose_print("Already processed")
                continue

            if i > self.max_entries_saved:
                break

            missing_attr = False
            for attr in ['title', 'title_detail', 'link', 'description']:
                if not hasattr(entry, attr):
                    msg = 'Feedcrawler refresh_feeds. Entry "%s" has no %s'
                    logger.error(msg % (entry.link, attr))
                    missing_attr = True
            if missing_attr:
                continue
            if entry.title == "":
                msg = 'Feedcrawler refresh_feeds. Entry "%s" has a blank title'
                logger.warning(msg % (entry.link))
                continue

            if entry.title_detail.type == 'text/plain':
                entry_title = html.escape(entry.title)
            else:
                entry_title = entry.title

            if not entry_title:
                entry_title = ""

            self.verbose_print("will create entry")

            slug = slugify(self.feed.slug + "-" + entry_title[:100])
            exists = self.entry_model.objects.filter(slug=slug).exists()
            if exists:
                slug = random.getrandbits(8) + "-" + slug
                self.verbose_print("Entry slug exists")

            if hasattr(entry, 'published_parsed'):
                published_time = datetime.fromtimestamp(
                    mktime(entry.published_parsed)
                )

                published_time = pytz.timezone(
                    settings.TIME_ZONE
                ).localize(
                    published_time,
                    is_dst=None
                )

                now = datetime.now()

                if published_time.date() > now.date():
                    self.verbose_print("Entry date is > now")
                    self.verbose_print(published_time)
                    self.verbose_print(now)
                    published_time = now
                elif published_time.date() < now.date():
                    self.verbose_print(
                        "Entry time is in the past, skipping: %s - %s"
                        % ( published_time.date(), now.date())
                    )
                    continue

            db_entry, created = self.entry_model.objects.get_or_create(
                entry_feed=self.feed,
                entry_link=entry.link,
                channel=self.feed.channel,
                title=entry_title[:150],
                slug=slug[:150],
                entry_title=entry_title[:150],
                site=self.feed.site,
                user=self.feed.user,
                published=True,
                show_on_root_channel=True
            )

            self.verbose_print("Entry found or created!!!")
            if created:
                if hasattr(entry, 'published_parsed'):
                    db_entry.entry_published_time = published_time

                # Lots of entries are missing description_detail attributes.
                # Escape their content by default
                if hasattr(entry, 'description_detail') and \
                        entry.description_detail.type != 'text/plain':
                    db_entry.entry_description = entry.description
                else:
                    db_entry.entry_description = html.escape(entry.description)

                try:
                    content = None
                    if hasattr(entry, 'content'):
                        content = entry.content
                        if isinstance(content, list) and content:
                            content = entry.content[0]

                    if content and content.type != 'text/plain':
                        db_entry.entry_content = content.value
                    elif hasattr(entry, 'content'):
                        db_entry.entry_content = html.escape(content.value)
                except Exception, e:
                    self.verbose_print(str(e))
                    msg = 'Feedcrawler refresh_feeds. Entry "%s" content error'
                    logger.warning(msg % (entry.link))

                try:
                    allowed = (str, unicode, dict, list, int, float, long)
                    entry_source = json.dumps(
                        {k: v for k, v in entry.iteritems()
                            if isinstance(v, allowed)}
                    )
                    db_entry.entry_json = entry_source
                    pass
                except Exception, e:
                    self.verbose_print(str(e))
                    msg = 'Feedcrawler refresh_feeds. Entry "%s" json error'
                    logger.warning(msg % (entry.link))

                # fill Article properties
                db_entry.headline = db_entry.entry_description
                db_entry.save()
                count += 1

                if self.verbose:
                    self.verbose_print("Entry fully created %s" % db_entry.title)
                    self.record_log(e_id)

                try:
                    self.verbose_print('creating post')
                    self.create_post(db_entry)
                except Exception as e:
                    self.verbose_print(str(e))

        self.verbose_print("%d entries created" % count)
        return count

    def delete_old_entries(self):
        entries = self.entry_model.objects.filter(entry_feed=self.feed)
        entries = entries[self.max_entries_saved:]
        # Cannot use 'limit' or 'offset' with delete.
        for entry in entries:
            entry.delete()

        if self.verbose:
            self.verbose_print("%d entries deleted" % len(entries))

    def process(self):
        # fetch and parse the feed

        self.max_entries_saved = self.feed.max_entries or 1000
        self.verbose_print("fetching")
        if not self.fetch():
            logger.warning("Feed cannot be parsed")
            return 0

        self.verbose_print("updating")
        if not self.update_feed():
            logger.info("No entry returned")
            return 0

        self.verbose_print("creating entries")
        created_count = self.create_entries()
        self.delete_old_entries()
        return created_count

    def get_channel_by_slug(self, slug):
        if not slug:
            return

        try:
            return Channel.objects.filter(long_slug=slug)[0]
        except:
            return

    def create_post(self, entry):
        # match category X channel
        channel_slug = CATEGORY_BRASIL.get(self.feed.source_url)
        channel = self.get_channel_by_slug(channel_slug) or entry.channel
        self.verbose_print(channel_slug)

        slug = slugify(entry.entry_title)
        if Post.objects.filter(channel=channel,
                               slug=slug,
                               site=entry.site).exists():
            slug = str(random.getrandbits(8)) + "-" + slug
            self.verbose_print("Post slug exists")

        post = Post(
            title=entry.entry_title[:150],
            slug=slug[:150],
            content=entry.entry_content or entry.entry_description,
            channel=channel,
            site=entry.site,
            user=entry.user,
            show_on_root_channel=True,
            published=True,
            # hat=entry.hat,
        )

        if self.feed.group:
            post.source = self.feed.group.name

        post.save()

        entry.post_created = True
        entry.save()

        self.verbose_print(u"Post {p.id}- {p.title} - {p.slug} created".format(p=post))

        return post
<commit_msg>Fix TypeError<commit_after>#coding: utf-8
import random
import feedparser
import logging
import pytz
import json
# import uuid

from datetime import datetime
from time import mktime

from django.conf import settings
from django.utils import html
from django.utils import timezone
from django.utils.text import slugify

from .base import BaseProcessor
from .category_brasil import CATEGORY_BRASIL

from opps.articles.models import Post
from opps.channels.models import Channel
from opps.contrib.db_backend.postgres.base import DatabaseError
logger = logging.getLogger()

class RSSProcessor(BaseProcessor):

    def fetch(self):
        self.parsed = feedparser.parse(self.feed.source_url)
        if hasattr(self.parsed.feed, 'bozo_exception'):
            msg = "Malformed feed %s" % self.feed.source_url
            logger.warning(msg)
            self.verbose_print(msg)
            return
        self.verbose_print("Feed succesfully parsed")
        return self.parsed

    def update_feed(self):
        self.verbose_print("updating feed")
        if hasattr(self.parsed, 'published_parsed'):
            published_time = datetime.fromtimestamp(
                mktime(self.parsed.feed.published_parsed)
            )
            published_time = pytz.timezone(
                settings.TIME_ZONE
            ).localize(
                published_time,
                is_dst=None
            )

            if (self.feed.published_time and
                    self.feed.published_time >= published_time):
                return

            self.feed.published_time = published_time

        for attr in ['title', 'title_detail', 'link']:
            if not hasattr(self.parsed.feed, attr):
                msg = 'refresh_feeds. Feed "%s" has no %s'
                logger.error(msg % (self.feed.source_url, attr))
                self.verbose_print(msg % (self.feed.source_url, attr))
                return

        if self.parsed.feed.title_detail.type == 'text/plain':
            self.feed.title = html.escape(self.parsed.feed.title)[:150]
        else:
            self.feed.title = self.parsed.feed.title[:150]

        self.feed.link = self.feed.link or self.parsed.feed.link

        try:
            if self.parsed.feed.description_detail.type == 'text/plain':
                self.feed.description = html.escape(self.parsed.feed.description)
            else:
                self.feed.description = self.parsed.feed.description
        except:
            pass

        self.feed.last_polled_time = datetime.now()

        self.feed.save()

        self.verbose_print("Feed obj updated %s" % self.feed.last_polled_time)

        return len(self.parsed.entries)

    def create_entries(self):
        self.verbose_print("creating entry")
        count = 0
        for i, entry in enumerate(self.parsed.entries):

            e_id = getattr(entry, 'id', getattr(entry, 'guid', None))
            if self.log_created(e_id):
                self.verbose_print("Already processed")
                continue

            if i > self.max_entries_saved:
                break

            missing_attr = False
            for attr in ['title', 'title_detail', 'link', 'description']:
                if not hasattr(entry, attr):
                    msg = 'Feedcrawler refresh_feeds. Entry "%s" has no %s'
                    logger.error(msg % (entry.link, attr))
                    missing_attr = True
            if missing_attr:
                continue
            if entry.title == "":
                msg = 'Feedcrawler refresh_feeds. Entry "%s" has a blank title'
                logger.warning(msg % (entry.link))
                continue

            if entry.title_detail.type == 'text/plain':
                entry_title = html.escape(entry.title)
            else:
                entry_title = entry.title

            if not entry_title:
                entry_title = ""

            self.verbose_print("will create entry")

            slug = slugify(self.feed.slug + "-" + entry_title[:100])
            exists = self.entry_model.objects.filter(slug=slug).exists()
            if exists:
                slug = str(random.getrandbits(8)) + "-" + slug
                self.verbose_print("Entry slug exists")

            if hasattr(entry, 'published_parsed'):
                published_time = datetime.fromtimestamp(
                    mktime(entry.published_parsed)
                )

                published_time = pytz.timezone(
                    settings.TIME_ZONE
                ).localize(
                    published_time,
                    is_dst=None
                )

                now = datetime.now()

                if published_time.date() > now.date():
                    self.verbose_print("Entry date is > now")
                    self.verbose_print(published_time)
                    self.verbose_print(now)
                    published_time = now
                elif published_time.date() < now.date():
                    self.verbose_print(
                        "Entry time is in the past, skipping: %s - %s"
                        % ( published_time.date(), now.date())
                    )
                    continue

            db_entry, created = self.entry_model.objects.get_or_create(
                entry_feed=self.feed,
                entry_link=entry.link,
                channel=self.feed.channel,
                title=entry_title[:150],
                slug=slug[:150],
                entry_title=entry_title[:150],
                site=self.feed.site,
                user=self.feed.user,
                published=True,
                show_on_root_channel=True
            )

            self.verbose_print("Entry found or created!!!")
            if created:
                if hasattr(entry, 'published_parsed'):
                    db_entry.entry_published_time = published_time

                # Lots of entries are missing description_detail attributes.
                # Escape their content by default
                if hasattr(entry, 'description_detail') and \
                        entry.description_detail.type != 'text/plain':
                    db_entry.entry_description = entry.description
                else:
                    db_entry.entry_description = html.escape(entry.description)

                try:
                    content = None
                    if hasattr(entry, 'content'):
                        content = entry.content
                        if isinstance(content, list) and content:
                            content = entry.content[0]

                    if content and content.type != 'text/plain':
                        db_entry.entry_content = content.value
                    elif hasattr(entry, 'content'):
                        db_entry.entry_content = html.escape(content.value)
                except Exception, e:
                    self.verbose_print(str(e))
                    msg = 'Feedcrawler refresh_feeds. Entry "%s" content error'
                    logger.warning(msg % (entry.link))

                try:
                    allowed = (str, unicode, dict, list, int, float, long)
                    entry_source = json.dumps(
                        {k: v for k, v in entry.iteritems()
                            if isinstance(v, allowed)}
                    )
                    db_entry.entry_json = entry_source
                    pass
                except Exception, e:
                    self.verbose_print(str(e))
                    msg = 'Feedcrawler refresh_feeds. Entry "%s" json error'
                    logger.warning(msg % (entry.link))

                # fill Article properties
                db_entry.headline = db_entry.entry_description
                db_entry.save()
                count += 1

                if self.verbose:
                    self.verbose_print("Entry fully created %s" % db_entry.title)
                    self.record_log(e_id)

                try:
                    self.verbose_print('creating post')
                    self.create_post(db_entry)
                except Exception as e:
                    self.verbose_print(str(e))

        self.verbose_print("%d entries created" % count)
        return count

    def delete_old_entries(self):
        entries = self.entry_model.objects.filter(entry_feed=self.feed)
        entries = entries[self.max_entries_saved:]
        # Cannot use 'limit' or 'offset' with delete.
        for entry in entries:
            entry.delete()

        if self.verbose:
            self.verbose_print("%d entries deleted" % len(entries))

    def process(self):
        # fetch and parse the feed

        self.max_entries_saved = self.feed.max_entries or 1000
        self.verbose_print("fetching")
        if not self.fetch():
            logger.warning("Feed cannot be parsed")
            return 0

        self.verbose_print("updating")
        if not self.update_feed():
            logger.info("No entry returned")
            return 0

        self.verbose_print("creating entries")
        created_count = self.create_entries()
        self.delete_old_entries()
        return created_count

    def get_channel_by_slug(self, slug):
        if not slug:
            return

        try:
            return Channel.objects.filter(long_slug=slug)[0]
        except:
            return

    def create_post(self, entry):
        # match category X channel
        channel_slug = CATEGORY_BRASIL.get(self.feed.source_url)
        channel = self.get_channel_by_slug(channel_slug) or entry.channel
        self.verbose_print(channel_slug)

        slug = slugify(entry.entry_title)
        if Post.objects.filter(channel=channel,
                               slug=slug,
                               site=entry.site).exists():
            slug = str(random.getrandbits(8)) + "-" + slug
            self.verbose_print("Post slug exists")

        post = Post(
            title=entry.entry_title[:150],
            slug=slug[:150],
            content=entry.entry_content or entry.entry_description,
            channel=channel,
            site=entry.site,
            user=entry.user,
            show_on_root_channel=True,
            published=True,
            # hat=entry.hat,
        )

        if self.feed.group:
            post.source = self.feed.group.name

        post.save()

        entry.post_created = True
        entry.save()

        self.verbose_print(u"Post {p.id}- {p.title} - {p.slug} created".format(p=post))

        return post
<|endoftext|>