<commit_before>from __future__ import unicode_literals
import re
import datetime
from django.core.exceptions import ValidationError
from django.db import models
from django.contrib.auth import get_user_model
from django.db.models import Q
from django.utils.translation import ugettext_lazy as _

User = get_user_model()
_DURATION_RX = re.compile(r'^\s*((\d+)w)?\s*((\d+)d)?\s*((\d+)h)?\s*((\d+)m)?\s*((\d+)s)?\s*$', re.IGNORECASE)

SPECIAL_USER_EXPIRER = "__ban_expirer__"
SPECIAL_USER_AUTOBAN = "__ban_autoban__"
SPECIAL_USER_CHECKER = "__ban_checker__"
SPECIAL_USERS = [SPECIAL_USER_AUTOBAN, SPECIAL_USER_EXPIRER, SPECIAL_USER_CHECKER]

def clean_duration(dur):
    """
    Transforms a duration string to a timedelta.
    """

    match = _DURATION_RX.match(dur)
    if match is None:
        raise ValidationError(_('Invalid duration. Expected format: w d <C>h <D>m <E>s '
                                'specifying weeks, days, hours, minutes, seconds. Specifying none'
                                'of them implies an infinite duration'), code="DURATION_STRING")
    interval = []
    for i, g in enumerate(match.groups()):
        if i % 2:
            try:
                interval.append(int(g))
            except (TypeError, ValueError):
                interval.append(0)
    if sum(interval):
        return datetime.timedelta(weeks=interval[0], days=interval[1],
                                  hours=interval[2], minutes=interval[3],
                                  seconds=interval[4])
    else:
        return None

class BanDataManager(models.Manager):
    """
    Erroneous entities are filtered by the manager (i.e. entities with no associated Active/Dead
    Ban entries - or both type of entries associated - are not retrieved).
    """

    def get_query_set(self):
        return super(BanDataManager, self).get_query_set().filter(Q(activeban__isnull=False, deadban__isnull=True) |
                                                                  Q(activeban__isnull=True, deadban__isnull=False))

class BanData(models.Model):
    """
    Ban data: first-level ban data (when was it created, by whom, to whom, reason, duration).
    This exists in it's own entity to track the data and the current state.

    Erroneous entities are filtered by the manager (i.e. entities with no associated Active/Dead
    Ban entries - or both type of entries associated - are not retrieved).
    """

    objects = BanDataManager()

    dictation_on = models.DateTimeField(null=False, editable=False, db_index=True, verbose_name=_("Ban creation date"))
    dictation_reason = models.TextField(null=False, max_length=1024, verbose_name=_("Ban reason"))
    duration = models.CharField(null=False, max_length=255, validators=[clean_duration], verbose_name=_("Ban duration"))
    dictated_to = models.ForeignKey(User, related_name="criminal_history", null=False, editable=False, verbose_name=_("Ban target"))
    dictated_by = models.ForeignKey(User, related_name="official_history", null=False, editable=False,verbose_name=_("Ban dictator"))

    def copy_from(self, ban):
        self.dictation_on = ban.dictation_on
        self.dictation_reason = ban.dictation_reason
        self.duration = ban.duration
        self.dictated_by = ban.dictated_by
        self.dictated_to = ban.dictated_to
        return self

class ActiveBanMixIn(models.Model):
    """
    Activa Ban Mix-in: tracks the ban's start date.
    """

    start_on = models.DateTimeField(null=True, editable=False, db_index=True, verbose_name=_("Ban start date"))

    class Meta:
        abstract = True

class DeadBanMixin(ActiveBanMixIn):
    """
    Dead Ban Mix-in: tracks ending data for the ban (and the active data as well).
    """

    DEATH_CHOICES = (
        ("E", _("Expired")),
        ("F", _("Forgiven")),
        ("R", _("Reverted")),
        ("I", _("Internal Error"))
    )

    death_on = models.DateTimeField(null=False, editable=False, db_index=True, verbose_name=_("Termination time"))
    death_by = models.ForeignKey(User, null=False, editable=False, verbose_name=_("Terminator"))
    death_reason = models.TextField(null=False, max_length=1024, verbose_name=_("Termination reason"))
    death_type = models.CharField(null=False, max_length=1, choices=DEATH_CHOICES, verbose_name=_("Termination type"))

    class Meta:
        abstract = True

class ActiveBan(BanData, ActiveBanMixIn):
    """
    Active Ban - implements the mix-in and links to BanData to get the generic data.
    """

    class Meta:
        verbose_name = _("Active Ban")
        verbose_name_plural = _("Active Bans")

class DeadBan(BanData, DeadBanMixin):
    """
    Dead Ban - implements the mix-in and links to BanData to get the generic data.
    """

    class Meta:
        verbose_name = _("Dead Ban")
        verbose_name_plural = _("Dead Bans")<commit_msg>Removing managers<commit_after>from __future__ import unicode_literals
import re
import datetime
from django.core.exceptions import ValidationError
from django.db import models
from django.contrib.auth import get_user_model
from django.db.models import Q
from django.utils.translation import ugettext_lazy as _

User = get_user_model()
_DURATION_RX = re.compile(r'^\s*((\d+)w)?\s*((\d+)d)?\s*((\d+)h)?\s*((\d+)m)?\s*((\d+)s)?\s*$', re.IGNORECASE)

SPECIAL_USER_EXPIRER = "__ban_expirer__"
SPECIAL_USER_AUTOBAN = "__ban_autoban__"
SPECIAL_USER_CHECKER = "__ban_checker__"
SPECIAL_USERS = [SPECIAL_USER_AUTOBAN, SPECIAL_USER_EXPIRER, SPECIAL_USER_CHECKER]

def clean_duration(dur):
    """
    Transforms a duration string to a timedelta.
    """

    match = _DURATION_RX.match(dur)
    if match is None:
        raise ValidationError(_('Invalid duration. Expected format: w d <C>h <D>m <E>s '
                                'specifying weeks, days, hours, minutes, seconds. Specifying none'
                                'of them implies an infinite duration'), code="DURATION_STRING")
    interval = []
    for i, g in enumerate(match.groups()):
        if i % 2:
            try:
                interval.append(int(g))
            except (TypeError, ValueError):
                interval.append(0)
    if sum(interval):
        return datetime.timedelta(weeks=interval[0], days=interval[1],
                                  hours=interval[2], minutes=interval[3],
                                  seconds=interval[4])
    else:
        return None

class BanDataManager(models.Manager):
    """
    Erroneous entities are filtered by the manager (i.e. entities with no associated Active/Dead
    Ban entries - or both type of entries associated - are not retrieved).
    """

    def get_query_set(self):
        return super(BanDataManager, self).get_query_set().filter(Q(activeban__isnull=False, deadban__isnull=True) |
                                                                  Q(activeban__isnull=True, deadban__isnull=False))

class BanData(models.Model):
    """
    Ban data: first-level ban data (when was it created, by whom, to whom, reason, duration).
    This exists in it's own entity to track the data and the current state.

    Erroneous entities are filtered by the manager (i.e. entities with no associated Active/Dead
    Ban entries - or both type of entries associated - are not retrieved).
    """

    objects = BanDataManager()

    dictation_on = models.DateTimeField(null=False, editable=False, db_index=True, verbose_name=_("Ban creation date"))
    dictation_reason = models.TextField(null=False, max_length=1024, verbose_name=_("Ban reason"))
    duration = models.CharField(null=False, max_length=255, validators=[clean_duration], verbose_name=_("Ban duration"))
    dictated_to = models.ForeignKey(User, related_name="+", null=False, editable=False, verbose_name=_("Ban target"))
    dictated_by = models.ForeignKey(User, related_name="+", null=False, editable=False,verbose_name=_("Ban dictator"))

    def copy_from(self, ban):
        self.dictation_on = ban.dictation_on
        self.dictation_reason = ban.dictation_reason
        self.duration = ban.duration
        self.dictated_by = ban.dictated_by
        self.dictated_to = ban.dictated_to
        return self

class ActiveBanMixIn(models.Model):
    """
    Activa Ban Mix-in: tracks the ban's start date.
    """

    start_on = models.DateTimeField(null=True, editable=False, db_index=True, verbose_name=_("Ban start date"))

    class Meta:
        abstract = True

class DeadBanMixin(ActiveBanMixIn):
    """
    Dead Ban Mix-in: tracks ending data for the ban (and the active data as well).
    """

    DEATH_CHOICES = (
        ("E", _("Expired")),
        ("F", _("Forgiven")),
        ("R", _("Reverted")),
        ("I", _("Internal Error"))
    )

    death_on = models.DateTimeField(null=False, editable=False, db_index=True, verbose_name=_("Termination time"))
    death_by = models.ForeignKey(User, null=False, editable=False, verbose_name=_("Terminator"))
    death_reason = models.TextField(null=False, max_length=1024, verbose_name=_("Termination reason"))
    death_type = models.CharField(null=False, max_length=1, choices=DEATH_CHOICES, verbose_name=_("Termination type"))

    class Meta:
        abstract = True

class ActiveBan(BanData, ActiveBanMixIn):
    """
    Active Ban - implements the mix-in and links to BanData to get the generic data.
    """

    class Meta:
        verbose_name = _("Active Ban")
        verbose_name_plural = _("Active Bans")

class DeadBan(BanData, DeadBanMixin):
    """
    Dead Ban - implements the mix-in and links to BanData to get the generic data.
    """

    class Meta:
        verbose_name = _("Dead Ban")
        verbose_name_plural = _("Dead Bans")<|endoftext|>


<commit_before>import os

from requests import HTTPError

def build_module_list(court_id):
    """Takes a string and builds up a list of modules to import.

    This is a simple recursive function that iteratively looks for __all__
    attributes in packages. If it finds one, it inspects each of the items in
    it to see if each of those has an __all__ attribute. If they lack it, the
    item is added to a list of modules.

    Returns either a list of modules or in the case of errors, an empty list.

    *N.B.*: For the most part, this re-creates the `walk` function from the
    pkgutil library. At some point, it's probably worth replacing this with
    that. Live and learn.

    https://docs.python.org/2/library/pkgutil.html#pkgutil.walk_packages
    """
    module_strings = []

    def find_all_attr_or_punt(court_id):
        """Checks that we have an __all__ attribute. If so, recurses. If not,
        adds the item to our list
        """
        try:
            # Test that we have an __all__ attribute (proving that it's a
            # package) something like: opinions.united_states.federal
            all_attr = __import__(court_id, globals(), locals(), ["*"]).__all__

            # Build the modules back up to full imports
            all_attr = [f"{court_id}.{item}" for item in all_attr]
            for module in all_attr:
                # If we've made it this far, we have an __all__ attribute, so
                # we should see if the items within that attribute do too. And
                # so forth, recursively...
                find_all_attr_or_punt(module)
        except AttributeError:
            # Lacks the __all__ attribute. Probably of the form:
            # juriscraper.opinions.united_states.federal_appellate.ca1,
            # therefore, we add it to our list!
            module_strings.append(court_id)
        # except ImportError as e:
        #    # Something has gone wrong with the import
        #    print("Import error: %s" % e)
        #    return []

    find_all_attr_or_punt(court_id)

    return module_strings

def get_module_by_name(name):
    db_root = os.path.abspath(
        os.path.join(os.path.dirname(__file__), "..", "opinions")
    )
    for dirName, subdirList, fileList in os.walk(db_root):
        for fname in fileList:
            if f"{name}.py" == fname:
                package, module = (
                    dirName.split("/juriscraper/", 1)[1].replace("/", "."),
                    fname[:-3],
                )
                juriscraper_module = __import__(
                    f"juriscraper.{package}.{module}",
                    globals(),
                    locals(),
                    [module],
                )
                return juriscraper_module.Site()

def site_yielder(iterable, mod):

    site = mod.Site()
    for i in iterable:
        try:
            site._download_backwards(i)
            yield site
        except HTTPError as e:
            continue
<commit_msg>refactor(importer): Make backscrapers generate new sites every iterable<commit_after>import os

from requests import HTTPError

def build_module_list(court_id):
    """Takes a string and builds up a list of modules to import.

    This is a simple recursive function that iteratively looks for __all__
    attributes in packages. If it finds one, it inspects each of the items in
    it to see if each of those has an __all__ attribute. If they lack it, the
    item is added to a list of modules.

    Returns either a list of modules or in the case of errors, an empty list.

    *N.B.*: For the most part, this re-creates the `walk` function from the
    pkgutil library. At some point, it's probably worth replacing this with
    that. Live and learn.

    https://docs.python.org/2/library/pkgutil.html#pkgutil.walk_packages
    """
    module_strings = []

    def find_all_attr_or_punt(court_id):
        """Checks that we have an __all__ attribute. If so, recurses. If not,
        adds the item to our list
        """
        try:
            # Test that we have an __all__ attribute (proving that it's a
            # package) something like: opinions.united_states.federal
            all_attr = __import__(court_id, globals(), locals(), ["*"]).__all__

            # Build the modules back up to full imports
            all_attr = [f"{court_id}.{item}" for item in all_attr]
            for module in all_attr:
                # If we've made it this far, we have an __all__ attribute, so
                # we should see if the items within that attribute do too. And
                # so forth, recursively...
                find_all_attr_or_punt(module)
        except AttributeError:
            # Lacks the __all__ attribute. Probably of the form:
            # juriscraper.opinions.united_states.federal_appellate.ca1,
            # therefore, we add it to our list!
            module_strings.append(court_id)
        # except ImportError as e:
        #    # Something has gone wrong with the import
        #    print("Import error: %s" % e)
        #    return []

    find_all_attr_or_punt(court_id)

    return module_strings

def get_module_by_name(name):
    db_root = os.path.abspath(
        os.path.join(os.path.dirname(__file__), "..", "opinions")
    )
    for dirName, subdirList, fileList in os.walk(db_root):
        for fname in fileList:
            if f"{name}.py" == fname:
                package, module = (
                    dirName.split("/juriscraper/", 1)[1].replace("/", "."),
                    fname[:-3],
                )
                juriscraper_module = __import__(
                    f"juriscraper.{package}.{module}",
                    globals(),
                    locals(),
                    [module],
                )
                return juriscraper_module.Site()

def site_yielder(iterable, mod):

    for i in iterable:
        site = mod.Site()
        try:
            site._download_backwards(i)
            yield site
        except HTTPError as e:
            continue
<|endoftext|>


<commit_before>import StringIO
import datetime
import io
import json
import logging
import re
import requests
import xlwt
from hashlib import md5
from optparse import make_option
from pyxform import xls2json_backends

from django.conf import settings
from django.contrib.auth.models import User
from django.core.exceptions import ImproperlyConfigured
from django.core.files.storage import get_storage_class
from django.core.management.base import BaseCommand
from django.db import models, transaction
from rest_framework.authtoken.models import Token

from formpack.utils.xls_to_ss_structure import xls_to_dicts
from hub.models import FormBuilderPreference
from ...deployment_backends.kobocat_backend import KobocatDeploymentBackend
from ...models import Asset
from .import_survey_drafts_from_dkobo import _set_auto_field_update

TIMESTAMP_DIFFERENCE_TOLERANCE = datetime.timedelta(seconds=30)

class SyncKCXFormsError(Exception):
    pass

class SyncKCXFormsWarning(Exception):
    pass

def _add_contents_to_sheet(sheet, contents):
    ''' Copied from dkobo/koboform/pyxform_utils.py '''
    cols = []
    for row in contents:
        for key in row.keys():
            if key not in cols:
                cols.append(key)
    for ci, col in enumerate(cols):
        sheet.write(0, ci, col)
    for ri, row in enumerate(contents):
        for ci, col in enumerate(cols):
            val = row.get(col, None)
            if val:
                sheet.write(ri+1, ci, val)

def _convert_dict_to_xls(ss_dict):
    ''' Copied from dkobo/koboform/pyxform_utils.py '''
    workbook = xlwt.Workbook()
    for sheet_name in ss_dict.keys():
        # pyxform.xls2json_backends adds "_header" items for each sheet.....
        if not re.match(r".*_header$", sheet_name):
            # Sheets with empty names are rejected by xlwt; omit them
            if not sheet_name:
                continue
            cur_sheet = workbook.add_sheet(sheet_name)
            _add_contents_to_sheet(cur_sheet, ss_dict[sheet_name])
    string_io = StringIO.StringIO()
    workbook.save(string_io)
    string_io.seek(0)
    return string_io

def _xlsform_to_kpi_content_schema(xlsform):
    '''
    parses xlsform structure into json representation
    of spreadsheet structure.
    '''
    content = xls_to_dicts(xlsform)
    # Remove the __version__ calculate question
    content['survey'] = [
        row for row in content['survey'] if not (
            'calculation' in row and row.get('type', None) == 'calculate' and
            row.get('name', None) == '__version__'
        )
    ]
    # a temporary fix to the problem of list_name alias
    # credit to @dorey
    return json.loads(re.sub('list name', 'list_name',
                  json.dumps(content, indent=4)))

def _kc_forms_api_request(token, xform_pk, xlsform=False):
    ''' Returns a `Response` object '''
    url = '{}/api/v1/forms/{}'.format(
        settings.KOBOCAT_INTERNAL_URL, xform_pk)
    if xlsform:
        url += '/form.xls'
    headers = {u'Authorization':'Token ' + token.key}
    return requests.get(url, headers=headers)

def _make_name_for_asset(asset, xform):
    desired_name = xform.title
    other_assets = Asset.objects.filter(owner=asset.owner)
    if asset.pk:
        other_assets.exclude(pk=asset.pk)
    if other_assets.filter(name=desired_name).exists():
        # The user already has an asset with this name. Append
        # `xform.id_string` in parentheses for clarification
        if desired_name and len(desired_name.strip()):
            desired_name = u'{} ({})'.format(
                desired_name, xform.id_string)
        else:
            desired_name = xform.id_string
    return desired_name

def _xform_to_asset_content(xform):
    # Load the xlsform from the KC API to avoid having to deal
    # with S3 credentials, etc.
    user = xform.user
    response = _kc_forms_api_request(user.auth_token, xform.pk, xlsform=True)
    if response.status_code == 404:
        raise SyncKCXFormsWarning([
            user.username,
            xform.id_string,
            u'unable to load xls ({})'.format(response.status_code)
        ])
    elif response.status_code != 200:
        raise SyncKCXFormsError([
            user.username,
            xform.id_string,
            u'unable to load xls ({})'.format(response.status_code)
        ])
    # Convert the xlsform to KPI JSON
    xls_io = io.BytesIO(response.content)
    if xform.xls.name.endswith('.csv'):
        dict_repr = xls2json_backends.csv_to_dict(xls_io)
        xls_io = _convert_dict_to_xls(dict_repr)
    asset_content = _xlsform_to_kpi_content_schema(xls_io)
    return asset_content

def _get_kc_backend_response(xform):
    # Get the form data from KC
    user = xform.user
    response = _kc_forms_api_request(user.auth_token, xform.pk)
    if response.status_code == 404:
        raise SyncKCXFormsWarning([
            user.username,
            xform.id_string,
            'unable to load form data ({})'.format(response.status_code)
        ])
    elif response.status_code != 200:
        raise SyncKCXFormsError([
            user.username,
            xform.id_string,
            'unable to load form data ({})'.format(response.status_code)
        ])
    backend_response = response.json()
    return backend_response

def _sync_form_content(asset, xform, changes):
    ''' Returns `True` and appends to `changes` if it modifies `asset`; does
    not save anything '''
    if not asset.has_deployment:
        # A brand-new asset
        asset.content = _xform_to_asset_content(xform)
        asset.date_created = xform.date_created
        asset.date_modified = xform.date_modified
        changes.append('CREATE CONTENT')
        return True

    modified = False
    # First, compare hashes to see if the KC form content
    # has changed since the last deployment
    backend_response = asset._deployment_data['backend_response']
    if 'hash' in backend_response:
        if backend_response['hash'] != xform.prefixed_hash:
            asset.content = _xform_to_asset_content(xform)
            asset.date_modified = xform.date_modified
            modified = True
            changes.append('UPDATE')
    else:
        # KC's `date_modified` is nearly useless (see
        # https://github.com/kobotoolbox/kpi/issues/661#issuecomment-218073765).
        # Still, in cases where KPI does not yet know the hash, comparing
        # timestamps can sometimes avoid creating duplicate asset versions
        time_diff = xform.date_modified - asset.date_modified
        # If KC timestamp is close enough to the KPI timestamp, we assume the
        # KC form content was not updated since the last KPI deployment
        if time_diff <= TIMESTAMP_DIFFERENCE_TOLERANCE:
            # We don't need an update, but we should copy the hash from KC to
            # KPI for future reference
            backend_response['hash'] = xform.prefixed_hash
            modified = True
            changes.append('HASH')
        else:
            asset.content = _xform_to_asset_content(xform)
            asset.date_modified = xform.date_modified
            modified = True
            changes.append('UPDATE')
    return modified

def _sync_form_metadata(asset, xform, changes):
    ''' Returns `True` and appends to `changes` if it modifies `asset`; does
    not save anything '''
    user = xform.user
    if not asset.has_deployment:
        # A brand-new asset
        asset.date_created = xform.date_created
        kc_deployment = KobocatDeploymentBackend(asset)
        kc_deployment.store_data({
            'backend': 'kobocat',
            'identifier': KobocatDeploymentBackend.make_identifier(
                user.username, xform.id_string),
            'active': xform.downloadable,
            'backend_response': _get_kc_backend_response(xform),
            'version': asset.version_id
        })
        changes.append('CREATE METADATA')
        return True

    modified = False
    fetch_backend_response = False
    deployment_data = asset._deployment_data
    backend_response = deployment_data['backend_response']

    if (
            deployment_data['active'] != xform.downloadable or
            backend_response['downloadable'] != xform.downloadable
    ):
        deployment_data['active'] = xform.downloadable
        modified = True
        fetch_backend_response = True
        changes.append('ACTIVE')

    if settings.KOBOCAT_URL not in deployment_data['identifier']:
        # Issue #1122
        deployment_data[
            'identifier'] = KobocatDeploymentBackend.make_identifier(
                user.username, xform.id_string)
        fetch_backend_response = True
        modified = True
        changes.append('IDENTIFIER')

    # Check to see if the asset name matches the xform title. Per #857, the
    # xform title takes priority.  The first check is a cheap one:
    if asset.name != xform.title:
        # Now do a full check of the name
        desired_name = _make_name_for_asset(asset, xform)
        if asset.name != desired_name:
            asset.name = desired_name
            modified = True
            changes.append('NAME')

    if fetch_backend_response:
        deployment_data[
            'backend_response'] = _get_kc_backend_response(xform)
        modified = True

    return modified

class XForm(models.Model):
    ''' A stripped-down version of `onadata.apps.logger.models.XForm`, included
    here so that we can access KC's data '''
    class Meta:
        app_label = 'logger'
    XFORM_TITLE_LENGTH = 255
    MAX_ID_LENGTH = 100
    xls = models.FileField(null=True)
    xml = models.TextField()
    user = models.ForeignKey(User, related_name='xforms', null=True)
    downloadable = models.BooleanField(default=True)
    id_string = models.SlugField(
        editable=False,
        max_length=MAX_ID_LENGTH
    )
    title = models.CharField(editable=False, max_length=XFORM_TITLE_LENGTH)
    date_created = models.DateTimeField(auto_now_add=True)
    date_modified = models.DateTimeField(auto_now=True)
    uuid = models.CharField(max_length=32, default=u'')

    @property
    def hash(self):
        return u'%s' % md5(self.xml.encode('utf8')).hexdigest()

    @property
    def prefixed_hash(self):
        ''' Matches what's returned by the KC API '''
        return u"md5:%s" % self.hash

class Command(BaseCommand):
    option_list = BaseCommand.option_list + (
        make_option('--all-users',
                    action='store_true',
                    dest='all_users',
                    default=False,
                    help='Import even when the user does not prefer KPI'),
        make_option('--username',
                    action='store',
                    dest='username',
                    default=False,
                    help="Import only a specific user's forms"),
        make_option('--quiet',
                    action='store_true',
                    dest='quiet',
                    default=False,
                    help='Do not output status messages'),
    )

    def _print_str(self, string):
        if not self._quiet:
            print string

    def _print_tabular(self, *args):
        self._print_str(u'\t'.join(map(lambda x: u'{}'.format(x), args)))

    def handle(self, *args, **options):
        if not settings.KOBOCAT_URL or not settings.KOBOCAT_INTERNAL_URL:
            raise ImproperlyConfigured(
                'Both KOBOCAT_URL and KOBOCAT_INTERNAL_URL must be '
                'configured before using this command'
            )
        self._quiet = options.get('quiet')
        users = User.objects.all()
        self._print_str('%d total users' % users.count())
        # A specific user or everyone?
        if options.get('username'):
            users = User.objects.filter(username=options.get('username'))
        self._print_str('%d users selected' % users.count())
        # Only users who prefer KPI or all users?
        if not options.get('all_users'):
            users = users.filter(
                models.Q(formbuilderpreference__preferred_builder=
                    FormBuilderPreference.KPI) |
                models.Q(formbuilderpreference=None) # KPI is the default now
            )
            self._print_str('%d of selected users prefer KPI' % users.count())

        # We'll be copying the date fields from KC, so don't auto-update them
        _set_auto_field_update(Asset, "date_created", False)
        _set_auto_field_update(Asset, "date_modified", False)

        for user in users:
            # Make sure the user has a token for access to KC's API
            Token.objects.get_or_create(user=user)

            existing_surveys = user.assets.filter(asset_type='survey')

            # Each asset that the user has already deployed to KC should have a
            # form uuid stored in its deployment data
            xform_uuids_to_asset_pks = {}
            for existing_survey in existing_surveys:
                dd = existing_survey._deployment_data
                try:
                    backend_response = dd['backend_response']
                except KeyError:
                    continue
                xform_uuids_to_asset_pks[backend_response['uuid']] = \
                    existing_survey.pk

            # Use our stub model to access KC's XForm objects
            xforms = user.xforms.all()
            for xform in xforms:
                with transaction.atomic():
                    if xform.uuid not in xform_uuids_to_asset_pks:
                        # This is an orphaned KC form. Build a new asset to
                        # match
                        asset = Asset(asset_type='survey', owner=user)
                        asset.name = _make_name_for_asset(asset, xform)
                    else:
                        asset = Asset.objects.get(
                            pk=xform_uuids_to_asset_pks[xform.uuid])

                    changes = []
                    try:
                        content_changed = _sync_form_content(
                            asset, xform, changes)
                        metadata_changed = _sync_form_metadata(
                            asset, xform, changes)
                    except SyncKCXFormsWarning as e:
                        error_information = [
                            'WARN',
                            user.username,
                            xform.id_string,
                            e.message
                        ]
                        self._print_tabular(*error_information)
                    except SyncKCXFormsError as e:
                        error_information = [
                            'FAIL',
                            user.username,
                            xform.id_string,
                            e.message
                        ]
                        self._print_tabular(*error_information)
                        logging.exception(u'sync_kobocat_xforms: {}'.format(
                            u', '.join(error_information)))

                    if content_changed or metadata_changed:
                        asset.save(adjust_content=False)
                        if content_changed:
                            asset._mark_latest_version_as_deployed()
                        self._print_tabular(
                            ','.join(changes),
                            user.username,
                            xform.id_string,
                            asset.uid
                        )
                    else:
                        self._print_tabular(
                            'NOOP',
                            user.username,
                            xform.id_string,
                            asset.uid
                        )

        _set_auto_field_update(Asset, "date_created", True)
        _set_auto_field_update(Asset, "date_modified", True)
<commit_msg>Fix error handling mistake<commit_after>import StringIO
import datetime
import io
import json
import logging
import re
import requests
import xlwt
from hashlib import md5
from optparse import make_option
from pyxform import xls2json_backends

from django.conf import settings
from django.contrib.auth.models import User
from django.core.exceptions import ImproperlyConfigured
from django.core.files.storage import get_storage_class
from django.core.management.base import BaseCommand
from django.db import models, transaction
from rest_framework.authtoken.models import Token

from formpack.utils.xls_to_ss_structure import xls_to_dicts
from hub.models import FormBuilderPreference
from ...deployment_backends.kobocat_backend import KobocatDeploymentBackend
from ...models import Asset
from .import_survey_drafts_from_dkobo import _set_auto_field_update

TIMESTAMP_DIFFERENCE_TOLERANCE = datetime.timedelta(seconds=30)

class SyncKCXFormsError(Exception):
    pass

class SyncKCXFormsWarning(Exception):
    pass

def _add_contents_to_sheet(sheet, contents):
    ''' Copied from dkobo/koboform/pyxform_utils.py '''
    cols = []
    for row in contents:
        for key in row.keys():
            if key not in cols:
                cols.append(key)
    for ci, col in enumerate(cols):
        sheet.write(0, ci, col)
    for ri, row in enumerate(contents):
        for ci, col in enumerate(cols):
            val = row.get(col, None)
            if val:
                sheet.write(ri+1, ci, val)

def _convert_dict_to_xls(ss_dict):
    ''' Copied from dkobo/koboform/pyxform_utils.py '''
    workbook = xlwt.Workbook()
    for sheet_name in ss_dict.keys():
        # pyxform.xls2json_backends adds "_header" items for each sheet.....
        if not re.match(r".*_header$", sheet_name):
            # Sheets with empty names are rejected by xlwt; omit them
            if not sheet_name:
                continue
            cur_sheet = workbook.add_sheet(sheet_name)
            _add_contents_to_sheet(cur_sheet, ss_dict[sheet_name])
    string_io = StringIO.StringIO()
    workbook.save(string_io)
    string_io.seek(0)
    return string_io

def _xlsform_to_kpi_content_schema(xlsform):
    '''
    parses xlsform structure into json representation
    of spreadsheet structure.
    '''
    content = xls_to_dicts(xlsform)
    # Remove the __version__ calculate question
    content['survey'] = [
        row for row in content['survey'] if not (
            'calculation' in row and row.get('type', None) == 'calculate' and
            row.get('name', None) == '__version__'
        )
    ]
    # a temporary fix to the problem of list_name alias
    # credit to @dorey
    return json.loads(re.sub('list name', 'list_name',
                  json.dumps(content, indent=4)))

def _kc_forms_api_request(token, xform_pk, xlsform=False):
    ''' Returns a `Response` object '''
    url = '{}/api/v1/forms/{}'.format(
        settings.KOBOCAT_INTERNAL_URL, xform_pk)
    if xlsform:
        url += '/form.xls'
    headers = {u'Authorization':'Token ' + token.key}
    return requests.get(url, headers=headers)

def _make_name_for_asset(asset, xform):
    desired_name = xform.title
    other_assets = Asset.objects.filter(owner=asset.owner)
    if asset.pk:
        other_assets.exclude(pk=asset.pk)
    if other_assets.filter(name=desired_name).exists():
        # The user already has an asset with this name. Append
        # `xform.id_string` in parentheses for clarification
        if desired_name and len(desired_name.strip()):
            desired_name = u'{} ({})'.format(
                desired_name, xform.id_string)
        else:
            desired_name = xform.id_string
    return desired_name

def _xform_to_asset_content(xform):
    # Load the xlsform from the KC API to avoid having to deal
    # with S3 credentials, etc.
    user = xform.user
    response = _kc_forms_api_request(user.auth_token, xform.pk, xlsform=True)
    if response.status_code == 404:
        raise SyncKCXFormsWarning(
            u'unable to load xls ({})'.format(response.status_code)
        )
    elif response.status_code != 200:
        raise SyncKCXFormsError(
            u'unable to load xls ({})'.format(response.status_code)
        )
    # Convert the xlsform to KPI JSON
    xls_io = io.BytesIO(response.content)
    if xform.xls.name.endswith('.csv'):
        dict_repr = xls2json_backends.csv_to_dict(xls_io)
        xls_io = _convert_dict_to_xls(dict_repr)
    asset_content = _xlsform_to_kpi_content_schema(xls_io)
    return asset_content

def _get_kc_backend_response(xform):
    # Get the form data from KC
    user = xform.user
    response = _kc_forms_api_request(user.auth_token, xform.pk)
    if response.status_code == 404:
        raise SyncKCXFormsWarning([
            user.username,
            xform.id_string,
            'unable to load form data ({})'.format(response.status_code)
        ])
    elif response.status_code != 200:
        raise SyncKCXFormsError([
            user.username,
            xform.id_string,
            'unable to load form data ({})'.format(response.status_code)
        ])
    backend_response = response.json()
    return backend_response

def _sync_form_content(asset, xform, changes):
    ''' Returns `True` and appends to `changes` if it modifies `asset`; does
    not save anything '''
    if not asset.has_deployment:
        # A brand-new asset
        asset.content = _xform_to_asset_content(xform)
        asset.date_created = xform.date_created
        asset.date_modified = xform.date_modified
        changes.append('CREATE CONTENT')
        return True

    modified = False
    # First, compare hashes to see if the KC form content
    # has changed since the last deployment
    backend_response = asset._deployment_data['backend_response']
    if 'hash' in backend_response:
        if backend_response['hash'] != xform.prefixed_hash:
            asset.content = _xform_to_asset_content(xform)
            asset.date_modified = xform.date_modified
            modified = True
            changes.append('UPDATE')
    else:
        # KC's `date_modified` is nearly useless (see
        # https://github.com/kobotoolbox/kpi/issues/661#issuecomment-218073765).
        # Still, in cases where KPI does not yet know the hash, comparing
        # timestamps can sometimes avoid creating duplicate asset versions
        time_diff = xform.date_modified - asset.date_modified
        # If KC timestamp is close enough to the KPI timestamp, we assume the
        # KC form content was not updated since the last KPI deployment
        if time_diff <= TIMESTAMP_DIFFERENCE_TOLERANCE:
            # We don't need an update, but we should copy the hash from KC to
            # KPI for future reference
            backend_response['hash'] = xform.prefixed_hash
            modified = True
            changes.append('HASH')
        else:
            asset.content = _xform_to_asset_content(xform)
            asset.date_modified = xform.date_modified
            modified = True
            changes.append('UPDATE')
    return modified

def _sync_form_metadata(asset, xform, changes):
    ''' Returns `True` and appends to `changes` if it modifies `asset`; does
    not save anything '''
    user = xform.user
    if not asset.has_deployment:
        # A brand-new asset
        asset.date_created = xform.date_created
        kc_deployment = KobocatDeploymentBackend(asset)
        kc_deployment.store_data({
            'backend': 'kobocat',
            'identifier': KobocatDeploymentBackend.make_identifier(
                user.username, xform.id_string),
            'active': xform.downloadable,
            'backend_response': _get_kc_backend_response(xform),
            'version': asset.version_id
        })
        changes.append('CREATE METADATA')
        return True

    modified = False
    fetch_backend_response = False
    deployment_data = asset._deployment_data
    backend_response = deployment_data['backend_response']

    if (
            deployment_data['active'] != xform.downloadable or
            backend_response['downloadable'] != xform.downloadable
    ):
        deployment_data['active'] = xform.downloadable
        modified = True
        fetch_backend_response = True
        changes.append('ACTIVE')

    if settings.KOBOCAT_URL not in deployment_data['identifier']:
        # Issue #1122
        deployment_data[
            'identifier'] = KobocatDeploymentBackend.make_identifier(
                user.username, xform.id_string)
        fetch_backend_response = True
        modified = True
        changes.append('IDENTIFIER')

    # Check to see if the asset name matches the xform title. Per #857, the
    # xform title takes priority.  The first check is a cheap one:
    if asset.name != xform.title:
        # Now do a full check of the name
        desired_name = _make_name_for_asset(asset, xform)
        if asset.name != desired_name:
            asset.name = desired_name
            modified = True
            changes.append('NAME')

    if fetch_backend_response:
        deployment_data[
            'backend_response'] = _get_kc_backend_response(xform)
        modified = True

    return modified

class XForm(models.Model):
    ''' A stripped-down version of `onadata.apps.logger.models.XForm`, included
    here so that we can access KC's data '''
    class Meta:
        app_label = 'logger'
    XFORM_TITLE_LENGTH = 255
    MAX_ID_LENGTH = 100
    xls = models.FileField(null=True)
    xml = models.TextField()
    user = models.ForeignKey(User, related_name='xforms', null=True)
    downloadable = models.BooleanField(default=True)
    id_string = models.SlugField(
        editable=False,
        max_length=MAX_ID_LENGTH
    )
    title = models.CharField(editable=False, max_length=XFORM_TITLE_LENGTH)
    date_created = models.DateTimeField(auto_now_add=True)
    date_modified = models.DateTimeField(auto_now=True)
    uuid = models.CharField(max_length=32, default=u'')

    @property
    def hash(self):
        return u'%s' % md5(self.xml.encode('utf8')).hexdigest()

    @property
    def prefixed_hash(self):
        ''' Matches what's returned by the KC API '''
        return u"md5:%s" % self.hash

class Command(BaseCommand):
    option_list = BaseCommand.option_list + (
        make_option('--all-users',
                    action='store_true',
                    dest='all_users',
                    default=False,
                    help='Import even when the user does not prefer KPI'),
        make_option('--username',
                    action='store',
                    dest='username',
                    default=False,
                    help="Import only a specific user's forms"),
        make_option('--quiet',
                    action='store_true',
                    dest='quiet',
                    default=False,
                    help='Do not output status messages'),
    )

    def _print_str(self, string):
        if not self._quiet:
            print string

    def _print_tabular(self, *args):
        self._print_str(u'\t'.join(map(lambda x: u'{}'.format(x), args)))

    def handle(self, *args, **options):
        if not settings.KOBOCAT_URL or not settings.KOBOCAT_INTERNAL_URL:
            raise ImproperlyConfigured(
                'Both KOBOCAT_URL and KOBOCAT_INTERNAL_URL must be '
                'configured before using this command'
            )
        self._quiet = options.get('quiet')
        users = User.objects.all()
        self._print_str('%d total users' % users.count())
        # A specific user or everyone?
        if options.get('username'):
            users = User.objects.filter(username=options.get('username'))
        self._print_str('%d users selected' % users.count())
        # Only users who prefer KPI or all users?
        if not options.get('all_users'):
            users = users.filter(
                models.Q(formbuilderpreference__preferred_builder=
                    FormBuilderPreference.KPI) |
                models.Q(formbuilderpreference=None) # KPI is the default now
            )
            self._print_str('%d of selected users prefer KPI' % users.count())

        # We'll be copying the date fields from KC, so don't auto-update them
        _set_auto_field_update(Asset, "date_created", False)
        _set_auto_field_update(Asset, "date_modified", False)

        for user in users:
            # Make sure the user has a token for access to KC's API
            Token.objects.get_or_create(user=user)

            existing_surveys = user.assets.filter(asset_type='survey')

            # Each asset that the user has already deployed to KC should have a
            # form uuid stored in its deployment data
            xform_uuids_to_asset_pks = {}
            for existing_survey in existing_surveys:
                dd = existing_survey._deployment_data
                try:
                    backend_response = dd['backend_response']
                except KeyError:
                    continue
                xform_uuids_to_asset_pks[backend_response['uuid']] = \
                    existing_survey.pk

            # Use our stub model to access KC's XForm objects
            xforms = user.xforms.all()
            for xform in xforms:
                with transaction.atomic():
                    if xform.uuid not in xform_uuids_to_asset_pks:
                        # This is an orphaned KC form. Build a new asset to
                        # match
                        asset = Asset(asset_type='survey', owner=user)
                        asset.name = _make_name_for_asset(asset, xform)
                    else:
                        asset = Asset.objects.get(
                            pk=xform_uuids_to_asset_pks[xform.uuid])

                    changes = []
                    try:
                        content_changed = _sync_form_content(
                            asset, xform, changes)
                        metadata_changed = _sync_form_metadata(
                            asset, xform, changes)
                    except SyncKCXFormsWarning as e:
                        error_information = [
                            'WARN',
                            user.username,
                            xform.id_string,
                            e.message
                        ]
                        self._print_tabular(*error_information)
                        continue
                    except SyncKCXFormsError as e:
                        error_information = [
                            'FAIL',
                            user.username,
                            xform.id_string,
                            e.message
                        ]
                        self._print_tabular(*error_information)
                        logging.exception(u'sync_kobocat_xforms: {}'.format(
                            u', '.join(error_information)))
                        continue

                    if content_changed or metadata_changed:
                        asset.save(adjust_content=False)
                        if content_changed:
                            asset._mark_latest_version_as_deployed()
                        self._print_tabular(
                            ','.join(changes),
                            user.username,
                            xform.id_string,
                            asset.uid
                        )
                    else:
                        self._print_tabular(
                            'NOOP',
                            user.username,
                            xform.id_string,
                            asset.uid
                        )

        _set_auto_field_update(Asset, "date_created", True)
        _set_auto_field_update(Asset, "date_modified", True)
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""  This script can be registered as a cron job on the development server.
The idea is to have the test battery executed automatically if the CPU load
allows.
"""

# standard library
import socket
import os

# Specify request
HOURS, NOTIFICATION = 12, True

# Get some basic information about the system.
HOSTNAME = socket.gethostname()
LOADAVG = os.getloadavg()[2]

PACKAGE_DIR = os.path.dirname(os.path.realpath(__file__))
PACKAGE_DIR = PACKAGE_DIR.replace('/tools', '')

# Move into the testing directory
os.chdir(PACKAGE_DIR + '/development/testing')

# Execute script on the development server.
if HOSTNAME == 'zeus':
    is_available = LOADAVG < 2.5

    if is_available:
        cmd = 'python run.py --compile --hours ' + str(HOURS)
        if NOTIFICATION:
            cmd += '  --notification'

        os.system(cmd)

<commit_msg>edits<commit_after>#!/usr/bin/env python
"""  This script can be registered as a cron job on the development server.
The idea is to have the test battery executed automatically if the CPU load
allows.
"""

# standard library
import socket
import os

# Specify request
HOURS, NOTIFICATION = 6, True

# Get some basic information about the system.
HOSTNAME = socket.gethostname()
LOADAVG = os.getloadavg()[2]

PACKAGE_DIR = os.path.dirname(os.path.realpath(__file__))
PACKAGE_DIR = PACKAGE_DIR.replace('/tools', '')

# Move into the testing directory
os.chdir(PACKAGE_DIR + '/development/testing')

# Execute script on the development server.
if HOSTNAME == 'zeus':
    is_available = LOADAVG < 2.5

    if is_available:
        cmd = 'python run.py --compile --hours ' + str(HOURS)
        if NOTIFICATION:
            cmd += '  --notification'

        os.system(cmd)

<|endoftext|>


<commit_before>import numpy as np
from transformations import euler_matrix

# defines calibration parameters for the Q50 setup.

# IMU orientation -> x forward, y out the left door, z up
# to convert from WGS84, roll, pitch, azimuth to IMU based cartesian coordinate sysem use:

# Lidar orientation -> x forward, y out the left door, z up

# camera orientation -> todo

##### LIDAR to IMU calibration parameters #####

def GetQ50LidarParams():
    params = { }
    params['R_from_i_to_l'] = euler_matrix(0.02, -0.0101, -0.016)[0:3,0:3]
    params['T_from_l_to_i'] = np.eye(4)
    params['T_from_l_to_i'][0:3,0:3] = params['R_from_i_to_l'].transpose()

    return params

##### LIDAR to Camera Calibration parameters #####

def GetQ50CameraParams():
    cam = [{}, {}]
    for i in [0, 1]:
        cam[i]['R_to_c_from_i'] = np.array([[-1, 0, 0],
                                         [0, 0, -1],
                                         [0, -1, 0]])

        if i == 0:
            R_to_c_from_l_in_camera_frame = euler_matrix(0.042, 0.0065, -0.002)[0:3,0:3]
            cam[i]['R_to_c_from_l_in_camera_frame'] = R_to_c_from_l_in_camera_frame
            cam[i]['displacement_from_l_to_c_in_lidar_frame'] = np.array([0.0, -0.33, 0.265]);
            cam[i]['fx'] = 2254.76 # these parameters for this camera are not updated
            cam[i]['fy'] = 2266.30
            cam[i]['cu'] = 655.55
            cam[i]['cv'] = 488.85

        elif i == 1:
            R_to_c_from_l_in_camera_frame = euler_matrix(0.0645,0.0236,0.0115)[0:3,0:3]
            cam[i]['R_to_c_from_l_in_camera_frame'] = R_to_c_from_l_in_camera_frame
            cam[i]['displacement_from_l_to_c_in_lidar_frame'] = np.array([-0.5,0.2875,0.34]);

            cam[i]['fx'] = 2254.76
            cam[i]['fy'] = 2266.30
            cam[i]['cu'] = 655.55
            cam[i]['cv'] = 488.85
            cam[i]['distort'] = np.array([-0.22146000368016028, 0.7987879799679538, -6.542034918087567e-05, 2.8680581938024014e-05, 0.0])

        cam[i]['KK'] = np.array([[cam[i]['fx'], 0.0, cam[i]['cu']],
                              [0.0, cam[i]['fy'], cam[i]['cv']],
                              [0.0, 0.0, 1.0]])
        cam[i]['f'] = (cam[i]['fx'] + cam[i]['fy']) / 2

    return cam

def GetQ50RadarParams():
    params = { }
    params['R_from_r_to_l'] = euler_matrix(0, 0, -.015)[0:3,0:3]
    params['T_from_r_to_l'] = [3.17, 0.4, -1.64]

    return params

def GetQ50Params():
    params = { }
    params['lidar'] = GetQ50LidarParams();
    params['cam'] = GetQ50CameraParams();
    params['radar'] = GetQ50RadarParams();
    return params
<commit_msg>bug fix: updating parameters to use appropriate intrinsics<commit_after>import numpy as np
from transformations import euler_matrix

# defines calibration parameters for the Q50 setup.

# IMU orientation -> x forward, y out the left door, z up
# to convert from WGS84, roll, pitch, azimuth to IMU based cartesian coordinate sysem use:

# Lidar orientation -> x forward, y out the left door, z up

# camera orientation -> todo

##### LIDAR to IMU calibration parameters #####

def GetQ50LidarParams():
    params = { }
    params['R_from_i_to_l'] = euler_matrix(-0.04, -0.0146, -0.0165)[0:3,0:3]
    params['T_from_l_to_i'] = np.eye(4)
    params['T_from_l_to_i'][0:3,0:3] = params['R_from_i_to_l'].transpose()

    return params

##### LIDAR to Camera Calibration parameters #####

def GetQ50CameraParams():
    cam = [{}, {}]
    for i in [0, 1]:
        cam[i]['R_to_c_from_i'] = np.array([[-1, 0, 0],
                                         [0, 0, -1],
                                         [0, -1, 0]])

        if i == 0: # left camera

            R_to_c_from_l_in_camera_frame = euler_matrix(0.044,0.0291,0.0115)[0:3,0:3]
            cam[i]['R_to_c_from_l_in_camera_frame'] = R_to_c_from_l_in_camera_frame
            cam[i]['displacement_from_l_to_c_in_lidar_frame'] = np.array([-0.5,0.31,0.34]);
            cam[i]['fx'] = 2254.76 # these parameters for this camera are not updated
            cam[i]['fy'] = 2266.30
            cam[i]['cu'] = 655.55
            cam[i]['cv'] = 488.85
            cam[i]['distort'] = np.array([-0.22146000368016028, 0.7987879799679538, -6.542034918087567e-05, 2.8680581938024014e-05, 0.0])

        elif i == 1: # right camera
            R_to_c_from_l_in_camera_frame = euler_matrix(0.044,0.0291,0.0115)[0:3,0:3]
            cam[i]['R_to_c_from_l_in_camera_frame'] = R_to_c_from_l_in_camera_frame
            cam[i]['displacement_from_l_to_c_in_lidar_frame'] = np.array([-0.5,0.31,0.34]);

            cam[i]['fx'] = 2250.72
            cam[i]['fy'] = 2263.75
            cam[i]['cu'] = 648.95
            cam[i]['cv'] = 450.24
            cam[i]['distort'] = np.array([-0.16879238412882028, 0.11971166628565273, -0.0017457365846050555, 0.0001853749033525837, 0.0])

        cam[i]['KK'] = np.array([[cam[i]['fx'], 0.0, cam[i]['cu']],
                              [0.0, cam[i]['fy'], cam[i]['cv']],
                              [0.0, 0.0, 1.0]])
        cam[i]['f'] = (cam[i]['fx'] + cam[i]['fy']) / 2

    return cam

def GetQ50RadarParams():
    params = { }
    params['R_from_r_to_l'] = euler_matrix(0, 0, -.015)[0:3,0:3]
    params['T_from_r_to_l'] = [3.17, 0.4, -1.64]

    return params

def GetQ50Params():
    params = { }
    params['lidar'] = GetQ50LidarParams();
    params['cam'] = GetQ50CameraParams();
    params['radar'] = GetQ50RadarParams();
    return params
<|endoftext|>


<commit_before>__version__ = "0.0.1a14"

default_app_config = "djes.apps.DJESConfig"
<commit_msg>Let's call this the first real version, huh? [ci skip]<commit_after>__version__ = "0.1.0"

default_app_config = "djes.apps.DJESConfig"
<|endoftext|>


<commit_before>#
# Copyright 2018 Analytics Zoo Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from pyspark.ml.param.shared import *
from pyspark.ml.wrapper import JavaModel, JavaEstimator, JavaTransformer
from bigdl.optim.optimizer import SGD
from bigdl.util.common import *

if sys.version >= '3':
    long = int
    unicode = str

class HasBatchSize(Params):
    """
    Mixin for param batchSize: batch size.
    """

    # a placeholder to make it appear in the generated doc
    batchSize = Param(Params._dummy(), "batchSize", "batchSize (>= 0).")

    def __init__(self):
        super(HasBatchSize, self).__init__()
        #: param for batch size.
        self.batchSize = Param(self, "batchSize", "batchSize")
        self._setDefault(batchSize=1)

    def setBatchSize(self, val):
        """
        Sets the value of :py:attr:`batchSize`.
        """
        self._paramMap[self.batchSize] = val
        return self

    def getBatchSize(self):
        """
        Gets the value of batchSize or its default value.
        """
        return self.getOrDefault(self.batchSize)

class HasOptimMethod:

    optimMethod = SGD()

    def __init__(self):
        super(HasOptimMethod, self).__init__()

    def setOptimMethod(self, val):
        """
        Sets optimization method. E.g. SGD, Adam, LBFGS etc. from bigdl.optim.optimizer.
        default: SGD()
        """
        pythonBigDL_method_name = "setOptimMethod"
        callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, val)
        self.optimMethod = val
        return self

    def getOptimMethod(self):
        """
        Gets the optimization method
        """
        return self.optimMethod

class NNEstimator(JavaEstimator, HasFeaturesCol, HasLabelCol, HasPredictionCol, HasBatchSize, HasOptimMethod, JavaValue):
    """
    NNEstimator provides DataFrame-based API that allows users to train a BigDL
    Model with the Spark ML Estimator/Transfomer pattern, thus Spark users can conveniently fit
    BigDL into Spark ML pipeline.

    NNEstimator supports feature and label data in the format of Array[Double], Array[Float],
    org.apache.spark.mllib.linalg.{Vector, VectorUDT}, org.apache.spark.ml.linalg.{Vector, VectorUDT},
    Double, Float and ImageSchema (refer to NNImageReader).
    """

    def __init__(self,  model, criterion, feature_transformer, label_transformer, jvalue=None, bigdl_type="float"):
        """
        create a NNEstimator with BigDL Model.
        :param model: BigDL Model to be trained.
        :param criterion: BigDL criterion.
        :param feature_size: The size (Tensor dimensions) of the feature data. e.g. an image may be with
                             width * height = 28 * 28, featureSize = Array(28, 28).
        :param label_size: The size (Tensor dimensions) of the feature data.
        :param jvalue: Java object create by Py4j
        :param bigdl_type: optional parameter. data type of model, "float"(default) or "double".
        """
        super(NNEstimator, self).__init__()
        self.value = jvalue if jvalue else callBigDlFunc(
            bigdl_type, self.jvm_class_constructor(), model, criterion, feature_transformer, label_transformer)
        self.bigdl_type = bigdl_type
        self._java_obj = self.value
        self.maxEpoch = Param(self, "maxEpoch", "number of max Epoch")
        self.learningRate = Param(self, "learningRate", "learning rate")
        self._setDefault(maxEpoch=50, learningRate=1e-3, batchSize=1)

    def setMaxEpoch(self, val):
        """
        Sets the value of :py:attr:`maxEpoch`.
        """
        self._paramMap[self.maxEpoch] = val
        return self

    def getMaxEpoch(self):
        """
        Gets the value of maxEpoch or its default value.
        """
        return self.getOrDefault(self.maxEpoch)

    def setLearningRate(self, val):
        """
        Sets the value of :py:attr:`learningRate`.
        """
        self._paramMap[self.learningRate] = val
        return self

    def getLearningRate(self):
        """
        Gets the value of learningRate or its default value.
        """
        return self.getOrDefault(self.learningRate)

    def _create_model(self, java_model):
        nnModel = NNModel.of(java_model, self.featureSize, self.bigdl_type)
        nnModel.setFeaturesCol(self.getFeaturesCol()) \
            .setPredictionCol(self.getPredictionCol()) \
            .setBatchSize(self.getBatchSize())
        return nnModel

class NNModel(JavaTransformer, HasFeaturesCol, HasPredictionCol, HasBatchSize, JavaValue):
    """
    NNModel helps embed a BigDL model into a Spark Transformer, thus Spark users can
    conveniently merge BigDL into Spark ML pipeline. NNModel supports feature data in
    the format of Array[Double], Array[Float], org.apache.spark.mllib.linalg.{Vector, VectorUDT},
    org.apache.spark.ml.linalg.{Vector, VectorUDT}, Double, Float and image schema.
    Internally NNModel use features column as storage of the feature data, and create
    Tensors according to the constructor parameter featureSize.
    """
    def __init__(self,  model, feature_transformer, jvalue=None, bigdl_type="float"):
        """
        create a NNModel with a BigDL model
        :param model: trained BigDL model to use in prediction.
        :param featureSize: The size (Tensor dimensions) of the feature data. (e.g. an
                            image may be with featureSize = 28 * 28).
        :param jvalue: Java object create by Py4j
        :param bigdl_type: optional parameter. data type of model, "float"(default) or "double".
        """
        super(NNModel, self).__init__()
        self.value = jvalue if jvalue else callBigDlFunc(
            bigdl_type, self.jvm_class_constructor(), model, feature_transformer)
        self._java_obj = self.value
        self.bigdl_type = bigdl_type

    @classmethod
    def of(self, jvalue, feature_transformer=None, bigdl_type="float"):
        model = NNModel(model=None, feature_transformer=feature_transformer, jvalue=jvalue, bigdl_type=bigdl_type)
        return model

class NNClassifier(NNEstimator):
    """
    NNClassifier is a specialized NNEstimator that simplifies the data format for
    classification tasks. It only supports label column of DoubleType, and the fitted
    NNClassifierModel will have the prediction column of DoubleType.
    """
    def __init__(self,  model, criterion, feature_size,  bigdl_type="float"):
        """
        :param model: BigDL module to be optimized
        :param criterion: BigDL criterion method
        :param feature_size: The size (Tensor dimensions) of the feature data. (e.g. an
                             image may be with featureSize = 28 * 28).
        :param bigdl_type(optional): Data type of BigDL model, "float"(default) or "double".
        """
        super(NNClassifier, self).__init__(model, criterion, feature_size, [1], None, bigdl_type)

class NNClassifierModel(NNModel):
    """
    NNClassifierModel is a specialized [[NNModel]] for classification tasks. The prediction
    column will have the datatype of Double.
    """
    def __init__(self,  model, featureSize, jvalue=None, bigdl_type="float"):
        """
        :param model: trained BigDL model to use in prediction.
        :param featureSize: The size (Tensor dimensions) of the feature data. (e.g. an
                            image may be with featureSize = 28 * 28).
        :param jvalue: Java object create by Py4j
        :param bigdl_type(optional): Data type of BigDL model, "float"(default) or "double".
        """
        super(NNClassifierModel, self).__init__(model, featureSize, jvalue, bigdl_type)

    @classmethod
    def of(self, jvalue, feature_size=None, bigdl_type="float"):
        model = NNClassifierModel(model=None, featureSize=feature_size, jvalue=jvalue, bigdl_type=bigdl_type)
        return model
<commit_msg>python support and image support<commit_after>#
# Copyright 2018 Analytics Zoo Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from pyspark.ml.param.shared import *
from pyspark.ml.wrapper import JavaModel, JavaEstimator, JavaTransformer
from bigdl.optim.optimizer import SGD
from bigdl.util.common import *
from zoo.pipeline.nnframes.nn_transformers import *

if sys.version >= '3':
    long = int
    unicode = str

class HasBatchSize(Params):
    """
    Mixin for param batchSize: batch size.
    """

    # a placeholder to make it appear in the generated doc
    batchSize = Param(Params._dummy(), "batchSize", "batchSize (>= 0).")

    def __init__(self):
        super(HasBatchSize, self).__init__()
        #: param for batch size.
        self.batchSize = Param(self, "batchSize", "batchSize")
        self._setDefault(batchSize=1)

    def setBatchSize(self, val):
        """
        Sets the value of :py:attr:`batchSize`.
        """
        self._paramMap[self.batchSize] = val
        return self

    def getBatchSize(self):
        """
        Gets the value of batchSize or its default value.
        """
        return self.getOrDefault(self.batchSize)

class HasOptimMethod:

    optimMethod = SGD()

    def __init__(self):
        super(HasOptimMethod, self).__init__()

    def setOptimMethod(self, val):
        """
        Sets optimization method. E.g. SGD, Adam, LBFGS etc. from bigdl.optim.optimizer.
        default: SGD()
        """
        pythonBigDL_method_name = "setOptimMethod"
        callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, val)
        self.optimMethod = val
        return self

    def getOptimMethod(self):
        """
        Gets the optimization method
        """
        return self.optimMethod

class NNEstimator(JavaEstimator, HasFeaturesCol, HasLabelCol, HasPredictionCol, HasBatchSize, HasOptimMethod, JavaValue):
    """
    NNEstimator provides DataFrame-based API that allows users to train a BigDL
    Model with the Spark ML Estimator/Transfomer pattern, thus Spark users can conveniently fit
    BigDL into Spark ML pipeline.

    NNEstimator supports feature and label data in the format of Array[Double], Array[Float],
    org.apache.spark.mllib.linalg.{Vector, VectorUDT}, org.apache.spark.ml.linalg.{Vector, VectorUDT},
    Double, Float and ImageSchema (refer to NNImageReader).
    """

    def __init__(self,  model, criterion, feature_transformer, label_transformer, jvalue=None, bigdl_type="float"):
        """
        create a NNEstimator with BigDL Model.
        :param model: BigDL Model to be trained.
        :param criterion: BigDL criterion.
        :param feature_size: The size (Tensor dimensions) of the feature data. e.g. an image may be with
                             width * height = 28 * 28, featureSize = Array(28, 28).
        :param label_size: The size (Tensor dimensions) of the feature data.
        :param jvalue: Java object create by Py4j
        :param bigdl_type: optional parameter. data type of model, "float"(default) or "double".
        """
        super(NNEstimator, self).__init__()
        self.value = jvalue if jvalue else callBigDlFunc(
            bigdl_type, self.jvm_class_constructor(), model, criterion, feature_transformer, label_transformer)
        self.bigdl_type = bigdl_type
        self._java_obj = self.value
        self.maxEpoch = Param(self, "maxEpoch", "number of max Epoch")
        self.learningRate = Param(self, "learningRate", "learning rate")
        self._setDefault(maxEpoch=50, learningRate=1e-3, batchSize=1)
        self.feature_transformer = feature_transformer

    def setMaxEpoch(self, val):
        """
        Sets the value of :py:attr:`maxEpoch`.
        """
        self._paramMap[self.maxEpoch] = val
        return self

    def getMaxEpoch(self):
        """
        Gets the value of maxEpoch or its default value.
        """
        return self.getOrDefault(self.maxEpoch)

    def setLearningRate(self, val):
        """
        Sets the value of :py:attr:`learningRate`.
        """
        self._paramMap[self.learningRate] = val
        return self

    def getLearningRate(self):
        """
        Gets the value of learningRate or its default value.
        """
        return self.getOrDefault(self.learningRate)

    def _create_model(self, java_model):
        nnModel = NNModel.of(java_model, self.feature_transformer, self.bigdl_type)
        nnModel.setFeaturesCol(self.getFeaturesCol()) \
            .setPredictionCol(self.getPredictionCol()) \
            .setBatchSize(self.getBatchSize())
        return nnModel

class NNModel(JavaTransformer, HasFeaturesCol, HasPredictionCol, HasBatchSize, JavaValue):
    """
    NNModel helps embed a BigDL model into a Spark Transformer, thus Spark users can
    conveniently merge BigDL into Spark ML pipeline. NNModel supports feature data in
    the format of Array[Double], Array[Float], org.apache.spark.mllib.linalg.{Vector, VectorUDT},
    org.apache.spark.ml.linalg.{Vector, VectorUDT}, Double, Float and image schema.
    Internally NNModel use features column as storage of the feature data, and create
    Tensors according to the constructor parameter featureSize.
    """
    def __init__(self,  model, feature_transformer, jvalue=None, bigdl_type="float"):
        """
        create a NNModel with a BigDL model
        :param model: trained BigDL model to use in prediction.
        :param jvalue: Java object create by Py4j
        :param bigdl_type: optional parameter. data type of model, "float"(default) or "double".
        """
        super(NNModel, self).__init__()
        self.value = jvalue if jvalue else callBigDlFunc(
            bigdl_type, self.jvm_class_constructor(), model, feature_transformer)
        self._java_obj = self.value
        self.bigdl_type = bigdl_type

    @classmethod
    def of(self, jvalue, feature_transformer=None, bigdl_type="float"):
        model = NNModel(model=None, feature_transformer=feature_transformer, jvalue=jvalue, bigdl_type=bigdl_type)
        return model

class NNClassifier(NNEstimator):
    """
    NNClassifier is a specialized NNEstimator that simplifies the data format for
    classification tasks. It only supports label column of DoubleType, and the fitted
    NNClassifierModel will have the prediction column of DoubleType.
    """
    def __init__(self,  model, criterion, feature_transformer, bigdl_type="float"):
        """
        :param model: BigDL module to be optimized
        :param criterion: BigDL criterion method
        :param feature_size: The size (Tensor dimensions) of the feature data. (e.g. an
                             image may be with featureSize = 28 * 28).
        :param bigdl_type(optional): Data type of BigDL model, "float"(default) or "double".
        """
        super(NNClassifier, self).__init__(model, criterion, feature_transformer, NumToTensor(), None, bigdl_type)

class NNClassifierModel(NNModel):
    """
    NNClassifierModel is a specialized [[NNModel]] for classification tasks. The prediction
    column will have the datatype of Double.
    """
    def __init__(self,  model, feature_transformer, jvalue=None, bigdl_type="float"):
        """
        :param model: trained BigDL model to use in prediction.
        :param featureSize: The size (Tensor dimensions) of the feature data. (e.g. an
                            image may be with featureSize = 28 * 28).
        :param jvalue: Java object create by Py4j
        :param bigdl_type(optional): Data type of BigDL model, "float"(default) or "double".
        """
        super(NNClassifierModel, self).__init__(model, feature_transformer, jvalue, bigdl_type)

    @classmethod
    def of(self, jvalue, feaTran=None, bigdl_type="float"):
        model = NNClassifierModel(model=None, feature_transformer=feaTran, jvalue=jvalue, bigdl_type=bigdl_type)
        return model
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-

# Dissemin: open access policy enforcement tool
# Copyright (C) 2014 Antonin Delpeuch
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
#

"""
Django settings for dissemin project.

See the doc for details of usage:
http://dissemin.readthedocs.org/en/latest/install.html

For the full list of Django settings and their values, see
https://docs.djangoproject.com/en/1.8/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
from datetime import timedelta

try:
    from .secret import SECRET_KEY, DATABASES, EMAIL_HOST, EMAIL_HOST_USER, EMAIL_HOST_PASSWORD, EMAIL_USE_TLS, ROMEO_API_KEY, CORE_API_KEY
except ImportError as e:
    raise RuntimeError('Secret file is missing, did you forget to add a secret.py in your settings folder?')

try:
    from .university import UNIVERSITY_BRANDING, CAS_SERVER_URL, CAS_LOGOUT_COMPLETELY, CAS_PROVIDE_URL_TO_LOGOUT, ENABLE_CAS
except ImportError as e:
    raise RuntimeError('University-specific file is missing, did you forget to add a university.py in your settings folder?')

# dirname(__file__) → repo/dissemin/settings/common.py
# .. → repo/dissemin/settings
# .. → repo/dissemin
# .. → repo/

BASE_DIR = os.path.dirname(os.path.join(os.path.dirname(__file__), '..', '..', '..'))

### DOI proxy ###
# The interface where to get DOI metadata from.
#
# This interface should at least support fetching metadata for one
# single DOI, like this:
# curl -LH "Accept: application/citeproc+json" http://DOI_PROXY_DOMAIN/10.1080/15568318.2012.660115
# (returns the citation as Citeproc+JSON)
#
DOI_PROXY_DOMAIN =  'doi-cache.dissem.in' # This acts as a caching proxy for dx.doi.org
#
# In addition, if the endpoint supports it, batch requests can be performed:
# curl -d 'dois=["10.1016/j.physletb.2015.01.010","10.5380/dp.v1i1.1922","10.1007/978-3-319-10936-7_9"]' \\
#        http://doi-cache.ulminfo.fr/batch
# (returns a list of citation in Citeproc+JSON format)
#
DOI_PROXY_SUPPORTS_BATCH = True

### RoMEO proxy ###
# Set this to 'sherpa.ac.uk' if our custom mirror is not up anymore.
# Otherwise our proxy caches results and is more reliable than the original endpoint.
ROMEO_API_DOMAIN = 'romeo-cache.dissem.in'

### Paper deposits ###
# Max size of the PDFs (in bytes)
# 2.5MB - 2621440
# 5MB - 5242880
# 10MB - 10485760
# 20MB - 20971520
# 50MB - 5242880
DEPOSIT_MAX_FILE_SIZE = 1024*1024*20 # 20 MB
# Max download time when the file is downloaded from an URL (in seconds)
URL_DEPOSIT_DOWNLOAD_TIMEOUT = 10

# Uncomment these settings if you rather want
# to fetch metadata directly from CrossRef (slower as not cached,
# and more requests as there is no batch support).
#DOI_PROXY_DOMAIN =  'dx.doi.org'
#DOI_PROXY_SUPPORTS_BATCH = False

### Paper freshness options ###
# On login of an user, minimum time between the last harvest to trigger
# a new harvest for that user.
PROFILE_REFRESH_ON_LOGIN = timedelta(days=1)

### Application definition ###
# You should not have to change anything in this section.

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.sites',
    'crispy_forms',
    'allauth',
    'allauth.account',
    'allauth.socialaccount',
    'allauth.socialaccount.providers.orcid',
    'statistics',
    'publishers',
    'papers',
    'upload',
    'deposit',
    'deposit.zenodo',
    'deposit.hal',
    'deposit.sword',
    'bootstrap_pagination',
    'solo',
    'rest_framework',
    'rest_framework_swagger',
)

CRISPY_TEMPLATE_PACK = 'bootstrap'

SITE_ID = 1

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.locale.LocaleMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

AUTHENTICATION_BACKENDS = (
    'django.contrib.auth.backends.ModelBackend',
    'allauth.account.auth_backends.AuthenticationBackend',
)

if ENABLE_CAS:
    MIDDLEWARE_CLASSES += (
        'django_cas_ng.middleware.CASMiddleware',
    )

    AUTHENTICATION_BACKENDS += (
        'django_cas_ng.backends.CASBackend',
    )

TEMPLATES = [
        {
            'BACKEND': 'django.template.backends.django.DjangoTemplates',
            'DIRS': [
                os.path.join(BASE_DIR, 'templates')
            ],
            'OPTIONS': {
                'loaders': (
                    ('django.template.loaders.cached.Loader', (
                        'django.template.loaders.filesystem.Loader',
                        'django.template.loaders.app_directories.Loader',
                    )),
                ),
                'context_processors': (
                    "django.contrib.auth.context_processors.auth",
                    "django.template.context_processors.debug",
                    "django.template.context_processors.i18n",
                    "django.template.context_processors.media",
                    "django.template.context_processors.static",
                    "django.template.context_processors.tz",
                    "django.template.context_processors.request"
                ),
                'debug': True
            }
        }
]

ROOT_URLCONF = 'dissemin.urls'

WSGI_APPLICATION = 'dissemin.wsgi.application'

### Static files (CSS, JavaScript, Images) ###
# This defines how static files are stored and accessed.
# https://docs.djangoproject.com/en/1.8/howto/static-files/
#
# Relative URL where static files are accessed (you don't have to change this).
STATIC_URL = '/static/'
# Relative path to the directory where we store user uploads (you don't have to change this).
MEDIA_URL = '/media/'

### Celery config ###
# Celery runs asynchronous tasks such as metadata harvesting or
# complex updates.
# To communicate with it, we need a "broker".
# This is an example broker with Redis.
BROKER_URL = 'redis://localhost:6379/0'
# We also use Redis as result backend.
CELERY_RESULT_BACKEND = BROKER_URL

CELERY_ACCEPT_CONTENT = ['pickle', 'json', 'msgpack', 'yaml']
CELERY_IMPORTS = ['backend.tasks']

CELERYBEAT_SCHEDULE = {
    'update_all_stats_but_researchers': {
        'task': 'update_all_stats_but_researchers',
        'schedule': timedelta(minutes=30),
    },
    'update_journal_stats': {
        'task':'update_journal_stats',
        'schedule': timedelta(days=1),
    },
    'remove_empty_profiles': {
        'task': 'remove_empty_profiles',
        'schedule': timedelta(hours=2),
    },
}

# This is the time in seconds before an unacknowledged task is re-sent to
# another worker. It should exceed the length of the longest task, otherwise
# it will be executed twice ! 43200 is one day.
BROKER_TRANSPORT_OPTIONS = {'visibility_timeout': 43200}

# Internationalization
# https://docs.djangoproject.com/en/1.8/topics/i18n/
LANGUAGE_CODE = 'en-us'
POSSIBLE_LANGUAGE_CODES = ['en','fr']
TIME_ZONE = 'Europe/Paris'
USE_I18N = True
USE_L10N = True
USE_TZ = True
LOCALE_PATHS = ('locale',)

# Login and athentication
LOGIN_URL = '/accounts/login'
LOGIN_REDIRECT_URL = '/'

# Settings for our own API
REST_FRAMEWORK = {
    'DEFAULT_PERMISSION_CLASSES': ('rest_framework.permissions.IsAdminUser',),
    'PAGE_SIZE': 10,
    'DEFAULT_RENDERER_CLASSES': (
                'rest_framework.renderers.JSONRenderer',
                'rest_framework.renderers.BrowsableAPIRenderer',
            ),
}
<commit_msg>dissemin(settings): add redis client connection<commit_after># -*- encoding: utf-8 -*-

# Dissemin: open access policy enforcement tool
# Copyright (C) 2014 Antonin Delpeuch
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
#

"""
Django settings for dissemin project.

See the doc for details of usage:
http://dissemin.readthedocs.org/en/latest/install.html

For the full list of Django settings and their values, see
https://docs.djangoproject.com/en/1.8/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
from datetime import timedelta

try:
    from .secret import SECRET_KEY, DATABASES, EMAIL_HOST, EMAIL_HOST_USER, EMAIL_HOST_PASSWORD, EMAIL_USE_TLS, ROMEO_API_KEY, CORE_API_KEY
except ImportError as e:
    raise RuntimeError('Secret file is missing, did you forget to add a secret.py in your settings folder?')

try:
    from .university import UNIVERSITY_BRANDING, CAS_SERVER_URL, CAS_LOGOUT_COMPLETELY, CAS_PROVIDE_URL_TO_LOGOUT, ENABLE_CAS
except ImportError as e:
    raise RuntimeError('University-specific file is missing, did you forget to add a university.py in your settings folder?')

# dirname(__file__) → repo/dissemin/settings/common.py
# .. → repo/dissemin/settings
# .. → repo/dissemin
# .. → repo/

BASE_DIR = os.path.dirname(os.path.join(os.path.dirname(__file__), '..', '..', '..'))

### DOI proxy ###
# The interface where to get DOI metadata from.
#
# This interface should at least support fetching metadata for one
# single DOI, like this:
# curl -LH "Accept: application/citeproc+json" http://DOI_PROXY_DOMAIN/10.1080/15568318.2012.660115
# (returns the citation as Citeproc+JSON)
#
DOI_PROXY_DOMAIN =  'doi-cache.dissem.in' # This acts as a caching proxy for dx.doi.org
#
# In addition, if the endpoint supports it, batch requests can be performed:
# curl -d 'dois=["10.1016/j.physletb.2015.01.010","10.5380/dp.v1i1.1922","10.1007/978-3-319-10936-7_9"]' \\
#        http://doi-cache.ulminfo.fr/batch
# (returns a list of citation in Citeproc+JSON format)
#
DOI_PROXY_SUPPORTS_BATCH = True

### RoMEO proxy ###
# Set this to 'sherpa.ac.uk' if our custom mirror is not up anymore.
# Otherwise our proxy caches results and is more reliable than the original endpoint.
ROMEO_API_DOMAIN = 'romeo-cache.dissem.in'

### Paper deposits ###
# Max size of the PDFs (in bytes)
# 2.5MB - 2621440
# 5MB - 5242880
# 10MB - 10485760
# 20MB - 20971520
# 50MB - 5242880
DEPOSIT_MAX_FILE_SIZE = 1024*1024*20 # 20 MB
# Max download time when the file is downloaded from an URL (in seconds)
URL_DEPOSIT_DOWNLOAD_TIMEOUT = 10

# Uncomment these settings if you rather want
# to fetch metadata directly from CrossRef (slower as not cached,
# and more requests as there is no batch support).
#DOI_PROXY_DOMAIN =  'dx.doi.org'
#DOI_PROXY_SUPPORTS_BATCH = False

### Paper freshness options ###
# On login of an user, minimum time between the last harvest to trigger
# a new harvest for that user.
PROFILE_REFRESH_ON_LOGIN = timedelta(days=1)

### Application definition ###
# You should not have to change anything in this section.

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.sites',
    'crispy_forms',
    'allauth',
    'allauth.account',
    'allauth.socialaccount',
    'allauth.socialaccount.providers.orcid',
    'statistics',
    'publishers',
    'papers',
    'upload',
    'deposit',
    'deposit.zenodo',
    'deposit.hal',
    'deposit.sword',
    'bootstrap_pagination',
    'solo',
    'rest_framework',
    'rest_framework_swagger',
)

CRISPY_TEMPLATE_PACK = 'bootstrap'

SITE_ID = 1

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.locale.LocaleMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

AUTHENTICATION_BACKENDS = (
    'django.contrib.auth.backends.ModelBackend',
    'allauth.account.auth_backends.AuthenticationBackend',
)

if ENABLE_CAS:
    MIDDLEWARE_CLASSES += (
        'django_cas_ng.middleware.CASMiddleware',
    )

    AUTHENTICATION_BACKENDS += (
        'django_cas_ng.backends.CASBackend',
    )

TEMPLATES = [
        {
            'BACKEND': 'django.template.backends.django.DjangoTemplates',
            'DIRS': [
                os.path.join(BASE_DIR, 'templates')
            ],
            'OPTIONS': {
                'loaders': (
                    ('django.template.loaders.cached.Loader', (
                        'django.template.loaders.filesystem.Loader',
                        'django.template.loaders.app_directories.Loader',
                    )),
                ),
                'context_processors': (
                    "django.contrib.auth.context_processors.auth",
                    "django.template.context_processors.debug",
                    "django.template.context_processors.i18n",
                    "django.template.context_processors.media",
                    "django.template.context_processors.static",
                    "django.template.context_processors.tz",
                    "django.template.context_processors.request"
                ),
                'debug': True
            }
        }
]

ROOT_URLCONF = 'dissemin.urls'

WSGI_APPLICATION = 'dissemin.wsgi.application'

### Static files (CSS, JavaScript, Images) ###
# This defines how static files are stored and accessed.
# https://docs.djangoproject.com/en/1.8/howto/static-files/
#
# Relative URL where static files are accessed (you don't have to change this).
STATIC_URL = '/static/'
# Relative path to the directory where we store user uploads (you don't have to change this).
MEDIA_URL = '/media/'

### Celery config ###
# Celery runs asynchronous tasks such as metadata harvesting or
# complex updates.
# To communicate with it, we need a "broker".
# This is an example broker with Redis.
BROKER_URL = 'redis://localhost:6379/0'
# We also use Redis as result backend.
CELERY_RESULT_BACKEND = BROKER_URL

# Redis is not mandatory, this client is reserved for deposits.
try:
    import redis
    redis_client = redis.StrictRedis(host='localhost', port=6379, db=0)
except ImportError:
    pass

CELERY_ACCEPT_CONTENT = ['pickle', 'json', 'msgpack', 'yaml']
CELERY_IMPORTS = ['backend.tasks']

CELERYBEAT_SCHEDULE = {
    'update_all_stats_but_researchers': {
        'task': 'update_all_stats_but_researchers',
        'schedule': timedelta(minutes=30),
    },
    'update_journal_stats': {
        'task':'update_journal_stats',
        'schedule': timedelta(days=1),
    },
    'remove_empty_profiles': {
        'task': 'remove_empty_profiles',
        'schedule': timedelta(hours=2),
    },
}

# This is the time in seconds before an unacknowledged task is re-sent to
# another worker. It should exceed the length of the longest task, otherwise
# it will be executed twice ! 43200 is one day.
BROKER_TRANSPORT_OPTIONS = {'visibility_timeout': 43200}

# Internationalization
# https://docs.djangoproject.com/en/1.8/topics/i18n/
LANGUAGE_CODE = 'en-us'
POSSIBLE_LANGUAGE_CODES = ['en','fr']
TIME_ZONE = 'Europe/Paris'
USE_I18N = True
USE_L10N = True
USE_TZ = True
LOCALE_PATHS = ('locale',)

# Login and athentication
LOGIN_URL = '/accounts/login'
LOGIN_REDIRECT_URL = '/'

# Settings for our own API
REST_FRAMEWORK = {
    'DEFAULT_PERMISSION_CLASSES': ('rest_framework.permissions.IsAdminUser',),
    'PAGE_SIZE': 10,
    'DEFAULT_RENDERER_CLASSES': (
                'rest_framework.renderers.JSONRenderer',
                'rest_framework.renderers.BrowsableAPIRenderer',
            ),
}
<|endoftext|>